chore: import upstream snapshot with attribution

2026-07-13 12:12:13 +08:00
commit 0446c45d8e
898 changed files with 328024 additions and 0 deletions
@@ -0,0 +1,100 @@
+[build-system]
+requires = ["setuptools>=64.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "Crawl4AI"
+dynamic = ["version"]
+description = "🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+authors = [
+    {name = "Unclecode", email = "unclecode@kidocode.com"}
+]
+dependencies = [
+    "aiofiles>=24.1.0",
+    "aiohttp>=3.11.11",
+    "aiosqlite~=0.20",
+    "anyio>=4.0.0",
+    "lxml>=5.3,<7",
+    "unclecode-litellm==1.81.13",
+    "numpy>=1.26.0,<3",
+    "pillow>=10.4",
+    "playwright>=1.49.0",
+    "patchright>=1.49.0",
+    "python-dotenv~=1.0",
+    "requests~=2.26",
+    "beautifulsoup4~=4.12",
+    "playwright-stealth>=2.0.0",
+    "xxhash~=3.4",
+    "rank-bm25~=0.2",
+    "snowballstemmer~=2.2",
+    "pydantic>=2.10",
+    "pyOpenSSL>=25.3.0",
+    "psutil>=6.1.1",
+    "PyYAML>=6.0",
+    "nltk>=3.9.1",
+    "rich>=13.9.4",
+    "cssselect>=1.2.0",
+    "httpx>=0.27.2",
+    "httpx[http2]>=0.27.2",
+    "fake-useragent>=2.0.3",
+    "click>=8.1.7",
+    "chardet>=5.2.0",
+    "brotli>=1.1.0",
+    "humanize>=4.10.0",
+    "lark>=1.2.2",
+    "alphashape>=1.3.1",
+    "shapely>=2.0.0"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+
+[project.optional-dependencies]
+pdf = ["pypdf"]  
+torch = ["torch", "nltk", "scikit-learn"]
+transformer = ["transformers", "tokenizers", "sentence-transformers"]
+cosine = ["torch", "transformers", "nltk", "sentence-transformers"]
+sync = ["selenium"]
+all = [
+    "pypdf",
+    "torch",
+    "nltk",
+    "scikit-learn",
+    "transformers",
+    "tokenizers",
+    "sentence-transformers",
+    "selenium"
+]
+
+[project.scripts]
+crawl4ai-download-models = "crawl4ai.model_loader:main"
+crawl4ai-migrate = "crawl4ai.migrations:main"
+crawl4ai-setup = "crawl4ai.install:post_install"
+crawl4ai-doctor = "crawl4ai.install:doctor"
+crwl = "crawl4ai.cli:main"
+
+[tool.setuptools]
+packages = {find = {where = ["."], include = ["crawl4ai*"]}}
+
+[tool.setuptools.package-data]
+crawl4ai = ["js_snippet/*.js", "crawlers/google_search/*.js"]
+
+[tool.setuptools.dynamic]
+version = {attr = "crawl4ai.__version__.__version__"}
+
+[tool.uv.sources]
+crawl4ai = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "crawl4ai",
+]