wehub-resource-sync
f36e2104d8
sep-tests / ragflow_preflight (push) Waiting to run
sep-tests / ragflow_tests_infinity (push) Blocked by required conditions
sep-tests / ragflow_tests_elasticsearch (push) Blocked by required conditions
tests / ragflow_preflight (push) Waiting to run
tests / ragflow_tests_infinity (push) Blocked by required conditions
tests / ragflow_tests_elasticsearch (push) Blocked by required conditions
183 lines
9.7 KiB
Python
183 lines
9.7 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# PEP 723 metadata
|
|
# /// script
|
|
# requires-python = ">=3.10"
|
|
# dependencies = [
|
|
# "nltk",
|
|
# "huggingface-hub"
|
|
# ]
|
|
# ///
|
|
|
|
# This script downloads every artifact that the `infiniflow/ragflow_deps`
|
|
# Docker image bakes in. Run it from anywhere — the `__main__` block
|
|
# chdir's into this file's own directory, so all outputs land under
|
|
# `ragflow_deps/` regardless of the caller's CWD.
|
|
#
|
|
# Build-context relationship: `ragflow_deps/Dockerfile` is built with
|
|
# `ragflow_deps/` as its build context, so the files written here MUST
|
|
# sit at the top of `ragflow_deps/`. The Dockerfile's COPY lines assume
|
|
# top-level paths (`huggingface.co`, `nltk_data`, `cl100k_base.tiktoken`,
|
|
# `*.deb`, `*.jar`, `*.tar.gz`, `stagehand-server-v3-linux-<arch>`).
|
|
#
|
|
# Typical workflow:
|
|
#
|
|
# uv run python3 ragflow_deps/download_deps.py # download
|
|
# cd ragflow_deps
|
|
# docker build -f Dockerfile -t infiniflow/ragflow_deps .
|
|
#
|
|
# The main `Dockerfile` (built from the project root) pulls this image
|
|
# via `--mount=type=bind,from=infiniflow/ragflow_deps:latest,...` and
|
|
# is unaffected by where these files live locally.
|
|
|
|
import argparse
|
|
import os
|
|
import urllib.request
|
|
from typing import Union
|
|
|
|
import nltk
|
|
from huggingface_hub import snapshot_download
|
|
|
|
|
|
def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]:
|
|
if use_china_mirrors:
|
|
return [
|
|
"http://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb",
|
|
"http://mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb",
|
|
"https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar",
|
|
"https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar.md5",
|
|
"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
|
|
["https://registry.npmmirror.com/-/binary/chrome-for-testing/121.0.6167.85/linux64/chrome-linux64.zip", "chrome-linux64-121-0-6167-85"],
|
|
["https://registry.npmmirror.com/-/binary/chrome-for-testing/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"],
|
|
"https://github.com/astral-sh/uv/releases/download/0.9.16/uv-x86_64-unknown-linux-gnu.tar.gz",
|
|
"https://github.com/astral-sh/uv/releases/download/0.9.16/uv-aarch64-unknown-linux-gnu.tar.gz",
|
|
# stagehand-server-v3 Node.js SEA binaries (used by Browser
|
|
# component in local mode).
|
|
#
|
|
# The stagehand-go Go module (pinned in go.mod) and the
|
|
# stagehand-server binary (this release) are LOOSELY
|
|
# MATCHED — both stay on the v3.x line and remain
|
|
# protocol-compatible. The two version numbers do NOT
|
|
# track each other: the Go SDK is at v3.21.0 while the
|
|
# current latest server release is v3.7.2.
|
|
#
|
|
# On every go.mod bump, refresh this URL to the current
|
|
# latest server release. There is no version
|
|
# correspondence to maintain; "both on v3.x" is the
|
|
# compatibility contract.
|
|
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64",
|
|
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64",
|
|
# Native static libraries for Go build (pdfium, pdf_oxide, office_oxide)
|
|
# Used by build.sh's check_*_deps functions — pre-downloaded to avoid
|
|
# network access during CI.
|
|
["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
|
|
["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.67/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
|
|
["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.2/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
|
|
]
|
|
else:
|
|
return [
|
|
"http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb",
|
|
"http://ports.ubuntu.com/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb",
|
|
"https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar",
|
|
"https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar.md5",
|
|
"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
|
|
["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chrome-linux64.zip", "chrome-linux64-121-0-6167-85"],
|
|
["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"],
|
|
"https://github.com/astral-sh/uv/releases/download/0.9.16/uv-x86_64-unknown-linux-gnu.tar.gz",
|
|
"https://github.com/astral-sh/uv/releases/download/0.9.16/uv-aarch64-unknown-linux-gnu.tar.gz",
|
|
# stagehand-server-v3 Node.js SEA binaries (used by Browser
|
|
# component in local mode).
|
|
#
|
|
# The stagehand-go Go module (pinned in go.mod) and the
|
|
# stagehand-server binary (this release) are LOOSELY
|
|
# MATCHED — both stay on the v3.x line and remain
|
|
# protocol-compatible. The two version numbers do NOT
|
|
# track each other: the Go SDK is at v3.21.0 while the
|
|
# current latest server release is v3.7.2.
|
|
#
|
|
# On every go.mod bump, refresh this URL to the current
|
|
# latest server release. There is no version
|
|
# correspondence to maintain; "both on v3.x" is the
|
|
# compatibility contract.
|
|
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64",
|
|
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64",
|
|
# Native static libraries for Go build (pdfium, pdf_oxide, office_oxide)
|
|
# Used by build.sh's check_*_deps functions — pre-downloaded to avoid
|
|
# network access during CI.
|
|
["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
|
|
["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.67/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
|
|
["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.2/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
|
|
]
|
|
|
|
|
|
repos = [
|
|
"InfiniFlow/text_concat_xgb_v1.0",
|
|
"InfiniFlow/deepdoc",
|
|
]
|
|
|
|
|
|
def download_model(repository_id):
|
|
local_directory = os.path.abspath(os.path.join("huggingface.co", repository_id))
|
|
os.makedirs(local_directory, exist_ok=True)
|
|
snapshot_download(repo_id=repository_id, local_dir=local_directory)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Anchor CWD to this file's directory so all relative outputs
|
|
# (huggingface.co/, nltk_data/, *.deb, *.jar, *.tar.gz, etc.) land
|
|
# at the top of ragflow_deps/ regardless of where the user invokes
|
|
# the script from. This is the build context for `ragflow_deps/Dockerfile`.
|
|
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
parser = argparse.ArgumentParser(description="Download dependencies with optional China mirror support")
|
|
parser.add_argument("--china-mirrors", action="store_true", help="Use China-accessible mirrors for downloads")
|
|
args = parser.parse_args()
|
|
|
|
urls = get_urls(args.china_mirrors)
|
|
|
|
# Some mirrors (e.g. archive.ubuntu.com) reject the default urllib
|
|
# User-Agent with HTTP 403, so install an opener with a browser-like UA.
|
|
opener = urllib.request.build_opener()
|
|
opener.addheaders = [("User-Agent", "Mozilla/5.0")]
|
|
urllib.request.install_opener(opener)
|
|
|
|
for url in urls:
|
|
download_url = url[0] if isinstance(url, list) else url
|
|
filename = url[1] if isinstance(url, list) else url.split("/")[-1]
|
|
print(f"Downloading {filename} from {download_url}...")
|
|
if not os.path.exists(filename):
|
|
urllib.request.urlretrieve(download_url, filename)
|
|
|
|
# Extract native static libraries to ~/ragflow-native-libs for Go build.
|
|
# Ensures build.sh can find them without network access.
|
|
native_deps_dir = os.path.expanduser("~/ragflow-native-libs")
|
|
extractions = [
|
|
("pdfium-linux-x64-static.tgz", "pdfium-static"),
|
|
("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"),
|
|
("office_oxide-linux-x86_64.tar.gz", "office_oxide"),
|
|
]
|
|
import tarfile
|
|
|
|
for archive, subdir in extractions:
|
|
archive_path = os.path.join(os.getcwd(), archive)
|
|
if not os.path.isfile(archive_path):
|
|
print(f" Skipping extraction: {archive} not found")
|
|
continue
|
|
target = os.path.join(native_deps_dir, subdir)
|
|
if os.path.isdir(target):
|
|
print(f" ✓ {subdir} already extracted to {target}")
|
|
continue
|
|
os.makedirs(target, exist_ok=True)
|
|
print(f" Extracting {archive} → {target}")
|
|
with tarfile.open(archive_path) as tf:
|
|
tf.extractall(target)
|
|
|
|
local_dir = os.path.abspath("nltk_data")
|
|
for data in ["wordnet", "punkt", "punkt_tab"]:
|
|
print(f"Downloading nltk {data}...")
|
|
nltk.download(data, download_dir=local_dir)
|
|
|
|
for repo_id in repos:
|
|
print(f"Downloading huggingface repo {repo_id}...")
|
|
download_model(repo_id)
|