wehub-resource-sync
95 lines
3.3 KiB
Docker
95 lines
3.3 KiB
Docker
# ML4T Benchmark Environment
|
|
# Storage benchmarks: Parquet, DuckDB, HDF5, database clients
|
|
#
|
|
# Usage:
|
|
# docker compose build benchmark
|
|
# docker compose run --rm benchmark python 02_financial_data_universe/18_storage_benchmark_database.py
|
|
#
|
|
# Note: ArcticDB excluded (no Linux ARM64 wheels on PyPI)
|
|
# Use benchmark-full profile on x86 systems for ArcticDB benchmarks
|
|
|
|
FROM python:3.14-slim
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
ENV TZ=UTC
|
|
|
|
# Install system dependencies.
|
|
# A modern Rust toolchain is required to build the questdb python client from
|
|
# sdist on linux/arm64 (PyPI ships no Linux arm64 wheel for it). Debian's
|
|
# packaged `cargo` is too old, so we install rustup so questdb-rs compiles
|
|
# successfully both on amd64 (where a wheel exists) and on arm64 (sdist build).
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
build-essential \
|
|
curl \
|
|
git \
|
|
libhdf5-dev \
|
|
pkg-config \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Install rustup so we have an up-to-date Rust toolchain for questdb-rs builds.
|
|
ENV CARGO_HOME=/opt/cargo \
|
|
RUSTUP_HOME=/opt/rustup \
|
|
PATH=/opt/cargo/bin:$PATH
|
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
|
|
sh -s -- -y --default-toolchain stable --profile minimal
|
|
|
|
# Install uv for fast package management
|
|
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
|
|
|
|
# Create non-root user
|
|
RUN useradd -m -s /bin/bash ml4t
|
|
|
|
# Create venv
|
|
RUN python -m venv /opt/ml4t
|
|
|
|
# Copy and install dependencies
|
|
WORKDIR /build
|
|
COPY envs/benchmark/pyproject.toml /build/
|
|
|
|
# Install all dependencies in one go. We deliberately do NOT have a fallback
|
|
# that strips questdb on failure — a successful build must include every
|
|
# database client declared in pyproject.toml so users never hit a runtime
|
|
# `ModuleNotFoundError`.
|
|
RUN /opt/ml4t/bin/pip install --no-cache-dir pip setuptools wheel && \
|
|
/opt/ml4t/bin/pip install --no-cache-dir /build && \
|
|
/opt/ml4t/bin/pip install --no-cache-dir "pytest>=8.0" "pytest-timeout>=2.3" "papermill>=2.6" "jupytext>=1.16"
|
|
|
|
WORKDIR /app
|
|
ENV PATH="/opt/ml4t/bin:$PATH"
|
|
ENV PYTHONPATH="/app"
|
|
|
|
# Jupyter configuration
|
|
ENV JUPYTER_ENABLE_LAB=yes
|
|
RUN mkdir -p /etc/jupyter && \
|
|
echo "c.ServerApp.token = ''" >> /etc/jupyter/jupyter_server_config.py && \
|
|
echo "c.ServerApp.password = ''" >> /etc/jupyter/jupyter_server_config.py && \
|
|
echo "c.ServerApp.allow_origin = '*'" >> /etc/jupyter/jupyter_server_config.py
|
|
|
|
# Platform detection script
|
|
RUN echo '#!/bin/bash\n\
|
|
ARCH=$(uname -m)\n\
|
|
echo "============================================"\n\
|
|
echo "ML4T Benchmark Environment"\n\
|
|
echo "============================================"\n\
|
|
echo "Platform: $(uname -s) $ARCH"\n\
|
|
echo ""\n\
|
|
if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then\n\
|
|
echo "Running on ARM64 (Apple Silicon compatible)"\n\
|
|
echo "ArcticDB: Not available (use benchmark-full on x86)"\n\
|
|
else\n\
|
|
echo "Running on x86_64"\n\
|
|
echo "ArcticDB: Use benchmark-full profile for full benchmarks"\n\
|
|
fi\n\
|
|
echo ""\n\
|
|
echo "Available benchmarks:"\n\
|
|
echo " - Parquet (PyArrow)"\n\
|
|
echo " - DuckDB"\n\
|
|
echo " - HDF5 (PyTables)"\n\
|
|
echo " - ClickHouse"\n\
|
|
echo " - TimescaleDB"\n\
|
|
echo " - QuestDB"\n\
|
|
echo " - InfluxDB"\n\
|
|
echo "============================================"' > /usr/local/bin/benchmark-status && chmod +x /usr/local/bin/benchmark-status
|
|
|
|
CMD ["jupyter", "lab", "--ip=0.0.0.0", "--no-browser"]
|