wehub-resource-sync
677 lines
26 KiB
YAML
677 lines
26 KiB
YAML
# ML4T 3rd Edition — CI Test Workflow
|
|
#
|
|
# Tests notebooks inside pre-built Docker images (ml4t/ml4t:latest).
|
|
# This ensures CI tests exactly what readers will use.
|
|
#
|
|
# Architecture:
|
|
# - All notebook tests run inside Docker containers (not native Python)
|
|
# - Incremental: dorny/paths-filter only tests changed chapters
|
|
# - Pre-built images from Docker Hub (built by docker-publish.yml)
|
|
# - Special images for py312 (signatory/gensim/tfcausalimpact), Neo4j, benchmarks
|
|
#
|
|
# Trigger policy:
|
|
# - PR: incremental (path-filtered, only changed chapters)
|
|
# - Weekly: full suite on schedule (Monday 6 AM UTC)
|
|
# - After Docker Publish: full suite (workflow_run trigger)
|
|
# - Manual: workflow_dispatch with optional run_all flag
|
|
# - Push to main: NOT triggered (conserves minutes)
|
|
|
|
name: Tests
|
|
|
|
on:
|
|
pull_request:
|
|
branches: [main]
|
|
schedule:
|
|
- cron: '0 6 * * 1' # Weekly Monday 6 AM UTC
|
|
workflow_run:
|
|
workflows: ["Docker Publish"]
|
|
types: [completed]
|
|
workflow_dispatch:
|
|
inputs:
|
|
run_all:
|
|
description: "Run all tests (ignore path filters)"
|
|
type: boolean
|
|
default: false
|
|
test_filter:
|
|
description: "pytest -k filter (e.g. 'us_equities_panel or nasdaq100' to run only those)"
|
|
type: string
|
|
default: ""
|
|
|
|
env:
|
|
ML4T_DATA_PATH: ${{ github.workspace }}/test-data/data
|
|
ML4T_OUTPUT_DIR: /tmp/ml4t-test-output
|
|
MPLBACKEND: Agg
|
|
PLOTLY_RENDERER: json
|
|
|
|
jobs:
|
|
# ---------------------------------------------------------------------------
|
|
# Detect which paths changed and build dynamic matrices
|
|
# ---------------------------------------------------------------------------
|
|
changes:
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
pull-requests: read
|
|
outputs:
|
|
chapter_matrix: ${{ steps.build.outputs.chapter_matrix }}
|
|
case_matrix: ${{ steps.build.outputs.case_matrix }}
|
|
run_py312: ${{ steps.build.outputs.run_py312 }}
|
|
run_neo4j: ${{ steps.build.outputs.run_neo4j }}
|
|
run_benchmark: ${{ steps.build.outputs.run_benchmark }}
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- uses: dorny/paths-filter@v4
|
|
id: filter
|
|
with:
|
|
filters: |
|
|
shared:
|
|
- 'utils/**'
|
|
- 'data/**'
|
|
- 'tests/**'
|
|
- 'pyproject.toml'
|
|
- '.github/workflows/test.yml'
|
|
- 'matplotlibrc'
|
|
ch01:
|
|
- '01_process_is_edge/**'
|
|
ch02-03:
|
|
- '02_financial_data_universe/**'
|
|
- '03_market_microstructure/**'
|
|
- 'data/**'
|
|
ch04:
|
|
- '04_fundamental_alternative_data/**'
|
|
ch05:
|
|
- '05_synthetic_data/**'
|
|
ch06-07:
|
|
- '06_strategy_definition/**'
|
|
- '07_defining_the_learning_task/**'
|
|
ch08-09:
|
|
- '08_financial_features/**'
|
|
- '09_model_based_features/**'
|
|
ch10:
|
|
- '10_text_feature_engineering/**'
|
|
ch11-12:
|
|
- '11_ml_pipeline/**'
|
|
- '12_gradient_boosting/**'
|
|
ch13:
|
|
- '13_dl_time_series/**'
|
|
ch14-15:
|
|
- '14_latent_factors/**'
|
|
- '15_causal_estimation/**'
|
|
ch16-17:
|
|
- '16_strategy_simulation/**'
|
|
- '17_portfolio_construction/**'
|
|
ch18-20:
|
|
- '18_transaction_costs/**'
|
|
- '19_risk_management/**'
|
|
- '20_strategy_synthesis/**'
|
|
ch21:
|
|
- '21_rl_execution_hedging/**'
|
|
ch22:
|
|
- '22_rag_financial_research/**'
|
|
ch23:
|
|
- '23_knowledge_graphs/**'
|
|
ch24:
|
|
- '24_autonomous_agents/**'
|
|
ch25:
|
|
- '25_live_trading/**'
|
|
ch26:
|
|
- '26_mlops_governance/**'
|
|
cs-etfs:
|
|
- 'case_studies/etfs/**'
|
|
cs-crypto:
|
|
- 'case_studies/crypto_perps_funding/**'
|
|
cs-nasdaq:
|
|
- 'case_studies/nasdaq100_microstructure/**'
|
|
cs-sp500eoa:
|
|
- 'case_studies/sp500_equity_option_analytics/**'
|
|
cs-firmchar:
|
|
- 'case_studies/us_firm_characteristics/**'
|
|
cs-fx:
|
|
- 'case_studies/fx_pairs/**'
|
|
cs-futures:
|
|
- 'case_studies/cme_futures/**'
|
|
cs-sp500opt:
|
|
- 'case_studies/sp500_options/**'
|
|
cs-usequity:
|
|
- 'case_studies/us_equities_panel/**'
|
|
|
|
- name: Build dynamic matrices
|
|
id: build
|
|
run: |
|
|
shared="${{ steps.filter.outputs.shared }}"
|
|
schedule="${{ github.event_name == 'schedule' }}"
|
|
after_docker="${{ github.event_name == 'workflow_run' }}"
|
|
run_all="${{ github.event.inputs.run_all }}"
|
|
all="false"
|
|
if [[ "$shared" == "true" ]] || [[ "$schedule" == "true" ]] || [[ "$run_all" == "true" ]] || [[ "$after_docker" == "true" ]]; then
|
|
all="true"
|
|
fi
|
|
|
|
# --- Chapter matrix (all chapters including Ch01 and Ch05) ---
|
|
chapters="[]"
|
|
# Public-subset scoping: include a chapter only if at least one of its
|
|
# NN_* directories is actually present. The public repo ships chapters
|
|
# incrementally (beat by beat), so this auto-scopes the matrix to what
|
|
# is shipped — no per-beat workflow edits, and unshipped chapters can't
|
|
# fail with "no tests collected".
|
|
add_ch() {
|
|
local n="$1" f="$2" p
|
|
for p in $(echo "$f" | grep -oE '[0-9]{2}_' || true); do
|
|
if compgen -G "${p}*" >/dev/null 2>&1; then
|
|
chapters=$(echo "$chapters" | jq -c --arg n "$n" --arg f "$f" '. + [{"name":$n,"filter":$f}]')
|
|
return
|
|
fi
|
|
done
|
|
}
|
|
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch01 }}" == "true" ]]; then
|
|
add_ch "ch01" "01_process"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch02-03 }}" == "true" ]]; then
|
|
add_ch "ch02-03" "02_financial or 03_market"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch04 }}" == "true" ]]; then
|
|
add_ch "ch04" "04_fundamental"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch05 }}" == "true" ]]; then
|
|
add_ch "ch05" "05_synthetic"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch06-07 }}" == "true" ]]; then
|
|
add_ch "ch06-07" "06_strategy or 07_defining"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch08-09 }}" == "true" ]]; then
|
|
add_ch "ch08-09" "08_financial or 09_model"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch10 }}" == "true" ]]; then
|
|
add_ch "ch10" "10_text"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch11-12 }}" == "true" ]]; then
|
|
add_ch "ch11-12" "11_ml_pipeline or 12_gradient"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch13 }}" == "true" ]]; then
|
|
add_ch "ch13" "13_dl_time"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch14-15 }}" == "true" ]]; then
|
|
add_ch "ch14-15" "14_latent or 15_causal"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch16-17 }}" == "true" ]]; then
|
|
add_ch "ch16-17" "16_strategy or 17_portfolio"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch18-20 }}" == "true" ]]; then
|
|
add_ch "ch18-20" "18_transaction or 19_risk or 20_strategy"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch21 }}" == "true" ]]; then
|
|
add_ch "ch21" "21_rl"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch22 }}" == "true" ]]; then
|
|
add_ch "ch22" "22_rag"
|
|
fi
|
|
# Ch23 skipped — requires Neo4j service (tested in test-neo4j when ready)
|
|
# if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch23 }}" == "true" ]]; then
|
|
# add_ch "ch23" "23_knowledge"
|
|
# fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch24 }}" == "true" ]]; then
|
|
add_ch "ch24" "24_autonomous"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch25 }}" == "true" ]]; then
|
|
add_ch "ch25" "25_live"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch26 }}" == "true" ]]; then
|
|
add_ch "ch26" "26_mlops"
|
|
fi
|
|
|
|
echo "chapter_matrix=$chapters" >> "$GITHUB_OUTPUT"
|
|
|
|
# --- Py312 flag ---
|
|
run_py312="false"
|
|
if [[ "$all" == "true" ]] \
|
|
|| [[ "${{ steps.filter.outputs.ch05 }}" == "true" ]] \
|
|
|| [[ "${{ steps.filter.outputs.ch08-09 }}" == "true" ]] \
|
|
|| [[ "${{ steps.filter.outputs.ch10 }}" == "true" ]] \
|
|
|| [[ "${{ steps.filter.outputs.ch14-15 }}" == "true" ]] \
|
|
|| [[ "${{ steps.filter.outputs.ch21 }}" == "true" ]]; then
|
|
run_py312="true"
|
|
fi
|
|
echo "run_py312=$run_py312" >> "$GITHUB_OUTPUT"
|
|
|
|
# --- Neo4j flag (needs populated Neo4j — skip until infra is ready) ---
|
|
run_neo4j="false"
|
|
echo "run_neo4j=$run_neo4j" >> "$GITHUB_OUTPUT"
|
|
|
|
# --- Benchmark flag (disabled — QuestDB container fails health check in CI) ---
|
|
run_benchmark="false"
|
|
echo "run_benchmark=$run_benchmark" >> "$GITHUB_OUTPUT"
|
|
|
|
# --- Case study matrix ---
|
|
cases="[]"
|
|
# Public repo ships case-study CONFIG but gates the pipeline code.
|
|
# test_case_studies.py auto-discovers stages by globbing
|
|
# case_studies/<cs>/[0-9][0-9]*.py, and the results registry is seeded
|
|
# from the test-data repo at run time (NOT shipped to this repo). So
|
|
# gate on the same signal the test collects: enroll a case study only
|
|
# once its pipeline notebooks are shipped here — currently only
|
|
# config/setup.yaml ships, so case-study jobs stay skipped. Future
|
|
# beats that ship the notebooks auto-enroll, no workflow edits.
|
|
add_cs() {
|
|
if compgen -G "case_studies/$1/[0-9][0-9]*.py" >/dev/null 2>&1; then
|
|
cases=$(echo "$cases" | jq -c --arg cs "$1" '. + [{"case-study":$cs}]')
|
|
fi
|
|
}
|
|
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-etfs }}" == "true" ]]; then
|
|
add_cs "etfs"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-crypto }}" == "true" ]]; then
|
|
add_cs "crypto_perps_funding"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-nasdaq }}" == "true" ]]; then
|
|
add_cs "nasdaq100_microstructure"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-sp500eoa }}" == "true" ]]; then
|
|
add_cs "sp500_equity_option_analytics"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-firmchar }}" == "true" ]]; then
|
|
add_cs "us_firm_characteristics"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-fx }}" == "true" ]]; then
|
|
add_cs "fx_pairs"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-futures }}" == "true" ]]; then
|
|
add_cs "cme_futures"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-sp500opt }}" == "true" ]]; then
|
|
add_cs "sp500_options"
|
|
fi
|
|
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-usequity }}" == "true" ]]; then
|
|
add_cs "us_equities_panel"
|
|
fi
|
|
|
|
echo "case_matrix=$cases" >> "$GITHUB_OUTPUT"
|
|
|
|
echo "Chapters: $chapters"
|
|
echo "Cases: $cases"
|
|
echo "Py312: $run_py312"
|
|
echo "Neo4j: $run_neo4j"
|
|
echo "Benchmark: $run_benchmark"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Lint (always runs, fast)
|
|
# ---------------------------------------------------------------------------
|
|
lint:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- uses: astral-sh/setup-uv@v6
|
|
with:
|
|
version: "latest"
|
|
- name: Ruff check
|
|
run: uvx ruff@0.15.8 check utils/ data/ tests/
|
|
- name: Ruff format
|
|
run: uvx ruff@0.15.8 format --check utils/ data/ tests/
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fast native unit tests — download-script logic (no Docker, no network,
|
|
# no test-data, no secrets). Seconds per PR.
|
|
#
|
|
# Notebook tests run against pre-seeded data, so the data/**/download.py code
|
|
# path is invisible to them — this is how the in-container free-data download
|
|
# bug (#361) reached readers. These tests exercise that logic directly:
|
|
# path resolution / output placement, download_all dispatch modes, the
|
|
# firm-characteristics archive flatten, download-script registry drift, the
|
|
# compose /data-mount contract (static regression-lock for #361), and the
|
|
# drift-coverage ratchet (every free dataset keeps a live external smoke).
|
|
# The networked container smoke (Phase 2b) and live drift pulls (Phase 3) run
|
|
# in container-smoke.yml / weekly-external.yml, not here.
|
|
# ---------------------------------------------------------------------------
|
|
test-unit:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- uses: astral-sh/setup-uv@v6
|
|
with:
|
|
version: "latest"
|
|
- name: Install minimal test deps
|
|
# Just the import surface of the download-logic tests (utils/downloading
|
|
# + data/download_all + firm-char extract_zip). Intentionally NOT the
|
|
# full project env (no torch/ml4t-*) so this stays a fast per-commit gate.
|
|
run: |
|
|
uv venv --python 3.14
|
|
uv pip install pytest polars pyyaml python-dotenv plotly
|
|
- name: Run download-logic unit tests
|
|
env:
|
|
PYTHONPATH: ${{ github.workspace }}
|
|
# Override the workflow-level test-data path: this job checks out no
|
|
# test-data. utils.config validates ML4T_DATA_PATH exists at import and
|
|
# reads the env var ahead of .env, so point both at the repo's own dirs.
|
|
ML4T_PATH: ${{ github.workspace }}
|
|
ML4T_DATA_PATH: ${{ github.workspace }}/data
|
|
run: |
|
|
# utils.config also requires a .env file to exist; create a minimal one.
|
|
printf 'ML4T_PATH=%s\nML4T_DATA_PATH=%s/data\n' \
|
|
"$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE" > .env
|
|
.venv/bin/python -m pytest \
|
|
tests/test_download_helpers.py \
|
|
tests/test_firm_characteristics_extract.py \
|
|
tests/test_download_all_dispatch.py \
|
|
tests/test_download_scripts_registry.py \
|
|
tests/test_compose_mounts.py \
|
|
tests/test_download_coverage.py \
|
|
-v --tb=short
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Chapter notebook tests — inside Docker (ml4t/ml4t:latest)
|
|
#
|
|
# Runs the same Python 3.14 environment that readers use.
|
|
# Notebooks needing signatory/gensim/esig/tfcausalimpact are skipped here,
|
|
# tested in test-py312 instead.
|
|
# ---------------------------------------------------------------------------
|
|
test-chapters:
|
|
name: ${{ matrix.name }}
|
|
timeout-minutes: 60
|
|
needs: [lint, changes]
|
|
if: needs.changes.outputs.chapter_matrix != '[]'
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include: ${{ fromJSON(needs.changes.outputs.chapter_matrix) }}
|
|
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: ml4t/ml4t:latest
|
|
|
|
steps:
|
|
- name: Install git (for checkout inside container)
|
|
run: apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null
|
|
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: Checkout test data
|
|
uses: actions/checkout@v6
|
|
with:
|
|
repository: ml4t/third-edition-test-data
|
|
path: test-data
|
|
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
|
|
lfs: false
|
|
|
|
- name: Seed case study intermediates
|
|
run: |
|
|
mkdir -p $ML4T_OUTPUT_DIR
|
|
if [ -d test-data/intermediates ]; then
|
|
cp -r test-data/intermediates/* $ML4T_OUTPUT_DIR/
|
|
echo "Seeded $(du -sh $ML4T_OUTPUT_DIR | cut -f1) of intermediates"
|
|
fi
|
|
# Ch20 outputs go to repo dir (get_chapter_dir doesn't redirect)
|
|
if [ -d test-data/intermediates/ch20_synthesis/output ]; then
|
|
mkdir -p 20_strategy_synthesis/output
|
|
cp test-data/intermediates/ch20_synthesis/output/* 20_strategy_synthesis/output/
|
|
echo "Seeded Ch20 synthesis outputs"
|
|
fi
|
|
# Ch24/11 research_operator: artifacts dir is gitignored in this repo;
|
|
# source them from test-data into the notebook dir.
|
|
if [ -d test-data/data/autonomous_agents/operator_artifacts ]; then
|
|
mkdir -p 24_autonomous_agents/operator_artifacts
|
|
cp test-data/data/autonomous_agents/operator_artifacts/*.json \
|
|
24_autonomous_agents/operator_artifacts/
|
|
echo "Seeded ch24/11 operator artifacts"
|
|
fi
|
|
|
|
- name: Run chapter notebooks (${{ matrix.name }})
|
|
env:
|
|
PYTHONPATH: ${{ github.workspace }}
|
|
TEST_FILTER_INPUT: ${{ github.event.inputs.test_filter }}
|
|
MATRIX_FILTER: ${{ matrix.filter }}
|
|
FRED_API_KEY: ${{ secrets.FRED_API_KEY }}
|
|
run: |
|
|
FILTER="${TEST_FILTER_INPUT:-$MATRIX_FILTER}"
|
|
python -m pytest tests/test_chapter_notebooks.py \
|
|
-v --tb=short \
|
|
-k "$FILTER"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Case study pipeline tests — inside Docker (ml4t/ml4t:latest)
|
|
# ---------------------------------------------------------------------------
|
|
test-case-studies:
|
|
name: cs-${{ matrix.case-study }}
|
|
timeout-minutes: 90
|
|
needs: [lint, changes]
|
|
if: needs.changes.outputs.case_matrix != '[]'
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include: ${{ fromJSON(needs.changes.outputs.case_matrix) }}
|
|
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: ml4t/ml4t:latest
|
|
|
|
steps:
|
|
- name: Install git
|
|
run: apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null
|
|
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: Checkout test data
|
|
uses: actions/checkout@v6
|
|
with:
|
|
repository: ml4t/third-edition-test-data
|
|
path: test-data
|
|
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
|
|
lfs: false
|
|
|
|
- name: Seed case study intermediates
|
|
run: |
|
|
mkdir -p $ML4T_OUTPUT_DIR
|
|
if [ -d test-data/intermediates ]; then
|
|
cp -r test-data/intermediates/* $ML4T_OUTPUT_DIR/
|
|
echo "Seeded $(du -sh $ML4T_OUTPUT_DIR | cut -f1) of intermediates"
|
|
fi
|
|
|
|
- name: Run pipeline (${{ matrix.case-study }})
|
|
env:
|
|
PYTHONPATH: ${{ github.workspace }}
|
|
TEST_FILTER_INPUT: ${{ github.event.inputs.test_filter }}
|
|
MATRIX_CASE_STUDY: ${{ matrix.case-study }}
|
|
FRED_API_KEY: ${{ secrets.FRED_API_KEY }}
|
|
run: |
|
|
FILTER="${TEST_FILTER_INPUT:-$MATRIX_CASE_STUDY}"
|
|
python -m pytest tests/test_case_studies.py \
|
|
-v --tb=short \
|
|
-k "$FILTER"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Py312 Docker image (Python 3.12: gensim, signatory, esig, pfhedge,
|
|
# tfcausalimpact, plus notebooks affected by the Py3.14 torch CUDA bug)
|
|
# ---------------------------------------------------------------------------
|
|
test-py312:
|
|
timeout-minutes: 45
|
|
needs: [lint, changes]
|
|
if: needs.changes.outputs.run_py312 == 'true'
|
|
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: ml4t/ml4t-py312:latest
|
|
|
|
steps:
|
|
- name: Install git
|
|
run: apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null
|
|
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: Checkout test data
|
|
uses: actions/checkout@v6
|
|
with:
|
|
repository: ml4t/third-edition-test-data
|
|
path: test-data
|
|
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
|
|
lfs: false
|
|
|
|
- name: Run py312 notebooks
|
|
env:
|
|
PYTHONPATH: ${{ github.workspace }}
|
|
run: |
|
|
python -m pytest tests/test_docker_notebooks.py \
|
|
-v --tb=short \
|
|
-k "03_sigcwgan or 06_path_signatures or 12_wasserstein \
|
|
or 01_word2vec or 02_asset_embeddings or 03_sentiment_evolution \
|
|
or 01_timegan or 07_dp_gan \
|
|
or 10_shap_nlp_sentiment or 06_conditional_autoencoder \
|
|
or 06_fed_announcement_bsts \
|
|
or deep_hedging_pfhedge"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Neo4j Docker service (Ch23 Knowledge Graphs)
|
|
# ---------------------------------------------------------------------------
|
|
test-neo4j:
|
|
timeout-minutes: 30
|
|
needs: [lint, changes]
|
|
if: needs.changes.outputs.run_neo4j == 'true'
|
|
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: ml4t/ml4t:latest
|
|
env:
|
|
NEO4J_URI: bolt://neo4j:7687
|
|
NEO4J_USER: neo4j
|
|
NEO4J_PASSWORD: password
|
|
|
|
services:
|
|
neo4j:
|
|
image: neo4j:2026.02.2
|
|
env:
|
|
NEO4J_AUTH: neo4j/password
|
|
ports:
|
|
- 7474:7474
|
|
- 7687:7687
|
|
options: >-
|
|
--health-cmd "cypher-shell -u neo4j -p password 'RETURN 1;'"
|
|
--health-interval 10s
|
|
--health-timeout 10s
|
|
--health-retries 10
|
|
|
|
steps:
|
|
- name: Install git
|
|
run: apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null
|
|
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: Checkout test data
|
|
uses: actions/checkout@v6
|
|
with:
|
|
repository: ml4t/third-edition-test-data
|
|
path: test-data
|
|
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
|
|
lfs: false
|
|
|
|
- name: Run Ch23 Knowledge Graph notebooks (pipeline order)
|
|
env:
|
|
PYTHONPATH: ${{ github.workspace }}
|
|
run: |
|
|
# Run in dependency order: upstream notebooks generate data for downstream
|
|
python -m pytest tests/test_docker_notebooks.py -v --tb=short \
|
|
-k "08_8k_event_extraction" && \
|
|
python -m pytest tests/test_docker_notebooks.py -v --tb=short \
|
|
-k "02_supply_chain_kg_construction" && \
|
|
python -m pytest tests/test_docker_notebooks.py -v --tb=short \
|
|
-k "07_dynamic_kg_temporal or 03_graph_rag_qa or 05_institutional_holdings_kg \
|
|
or 09_knowledge_graph_features"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Benchmark Docker image (Ch02 storage benchmarks + database services)
|
|
# ---------------------------------------------------------------------------
|
|
test-benchmark:
|
|
timeout-minutes: 30
|
|
needs: [lint, changes]
|
|
if: needs.changes.outputs.run_benchmark == 'true'
|
|
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: ml4t/ml4t-benchmark:latest
|
|
env:
|
|
CLICKHOUSE_HOST: clickhouse
|
|
CLICKHOUSE_PORT: "8123"
|
|
QUESTDB_HOST: questdb
|
|
QUESTDB_HTTP_PORT: "9000"
|
|
TIMESCALE_HOST: timescaledb
|
|
TIMESCALE_PORT: "5432"
|
|
TIMESCALE_PASSWORD: benchmark
|
|
INFLUXDB_HOST: influxdb
|
|
INFLUXDB_PORT: "8086"
|
|
INFLUXDB_ORG: ml4t
|
|
INFLUXDB_TOKEN: benchmark-token
|
|
INFLUXDB_BUCKET: market_data
|
|
|
|
services:
|
|
timescaledb:
|
|
image: timescale/timescaledb:latest-pg16
|
|
env:
|
|
POSTGRES_PASSWORD: benchmark
|
|
POSTGRES_DB: ml4t
|
|
POSTGRES_USER: postgres
|
|
options: >-
|
|
--health-cmd "pg_isready -U postgres"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
|
|
clickhouse:
|
|
image: clickhouse/clickhouse-server:latest
|
|
env:
|
|
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
|
|
CLICKHOUSE_USER: default
|
|
CLICKHOUSE_PASSWORD: ""
|
|
options: >-
|
|
--health-cmd "wget --spider -q http://localhost:8123/ping"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
|
|
questdb:
|
|
image: questdb/questdb:latest
|
|
env:
|
|
QDB_METRICS_ENABLED: "true"
|
|
options: >-
|
|
--health-cmd "curl -sf http://localhost:9003/status || curl -sf http://localhost:9000/exec?query=SELECT%201"
|
|
--health-interval 10s
|
|
--health-timeout 10s
|
|
--health-retries 10
|
|
|
|
influxdb:
|
|
image: influxdb:2.7
|
|
env:
|
|
DOCKER_INFLUXDB_INIT_MODE: setup
|
|
DOCKER_INFLUXDB_INIT_USERNAME: admin
|
|
DOCKER_INFLUXDB_INIT_PASSWORD: benchmark123
|
|
DOCKER_INFLUXDB_INIT_ORG: ml4t
|
|
DOCKER_INFLUXDB_INIT_BUCKET: market_data
|
|
DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: benchmark-token
|
|
options: >-
|
|
--health-cmd "curl -f http://localhost:8086/health"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
|
|
steps:
|
|
- name: Install git
|
|
run: |
|
|
apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null 2>&1 || true
|
|
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: Checkout test data
|
|
uses: actions/checkout@v6
|
|
with:
|
|
repository: ml4t/third-edition-test-data
|
|
path: test-data
|
|
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
|
|
lfs: false
|
|
|
|
- name: Run Ch02 database benchmark notebooks
|
|
env:
|
|
PYTHONPATH: ${{ github.workspace }}
|
|
run: |
|
|
python -m pytest tests/test_docker_notebooks.py \
|
|
-v --tb=short \
|
|
-k "18_storage_benchmark_database or 20_storage_benchmark_database"
|