mirrors/stefan-jansen--machine-learning-for-trading
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:26:28 +08:00
commit 0249a267e1
495 changed files with 1277094 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/container-smoke.yml
+100
View File
@@ -0,0 +1,100 @@
# ML4T 3rd Edition — Container first-contact smoke (Phase 2b)
#
# Runs the reader's ACTUAL first command through docker compose, in the reader's
# real image, and proves the free-data download can write to /data. This is the
# only layer that catches compose/mount/env regressions like #361 (the `/data`
# read-only mount) — a unit test can't, because it never composes the container.
#
# Cost control: the ml4t image is ~12 GB, so per-PR we run this ONLY when the
# compose/env surface changes (docker-compose.yml, envs/**). The download-script
# *logic* is covered deterministically by the native `test-unit` job on every PR
# (tests/test_download_helpers.py et al.) and the static mount contract by
# tests/test_compose_mounts.py — neither needs the image. The weekly run exercises
# the full path regardless and files an issue if the reader's first contact breaks.
name: Container Smoke
on:
schedule:
- cron: '0 6 * * 1' # Monday 06:00 UTC
pull_request:
branches: [main]
paths:
- 'docker-compose.yml'
- 'envs/**'
- '.github/workflows/container-smoke.yml'
workflow_dispatch:
jobs:
container-smoke:
name: container-smoke
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@v6
- name: First-contact download in the composed container
id: smoke
env:
# Point the compose data mount at a workspace dir we own, so we can
# assert the downloaded files actually land on the host side of the
# bind mount (proves the mount is writable AND correctly wired).
ML4T_DATA_PATH: ${{ github.workspace }}/_smoke_data
run: |
set -o pipefail
mkdir -p "$ML4T_DATA_PATH"
# The reader's real first command, scoped to a single symbol so the
# pull is tiny. --user root mirrors the documented in-container
# download invocation and still fails hard on a read-only mount (even
# root cannot write to a `:ro` bind), so this catches the #361 class.
docker compose run --rm --user root ml4t \
python data/etfs/market/download.py --symbol SPY 2>&1 | tee smoke.log
echo "----- assertions -----"
# 1. No read-only-mount failure (the #361 signature).
if grep -qi "Read-only file system" smoke.log; then
echo "FAIL: container hit a read-only /data mount (#361 regression)"
exit 1
fi
# 2. The download actually populated the host side of the /data mount.
out="$ML4T_DATA_PATH/etfs/market/etf_universe.parquet"
if [ ! -f "$out" ]; then
echo "FAIL: expected $out on the host mount, but it is missing"
echo "Contents of $ML4T_DATA_PATH:"
find "$ML4T_DATA_PATH" -maxdepth 3 -print || true
exit 1
fi
echo "OK: $out present ($(du -h "$out" | cut -f1)) — /data mount is writable and wired"
- name: Open issue on weekly failure
if: failure() && github.event_name == 'schedule'
uses: actions/github-script@v7
with:
script: |
const owner = context.repo.owner;
const repo = context.repo.repo;
const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
const title = 'container-smoke: first-contact download failed';
const q = `repo:${owner}/${repo} is:issue is:open in:title "${title}"`;
const existing = await github.rest.search.issuesAndPullRequests({ q });
if (existing.data.total_count > 0) {
const num = existing.data.items[0].number;
await github.rest.issues.createComment({
owner, repo, issue_number: num,
body: `Failed again in the weekly container smoke: ${runUrl}`,
});
} else {
await github.rest.issues.create({
owner, repo, title,
body: `The weekly container first-contact smoke failed.\n\n` +
`The reader's \`docker compose run … data/etfs/market/download.py --symbol SPY\` ` +
`did not populate \`/data\` in the composed \`ml4t\` container.\n\n` +
`This is the #361 class (compose mount / env wiring). Run: ${runUrl}`,
labels: ['weekly-flake'],
});
}
.github/workflows/full-test.yml
+198
View File
@@ -0,0 +1,198 @@
# ML4T Full Test Suite — Release Gate
#
# Runs ALL 443 notebooks inside Docker. Manual dispatch only.
# Expected runtime: 60-90 minutes.
#
# Use this before releases to verify everything works end-to-end.
# For incremental testing on push/PR, use test.yml instead.
name: Full Test Suite
on:
workflow_dispatch:
inputs:
scope:
description: "Test scope"
type: choice
options:
- all
- chapters-only
- case-studies-only
default: all
env:
ML4T_DATA_PATH: ${{ github.workspace }}/test-data/data
ML4T_OUTPUT_DIR: /tmp/ml4t-test-output
MPLBACKEND: Agg
PLOTLY_RENDERER: json
jobs:
# ---------------------------------------------------------------------------
# All chapter notebooks (Ch01-Ch26)
# ---------------------------------------------------------------------------
chapters:
if: >-
github.event.inputs.scope == 'all' ||
github.event.inputs.scope == 'chapters-only'
timeout-minutes: 120
runs-on: ubuntu-latest
container:
image: ml4t/ml4t:latest
steps:
- name: Install git
run: apt-get update -qq && apt-get install -y -qq git openssh-client >/dev/null
- uses: actions/checkout@v6
- name: Checkout test data
uses: actions/checkout@v6
with:
repository: ml4t/third-edition-test-data
path: test-data
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
lfs: false
- name: Seed intermediates
run: |
mkdir -p $ML4T_OUTPUT_DIR
if [ -d test-data/intermediates ]; then
cp -r test-data/intermediates/* $ML4T_OUTPUT_DIR/
fi
- name: Run ALL chapter notebooks
env:
PYTHONPATH: ${{ github.workspace }}
run: |
python -m pytest tests/test_chapter_notebooks.py \
-v --tb=short \
--timeout=600
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: chapter-results
path: /tmp/ml4t-test-results.jsonl
if-no-files-found: ignore
# ---------------------------------------------------------------------------
# All case study pipelines (9 case studies)
# ---------------------------------------------------------------------------
case-studies:
if: >-
github.event.inputs.scope == 'all' ||
github.event.inputs.scope == 'case-studies-only'
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
case-study:
- etfs
- crypto_perps_funding
- nasdaq100_microstructure
- sp500_equity_option_analytics
- us_firm_characteristics
- fx_pairs
- cme_futures
- sp500_options
- us_equities_panel
runs-on: ubuntu-latest
container:
image: ml4t/ml4t:latest
steps:
- name: Install git
run: apt-get update -qq && apt-get install -y -qq git openssh-client >/dev/null
- uses: actions/checkout@v6
- name: Checkout test data
uses: actions/checkout@v6
with:
repository: ml4t/third-edition-test-data
path: test-data
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
lfs: false
- name: Run pipeline (${{ matrix.case-study }})
env:
PYTHONPATH: ${{ github.workspace }}
run: |
python -m pytest tests/test_case_studies.py \
-v --tb=short \
-k "${{ matrix.case-study }}" \
--timeout=600
# ---------------------------------------------------------------------------
# Py312 notebooks (signatory, gensim, esig, pfhedge, tfcausalimpact, torch CUDA bug)
# ---------------------------------------------------------------------------
py312:
if: github.event.inputs.scope == 'all'
timeout-minutes: 45
runs-on: ubuntu-latest
container:
image: ml4t/ml4t-py312:latest
steps:
- name: Install git
run: apt-get update -qq && apt-get install -y -qq git openssh-client >/dev/null
- uses: actions/checkout@v6
- name: Checkout test data
uses: actions/checkout@v6
with:
repository: ml4t/third-edition-test-data
path: test-data
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
- name: Run py312 notebooks
env:
PYTHONPATH: ${{ github.workspace }}
run: |
python -m pytest tests/test_docker_notebooks.py \
-v --tb=short \
-k "03_sigcwgan or 06_path_signatures or 12_wasserstein \
or 01_word2vec or 02_asset_embeddings or 03_sentiment_evolution \
or 01_timegan or 07_dp_gan \
or 10_shap_nlp_sentiment or 06_conditional_autoencoder \
or 06_fed_announcement_bsts \
or deep_hedging_pfhedge"
# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------
summary:
runs-on: ubuntu-latest
needs: [chapters, case-studies, py312]
if: always()
steps:
- name: Download results
uses: actions/download-artifact@v4
with:
pattern: "*-results"
merge-multiple: true
path: results/
continue-on-error: true
- name: Report
run: |
echo "============================================"
echo "ML4T Full Test Suite — Summary"
echo "============================================"
echo "Chapters: ${{ needs.chapters.result }}"
echo "Case Studies: ${{ needs.case-studies.result }}"
echo "Py312: ${{ needs.py312.result }}"
echo "============================================"
if [[ "${{ needs.chapters.result }}" == "failure" ]] || \
[[ "${{ needs.case-studies.result }}" == "failure" ]] || \
[[ "${{ needs.py312.result }}" == "failure" ]]; then
echo "RESULT: SOME TESTS FAILED"
exit 1
else
echo "RESULT: ALL TESTS PASSED"
fi
.github/workflows/test.yml
+676
View File
@@ -0,0 +1,676 @@
# ML4T 3rd Edition — CI Test Workflow
#
# Tests notebooks inside pre-built Docker images (ml4t/ml4t:latest).
# This ensures CI tests exactly what readers will use.
#
# Architecture:
# - All notebook tests run inside Docker containers (not native Python)
# - Incremental: dorny/paths-filter only tests changed chapters
# - Pre-built images from Docker Hub (built by docker-publish.yml)
# - Special images for py312 (signatory/gensim/tfcausalimpact), Neo4j, benchmarks
#
# Trigger policy:
# - PR: incremental (path-filtered, only changed chapters)
# - Weekly: full suite on schedule (Monday 6 AM UTC)
# - After Docker Publish: full suite (workflow_run trigger)
# - Manual: workflow_dispatch with optional run_all flag
# - Push to main: NOT triggered (conserves minutes)
name: Tests
on:
pull_request:
branches: [main]
schedule:
- cron: '0 6 * * 1' # Weekly Monday 6 AM UTC
workflow_run:
workflows: ["Docker Publish"]
types: [completed]
workflow_dispatch:
inputs:
run_all:
description: "Run all tests (ignore path filters)"
type: boolean
default: false
test_filter:
description: "pytest -k filter (e.g. 'us_equities_panel or nasdaq100' to run only those)"
type: string
default: ""
env:
ML4T_DATA_PATH: ${{ github.workspace }}/test-data/data
ML4T_OUTPUT_DIR: /tmp/ml4t-test-output
MPLBACKEND: Agg
PLOTLY_RENDERER: json
jobs:
# ---------------------------------------------------------------------------
# Detect which paths changed and build dynamic matrices
# ---------------------------------------------------------------------------
changes:
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read
outputs:
chapter_matrix: ${{ steps.build.outputs.chapter_matrix }}
case_matrix: ${{ steps.build.outputs.case_matrix }}
run_py312: ${{ steps.build.outputs.run_py312 }}
run_neo4j: ${{ steps.build.outputs.run_neo4j }}
run_benchmark: ${{ steps.build.outputs.run_benchmark }}
steps:
- uses: actions/checkout@v6
- uses: dorny/paths-filter@v4
id: filter
with:
filters: |
shared:
- 'utils/**'
- 'data/**'
- 'tests/**'
- 'pyproject.toml'
- '.github/workflows/test.yml'
- 'matplotlibrc'
ch01:
- '01_process_is_edge/**'
ch02-03:
- '02_financial_data_universe/**'
- '03_market_microstructure/**'
- 'data/**'
ch04:
- '04_fundamental_alternative_data/**'
ch05:
- '05_synthetic_data/**'
ch06-07:
- '06_strategy_definition/**'
- '07_defining_the_learning_task/**'
ch08-09:
- '08_financial_features/**'
- '09_model_based_features/**'
ch10:
- '10_text_feature_engineering/**'
ch11-12:
- '11_ml_pipeline/**'
- '12_gradient_boosting/**'
ch13:
- '13_dl_time_series/**'
ch14-15:
- '14_latent_factors/**'
- '15_causal_estimation/**'
ch16-17:
- '16_strategy_simulation/**'
- '17_portfolio_construction/**'
ch18-20:
- '18_transaction_costs/**'
- '19_risk_management/**'
- '20_strategy_synthesis/**'
ch21:
- '21_rl_execution_hedging/**'
ch22:
- '22_rag_financial_research/**'
ch23:
- '23_knowledge_graphs/**'
ch24:
- '24_autonomous_agents/**'
ch25:
- '25_live_trading/**'
ch26:
- '26_mlops_governance/**'
cs-etfs:
- 'case_studies/etfs/**'
cs-crypto:
- 'case_studies/crypto_perps_funding/**'
cs-nasdaq:
- 'case_studies/nasdaq100_microstructure/**'
cs-sp500eoa:
- 'case_studies/sp500_equity_option_analytics/**'
cs-firmchar:
- 'case_studies/us_firm_characteristics/**'
cs-fx:
- 'case_studies/fx_pairs/**'
cs-futures:
- 'case_studies/cme_futures/**'
cs-sp500opt:
- 'case_studies/sp500_options/**'
cs-usequity:
- 'case_studies/us_equities_panel/**'
- name: Build dynamic matrices
id: build
run: |
shared="${{ steps.filter.outputs.shared }}"
schedule="${{ github.event_name == 'schedule' }}"
after_docker="${{ github.event_name == 'workflow_run' }}"
run_all="${{ github.event.inputs.run_all }}"
all="false"
if [[ "$shared" == "true" ]] || [[ "$schedule" == "true" ]] || [[ "$run_all" == "true" ]] || [[ "$after_docker" == "true" ]]; then
all="true"
fi
# --- Chapter matrix (all chapters including Ch01 and Ch05) ---
chapters="[]"
# Public-subset scoping: include a chapter only if at least one of its
# NN_* directories is actually present. The public repo ships chapters
# incrementally (beat by beat), so this auto-scopes the matrix to what
# is shipped — no per-beat workflow edits, and unshipped chapters can't
# fail with "no tests collected".
add_ch() {
local n="$1" f="$2" p
for p in $(echo "$f" | grep -oE '[0-9]{2}_' || true); do
if compgen -G "${p}*" >/dev/null 2>&1; then
chapters=$(echo "$chapters" | jq -c --arg n "$n" --arg f "$f" '. + [{"name":$n,"filter":$f}]')
return
fi
done
}
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch01 }}" == "true" ]]; then
add_ch "ch01" "01_process"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch02-03 }}" == "true" ]]; then
add_ch "ch02-03" "02_financial or 03_market"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch04 }}" == "true" ]]; then
add_ch "ch04" "04_fundamental"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch05 }}" == "true" ]]; then
add_ch "ch05" "05_synthetic"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch06-07 }}" == "true" ]]; then
add_ch "ch06-07" "06_strategy or 07_defining"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch08-09 }}" == "true" ]]; then
add_ch "ch08-09" "08_financial or 09_model"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch10 }}" == "true" ]]; then
add_ch "ch10" "10_text"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch11-12 }}" == "true" ]]; then
add_ch "ch11-12" "11_ml_pipeline or 12_gradient"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch13 }}" == "true" ]]; then
add_ch "ch13" "13_dl_time"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch14-15 }}" == "true" ]]; then
add_ch "ch14-15" "14_latent or 15_causal"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch16-17 }}" == "true" ]]; then
add_ch "ch16-17" "16_strategy or 17_portfolio"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch18-20 }}" == "true" ]]; then
add_ch "ch18-20" "18_transaction or 19_risk or 20_strategy"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch21 }}" == "true" ]]; then
add_ch "ch21" "21_rl"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch22 }}" == "true" ]]; then
add_ch "ch22" "22_rag"
fi
# Ch23 skipped — requires Neo4j service (tested in test-neo4j when ready)
# if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch23 }}" == "true" ]]; then
# add_ch "ch23" "23_knowledge"
# fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch24 }}" == "true" ]]; then
add_ch "ch24" "24_autonomous"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch25 }}" == "true" ]]; then
add_ch "ch25" "25_live"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.ch26 }}" == "true" ]]; then
add_ch "ch26" "26_mlops"
fi
echo "chapter_matrix=$chapters" >> "$GITHUB_OUTPUT"
# --- Py312 flag ---
run_py312="false"
if [[ "$all" == "true" ]] \
|| [[ "${{ steps.filter.outputs.ch05 }}" == "true" ]] \
|| [[ "${{ steps.filter.outputs.ch08-09 }}" == "true" ]] \
|| [[ "${{ steps.filter.outputs.ch10 }}" == "true" ]] \
|| [[ "${{ steps.filter.outputs.ch14-15 }}" == "true" ]] \
|| [[ "${{ steps.filter.outputs.ch21 }}" == "true" ]]; then
run_py312="true"
fi
echo "run_py312=$run_py312" >> "$GITHUB_OUTPUT"
# --- Neo4j flag (needs populated Neo4j — skip until infra is ready) ---
run_neo4j="false"
echo "run_neo4j=$run_neo4j" >> "$GITHUB_OUTPUT"
# --- Benchmark flag (disabled — QuestDB container fails health check in CI) ---
run_benchmark="false"
echo "run_benchmark=$run_benchmark" >> "$GITHUB_OUTPUT"
# --- Case study matrix ---
cases="[]"
# Public repo ships case-study CONFIG but gates the pipeline code.
# test_case_studies.py auto-discovers stages by globbing
# case_studies/<cs>/[0-9][0-9]*.py, and the results registry is seeded
# from the test-data repo at run time (NOT shipped to this repo). So
# gate on the same signal the test collects: enroll a case study only
# once its pipeline notebooks are shipped here — currently only
# config/setup.yaml ships, so case-study jobs stay skipped. Future
# beats that ship the notebooks auto-enroll, no workflow edits.
add_cs() {
if compgen -G "case_studies/$1/[0-9][0-9]*.py" >/dev/null 2>&1; then
cases=$(echo "$cases" | jq -c --arg cs "$1" '. + [{"case-study":$cs}]')
fi
}
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-etfs }}" == "true" ]]; then
add_cs "etfs"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-crypto }}" == "true" ]]; then
add_cs "crypto_perps_funding"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-nasdaq }}" == "true" ]]; then
add_cs "nasdaq100_microstructure"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-sp500eoa }}" == "true" ]]; then
add_cs "sp500_equity_option_analytics"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-firmchar }}" == "true" ]]; then
add_cs "us_firm_characteristics"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-fx }}" == "true" ]]; then
add_cs "fx_pairs"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-futures }}" == "true" ]]; then
add_cs "cme_futures"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-sp500opt }}" == "true" ]]; then
add_cs "sp500_options"
fi
if [[ "$all" == "true" ]] || [[ "${{ steps.filter.outputs.cs-usequity }}" == "true" ]]; then
add_cs "us_equities_panel"
fi
echo "case_matrix=$cases" >> "$GITHUB_OUTPUT"
echo "Chapters: $chapters"
echo "Cases: $cases"
echo "Py312: $run_py312"
echo "Neo4j: $run_neo4j"
echo "Benchmark: $run_benchmark"
# ---------------------------------------------------------------------------
# Lint (always runs, fast)
# ---------------------------------------------------------------------------
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: astral-sh/setup-uv@v6
with:
version: "latest"
- name: Ruff check
run: uvx ruff@0.15.8 check utils/ data/ tests/
- name: Ruff format
run: uvx ruff@0.15.8 format --check utils/ data/ tests/
# ---------------------------------------------------------------------------
# Fast native unit tests — download-script logic (no Docker, no network,
# no test-data, no secrets). Seconds per PR.
#
# Notebook tests run against pre-seeded data, so the data/**/download.py code
# path is invisible to them — this is how the in-container free-data download
# bug (#361) reached readers. These tests exercise that logic directly:
# path resolution / output placement, download_all dispatch modes, the
# firm-characteristics archive flatten, download-script registry drift, the
# compose /data-mount contract (static regression-lock for #361), and the
# drift-coverage ratchet (every free dataset keeps a live external smoke).
# The networked container smoke (Phase 2b) and live drift pulls (Phase 3) run
# in container-smoke.yml / weekly-external.yml, not here.
# ---------------------------------------------------------------------------
test-unit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: astral-sh/setup-uv@v6
with:
version: "latest"
- name: Install minimal test deps
# Just the import surface of the download-logic tests (utils/downloading
# + data/download_all + firm-char extract_zip). Intentionally NOT the
# full project env (no torch/ml4t-*) so this stays a fast per-commit gate.
run: |
uv venv --python 3.14
uv pip install pytest polars pyyaml python-dotenv plotly
- name: Run download-logic unit tests
env:
PYTHONPATH: ${{ github.workspace }}
# Override the workflow-level test-data path: this job checks out no
# test-data. utils.config validates ML4T_DATA_PATH exists at import and
# reads the env var ahead of .env, so point both at the repo's own dirs.
ML4T_PATH: ${{ github.workspace }}
ML4T_DATA_PATH: ${{ github.workspace }}/data
run: |
# utils.config also requires a .env file to exist; create a minimal one.
printf 'ML4T_PATH=%s\nML4T_DATA_PATH=%s/data\n' \
"$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE" > .env
.venv/bin/python -m pytest \
tests/test_download_helpers.py \
tests/test_firm_characteristics_extract.py \
tests/test_download_all_dispatch.py \
tests/test_download_scripts_registry.py \
tests/test_compose_mounts.py \
tests/test_download_coverage.py \
-v --tb=short
# ---------------------------------------------------------------------------
# Chapter notebook tests — inside Docker (ml4t/ml4t:latest)
#
# Runs the same Python 3.14 environment that readers use.
# Notebooks needing signatory/gensim/esig/tfcausalimpact are skipped here,
# tested in test-py312 instead.
# ---------------------------------------------------------------------------
test-chapters:
name: ${{ matrix.name }}
timeout-minutes: 60
needs: [lint, changes]
if: needs.changes.outputs.chapter_matrix != '[]'
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.changes.outputs.chapter_matrix) }}
runs-on: ubuntu-latest
container:
image: ml4t/ml4t:latest
steps:
- name: Install git (for checkout inside container)
run: apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null
- uses: actions/checkout@v6
- name: Checkout test data
uses: actions/checkout@v6
with:
repository: ml4t/third-edition-test-data
path: test-data
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
lfs: false
- name: Seed case study intermediates
run: |
mkdir -p $ML4T_OUTPUT_DIR
if [ -d test-data/intermediates ]; then
cp -r test-data/intermediates/* $ML4T_OUTPUT_DIR/
echo "Seeded $(du -sh $ML4T_OUTPUT_DIR | cut -f1) of intermediates"
fi
# Ch20 outputs go to repo dir (get_chapter_dir doesn't redirect)
if [ -d test-data/intermediates/ch20_synthesis/output ]; then
mkdir -p 20_strategy_synthesis/output
cp test-data/intermediates/ch20_synthesis/output/* 20_strategy_synthesis/output/
echo "Seeded Ch20 synthesis outputs"
fi
# Ch24/11 research_operator: artifacts dir is gitignored in this repo;
# source them from test-data into the notebook dir.
if [ -d test-data/data/autonomous_agents/operator_artifacts ]; then
mkdir -p 24_autonomous_agents/operator_artifacts
cp test-data/data/autonomous_agents/operator_artifacts/*.json \
24_autonomous_agents/operator_artifacts/
echo "Seeded ch24/11 operator artifacts"
fi
- name: Run chapter notebooks (${{ matrix.name }})
env:
PYTHONPATH: ${{ github.workspace }}
TEST_FILTER_INPUT: ${{ github.event.inputs.test_filter }}
MATRIX_FILTER: ${{ matrix.filter }}
FRED_API_KEY: ${{ secrets.FRED_API_KEY }}
run: |
FILTER="${TEST_FILTER_INPUT:-$MATRIX_FILTER}"
python -m pytest tests/test_chapter_notebooks.py \
-v --tb=short \
-k "$FILTER"
# ---------------------------------------------------------------------------
# Case study pipeline tests — inside Docker (ml4t/ml4t:latest)
# ---------------------------------------------------------------------------
test-case-studies:
name: cs-${{ matrix.case-study }}
timeout-minutes: 90
needs: [lint, changes]
if: needs.changes.outputs.case_matrix != '[]'
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.changes.outputs.case_matrix) }}
runs-on: ubuntu-latest
container:
image: ml4t/ml4t:latest
steps:
- name: Install git
run: apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null
- uses: actions/checkout@v6
- name: Checkout test data
uses: actions/checkout@v6
with:
repository: ml4t/third-edition-test-data
path: test-data
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
lfs: false
- name: Seed case study intermediates
run: |
mkdir -p $ML4T_OUTPUT_DIR
if [ -d test-data/intermediates ]; then
cp -r test-data/intermediates/* $ML4T_OUTPUT_DIR/
echo "Seeded $(du -sh $ML4T_OUTPUT_DIR | cut -f1) of intermediates"
fi
- name: Run pipeline (${{ matrix.case-study }})
env:
PYTHONPATH: ${{ github.workspace }}
TEST_FILTER_INPUT: ${{ github.event.inputs.test_filter }}
MATRIX_CASE_STUDY: ${{ matrix.case-study }}
FRED_API_KEY: ${{ secrets.FRED_API_KEY }}
run: |
FILTER="${TEST_FILTER_INPUT:-$MATRIX_CASE_STUDY}"
python -m pytest tests/test_case_studies.py \
-v --tb=short \
-k "$FILTER"
# ---------------------------------------------------------------------------
# Py312 Docker image (Python 3.12: gensim, signatory, esig, pfhedge,
# tfcausalimpact, plus notebooks affected by the Py3.14 torch CUDA bug)
# ---------------------------------------------------------------------------
test-py312:
timeout-minutes: 45
needs: [lint, changes]
if: needs.changes.outputs.run_py312 == 'true'
runs-on: ubuntu-latest
container:
image: ml4t/ml4t-py312:latest
steps:
- name: Install git
run: apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null
- uses: actions/checkout@v6
- name: Checkout test data
uses: actions/checkout@v6
with:
repository: ml4t/third-edition-test-data
path: test-data
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
lfs: false
- name: Run py312 notebooks
env:
PYTHONPATH: ${{ github.workspace }}
run: |
python -m pytest tests/test_docker_notebooks.py \
-v --tb=short \
-k "03_sigcwgan or 06_path_signatures or 12_wasserstein \
or 01_word2vec or 02_asset_embeddings or 03_sentiment_evolution \
or 01_timegan or 07_dp_gan \
or 10_shap_nlp_sentiment or 06_conditional_autoencoder \
or 06_fed_announcement_bsts \
or deep_hedging_pfhedge"
# ---------------------------------------------------------------------------
# Neo4j Docker service (Ch23 Knowledge Graphs)
# ---------------------------------------------------------------------------
test-neo4j:
timeout-minutes: 30
needs: [lint, changes]
if: needs.changes.outputs.run_neo4j == 'true'
runs-on: ubuntu-latest
container:
image: ml4t/ml4t:latest
env:
NEO4J_URI: bolt://neo4j:7687
NEO4J_USER: neo4j
NEO4J_PASSWORD: password
services:
neo4j:
image: neo4j:2026.02.2
env:
NEO4J_AUTH: neo4j/password
ports:
- 7474:7474
- 7687:7687
options: >-
--health-cmd "cypher-shell -u neo4j -p password 'RETURN 1;'"
--health-interval 10s
--health-timeout 10s
--health-retries 10
steps:
- name: Install git
run: apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null
- uses: actions/checkout@v6
- name: Checkout test data
uses: actions/checkout@v6
with:
repository: ml4t/third-edition-test-data
path: test-data
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
lfs: false
- name: Run Ch23 Knowledge Graph notebooks (pipeline order)
env:
PYTHONPATH: ${{ github.workspace }}
run: |
# Run in dependency order: upstream notebooks generate data for downstream
python -m pytest tests/test_docker_notebooks.py -v --tb=short \
-k "08_8k_event_extraction" && \
python -m pytest tests/test_docker_notebooks.py -v --tb=short \
-k "02_supply_chain_kg_construction" && \
python -m pytest tests/test_docker_notebooks.py -v --tb=short \
-k "07_dynamic_kg_temporal or 03_graph_rag_qa or 05_institutional_holdings_kg \
or 09_knowledge_graph_features"
# ---------------------------------------------------------------------------
# Benchmark Docker image (Ch02 storage benchmarks + database services)
# ---------------------------------------------------------------------------
test-benchmark:
timeout-minutes: 30
needs: [lint, changes]
if: needs.changes.outputs.run_benchmark == 'true'
runs-on: ubuntu-latest
container:
image: ml4t/ml4t-benchmark:latest
env:
CLICKHOUSE_HOST: clickhouse
CLICKHOUSE_PORT: "8123"
QUESTDB_HOST: questdb
QUESTDB_HTTP_PORT: "9000"
TIMESCALE_HOST: timescaledb
TIMESCALE_PORT: "5432"
TIMESCALE_PASSWORD: benchmark
INFLUXDB_HOST: influxdb
INFLUXDB_PORT: "8086"
INFLUXDB_ORG: ml4t
INFLUXDB_TOKEN: benchmark-token
INFLUXDB_BUCKET: market_data
services:
timescaledb:
image: timescale/timescaledb:latest-pg16
env:
POSTGRES_PASSWORD: benchmark
POSTGRES_DB: ml4t
POSTGRES_USER: postgres
options: >-
--health-cmd "pg_isready -U postgres"
--health-interval 10s
--health-timeout 5s
--health-retries 5
clickhouse:
image: clickhouse/clickhouse-server:latest
env:
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
CLICKHOUSE_USER: default
CLICKHOUSE_PASSWORD: ""
options: >-
--health-cmd "wget --spider -q http://localhost:8123/ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
questdb:
image: questdb/questdb:latest
env:
QDB_METRICS_ENABLED: "true"
options: >-
--health-cmd "curl -sf http://localhost:9003/status || curl -sf http://localhost:9000/exec?query=SELECT%201"
--health-interval 10s
--health-timeout 10s
--health-retries 10
influxdb:
image: influxdb:2.7
env:
DOCKER_INFLUXDB_INIT_MODE: setup
DOCKER_INFLUXDB_INIT_USERNAME: admin
DOCKER_INFLUXDB_INIT_PASSWORD: benchmark123
DOCKER_INFLUXDB_INIT_ORG: ml4t
DOCKER_INFLUXDB_INIT_BUCKET: market_data
DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: benchmark-token
options: >-
--health-cmd "curl -f http://localhost:8086/health"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Install git
run: |
apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null 2>&1 || true
- uses: actions/checkout@v6
- name: Checkout test data
uses: actions/checkout@v6
with:
repository: ml4t/third-edition-test-data
path: test-data
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
lfs: false
- name: Run Ch02 database benchmark notebooks
env:
PYTHONPATH: ${{ github.workspace }}
run: |
python -m pytest tests/test_docker_notebooks.py \
-v --tb=short \
-k "18_storage_benchmark_database or 20_storage_benchmark_database"
.github/workflows/weekly-external.yml
+296
View File
@@ -0,0 +1,296 @@
# ML4T 3rd Edition — Weekly External Workflow (Tier 2)
#
# Runs notebooks tagged `tier: weekly` in tests/overrides.yaml — these have
# external dependencies (IB Gateway, OKX, chromadb, real LLM APIs, real HTTP)
# that aren't available in the per-commit Tier 1 workflow.
#
# Routing: ML4T_TEST_TIER=weekly env var → conftest tier-skip allows weekly NBs
# to run (per-commit run skips them automatically with the inverse rule).
#
# Today most still skip via existing `skip: true` because the underlying
# infrastructure (IB Gateway, OKX, etc.) isn't provisioned yet. The job's
# purpose is to exercise the routing/flake-retry/issue-opening plumbing so
# that when fixtures land (Steps 4-9 of the test-infra coverage plan) the
# weekly job auto-picks them up with no workflow change.
#
# Failure handling: any failing NB opens a `weekly-flake` issue (idempotent
# by NB id in title); pytest-rerunfailures retries flaky NBs per-NB based on
# `reruns: N` in overrides.yaml.
name: Weekly External
on:
schedule:
- cron: '0 6 * * 1' # Monday 06:00 UTC
workflow_dispatch:
inputs:
test_filter:
description: "Optional pytest -k filter to narrow scope"
type: string
default: ""
env:
ML4T_TEST_TIER: weekly
ML4T_DATA_PATH: ${{ github.workspace }}/test-data/data
ML4T_OUTPUT_DIR: /tmp/ml4t-test-output
MPLBACKEND: Agg
PLOTLY_RENDERER: json
jobs:
weekly-chapters:
name: weekly-chapters
runs-on: ubuntu-latest
timeout-minutes: 60
container:
image: ml4t/ml4t:latest
permissions:
contents: read
issues: write
steps:
- name: Install git
run: apt-get update -qq && apt-get install -y -qq git git-lfs openssh-client >/dev/null
- uses: actions/checkout@v6
- name: Checkout test data
uses: actions/checkout@v6
with:
repository: ml4t/third-edition-test-data
path: test-data
ssh-key: ${{ secrets.TEST_DATA_DEPLOY_KEY }}
lfs: false
- name: Seed case study intermediates
run: |
mkdir -p $ML4T_OUTPUT_DIR
if [ -d test-data/intermediates ]; then
cp -r test-data/intermediates/* $ML4T_OUTPUT_DIR/
fi
- name: Run weekly chapter notebooks
id: pytest_chapters
env:
PYTHONPATH: ${{ github.workspace }}
TEST_FILTER_INPUT: ${{ github.event.inputs.test_filter }}
# NO-SPEND POLICY: only FREE providers are wired here. Billed APIs
# (OpenAI, OKX authenticated, Databento) are intentionally NOT passed,
# so any notebook needing them skips rather than incurs a charge.
FRED_API_KEY: ${{ secrets.FRED_API_KEY }}
EDGAR_IDENTITY: ${{ secrets.EDGAR_IDENTITY }}
run: |
set +e
# Build -k as an array so a multi-word filter (e.g. "foo or bar")
# stays a single pytest argument instead of being word-split.
FILTER_ARGS=()
if [ -n "${TEST_FILTER_INPUT:-}" ]; then
FILTER_ARGS=(-k "${TEST_FILTER_INPUT}")
fi
# Public repo: chapter notebooks only. Case-study pipeline tests need
# gated results/registries that aren't shipped, so they're excluded.
python -m pytest tests/test_chapter_notebooks.py \
-v --tb=short -m weekly "${FILTER_ARGS[@]}" \
--junitxml=junit-weekly.xml
echo "exit_code=$?" >> "$GITHUB_OUTPUT"
continue-on-error: true
- name: Upload junit report
if: always()
uses: actions/upload-artifact@v4
with:
name: junit-weekly
path: junit-weekly.xml
retention-days: 30
- name: Open issues for failed notebooks
if: always() && steps.pytest_chapters.outputs.exit_code != '0'
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const { execSync } = require('child_process');
let xml;
try {
xml = fs.readFileSync('junit-weekly.xml', 'utf8');
} catch (e) {
core.info('No junit-weekly.xml found; nothing to file.');
return;
}
// Crude but dependency-free junit parse: pull <testcase ...> blocks
// and check each for a child <failure> or <error>.
const testcaseRe = /<testcase\b[^>]*>([\s\S]*?)<\/testcase>|<testcase\b[^>]*\/>/g;
const attrRe = (name) => new RegExp(`${name}="([^"]*)"`);
const failures = [];
let m;
while ((m = testcaseRe.exec(xml)) !== null) {
const block = m[0];
const body = m[1] || '';
if (!/<failure\b|<error\b/.test(body)) continue;
const nameMatch = block.match(attrRe('name'));
const classMatch = block.match(attrRe('classname'));
if (nameMatch) {
failures.push({
name: nameMatch[1],
classname: classMatch ? classMatch[1] : '',
});
}
}
core.info(`Found ${failures.length} failing testcases.`);
const owner = context.repo.owner;
const repo = context.repo.repo;
const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
for (const f of failures) {
const nbId = f.name.replace(/^test_[a-z_]+\[/, '').replace(/\]$/, '');
const title = `weekly-flake: ${nbId}`;
// Search for existing open issue with same title to dedupe.
const q = `repo:${owner}/${repo} is:issue is:open in:title "weekly-flake: ${nbId}"`;
const existing = await github.rest.search.issuesAndPullRequests({ q });
if (existing.data.total_count > 0) {
const num = existing.data.items[0].number;
core.info(`Existing issue #${num} for ${nbId}; commenting.`);
await github.rest.issues.createComment({
owner, repo, issue_number: num,
body: `Failed again in weekly run: ${runUrl}`,
});
} else {
core.info(`Opening new issue for ${nbId}.`);
await github.rest.issues.create({
owner, repo, title,
body: `Weekly external workflow failed for \`${nbId}\`.\n\n` +
`Run: ${runUrl}\n\n` +
`Test class: \`${f.classname}\`\n\n` +
`Dedupes by NB id — comments will append on subsequent failures.`,
labels: ['weekly-flake'],
});
}
}
- name: Fail job if pytest failed
if: steps.pytest_chapters.outputs.exit_code != '0'
run: |
echo "Weekly pytest exited with code ${{ steps.pytest_chapters.outputs.exit_code }}"
exit 1
# ---------------------------------------------------------------------------
# Phase 3 — live external drift smoke.
#
# One tiny REAL pull per free data source (tests/test_external_drift.py, marked
# `drift`). Catches upstream provider/schema drift: when a source moves,
# renames a file, or changes its payload shape, the matching test fails and we
# file a deduped issue. No billed APIs — key-gated sources (FRED, OANDA) skip
# when their free key is absent; geo-restricted sources (Kalshi/Polymarket)
# skip when blocked. Flake-tolerant via pytest-rerunfailures when available.
# ---------------------------------------------------------------------------
weekly-drift:
name: weekly-drift
runs-on: ubuntu-latest
timeout-minutes: 20
container:
image: ml4t/ml4t:latest
permissions:
contents: read
issues: write
steps:
- name: Install git
run: apt-get update -qq && apt-get install -y -qq git >/dev/null
- uses: actions/checkout@v6
- name: Run external drift smoke
id: pytest_drift
env:
PYTHONPATH: ${{ github.workspace }}
# NO-SPEND POLICY: only FREE keys. Billed APIs stay unset → their
# drift tests skip rather than charge.
FRED_API_KEY: ${{ secrets.FRED_API_KEY }}
EDGAR_IDENTITY: ${{ secrets.EDGAR_IDENTITY }}
run: |
set +e
# utils.config requires a .env file to exist; create a minimal one and
# point the data path at the repo's own data/ dir (no test-data here).
printf 'ML4T_PATH=%s\nML4T_DATA_PATH=%s/data\n' \
"$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE" > .env
# Flake tolerance only if pytest-rerunfailures is installed in the image.
RERUN_ARGS=""
if python -c "import pytest_rerunfailures" 2>/dev/null; then
RERUN_ARGS="--reruns 2 --reruns-delay 30"
fi
python -m pytest tests/test_external_drift.py -m drift \
-v --tb=short $RERUN_ARGS \
--junitxml=junit-drift.xml
echo "exit_code=$?" >> "$GITHUB_OUTPUT"
continue-on-error: true
- name: Upload junit report
if: always()
uses: actions/upload-artifact@v4
with:
name: junit-drift
path: junit-drift.xml
retention-days: 30
- name: Open issues for drifted sources
if: always() && steps.pytest_drift.outputs.exit_code != '0'
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
let xml;
try {
xml = fs.readFileSync('junit-drift.xml', 'utf8');
} catch (e) {
core.info('No junit-drift.xml found; nothing to file.');
return;
}
// Dependency-free junit parse: <testcase> blocks with a child
// <failure>/<error> are real drift (skips are not failures).
const testcaseRe = /<testcase\b[^>]*>([\s\S]*?)<\/testcase>|<testcase\b[^>]*\/>/g;
const attrRe = (name) => new RegExp(`${name}="([^"]*)"`);
const failures = [];
let m;
while ((m = testcaseRe.exec(xml)) !== null) {
const block = m[0];
const body = m[1] || '';
if (!/<failure\b|<error\b/.test(body)) continue;
const nameMatch = block.match(attrRe('name'));
if (nameMatch) failures.push(nameMatch[1]);
}
core.info(`Found ${failures.length} drifted source(s).`);
const owner = context.repo.owner;
const repo = context.repo.repo;
const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
for (const name of failures) {
// test_etf_yahoo_reachable -> etf_yahoo
const source = name.replace(/^test_/, '').replace(/_reachable$/, '');
const title = `external-drift: ${source}`;
const q = `repo:${owner}/${repo} is:issue is:open in:title "external-drift: ${source}"`;
const existing = await github.rest.search.issuesAndPullRequests({ q });
if (existing.data.total_count > 0) {
const num = existing.data.items[0].number;
await github.rest.issues.createComment({
owner, repo, issue_number: num,
body: `Drifted again in the weekly run: ${runUrl}`,
});
} else {
await github.rest.issues.create({
owner, repo, title,
body: `The weekly external drift smoke failed for \`${name}\`.\n\n` +
`An upstream free data source appears to have moved, renamed a ` +
`file, or changed its payload schema.\n\n` +
`Run: ${runUrl}\n\n` +
`Dedupes by source — comments append on subsequent failures.`,
labels: ['external-drift', 'weekly-flake'],
});
}
}
- name: Fail job if drift smoke failed
if: steps.pytest_drift.outputs.exit_code != '0'
run: |
echo "Drift smoke exited with code ${{ steps.pytest_drift.outputs.exit_code }}"
exit 1