wehub-resource-sync
60 lines
1.7 KiB
Bash
Executable File
60 lines
1.7 KiB
Bash
Executable File
#!/bin/bash
|
|
# Regenerate all test data (raw + intermediates) for CI.
|
|
#
|
|
# Run from the review repo root. Requires:
|
|
# - ML4T_DATA_PATH to point to full production data (default: ~/Dropbox/ml4t/data)
|
|
# - The working repo at ~/ml4t/third-edition
|
|
# - The review repo at ~/ml4t/technical_review
|
|
#
|
|
# Usage:
|
|
# cd ~/ml4t/technical_review
|
|
# bash tests/generate_test_data.sh [TEST_DATA_DIR]
|
|
|
|
set -euo pipefail
|
|
|
|
TEST_DATA_DIR="${1:-$HOME/ml4t/test-data}"
|
|
WORKING_REPO="$HOME/ml4t/third-edition"
|
|
REVIEW_REPO="$HOME/ml4t/technical_review"
|
|
|
|
echo "=== ML4T Test Data Generator ==="
|
|
echo "Output: $TEST_DATA_DIR"
|
|
echo "Working repo: $WORKING_REPO"
|
|
echo "Review repo: $REVIEW_REPO"
|
|
echo ""
|
|
|
|
# Step 1: Generate subsampled raw data
|
|
echo "=== Step 1: Generating subsampled data ==="
|
|
cd "$WORKING_REPO"
|
|
uv run python tests/create_test_data.py \
|
|
--source "${ML4T_DATA_PATH:-$HOME/Dropbox/ml4t/data}" \
|
|
--output "$TEST_DATA_DIR/data" \
|
|
--clean
|
|
echo ""
|
|
|
|
# Step 2: Deploy latest notebooks from third-edition -> review repo
|
|
echo "=== Step 2: Deploying latest notebooks ==="
|
|
cd "$WORKING_REPO"
|
|
uv run python scripts/deploy_to_review.py --all
|
|
echo ""
|
|
|
|
# Step 3: Generate intermediates (runs pipeline notebooks via Papermill)
|
|
echo "=== Step 3: Generating intermediates ==="
|
|
cd "$REVIEW_REPO"
|
|
ML4T_DATA_PATH="$TEST_DATA_DIR/data" \
|
|
MPLBACKEND=Agg \
|
|
PLOTLY_RENDERER=json \
|
|
uv run python tests/generate_intermediates.py \
|
|
--output "$TEST_DATA_DIR/intermediates"
|
|
echo ""
|
|
|
|
# Summary
|
|
echo "=== Done ==="
|
|
echo "Test data directory: $TEST_DATA_DIR"
|
|
du -sh "$TEST_DATA_DIR/data" "$TEST_DATA_DIR/intermediates" 2>/dev/null || true
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " cd $TEST_DATA_DIR"
|
|
echo " git add -A"
|
|
echo " git commit -m 'update: regenerate test data'"
|
|
echo " git push"
|