chore: import upstream snapshot with attribution

bin/distributed-test/scripts/benchmark-sizing.sh

@@ -0,0 +1,714 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PERF_TEST="$SCRIPT_DIR/perf-test.sh"
+
+# ─── Defaults ───────────────────────────────────────────────────────────────────
+SERVER="http://localhost:8585"
+WORKERS=30
+OUTPUT_DIR="./sizing-results"
+START_SCALE="10k"
+END_SCALE="xlarge"
+MODES="seq,realistic"
+ADMIN_PORT=""
+TOKEN=""
+RAMP=false
+SKIP_EXISTING=false
+MIXED=false
+MIXED_DURATION=60
+NO_BREAK=false
+
+# ─── Scale Ladder (ordered) ────────────────────────────────────────────────────
+ALL_SCALES=(10k 50k 100k 200k 500k large xlarge)
+
+# ─── Colors ─────────────────────────────────────────────────────────────────────
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+BOLD='\033[1m'
+NC='\033[0m'
+
+# ─── Usage ──────────────────────────────────────────────────────────────────────
+usage() {
+    cat <<'EOF'
+Usage: benchmark-sizing.sh [OPTIONS]
+
+Progressive benchmark runner that loops through scale tiers, runs perf-test.sh
+at each tier, captures JSON outputs, and generates a final comparison report.
+
+Scale Ladder: 10k → 50k → 100k → 200k → 500k → large (~2M) → xlarge (~5M)
+
+OPTIONS:
+  --server URL            Target server (default: http://localhost:8585)
+  --workers NUM           Worker count for all runs (default: 30)
+  --output-dir DIR        Directory for JSON reports + final summary (default: ./sizing-results)
+  --start-scale PRESET    First scale tier to run (default: 10k)
+  --end-scale PRESET      Last scale tier to run (default: xlarge)
+  --modes MODES           Comma-separated: seq, realistic, or both (default: seq,realistic)
+  --admin-port PORT       Pass through to perf-test.sh for server diagnostics
+  --token TOKEN           Auth token pass-through
+  --ramp                  Run ramp test at first tier to find optimal workers
+  --skip-existing         Skip tiers that already have JSON output in output-dir
+  --mixed                 Also run mixed read/write workload at each tier
+  --mixed-duration SECS   Duration for mixed workload (default: 60)
+  --no-break              Don't stop at break-points, run all tiers regardless
+  -h, --help              Show this help message
+
+EXAMPLES:
+  # Quick 2-tier test
+  benchmark-sizing.sh --start-scale 10k --end-scale 50k
+
+  # Full progressive benchmark with ramp and mixed workloads
+  benchmark-sizing.sh --ramp --mixed --output-dir /data/sizing
+
+  # Resume after failure (skips completed tiers)
+  benchmark-sizing.sh --skip-existing --output-dir ./sizing-results
+
+  # Single mode only
+  benchmark-sizing.sh --modes realistic --end-scale 500k
+EOF
+    exit 0
+}
+
+# ─── Parse Arguments ────────────────────────────────────────────────────────────
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --server)       SERVER="$2"; shift 2 ;;
+        --workers)      WORKERS="$2"; shift 2 ;;
+        --output-dir)   OUTPUT_DIR="$2"; shift 2 ;;
+        --start-scale)  START_SCALE="$2"; shift 2 ;;
+        --end-scale)    END_SCALE="$2"; shift 2 ;;
+        --modes)        MODES="$2"; shift 2 ;;
+        --admin-port)   ADMIN_PORT="$2"; shift 2 ;;
+        --token)        TOKEN="$2"; shift 2 ;;
+        --ramp)         RAMP=true; shift ;;
+        --skip-existing) SKIP_EXISTING=true; shift ;;
+        --mixed)        MIXED=true; shift ;;
+        --mixed-duration) MIXED_DURATION="$2"; shift 2 ;;
+        --no-break)     NO_BREAK=true; shift ;;
+        -h|--help)      usage ;;
+        *) echo "Unknown option: $1"; usage ;;
+    esac
+done
+
+# ─── Validate ───────────────────────────────────────────────────────────────────
+if [[ ! -f "$PERF_TEST" ]]; then
+    echo -e "${RED}Error: perf-test.sh not found at $PERF_TEST${NC}"
+    exit 1
+fi
+
+validate_scale() {
+    local scale="$1"
+    for s in "${ALL_SCALES[@]}"; do
+        [[ "$s" == "$scale" ]] && return 0
+    done
+    echo -e "${RED}Error: Invalid scale '$scale'. Valid: ${ALL_SCALES[*]}${NC}"
+    exit 1
+}
+
+validate_scale "$START_SCALE"
+validate_scale "$END_SCALE"
+
+# ─── Resolve Scale Range ───────────────────────────────────────────────────────
+resolve_scales() {
+    local start="$1" end="$2"
+    local collecting=false
+    SCALES=()
+    for s in "${ALL_SCALES[@]}"; do
+        [[ "$s" == "$start" ]] && collecting=true
+        if $collecting; then
+            SCALES+=("$s")
+        fi
+        [[ "$s" == "$end" ]] && break
+    done
+    if [[ ${#SCALES[@]} -eq 0 ]]; then
+        echo -e "${RED}Error: No scales resolved between $start and $end${NC}"
+        exit 1
+    fi
+}
+
+resolve_scales "$START_SCALE" "$END_SCALE"
+
+# ─── Parse Modes ────────────────────────────────────────────────────────────────
+IFS=',' read -ra MODE_LIST <<< "$MODES"
+for m in "${MODE_LIST[@]}"; do
+    if [[ "$m" != "seq" && "$m" != "realistic" ]]; then
+        echo -e "${RED}Error: Invalid mode '$m'. Valid: seq, realistic${NC}"
+        exit 1
+    fi
+done
+
+# ─── Setup Output Directory ────────────────────────────────────────────────────
+mkdir -p "$OUTPUT_DIR"
+SUMMARY_CSV="$OUTPUT_DIR/.sizing-progress.csv"
+SUMMARY_MD="$OUTPUT_DIR/SIZING-SUMMARY.md"
+LOG_FILE="$OUTPUT_DIR/benchmark-sizing.log"
+
+log() {
+    local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
+    echo "$msg" >> "$LOG_FILE"
+    echo -e "$msg"
+}
+
+# ─── Print Banner ──────────────────────────────────────────────────────────────
+echo ""
+echo -e "${BOLD}══════════════════════════════════════════════════════════════════${NC}"
+echo -e "${BOLD}  OpenMetadata Progressive Cluster Sizing Benchmark${NC}"
+echo -e "${BOLD}══════════════════════════════════════════════════════════════════${NC}"
+echo ""
+echo -e "  Server:       ${CYAN}$SERVER${NC}"
+echo -e "  Workers:      ${CYAN}$WORKERS${NC}"
+echo -e "  Scale range:  ${CYAN}${SCALES[*]}${NC}"
+echo -e "  Modes:        ${CYAN}${MODE_LIST[*]}${NC}"
+echo -e "  Output:       ${CYAN}$OUTPUT_DIR${NC}"
+echo -e "  Ramp test:    ${CYAN}$RAMP${NC}"
+echo -e "  Mixed:        ${CYAN}$MIXED${NC}"
+echo -e "  No-break:     ${CYAN}$NO_BREAK${NC}"
+[[ -n "$ADMIN_PORT" ]] && echo -e "  Admin port:   ${CYAN}$ADMIN_PORT${NC}"
+echo ""
+
+# ─── Connectivity Check ────────────────────────────────────────────────────────
+log "Checking server connectivity..."
+HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$SERVER/api/v1/system/version" 2>/dev/null || echo "000")
+if [[ "$HTTP_CODE" == "000" ]]; then
+    echo -e "${RED}Error: Cannot reach $SERVER (connection refused or timeout)${NC}"
+    exit 1
+elif [[ "$HTTP_CODE" -ge 400 ]]; then
+    echo -e "${YELLOW}Warning: Server returned HTTP $HTTP_CODE (may need auth token)${NC}"
+fi
+log "Server reachable (HTTP $HTTP_CODE)"
+
+# ─── Build perf-test.sh Base Args ──────────────────────────────────────────────
+build_perf_args() {
+    local scale="$1" mode="$2" output_file="$3"
+    local args=()
+    args+=(--server "$SERVER")
+    args+=(--workers "$WORKERS")
+    args+=(--scale "$scale")
+    args+=(--output "$output_file")
+    [[ -n "$ADMIN_PORT" ]] && args+=(--admin-port "$ADMIN_PORT")
+    [[ -n "$TOKEN" ]] && args+=(--token "$TOKEN")
+    [[ "$mode" == "realistic" ]] && args+=(--realistic)
+    if $MIXED; then
+        args+=(--mixed)
+        args+=(--mixed-duration "$MIXED_DURATION")
+    fi
+    echo "${args[@]}"
+}
+
+# ─── Extract Metrics from JSON ──────────────────────────────────────────────────
+extract_metrics() {
+    local json_file="$1"
+    if [[ ! -f "$json_file" ]]; then
+        echo "N/A,0,0,0,0,0,N/A"
+        return
+    fi
+
+    python3 -c "
+import json, sys
+
+with open('$json_file') as f:
+    data = json.load(f)
+
+overall = data.get('overall', {})
+sizing = data.get('cluster_sizing', {})
+server = data.get('server_info', {})
+
+total = overall.get('total_entities_created', 0)
+rps = overall.get('overall_throughput_rps', 0)
+error_rate = overall.get('overall_error_rate_pct', 0)
+wall_clock = overall.get('total_wall_clock_s', 0)
+assessment = sizing.get('assessment', 'unknown')
+
+# Find max p95 and p99 across write entities
+max_p95 = 0
+max_p99 = 0
+entities = data.get('entities', {})
+for name, ent in entities.items():
+    if name.startswith('read_') or name.startswith('mixed_'):
+        continue
+    lat = ent.get('latency_ms', {})
+    p95 = lat.get('p95', 0)
+    p99 = lat.get('p99', 0)
+    if p95 > max_p95:
+        max_p95 = p95
+    if p99 > max_p99:
+        max_p99 = p99
+
+print(f'{total},{rps:.1f},{max_p95:.0f},{max_p99:.0f},{error_rate:.2f},{wall_clock:.0f},{assessment}')
+" 2>/dev/null || echo "N/A,0,0,0,0,0,error"
+}
+
+# ─── Extract Config Summary from JSON ───────────────────────────────────────────
+extract_config() {
+    local json_file="$1"
+    if [[ ! -f "$json_file" ]]; then
+        echo "N/A"
+        return
+    fi
+
+    python3 -c "
+import json
+with open('$json_file') as f:
+    data = json.load(f)
+cs = data.get('cluster_sizing', {}).get('config_summary', [])
+print('; '.join(cs) if cs else 'N/A')
+" 2>/dev/null || echo "N/A"
+}
+
+# ─── Extract Server Info ────────────────────────────────────────────────────────
+extract_server_info() {
+    local json_file="$1"
+    python3 -c "
+import json
+with open('$json_file') as f:
+    data = json.load(f)
+si = data.get('server_info', {})
+version = si.get('version', 'unknown')
+diag = data.get('diagnostics_before', {})
+jvm = diag.get('jvm', {})
+heap_max = jvm.get('heap_max_bytes', 0)
+heap_gb = heap_max / (1024**3) if heap_max else 0
+jetty = diag.get('jetty', {})
+max_threads = jetty.get('threads_max', 'unknown')
+db = diag.get('database', {})
+db_pool = db.get('pool_max', 'unknown')
+print(f'Version: {version}')
+print(f'Heap: {heap_gb:.1f}GB')
+print(f'Threads: {max_threads}')
+print(f'DB Pool: {db_pool}')
+" 2>/dev/null || echo "Version: unknown"
+}
+
+# ─── Break-point Detection ──────────────────────────────────────────────────────
+PREV_RPS_PER_ENTITY=""
+
+is_broken() {
+    local metrics="$1"
+    IFS=',' read -r total rps p95 p99 error_rate wall_clock assessment <<< "$metrics"
+
+    [[ "$assessment" == "undersized" ]] && return 0
+
+    if python3 -c "exit(0 if float('$error_rate') > 10 else 1)" 2>/dev/null; then
+        return 0
+    fi
+
+    if python3 -c "exit(0 if float('$p95') > 10000 else 1)" 2>/dev/null; then
+        return 0
+    fi
+
+    if [[ -n "$PREV_RPS_PER_ENTITY" && "$total" != "0" && "$total" != "N/A" ]]; then
+        if python3 -c "
+curr_rpe = float('$rps') / max(float('$total'), 1)
+prev_rpe = float('$PREV_RPS_PER_ENTITY')
+exit(0 if prev_rpe > 0 and curr_rpe / prev_rpe < 0.5 else 1)
+" 2>/dev/null; then
+            return 0
+        fi
+    fi
+
+    return 1
+}
+
+# ─── Results Tracking ──────────────────────────────────────────────────────────
+declare -a RESULT_LINES=()
+BREAK_SCALE=""
+BREAK_MODE=""
+BREAK_REASON=""
+
+add_result() {
+    local scale="$1" mode="$2" metrics="$3" config="$4"
+    RESULT_LINES+=("$scale,$mode,$metrics,$config")
+}
+
+# ─── Optional Ramp Test ────────────────────────────────────────────────────────
+if $RAMP; then
+    FIRST_SCALE="${SCALES[0]}"
+    log "${BOLD}Running ramp test at scale $FIRST_SCALE to find optimal workers...${NC}"
+    RAMP_OUTPUT="$OUTPUT_DIR/sizing-${FIRST_SCALE}-ramp.json"
+
+    RAMP_ARGS=(--server "$SERVER" --workers "$WORKERS" --scale "$FIRST_SCALE" --ramp --output "$RAMP_OUTPUT")
+    [[ -n "$ADMIN_PORT" ]] && RAMP_ARGS+=(--admin-port "$ADMIN_PORT")
+    [[ -n "$TOKEN" ]] && RAMP_ARGS+=(--token "$TOKEN")
+
+    if $SKIP_EXISTING && [[ -f "$RAMP_OUTPUT" ]]; then
+        log "Ramp output exists, skipping (--skip-existing)"
+    else
+        log "Running: perf-test.sh ${RAMP_ARGS[*]}"
+        if "$PERF_TEST" "${RAMP_ARGS[@]}" 2>&1 | tee -a "$LOG_FILE"; then
+            OPTIMAL=$(python3 -c "
+import json
+with open('$RAMP_OUTPUT') as f:
+    data = json.load(f)
+rt = data.get('ramp_test', {})
+print(rt.get('optimal_workers', $WORKERS))
+" 2>/dev/null || echo "$WORKERS")
+            log "${GREEN}Ramp test complete. Optimal workers: $OPTIMAL${NC}"
+            echo -e "\n${YELLOW}Ramp test suggests $OPTIMAL workers. Current setting: $WORKERS${NC}"
+            echo -e "${YELLOW}To use the suggested value, re-run with --workers $OPTIMAL${NC}\n"
+        else
+            log "${YELLOW}Ramp test failed, continuing with $WORKERS workers${NC}"
+        fi
+    fi
+fi
+
+# ─── Main Benchmark Loop ──────────────────────────────────────────────────────
+BENCHMARK_START=$(date +%s)
+TOTAL_TIERS=${#SCALES[@]}
+TIER_NUM=0
+BROKEN=false
+
+for scale in "${SCALES[@]}"; do
+    TIER_NUM=$((TIER_NUM + 1))
+
+    echo ""
+    echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+    echo -e "${BOLD}  Tier $TIER_NUM/$TOTAL_TIERS: scale=$scale${NC}"
+    echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+
+    for mode in "${MODE_LIST[@]}"; do
+        OUTPUT_FILE="$OUTPUT_DIR/sizing-${scale}-${mode}.json"
+        MODE_LABEL=$([[ "$mode" == "seq" ]] && echo "sequential" || echo "realistic")
+
+        log "  [$scale / $MODE_LABEL] Starting..."
+
+        if $SKIP_EXISTING && [[ -f "$OUTPUT_FILE" ]]; then
+            log "  [$scale / $MODE_LABEL] Output exists, skipping (--skip-existing)"
+            METRICS=$(extract_metrics "$OUTPUT_FILE")
+            CONFIG=$(extract_config "$OUTPUT_FILE")
+            add_result "$scale" "$mode" "$METRICS" "$CONFIG"
+
+            IFS=',' read -r _total _rps _p95 _p99 _err _wall _assess <<< "$METRICS"
+            echo -e "  ${GREEN}[SKIP]${NC} $scale/$MODE_LABEL: ${_total} entities, ${_rps} RPS, p95=${_p95}ms, err=${_err}%, ${_assess}"
+            continue
+        fi
+
+        PERF_ARGS=$(build_perf_args "$scale" "$mode" "$OUTPUT_FILE")
+        log "  Running: perf-test.sh $PERF_ARGS"
+
+        RUN_START=$(date +%s)
+        EXIT_CODE=0
+        # shellcheck disable=SC2086
+        "$PERF_TEST" $PERF_ARGS 2>&1 | tee -a "$LOG_FILE" || EXIT_CODE=$?
+        RUN_END=$(date +%s)
+        RUN_DURATION=$((RUN_END - RUN_START))
+
+        if [[ $EXIT_CODE -ne 0 ]]; then
+            log "  ${RED}[$scale / $MODE_LABEL] FAILED (exit code $EXIT_CODE) after ${RUN_DURATION}s${NC}"
+            add_result "$scale" "$mode" "FAILED,0,0,0,100,${RUN_DURATION},failed" "N/A"
+            BREAK_SCALE="$scale"
+            BREAK_MODE="$mode"
+            BREAK_REASON="perf-test.sh exited with code $EXIT_CODE"
+            BROKEN=true
+            break
+        fi
+
+        METRICS=$(extract_metrics "$OUTPUT_FILE")
+        CONFIG=$(extract_config "$OUTPUT_FILE")
+        add_result "$scale" "$mode" "$METRICS" "$CONFIG"
+
+        IFS=',' read -r _total _rps _p95 _p99 _err _wall _assess <<< "$METRICS"
+        log "  ${GREEN}[$scale / $MODE_LABEL] Done:${NC} ${_total} entities, ${_rps} RPS, p95=${_p95}ms, err=${_err}%, ${_assess} (${RUN_DURATION}s)"
+
+        if is_broken "$METRICS"; then
+            if [[ "$_assess" == "undersized" ]]; then
+                reason="Cluster assessed as undersized"
+            elif python3 -c "exit(0 if float('$_err') > 10 else 1)" 2>/dev/null; then
+                reason="Error rate ${_err}% exceeds 10% threshold"
+            elif python3 -c "exit(0 if float('$_p95') > 10000 else 1)" 2>/dev/null; then
+                reason="p95 latency ${_p95}ms exceeds 10s threshold"
+            else
+                reason="Throughput degraded >50% from previous tier"
+            fi
+
+            if $NO_BREAK; then
+                echo -e "\n  ${YELLOW}${BOLD}BREAK-POINT WOULD FIRE at $scale/$MODE_LABEL: $reason (--no-break, continuing)${NC}\n"
+                log "BREAK-POINT SUPPRESSED at $scale/$MODE_LABEL: $reason (--no-break)"
+                if [[ -z "$BREAK_SCALE" ]]; then
+                    BREAK_SCALE="$scale"
+                    BREAK_MODE="$mode"
+                    BREAK_REASON="$reason (continued with --no-break)"
+                fi
+            else
+                BREAK_SCALE="$scale"
+                BREAK_MODE="$mode"
+                BREAK_REASON="$reason"
+                echo -e "\n  ${RED}${BOLD}BREAK-POINT DETECTED at $scale/$MODE_LABEL: $BREAK_REASON${NC}\n"
+                log "BREAK-POINT DETECTED at $scale/$MODE_LABEL: $BREAK_REASON"
+                BROKEN=true
+                break
+            fi
+        fi
+
+        if [[ "$_total" != "0" && "$_total" != "N/A" ]]; then
+            PREV_RPS_PER_ENTITY=$(python3 -c "print(float('$_rps') / max(float('$_total'), 1))" 2>/dev/null || echo "")
+        fi
+    done
+
+    if $BROKEN; then
+        break
+    fi
+
+    # Tier comparison
+    if [[ ${#MODE_LIST[@]} -gt 1 ]]; then
+        SEQ_FILE="$OUTPUT_DIR/sizing-${scale}-seq.json"
+        REAL_FILE="$OUTPUT_DIR/sizing-${scale}-realistic.json"
+        if [[ -f "$SEQ_FILE" && -f "$REAL_FILE" ]]; then
+            echo ""
+            echo -e "  ${CYAN}Tier $scale comparison:${NC}"
+            SEQ_M=$(extract_metrics "$SEQ_FILE")
+            REAL_M=$(extract_metrics "$REAL_FILE")
+            IFS=',' read -r s_total s_rps s_p95 s_p99 s_err s_wall s_assess <<< "$SEQ_M"
+            IFS=',' read -r r_total r_rps r_p95 r_p99 r_err r_wall r_assess <<< "$REAL_M"
+            printf "  %-12s  %8s  %8s  %8s  %8s  %10s\n" "" "Entities" "RPS" "p95ms" "Errors%" "Assessment"
+            printf "  %-12s  %8s  %8s  %8s  %8s  %10s\n" "Sequential" "$s_total" "$s_rps" "$s_p95" "${s_err}%" "$s_assess"
+            printf "  %-12s  %8s  %8s  %8s  %8s  %10s\n" "Realistic" "$r_total" "$r_rps" "$r_p95" "${r_err}%" "$r_assess"
+            echo ""
+        fi
+    fi
+done
+
+BENCHMARK_END=$(date +%s)
+BENCHMARK_DURATION=$((BENCHMARK_END - BENCHMARK_START))
+
+# ─── Generate SIZING-SUMMARY.md ────────────────────────────────────────────────
+log "Generating $SUMMARY_MD..."
+
+# Try to get server info from the first available JSON
+SERVER_INFO=""
+for scale in "${SCALES[@]}"; do
+    for mode in "${MODE_LIST[@]}"; do
+        F="$OUTPUT_DIR/sizing-${scale}-${mode}.json"
+        if [[ -f "$F" ]]; then
+            SERVER_INFO=$(extract_server_info "$F")
+            break 2
+        fi
+    done
+done
+
+{
+    echo "# OpenMetadata Cluster Sizing Summary"
+    echo ""
+    echo "Generated: $(date '+%Y-%m-%d %H:%M:%S')"
+    echo "Total benchmark time: ${BENCHMARK_DURATION}s ($((BENCHMARK_DURATION / 60))m $((BENCHMARK_DURATION % 60))s)"
+    echo ""
+    echo "## Server Configuration"
+    echo ""
+    echo '```'
+    echo "Server: $SERVER"
+    echo "Workers: $WORKERS"
+    if [[ -n "$SERVER_INFO" ]]; then
+        echo "$SERVER_INFO"
+    fi
+    echo '```'
+    echo ""
+    echo "## Progressive Results"
+    echo ""
+    echo "| Scale | Mode | Entities | RPS | p95 (ms) | p99 (ms) | Errors % | Assessment | Duration |"
+    echo "|-------|------|----------|-----|----------|----------|----------|------------|----------|"
+
+    LAST_ADEQUATE_SCALE=""
+    LAST_ADEQUATE_CONFIG=""
+
+    for line in "${RESULT_LINES[@]}"; do
+        IFS=',' read -r scale mode total rps p95 p99 err wall assess config <<< "$line"
+        mode_label=$([[ "$mode" == "seq" ]] && echo "Sequential" || echo "Realistic")
+
+        # Format duration
+        if [[ "$wall" =~ ^[0-9]+$ ]]; then
+            duration="${wall}s"
+        else
+            duration="$wall"
+        fi
+
+        # Assessment emoji
+        case "$assess" in
+            adequate)  assess_fmt="$assess" ;;
+            marginal)  assess_fmt="$assess" ;;
+            undersized) assess_fmt="$assess" ;;
+            failed)    assess_fmt="FAILED" ;;
+            *)         assess_fmt="$assess" ;;
+        esac
+
+        echo "| $scale | $mode_label | $total | $rps | $p95 | $p99 | $err | $assess_fmt | $duration |"
+
+        if [[ "$assess" == "adequate" || "$assess" == "marginal" ]]; then
+            LAST_ADEQUATE_SCALE="$scale"
+            LAST_ADEQUATE_CONFIG="$config"
+        fi
+    done
+
+    echo ""
+
+    # Break-point section
+    if [[ -n "$BREAK_SCALE" ]]; then
+        echo "## Break-Point Detected"
+        echo ""
+        echo "**Scale:** $BREAK_SCALE  "
+        echo "**Mode:** $([[ "$BREAK_MODE" == "seq" ]] && echo "Sequential" || echo "Realistic")  "
+        echo "**Reason:** $BREAK_REASON"
+        echo ""
+        if [[ -n "$LAST_ADEQUATE_SCALE" ]]; then
+            echo "The cluster handled **$LAST_ADEQUATE_SCALE** scale adequately but broke at **$BREAK_SCALE**."
+            echo ""
+        fi
+    else
+        echo "## Result"
+        echo ""
+        echo "No break-point detected. The cluster handled all tested scales up to **${SCALES[${#SCALES[@]}-1]}**."
+        echo ""
+    fi
+
+    # Recommended configuration
+    if [[ -n "$LAST_ADEQUATE_CONFIG" && "$LAST_ADEQUATE_CONFIG" != "N/A" ]]; then
+        echo "## Recommended Configuration"
+        echo ""
+        echo "Based on the last adequate tier ($LAST_ADEQUATE_SCALE):"
+        echo ""
+        echo '```bash'
+        IFS=';' read -ra CONFIGS <<< "$LAST_ADEQUATE_CONFIG"
+        for cfg in "${CONFIGS[@]}"; do
+            cfg_trimmed=$(echo "$cfg" | xargs)
+            [[ -n "$cfg_trimmed" ]] && echo "export $cfg_trimmed"
+        done
+        echo '```'
+        echo ""
+    fi
+
+    # Sequential vs Realistic comparison
+    if [[ ${#MODE_LIST[@]} -gt 1 ]]; then
+        echo "## Sequential vs Realistic Comparison"
+        echo ""
+        echo "| Scale | Seq RPS | Real RPS | RPS Diff | Seq p95 | Real p95 | p95 Diff |"
+        echo "|-------|---------|----------|----------|---------|----------|----------|"
+
+        for scale in "${SCALES[@]}"; do
+            SEQ_FILE="$OUTPUT_DIR/sizing-${scale}-seq.json"
+            REAL_FILE="$OUTPUT_DIR/sizing-${scale}-realistic.json"
+            if [[ -f "$SEQ_FILE" && -f "$REAL_FILE" ]]; then
+                python3 -c "
+import json
+with open('$SEQ_FILE') as f:
+    seq = json.load(f)
+with open('$REAL_FILE') as f:
+    real = json.load(f)
+s_rps = seq.get('overall', {}).get('overall_throughput_rps', 0)
+r_rps = real.get('overall', {}).get('overall_throughput_rps', 0)
+rps_diff = ((r_rps - s_rps) / s_rps * 100) if s_rps > 0 else 0
+
+# Max p95
+def max_p95(data):
+    mx = 0
+    for name, ent in data.get('entities', {}).items():
+        if name.startswith('read_') or name.startswith('mixed_'):
+            continue
+        p = ent.get('latency_ms', {}).get('p95', 0)
+        if p > mx:
+            mx = p
+    return mx
+
+s_p95 = max_p95(seq)
+r_p95 = max_p95(real)
+p95_diff = ((r_p95 - s_p95) / s_p95 * 100) if s_p95 > 0 else 0
+
+print(f'| $scale | {s_rps:.1f} | {r_rps:.1f} | {rps_diff:+.1f}% | {s_p95:.0f} | {r_p95:.0f} | {p95_diff:+.1f}% |')
+" 2>/dev/null || true
+            fi
+        done
+
+        echo ""
+        echo "> **Sequential** runs entity types one at a time. **Realistic** runs all concurrently through a shared worker pool, exposing cross-entity contention."
+        echo ""
+    fi
+
+    # Individual tier details
+    echo "## Tier Details"
+    echo ""
+    for scale in "${SCALES[@]}"; do
+        for mode in "${MODE_LIST[@]}"; do
+            F="$OUTPUT_DIR/sizing-${scale}-${mode}.json"
+            [[ ! -f "$F" ]] && continue
+            mode_label=$([[ "$mode" == "seq" ]] && echo "Sequential" || echo "Realistic")
+            echo "### $scale ($mode_label)"
+            echo ""
+            python3 -c "
+import json
+with open('$F') as f:
+    data = json.load(f)
+
+entities = data.get('entities', {})
+print('| Entity | Count | RPS | p50 | p95 | p99 | Errors % |')
+print('|--------|-------|-----|-----|-----|-----|----------|')
+for name, ent in sorted(entities.items()):
+    count = ent.get('created', ent.get('total_requests', 0))
+    rps = ent.get('throughput_rps', 0)
+    lat = ent.get('latency_ms', {})
+    p50 = lat.get('p50', 0)
+    p95 = lat.get('p95', 0)
+    p99 = lat.get('p99', 0)
+    err = ent.get('error_rate_pct', 0)
+    print(f'| {name} | {count} | {rps:.1f} | {p50:.0f} | {p95:.0f} | {p99:.0f} | {err:.2f} |')
+
+# Sizing findings
+sizing = data.get('cluster_sizing', {})
+findings = sizing.get('findings', [])
+if findings:
+    print()
+    print('**Findings:**')
+    for f in findings:
+        print(f'- {f}')
+" 2>/dev/null || echo "_Failed to parse results_"
+            echo ""
+        done
+    done
+
+    echo "---"
+    echo ""
+    echo "Generated by \`benchmark-sizing.sh\` on $(date '+%Y-%m-%d %H:%M:%S')"
+
+} > "$SUMMARY_MD"
+
+# ─── Final Console Summary ─────────────────────────────────────────────────────
+echo ""
+echo -e "${BOLD}══════════════════════════════════════════════════════════════════${NC}"
+echo -e "${BOLD}  SIZING BENCHMARK COMPLETE${NC}"
+echo -e "${BOLD}══════════════════════════════════════════════════════════════════${NC}"
+echo ""
+
+printf "  %-8s  %-12s  %8s  %8s  %8s  %8s  %10s\n" "Scale" "Mode" "Entities" "RPS" "p95ms" "Errors%" "Assessment"
+printf "  %-8s  %-12s  %8s  %8s  %8s  %8s  %10s\n" "────────" "────────────" "────────" "────────" "────────" "────────" "──────────"
+
+for line in "${RESULT_LINES[@]}"; do
+    IFS=',' read -r scale mode total rps p95 p99 err wall assess config <<< "$line"
+    mode_label=$([[ "$mode" == "seq" ]] && echo "Sequential" || echo "Realistic")
+
+    case "$assess" in
+        adequate)   color="$GREEN" ;;
+        marginal)   color="$YELLOW" ;;
+        undersized) color="$RED" ;;
+        failed)     color="$RED" ;;
+        *)          color="$NC" ;;
+    esac
+
+    printf "  %-8s  %-12s  %8s  %8s  %8s  %8s  ${color}%10s${NC}\n" \
+        "$scale" "$mode_label" "$total" "$rps" "$p95" "${err}%" "$assess"
+done
+
+echo ""
+
+if [[ -n "$BREAK_SCALE" ]]; then
+    echo -e "  ${RED}${BOLD}Break-point: $BREAK_SCALE ($BREAK_MODE) — $BREAK_REASON${NC}"
+    if [[ -n "$LAST_ADEQUATE_SCALE" ]]; then
+        echo -e "  ${GREEN}Last adequate scale: $LAST_ADEQUATE_SCALE${NC}"
+    fi
+else
+    echo -e "  ${GREEN}No break-point detected — cluster handled all scales through ${SCALES[${#SCALES[@]}-1]}${NC}"
+fi
+
+echo ""
+echo -e "  Total time: ${BENCHMARK_DURATION}s ($((BENCHMARK_DURATION / 60))m $((BENCHMARK_DURATION % 60))s)"
+echo -e "  Summary:    ${CYAN}$SUMMARY_MD${NC}"
+echo -e "  Log:        ${CYAN}$LOG_FILE${NC}"
+echo ""

bin/distributed-test/scripts/gc-reindex-report.sh

@@ -0,0 +1,370 @@
+#!/bin/bash
+# Captures GC pause statistics during a reindex run and compares them to a saved baseline.
+#
+# Usage:
+#   ./gc-reindex-report.sh --token TOKEN [--server URL] [--gclog PATH] [--save-baseline]
+#
+# What it does:
+#   1. Reads the server's GC log (or polls /api/v1/system/metrics for JVM stats if no GC log)
+#   2. Triggers a full reindex
+#   3. Polls until the reindex finishes
+#   4. Summarises: total STW pause time, max pause, p99 pause, probe failures, throughput
+#   5. Optionally saves the result as a baseline for future comparison
+#
+# GC log note:
+#   Start the server with:  -Xlog:gc*:file=/tmp/om-gc.log:time,uptime,tags:filecount=5,filesize=20m
+#   Then pass --gclog /tmp/om-gc.log to this script.
+#
+# Without --gclog the script uses the Prometheus /metrics endpoint to estimate GC pressure.
+
+set -euo pipefail
+
+SERVER_URL="http://localhost:8585"
+TOKEN=""
+GC_LOG=""
+SAVE_BASELINE=false
+BASELINE_FILE="$(dirname "$0")/../gc-baseline.json"
+POLL_INTERVAL=10   # seconds between status polls
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --server)       SERVER_URL="$2";  shift 2 ;;
+    --token)        TOKEN="$2";       shift 2 ;;
+    --gclog)        GC_LOG="$2";      shift 2 ;;
+    --save-baseline) SAVE_BASELINE=true; shift ;;
+    --baseline)     BASELINE_FILE="$2"; shift 2 ;;
+    -h|--help)
+      sed -n '2,20p' "$0" | sed 's/^# //;s/^#//'
+      exit 0
+      ;;
+    *) echo "Unknown arg: $1"; exit 1 ;;
+  esac
+done
+
+if [[ -z "$TOKEN" ]]; then
+  echo "ERROR: --token is required"; exit 1
+fi
+
+AUTH_HEADER="Authorization: Bearer $TOKEN"
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+
+jq_required() {
+  if ! command -v jq &>/dev/null; then
+    echo "ERROR: jq is required. Install with: brew install jq"; exit 1
+  fi
+}
+
+curl_json() {
+  curl -sf -H "$AUTH_HEADER" -H "Content-Type: application/json" "$@"
+}
+
+# ── capture GC snapshot via /metrics ─────────────────────────────────────────
+
+gc_metrics_snapshot() {
+  local label="$1"
+  local result
+  # Try Prometheus endpoint first (if enabled)
+  if result=$(curl -sf -H "$AUTH_HEADER" "${SERVER_URL}/api/v1/system/metrics" 2>/dev/null); then
+    local gc_count gc_time
+    gc_count=$(echo "$result" | grep '^jvm_gc_pause_seconds_count' | awk '{s+=$NF} END{print int(s)}' || echo 0)
+    gc_time=$(echo  "$result" | grep '^jvm_gc_pause_seconds_sum'   | awk '{s+=$NF} END{printf "%.3f", s}' || echo "0.000")
+    echo "${label}:gc_count=${gc_count},gc_time_s=${gc_time}"
+  else
+    echo "${label}:gc_count=N/A,gc_time_s=N/A"
+  fi
+}
+
+# ── parse GC log (G1GC format) ────────────────────────────────────────────────
+
+parse_gc_log() {
+  local logfile="$1"
+  if [[ ! -f "$logfile" ]]; then
+    echo "GC_LOG_NOT_FOUND"; return
+  fi
+
+  python3 - "$logfile" <<'PYEOF'
+import re, sys, statistics
+
+log_file = sys.argv[1]
+pauses = []
+
+# Match G1GC pause lines: [0.123s][info][gc] GC(0) Pause ... 12.345ms
+pause_re = re.compile(r'Pause (?:Young|Full|Initial Mark|Remark|Cleanup)[^0-9]*(\d+\.\d+)ms', re.IGNORECASE)
+# Also match: GC(N) Pause ... Nms
+generic_re = re.compile(r'\bPause\b.*?(\d+\.\d+)ms')
+
+with open(log_file) as f:
+    for line in f:
+        m = pause_re.search(line) or generic_re.search(line)
+        if m:
+            pauses.append(float(m.group(1)))
+
+if not pauses:
+    print("no_pauses_found")
+    sys.exit(0)
+
+pauses.sort()
+p99_idx = max(0, int(len(pauses) * 0.99) - 1)
+print(f"count={len(pauses)}")
+print(f"total_ms={sum(pauses):.1f}")
+print(f"max_ms={max(pauses):.1f}")
+print(f"p99_ms={pauses[p99_idx]:.1f}")
+print(f"mean_ms={statistics.mean(pauses):.1f}")
+PYEOF
+}
+
+# ── probe health once ─────────────────────────────────────────────────────────
+
+probe_health() {
+  local http_code
+  http_code=$(curl -o /dev/null -sf -w "%{http_code}" \
+    --max-time 1 \
+    -H "$AUTH_HEADER" \
+    "${SERVER_URL}/api/v1/system/health" 2>/dev/null || echo "000")
+  echo "$http_code"
+}
+
+# ── trigger reindex ───────────────────────────────────────────────────────────
+
+trigger_reindex() {
+  local payload='{"recreateIndex":false}'
+  curl_json -X POST \
+    "${SERVER_URL}/api/v1/apps/trigger/SearchIndexingApplication" \
+    -d "$payload" > /dev/null 2>&1 || true
+  echo "Reindex triggered"
+}
+
+# ── poll reindex status ───────────────────────────────────────────────────────
+
+STATS_FILE="/tmp/om-reindex-stats-$$.txt"
+
+poll_until_done() {
+  local start_ts
+  start_ts=$(date +%s)
+  local probe_failures=0
+  local probe_total=0
+  local max_latency_ms=0
+  local probes_over_1000=0
+  local consecutive_slow=0
+  local max_consecutive_slow=0
+  local last_probe_ts=0
+
+  echo ""
+  echo "Polling reindex status every ${POLL_INTERVAL}s ..."
+  echo "  (probing /system/health every 10s — k8s periodSeconds=10, failureThreshold=3)"
+  echo ""
+
+  while true; do
+    sleep "$POLL_INTERVAL"
+    local now
+    now=$(date +%s)
+
+    # Probe health every 10s (k8s default periodSeconds)
+    if (( now - last_probe_ts >= 10 )); then
+      local t0 code latency
+      t0=$(python3 -c "import time; print(int(time.time()*1000))")
+      code=$(probe_health)
+      latency=$(( $(python3 -c "import time; print(int(time.time()*1000))") - t0 ))
+      probe_total=$(( probe_total + 1 ))
+      last_probe_ts=$now
+
+      if (( latency > max_latency_ms )); then max_latency_ms=$latency; fi
+      if (( latency > 1000 )); then
+        probes_over_1000=$(( probes_over_1000 + 1 ))
+        consecutive_slow=$(( consecutive_slow + 1 ))
+        if (( consecutive_slow > max_consecutive_slow )); then
+          max_consecutive_slow=$consecutive_slow
+        fi
+      else
+        consecutive_slow=0
+      fi
+
+      if [[ "$code" != "200" ]]; then
+        probe_failures=$(( probe_failures + 1 ))
+        echo "  [$(date +%H:%M:%S)] PROBE FAIL  code=$code latency=${latency}ms  consecutive=${consecutive_slow}" >&2
+      elif (( latency > 1000 )); then
+        echo "  [$(date +%H:%M:%S)] SLOW PROBE  latency=${latency}ms  *** >1000ms (k8s timeout risk) consecutive=${consecutive_slow}" >&2
+      else
+        echo "  [$(date +%H:%M:%S)] probe ok    latency=${latency}ms" >&2
+      fi
+
+      if (( max_consecutive_slow >= 3 )); then
+        echo "  *** WARNING: ${max_consecutive_slow} consecutive slow probes — k8s WOULD RESTART POD ***" >&2
+      fi
+    fi
+
+    # Check app status
+    local status_json
+    status_json=$(curl_json "${SERVER_URL}/api/v1/apps/name/SearchIndexingApplication/status?offset=0&limit=1" 2>/dev/null || echo '{}')
+    local app_status
+    app_status=$(echo "$status_json" | python3 -c "
+import json,sys
+try:
+  d=json.load(sys.stdin)
+  runs=d.get('data',[])
+  if runs: print(runs[0].get('status','unknown'))
+  else: print('pending')
+except: print('unknown')
+" 2>/dev/null || echo "unknown")
+
+    if [[ "$app_status" == "success" || "$app_status" == "failed" ]]; then
+      local elapsed=$(( now - start_ts ))
+      echo "" >&2
+      echo "  Reindex finished: status=$app_status  elapsed=${elapsed}s" >&2
+      # Write structured results to stats file (not stdout)
+      {
+        echo "PROBE_FAILURES=$probe_failures"
+        echo "PROBE_TOTAL=$probe_total"
+        echo "MAX_LATENCY_MS=$max_latency_ms"
+        echo "PROBES_OVER_1000=$probes_over_1000"
+        echo "MAX_CONSECUTIVE_SLOW=$max_consecutive_slow"
+        echo "ELAPSED_S=$elapsed"
+      } > "$STATS_FILE"
+      return
+    fi
+  done
+}
+
+# ── main ──────────────────────────────────────────────────────────────────────
+
+jq_required
+
+echo "═══════════════════════════════════════════════════════════════"
+echo "  GC / Reindex Report  —  $(date)"
+echo "  Server: $SERVER_URL"
+echo "═══════════════════════════════════════════════════════════════"
+
+# Snapshot before
+METRICS_BEFORE=$(gc_metrics_snapshot "before")
+GC_LOG_LINES_BEFORE=0
+if [[ -n "$GC_LOG" && -f "$GC_LOG" ]]; then
+  GC_LOG_LINES_BEFORE=$(wc -l < "$GC_LOG")
+  echo "GC log: $GC_LOG  (current lines: $GC_LOG_LINES_BEFORE)"
+fi
+
+trigger_reindex
+
+# Poll — probe lines print live to stderr, structured results written to STATS_FILE
+poll_until_done
+
+PROBE_FAILURES=$(grep '^PROBE_FAILURES='       "$STATS_FILE" | cut -d= -f2)
+PROBE_TOTAL=$(grep    '^PROBE_TOTAL='          "$STATS_FILE" | cut -d= -f2)
+MAX_LATENCY=$(grep    '^MAX_LATENCY_MS='       "$STATS_FILE" | cut -d= -f2)
+PROBES_OVER_1000=$(grep '^PROBES_OVER_1000='   "$STATS_FILE" | cut -d= -f2)
+MAX_CONSECUTIVE=$(grep '^MAX_CONSECUTIVE_SLOW=' "$STATS_FILE" | cut -d= -f2)
+ELAPSED_S=$(grep      '^ELAPSED_S='            "$STATS_FILE" | cut -d= -f2)
+rm -f "$STATS_FILE"
+
+# Snapshot after
+METRICS_AFTER=$(gc_metrics_snapshot "after")
+
+# GC log delta
+GC_LOG_STATS=""
+if [[ -n "$GC_LOG" && -f "$GC_LOG" ]]; then
+  # Extract only lines added since reindex started
+  TMPLOG="/tmp/om-gc-run-${TIMESTAMP}.log"
+  tail -n "+${GC_LOG_LINES_BEFORE}" "$GC_LOG" > "$TMPLOG"
+  GC_LOG_STATS=$(parse_gc_log "$TMPLOG")
+  rm -f "$TMPLOG"
+fi
+
+# Parse metrics delta
+GC_COUNT_BEFORE=$(echo "$METRICS_BEFORE" | sed 's/.*gc_count=\([^,]*\).*/\1/')
+GC_TIME_BEFORE=$(echo  "$METRICS_BEFORE" | sed 's/.*gc_time_s=\([^,]*\).*/\1/')
+GC_COUNT_AFTER=$(echo  "$METRICS_AFTER"  | sed 's/.*gc_count=\([^,]*\).*/\1/')
+GC_TIME_AFTER=$(echo   "$METRICS_AFTER"  | sed 's/.*gc_time_s=\([^,]*\).*/\1/')
+
+# Compute deltas if numeric
+GC_COUNT_DELTA="N/A"
+GC_TIME_DELTA="N/A"
+if [[ "$GC_COUNT_BEFORE" =~ ^[0-9]+$ && "$GC_COUNT_AFTER" =~ ^[0-9]+$ ]]; then
+  GC_COUNT_DELTA=$(( GC_COUNT_AFTER - GC_COUNT_BEFORE ))
+fi
+if [[ "$GC_TIME_BEFORE" =~ ^[0-9.]+$ && "$GC_TIME_AFTER" =~ ^[0-9.]+$ ]]; then
+  GC_TIME_DELTA=$(python3 -c "print(f'{float('$GC_TIME_AFTER') - float('$GC_TIME_BEFORE'):.3f}')")
+fi
+
+echo ""
+echo "══════════════════ RESULTS ════════════════════════════════════"
+echo ""
+printf "  %-30s  %s\n" "Reindex duration"          "${ELAPSED_S}s"
+printf "  %-30s  %s\n" "Health probe total"        "${PROBE_TOTAL}"
+printf "  %-30s  %s\n" "Probe failures (non-200)"  "${PROBE_FAILURES:-0}  $([ "${PROBE_FAILURES:-0}" -eq 0 ] && echo '✓ none' || echo '⚠ k8s would have restarted pod')"
+printf "  %-30s  %s\n" "Probes >1000ms"            "${PROBES_OVER_1000:-0}  $([ "${PROBES_OVER_1000:-0}" -eq 0 ] && echo '✓ none' || echo '⚠ GC pressure detected')"
+printf "  %-30s  %s\n" "Max consecutive >1000ms"   "${MAX_CONSECUTIVE:-0}  $([ "${MAX_CONSECUTIVE:-0}" -lt 3 ] && echo '✓ ok' || echo '*** k8s WOULD RESTART POD ***')"
+printf "  %-30s  %s\n" "Max probe latency"         "${MAX_LATENCY:-0}ms  $([ "${MAX_LATENCY:-0}" -lt 1000 ] && echo '✓ ok' || echo '⚠ high')"
+echo ""
+printf "  %-30s  %s\n" "GC collections (delta)"  "$GC_COUNT_DELTA"
+printf "  %-30s  %s\n" "GC pause time (delta)"   "${GC_TIME_DELTA}s"
+echo ""
+
+if [[ -n "$GC_LOG_STATS" && "$GC_LOG_STATS" != "no_pauses_found" && "$GC_LOG_STATS" != "GC_LOG_NOT_FOUND" ]]; then
+  echo "  GC log pause breakdown:"
+  echo "$GC_LOG_STATS" | sed 's/^/    /'
+  echo ""
+fi
+
+# Build JSON result
+RESULT_JSON=$(python3 -c "
+import json, sys
+result = {
+  'timestamp': '$(date -u +%Y-%m-%dT%H:%M:%SZ)',
+  'server': '$SERVER_URL',
+  'elapsed_s': int('${ELAPSED_S:-0}'),
+  'probe_total': int('${PROBE_TOTAL:-0}'),
+  'probe_failures': int('${PROBE_FAILURES:-0}'),
+  'probes_over_1000ms': int('${PROBES_OVER_1000:-0}'),
+  'max_consecutive_slow': int('${MAX_CONSECUTIVE:-0}'),
+  'max_probe_latency_ms': int('${MAX_LATENCY:-0}'),
+  'gc_collections_delta': '${GC_COUNT_DELTA}',
+  'gc_pause_time_delta_s': '${GC_TIME_DELTA}',
+  'gc_log_stats': '${GC_LOG_STATS:-none}'
+}
+print(json.dumps(result, indent=2))
+")
+
+# Compare to baseline
+if [[ -f "$BASELINE_FILE" ]]; then
+  echo "  Comparison to baseline ($BASELINE_FILE):"
+  python3 - "$BASELINE_FILE" <<PYEOF
+import json, sys
+
+with open(sys.argv[1]) as f:
+    baseline = json.load(f)
+
+current = $RESULT_JSON
+
+def pct(new, old):
+    if old == 0: return '+∞%'
+    delta = new - old
+    return f'{delta/old*100:+.1f}%'
+
+print(f"    {'Metric':<30}  {'Baseline':>12}  {'Current':>12}  {'Change':>10}")
+print(f"    {'-'*30}  {'-'*12}  {'-'*12}  {'-'*10}")
+for k in ['elapsed_s','probe_failures','probes_over_1000ms','max_consecutive_slow','max_probe_latency_ms']:
+    b = baseline.get(k, 'N/A')
+    c = current.get(k, 'N/A')
+    change = pct(c, b) if isinstance(b, (int,float)) and isinstance(c, (int,float)) else 'N/A'
+    print(f"    {k:<30}  {str(b):>12}  {str(c):>12}  {change:>10}")
+PYEOF
+  echo ""
+fi
+
+# Save result
+OUT_FILE="gc-report-${TIMESTAMP}.json"
+echo "$RESULT_JSON" > "$OUT_FILE"
+echo "  Report saved to: $OUT_FILE"
+
+if [[ "$SAVE_BASELINE" == "true" ]]; then
+  cp "$OUT_FILE" "$BASELINE_FILE"
+  echo "  Saved as baseline: $BASELINE_FILE"
+fi
+
+echo ""
+echo "══════════════════════════════════════════════════════════════"
+echo ""
+echo "  Tip: Run once with --save-baseline to record a baseline, then"
+echo "       run again after code changes to see the delta."
+echo ""

bin/distributed-test/scripts/logs.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# View aggregated logs from all OM servers with color coding
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+cd "$PROJECT_DIR"
+
+# Colors for different servers
+COLOR_SERVER1="\033[1;32m"  # Green
+COLOR_SERVER2="\033[1;34m"  # Blue
+COLOR_SERVER3="\033[1;35m"  # Magenta
+COLOR_MYSQL="\033[1;33m"    # Yellow
+COLOR_OPENSEARCH="\033[1;36m"  # Cyan
+COLOR_RESET="\033[0m"
+
+# Default values
+FOLLOW=false
+SERVER_FILTER=""
+GREP_PATTERN=""
+TAIL_LINES=100
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -f|--follow)
+            FOLLOW=true
+            shift
+            ;;
+        --server)
+            SERVER_FILTER="$2"
+            shift 2
+            ;;
+        --grep)
+            GREP_PATTERN="$2"
+            shift 2
+            ;;
+        --tail)
+            TAIL_LINES="$2"
+            shift 2
+            ;;
+        -h|--help)
+            echo "Usage: $0 [OPTIONS]"
+            echo ""
+            echo "Options:"
+            echo "  -f, --follow       Follow log output (like tail -f)"
+            echo "  --server NUM       Filter to specific server (1, 2, or 3)"
+            echo "  --grep PATTERN     Filter logs by pattern"
+            echo "  --tail NUM         Number of lines to show (default: 100)"
+            echo "  -h, --help         Show this help message"
+            echo ""
+            echo "Examples:"
+            echo "  $0 -f                           # Follow all server logs"
+            echo "  $0 --server 1 -f                # Follow only server 1 logs"
+            echo "  $0 --grep 'partition' -f        # Follow logs containing 'partition'"
+            echo "  $0 --grep 'SearchIndex' --tail 500"
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+done
+
+# Build the docker compose logs command
+COMPOSE_ARGS=""
+
+if [ "$FOLLOW" == "true" ]; then
+    COMPOSE_ARGS="$COMPOSE_ARGS -f"
+fi
+
+COMPOSE_ARGS="$COMPOSE_ARGS --tail=$TAIL_LINES"
+
+# Determine which services to show
+SERVICES=""
+if [ -n "$SERVER_FILTER" ]; then
+    case $SERVER_FILTER in
+        1) SERVICES="openmetadata-server-1" ;;
+        2) SERVICES="openmetadata-server-2" ;;
+        3) SERVICES="openmetadata-server-3" ;;
+        mysql) SERVICES="mysql" ;;
+        opensearch) SERVICES="opensearch" ;;
+        *)
+            echo "Invalid server filter: $SERVER_FILTER"
+            echo "Use 1, 2, 3, mysql, or opensearch"
+            exit 1
+            ;;
+    esac
+else
+    SERVICES="openmetadata-server-1 openmetadata-server-2 openmetadata-server-3"
+fi
+
+# Function to colorize output
+colorize_logs() {
+    while IFS= read -r line; do
+        if [[ $line == *"openmetadata-server-1"* ]] || [[ $line == *"om-server-1"* ]]; then
+            echo -e "${COLOR_SERVER1}[SERVER-1]${COLOR_RESET} $line"
+        elif [[ $line == *"openmetadata-server-2"* ]] || [[ $line == *"om-server-2"* ]]; then
+            echo -e "${COLOR_SERVER2}[SERVER-2]${COLOR_RESET} $line"
+        elif [[ $line == *"openmetadata-server-3"* ]] || [[ $line == *"om-server-3"* ]]; then
+            echo -e "${COLOR_SERVER3}[SERVER-3]${COLOR_RESET} $line"
+        elif [[ $line == *"mysql"* ]]; then
+            echo -e "${COLOR_MYSQL}[MYSQL]${COLOR_RESET} $line"
+        elif [[ $line == *"opensearch"* ]]; then
+            echo -e "${COLOR_OPENSEARCH}[OPENSEARCH]${COLOR_RESET} $line"
+        else
+            echo "$line"
+        fi
+    done
+}
+
+echo "======================================"
+echo "Distributed Test Logs"
+echo "======================================"
+echo "Services: $SERVICES"
+if [ -n "$GREP_PATTERN" ]; then
+    echo "Filter: $GREP_PATTERN"
+fi
+echo "--------------------------------------"
+echo ""
+
+# Run the logs command with optional grep filter
+if [ -n "$GREP_PATTERN" ]; then
+    docker compose logs $COMPOSE_ARGS $SERVICES 2>&1 | grep --line-buffered -i "$GREP_PATTERN" | colorize_logs
+else
+    docker compose logs $COMPOSE_ARGS $SERVICES 2>&1 | colorize_logs
+fi

bin/distributed-test/scripts/start.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+# Start the distributed test environment
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+ROOT_DIR="$(cd "$PROJECT_DIR/../../.." && pwd)"
+
+cd "$PROJECT_DIR"
+
+# Load environment variables (handle values with spaces properly)
+if [ -f .env ]; then
+    set -a
+    source .env
+    set +a
+fi
+
+echo "======================================"
+echo "Distributed Search Indexing Test Setup"
+echo "======================================"
+echo ""
+echo "Configuration:"
+echo "  - MySQL Port: ${MYSQL_PORT:-3306}"
+echo "  - OpenSearch Port: ${OPENSEARCH_PORT:-9200}"
+echo "  - Server 1: http://localhost:8585"
+echo "  - Server 2: http://localhost:8587"
+echo "  - Server 3: http://localhost:8589"
+echo ""
+
+# Parse arguments
+BUILD_FLAG=""
+SKIP_MVN=false
+for arg in "$@"; do
+    case $arg in
+        --build|-b)
+            BUILD_FLAG="--build"
+            ;;
+        --skip-mvn|-s)
+            SKIP_MVN=true
+            ;;
+    esac
+done
+
+# Check if distribution exists, if not build with Maven
+DIST_TAR=$(find "$ROOT_DIR/openmetadata-dist/target" -name "openmetadata-*.tar.gz" 2>/dev/null | head -1)
+if [ -z "$DIST_TAR" ] && [ "$SKIP_MVN" != "true" ]; then
+    echo "OpenMetadata distribution not found. Building with Maven..."
+    echo "This may take several minutes on first run."
+    echo ""
+    cd "$ROOT_DIR"
+    mvn clean install -DskipTests -Pquickstart -pl '!openmetadata-ui' -am
+    cd "$PROJECT_DIR"
+    BUILD_FLAG="--build"
+    echo ""
+    echo "Maven build complete."
+elif [ "$SKIP_MVN" == "true" ]; then
+    echo "Skipping Maven build (--skip-mvn flag)"
+fi
+
+if [ -n "$BUILD_FLAG" ]; then
+    echo "Building Docker images..."
+fi
+
+# Start the services
+echo "Starting services..."
+docker compose up -d $BUILD_FLAG
+
+echo ""
+echo "Waiting for services to be healthy..."
+
+# Wait for MySQL
+echo -n "  MySQL: "
+until docker compose exec -T mysql mysqladmin ping -h localhost -uroot -p${MYSQL_ROOT_PASSWORD:-password} --silent 2>/dev/null; do
+    echo -n "."
+    sleep 2
+done
+echo " Ready"
+
+# Wait for OpenSearch
+echo -n "  OpenSearch: "
+until curl -s http://localhost:${OPENSEARCH_PORT:-9200}/_cluster/health 2>/dev/null | grep -qE '"status":"(green|yellow)"'; do
+    echo -n "."
+    sleep 2
+done
+echo " Ready"
+
+# Wait for each OM server
+for server_num in 1 2 3; do
+    case $server_num in
+        1) port=8586 ;;
+        2) port=8588 ;;
+        3) port=8590 ;;
+    esac
+
+    echo -n "  Server $server_num (admin port $port): "
+    timeout=120
+    elapsed=0
+    until curl -s "http://localhost:$port/healthcheck" >/dev/null 2>&1; do
+        echo -n "."
+        sleep 3
+        elapsed=$((elapsed + 3))
+        if [ $elapsed -ge $timeout ]; then
+            echo " Timeout waiting for server $server_num"
+            echo "Check logs with: ./scripts/logs.sh -f --server $server_num"
+            exit 1
+        fi
+    done
+    echo " Ready"
+done
+
+echo ""
+echo "======================================"
+echo "All services are up and running!"
+echo "======================================"
+echo ""
+echo "Next steps:"
+echo "  1. Load test data:     ./scripts/perf-test.sh --tables 10000"
+echo "  2. Trigger reindexing: ./scripts/trigger-reindex.sh"
+echo "  3. Watch logs:         ./scripts/logs.sh -f"
+echo ""
+echo "Server endpoints:"
+echo "  - Server 1: http://localhost:8585 (trigger reindex here)"
+echo "  - Server 2: http://localhost:8587"
+echo "  - Server 3: http://localhost:8589"
+echo ""

bin/distributed-test/scripts/stop.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Stop the distributed test environment
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+cd "$PROJECT_DIR"
+
+echo "Stopping distributed test environment..."
+
+# Check for cleanup flag
+if [ "$1" == "--clean" ] || [ "$1" == "-c" ]; then
+    echo "Removing containers and volumes..."
+    docker compose down -v --remove-orphans
+    echo "Cleaned up all containers and volumes."
+else
+    docker compose down --remove-orphans
+    echo "Containers stopped. Data volumes preserved."
+    echo "Use --clean or -c to also remove volumes."
+fi
+
+echo "Done."

bin/distributed-test/scripts/trigger-reindex.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+# Trigger reindexing via REST API
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# Default values
+SERVER_URL="http://localhost:8585"
+ENTITY_TYPES=""
+BATCH_SIZE=100
+PARTITION_SIZE=10000
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --server)
+            SERVER_URL="$2"
+            shift 2
+            ;;
+        --entities)
+            ENTITY_TYPES="$2"
+            shift 2
+            ;;
+        --batch-size)
+            BATCH_SIZE="$2"
+            shift 2
+            ;;
+        --partition-size)
+            PARTITION_SIZE="$2"
+            shift 2
+            ;;
+        -h|--help)
+            echo "Usage: $0 [OPTIONS]"
+            echo ""
+            echo "Options:"
+            echo "  --server URL          Target server URL (default: http://localhost:8585)"
+            echo "  --entities TYPES      Comma-separated entity types to reindex (default: all)"
+            echo "  --batch-size NUM      Batch size for indexing (default: 100)"
+            echo "  --partition-size NUM  Partition size for distributed indexing (default: 10000, range: 1000-50000)"
+            echo "                        Smaller values = more partitions = better distribution across servers"
+            echo "  -h, --help            Show this help message"
+            echo ""
+            echo "Examples:"
+            echo "  $0                                    # Reindex all on server 1"
+            echo "  $0 --server http://localhost:8587    # Trigger on server 2"
+            echo "  $0 --entities table,dashboard        # Reindex only tables and dashboards"
+            echo "  $0 --partition-size 2000             # Use smaller partitions for better distribution"
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+done
+
+echo "======================================"
+echo "Triggering Search Reindexing"
+echo "======================================"
+echo "Server: $SERVER_URL"
+echo "Indexing mode: staged indexes with alias promotion"
+echo "Batch size: $BATCH_SIZE"
+echo "Partition size: $PARTITION_SIZE"
+if [ -n "$ENTITY_TYPES" ]; then
+    echo "Entity types: $ENTITY_TYPES"
+else
+    echo "Entity types: all"
+fi
+echo ""
+
+# First, get a JWT token (using admin user)
+echo "Authenticating..."
+TOKEN_RESPONSE=$(curl -s -X POST "${SERVER_URL}/api/v1/users/login" \
+    -H "Content-Type: application/json" \
+    -d '{"email": "admin@open-metadata.org", "password": "admin"}')
+
+ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | grep -o '"accessToken":"[^"]*"' | cut -d'"' -f4)
+
+if [ -z "$ACCESS_TOKEN" ]; then
+    echo "Failed to authenticate. Response: $TOKEN_RESPONSE"
+    echo ""
+    echo "Trying with basic auth token..."
+    # Try to get the bot token instead
+    ACCESS_TOKEN="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJvcGVuLW1ldGFkYXRhLm9yZyIsInN1YiI6ImFkbWluIiwiZW1haWwiOiJhZG1pbkBvcGVuLW1ldGFkYXRhLm9yZyIsImlzQm90IjpmYWxzZSwiaXNBZG1pbiI6dHJ1ZSwidG9rZW5UeXBlIjoiUEVSU09OQUxfQUNDRVNTIiwiaWF0IjoxNjk1MjM3MzY2LCJleHAiOjE2OTc4MjkzNjZ9.placeholder"
+fi
+
+echo "Authenticated successfully."
+echo ""
+
+# Build entities array
+if [ -n "$ENTITY_TYPES" ]; then
+    # Convert comma-separated to JSON array
+    ENTITIES_JSON=$(echo "$ENTITY_TYPES" | sed 's/,/","/g' | sed 's/^/["/' | sed 's/$/"]/')
+else
+    ENTITIES_JSON='["all"]'
+fi
+
+REQUEST_BODY=$(cat <<EOF
+{
+  "entities": $ENTITIES_JSON,
+  "batchSize": $BATCH_SIZE,
+  "partitionSize": $PARTITION_SIZE,
+  "runMode": "BATCH"
+}
+EOF
+)
+
+echo "Request body:"
+echo "$REQUEST_BODY"
+echo ""
+
+# Trigger the reindex
+echo "Triggering reindex..."
+RESPONSE=$(curl -s -X POST "${SERVER_URL}/api/v1/apps/name/SearchIndexingApplication/trigger" \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -d "$REQUEST_BODY")
+
+echo "Response:"
+echo "$RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$RESPONSE"
+echo ""
+
+# Check job status
+echo "Checking job status..."
+sleep 2
+
+STATUS_RESPONSE=$(curl -s -X GET "${SERVER_URL}/api/v1/apps/name/SearchIndexingApplication/status" \
+    -H "Authorization: Bearer $ACCESS_TOKEN")
+
+echo "Job status:"
+echo "$STATUS_RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$STATUS_RESPONSE"
+echo ""
+
+echo "======================================"
+echo "Reindex triggered!"
+echo "======================================"
+echo ""
+echo "Monitor progress with:"
+echo "  ./scripts/logs.sh -f --grep 'partition\\|SearchIndex\\|Distributed'"
+echo ""
+echo "Check job status:"
+echo "  curl -s '${SERVER_URL}/api/v1/apps/name/SearchIndexingApplication/status'"
+echo ""