wehub/open-metadata--openmetadata
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Details
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Details
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Details
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Details
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Details
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Details
Java Checkstyle / java-checkstyle (push) Has been cancelled
Details
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Details
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:35:45 +08:00
commit bf2343b7e4
16049 changed files with 3531137 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/debug-latency.sh
Executable
+439
View File
@@ -0,0 +1,439 @@
#!/usr/bin/env bash
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Script to analyze request latency metrics for a specific endpoint from OpenMetadata Prometheus metrics
# Default values
HOST="localhost"
PORT="8586"
ENDPOINT=""
SHOW_RAW=false
PROTOCOL="http"
LIST_MODE=false
SORT_BY="count"
TOP_N=0
# Colors for output
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color
# Function to show usage
usage() {
echo "Usage: $0 [OPTIONS]"
echo ""
echo "Modes:"
echo " Analyze mode: $0 -e <endpoint> [options]"
echo " List mode: $0 -l [options]"
echo ""
echo "Common Options:"
echo " -h <host> Prometheus host (default: localhost)"
echo " -p <port> Prometheus port (default: 8586)"
echo " -s Use HTTPS instead of HTTP"
echo ""
echo "Analyze Mode Options:"
echo " -e <endpoint> The endpoint to analyze (e.g., 'v1/tables/123-456-789')"
echo " -r Show raw metrics data"
echo ""
echo "List Mode Options:"
echo " -l List all endpoints with metrics"
echo " -t <top_n> Show only top N endpoints (default: all)"
echo " -o <sort_by> Sort by: count, time, or name (default: count)"
echo ""
echo "Examples:"
echo " # Analyze specific endpoint"
echo " $0 -e 'v1/tables/0a4cd328-50c3-44c8-a7d2-ba71ba53bbe3'"
echo " $0 -e 'v1/users' -h 'myserver.com' -p 9090 -r"
echo ""
echo " # List all endpoints"
echo " $0 -l"
echo " $0 -l -t 10 # Top 10 by request count"
echo " $0 -l -o time -t 5 # Top 5 by total time"
exit 1
}
# Parse command line arguments
while getopts "e:h:p:srlt:o:" opt; do
case $opt in
e)
ENDPOINT="$OPTARG"
;;
h)
HOST="$OPTARG"
;;
p)
PORT="$OPTARG"
;;
s)
PROTOCOL="https"
;;
r)
SHOW_RAW=true
;;
l)
LIST_MODE=true
;;
t)
TOP_N="$OPTARG"
;;
o)
SORT_BY="$OPTARG"
if [[ ! "$SORT_BY" =~ ^(count|time|name)$ ]]; then
echo -e "${RED}Error: Invalid sort option. Must be: count, time, or name${NC}"
usage
fi
;;
*)
usage
;;
esac
done
# Validate mode selection
if [ "$LIST_MODE" = true ] && [ -n "$ENDPOINT" ]; then
echo -e "${RED}Error: Cannot use both -l and -e options together${NC}"
usage
fi
if [ "$LIST_MODE" = false ] && [ -z "$ENDPOINT" ]; then
echo -e "${RED}Error: Either -e <endpoint> or -l must be specified${NC}"
usage
fi
# Construct metrics URL
METRICS_URL="${PROTOCOL}://${HOST}:${PORT}/prometheus"
# Fetch metrics
echo -e "${YELLOW}Fetching metrics...${NC}"
METRICS=$(curl -s "$METRICS_URL")
if [ -z "$METRICS" ]; then
echo -e "${RED}Error: Failed to fetch metrics from $METRICS_URL${NC}"
exit 1
fi
# Branch based on mode
if [ "$LIST_MODE" = true ]; then
# List mode
echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════${NC}"
echo -e "${BLUE}OpenMetadata Endpoints with Metrics${NC}"
echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════${NC}"
echo -e "${GREEN}Host:${NC} $HOST"
echo -e "${GREEN}Port:${NC} $PORT"
echo -e "${GREEN}Protocol:${NC} $PROTOCOL"
echo -e "${GREEN}Metrics URL:${NC} $METRICS_URL"
echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════${NC}"
echo ""
echo -e "${YELLOW}Processing endpoint data...${NC}"
echo ""
# Create temporary file for processing
TEMP_FILE=$(mktemp)
# Extract endpoint data
echo "$METRICS" | grep -E "^request_latency_total_seconds_count" | while read -r line; do
endpoint=$(echo "$line" | sed -n 's/.*endpoint="\([^"]*\)".*/\1/p')
count=$(echo "$line" | awk '{print $NF}')
# Get total time for this endpoint
total_time=$(echo "$METRICS" | grep -E "^request_latency_total_seconds_sum.*endpoint=\"${endpoint}\"" | head -1 | awk '{print $NF}')
# Get percentages
db_pct=$(echo "$METRICS" | grep -E "^request_percentage_database.*endpoint=\"${endpoint}\"" | head -1 | awk '{print $NF}')
search_pct=$(echo "$METRICS" | grep -E "^request_percentage_search.*endpoint=\"${endpoint}\"" | head -1 | awk '{print $NF}')
internal_pct=$(echo "$METRICS" | grep -E "^request_percentage_internal.*endpoint=\"${endpoint}\"" | head -1 | awk '{print $NF}')
# Calculate average time in ms
if [ -n "$total_time" ] && [ -n "$count" ] && [ "$count" != "0" ]; then
avg_time=$(echo "scale=3; $total_time * 1000 / $count" | bc)
else
avg_time="0"
fi
# Format percentages
db_pct=${db_pct:-0}
search_pct=${search_pct:-0}
internal_pct=${internal_pct:-0}
# Output in tab-separated format
echo -e "$endpoint\t$count\t$total_time\t$avg_time\t$db_pct\t$search_pct\t$internal_pct"
done > "$TEMP_FILE"
# Sort based on option
case "$SORT_BY" in
count)
SORTED=$(sort -t$'\t' -k2 -nr "$TEMP_FILE")
;;
time)
SORTED=$(sort -t$'\t' -k3 -nr "$TEMP_FILE")
;;
name)
SORTED=$(sort -t$'\t' -k1 "$TEMP_FILE")
;;
esac
# Apply top N filter if specified
if [ "$TOP_N" -gt 0 ]; then
SORTED=$(echo "$SORTED" | head -n "$TOP_N")
fi
# Display header
printf "${GREEN}%-60s %10s %15s %12s %10s %10s %10s${NC}\n" \
"Endpoint" "Requests" "Total Time(s)" "Avg(ms)" "DB%" "Search%" "Internal%"
echo -e "${BLUE}$(printf '%.0s─' {1..130})${NC}"
# Display data
echo "$SORTED" | while IFS=$'\t' read -r endpoint count total_time avg_time db_pct search_pct internal_pct; do
# Truncate endpoint if too long
if [ ${#endpoint} -gt 57 ]; then
display_endpoint="${endpoint:0:54}..."
else
display_endpoint="$endpoint"
fi
# Format numbers
total_time_fmt=$(printf "%.3f" "$total_time" 2>/dev/null || echo "0.000")
avg_time_fmt=$(printf "%.1f" "$avg_time" 2>/dev/null || echo "0.0")
db_pct_fmt=$(printf "%.1f" "$db_pct" 2>/dev/null || echo "0.0")
search_pct_fmt=$(printf "%.1f" "$search_pct" 2>/dev/null || echo "0.0")
internal_pct_fmt=$(printf "%.1f" "$internal_pct" 2>/dev/null || echo "0.0")
printf "%-60s %10s %15s %12s %10s %10s %10s\n" \
"$display_endpoint" "$count" "$total_time_fmt" "$avg_time_fmt" \
"$db_pct_fmt" "$search_pct_fmt" "$internal_pct_fmt"
done
# Summary
echo -e "${BLUE}$(printf '%.0s─' {1..130})${NC}"
TOTAL_ENDPOINTS=$(echo "$SORTED" | wc -l | tr -d ' ')
TOTAL_REQUESTS=$(echo "$SORTED" | awk -F$'\t' '{sum+=$2} END {print sum}')
echo -e "${GREEN}Total Endpoints:${NC} $TOTAL_ENDPOINTS"
echo -e "${GREEN}Total Requests:${NC} $TOTAL_REQUESTS"
# Cleanup
rm -f "$TEMP_FILE"
echo ""
echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════${NC}"
exit 0
fi
# Analyze mode
echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════${NC}"
echo -e "${BLUE}OpenMetadata Endpoint Metrics Analysis${NC}"
echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════${NC}"
echo -e "${GREEN}Endpoint:${NC} $ENDPOINT"
echo -e "${GREEN}Host:${NC} $HOST"
echo -e "${GREEN}Port:${NC} $PORT"
echo -e "${GREEN}Protocol:${NC} $PROTOCOL"
echo -e "${GREEN}Metrics URL:${NC} $METRICS_URL"
echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════${NC}"
echo ""
# Escape special characters in endpoint for regex
ENDPOINT_ESCAPED=$(echo "$ENDPOINT" | sed 's/[[\.*^$()+?{|]/\\&/g')
# Function to extract metric value
get_metric_value() {
local metric_name=$1
local metric_type=$2
if [ "$metric_type" = "counter" ]; then
# For counters, get the total count
echo "$METRICS" | grep -E "^${metric_name}.*endpoint=\"${ENDPOINT_ESCAPED}\"[^}]*} " | grep -v "_total" | grep -v "_created" | head -1 | awk '{print $NF}'
elif [ "$metric_type" = "histogram_count" ]; then
# For histograms, get the count
echo "$METRICS" | grep -E "^${metric_name}_count.*endpoint=\"${ENDPOINT_ESCAPED}\"" | head -1 | awk '{print $NF}'
elif [ "$metric_type" = "histogram_sum" ]; then
# For histograms, get the sum (total time)
echo "$METRICS" | grep -E "^${metric_name}_sum.*endpoint=\"${ENDPOINT_ESCAPED}\"" | head -1 | awk '{print $NF}'
elif [ "$metric_type" = "gauge" ]; then
# For gauges, get the value
echo "$METRICS" | grep -E "^${metric_name}.*endpoint=\"${ENDPOINT_ESCAPED}\"" | grep -v "#" | head -1 | awk '{print $NF}'
fi
}
# Function to calculate mean from histogram
calculate_mean() {
local sum=$1
local count=$2
if [ -n "$sum" ] && [ -n "$count" ] && [ "$count" != "0" ]; then
echo "scale=6; $sum / $count" | bc
else
echo "0"
fi
}
# Function to format time
format_time() {
local seconds=$1
if [ -z "$seconds" ] || [ "$seconds" = "0" ]; then
echo "N/A"
else
local ms=$(echo "scale=3; $seconds * 1000" | bc)
echo "${ms}ms"
fi
}
# Function to get percentile from histogram
get_percentile() {
local metric_name=$1
local percentile=$2
# Get all bucket values for this metric
local buckets=$(echo "$METRICS" | grep -E "^${metric_name}_bucket.*endpoint=\"${ENDPOINT_ESCAPED}\"" | grep -v "#")
if [ -z "$buckets" ]; then
echo "N/A"
return
fi
# Get total count
local total_count=$(echo "$buckets" | grep 'le="+Inf"' | awk '{print $NF}')
if [ -z "$total_count" ] || [ "$total_count" = "0" ]; then
echo "N/A"
return
fi
# Calculate target count for percentile
local target_count=$(echo "scale=0; $total_count * $percentile / 100" | bc)
# Find the bucket that contains the percentile
local prev_le="0"
echo "$buckets" | sort -t'=' -k3 -g | while IFS= read -r line; do
local le=$(echo "$line" | sed -n 's/.*le="\([^"]*\)".*/\1/p')
local count=$(echo "$line" | awk '{print $NF}')
if [ "$le" != "+Inf" ] && [ "$(echo "$count >= $target_count" | bc)" -eq 1 ]; then
format_time "$le"
break
fi
prev_le="$le"
done
}
# Collect metrics
echo -e "${YELLOW}Analyzing metrics for endpoint: ${NC}${ENDPOINT}"
echo ""
# Total request metrics
TOTAL_COUNT=$(get_metric_value "request_latency_total_seconds" "histogram_count")
TOTAL_SUM=$(get_metric_value "request_latency_total_seconds" "histogram_sum")
TOTAL_MEAN=$(calculate_mean "$TOTAL_SUM" "$TOTAL_COUNT")
# Database metrics
DB_COUNT=$(get_metric_value "request_latency_database_seconds" "histogram_count")
DB_SUM=$(get_metric_value "request_latency_database_seconds" "histogram_sum")
DB_MEAN=$(calculate_mean "$DB_SUM" "$DB_COUNT")
DB_OPERATIONS=$(get_metric_value "request_operations_database" "counter")
DB_PERCENTAGE=$(get_metric_value "request_percentage_database" "gauge")
# Search metrics
SEARCH_COUNT=$(get_metric_value "request_latency_search_seconds" "histogram_count")
SEARCH_SUM=$(get_metric_value "request_latency_search_seconds" "histogram_sum")
SEARCH_MEAN=$(calculate_mean "$SEARCH_SUM" "$SEARCH_COUNT")
SEARCH_OPERATIONS=$(get_metric_value "request_operations_search" "counter")
SEARCH_PERCENTAGE=$(get_metric_value "request_percentage_search" "gauge")
# Internal processing metrics
INTERNAL_COUNT=$(get_metric_value "request_latency_internal_seconds" "histogram_count")
INTERNAL_SUM=$(get_metric_value "request_latency_internal_seconds" "histogram_sum")
INTERNAL_MEAN=$(calculate_mean "$INTERNAL_SUM" "$INTERNAL_COUNT")
INTERNAL_PERCENTAGE=$(get_metric_value "request_percentage_internal" "gauge")
# Calculate average time per request (not per operation)
if [ -n "$TOTAL_COUNT" ] && [ "$TOTAL_COUNT" != "0" ]; then
DB_TIME_PER_REQUEST=$(echo "scale=6; $DB_SUM / $TOTAL_COUNT" | bc)
SEARCH_TIME_PER_REQUEST=$(echo "scale=6; $SEARCH_SUM / $TOTAL_COUNT" | bc)
INTERNAL_TIME_PER_REQUEST=$(echo "scale=6; $INTERNAL_SUM / $TOTAL_COUNT" | bc)
else
DB_TIME_PER_REQUEST="0"
SEARCH_TIME_PER_REQUEST="0"
INTERNAL_TIME_PER_REQUEST="0"
fi
# Check if we have any data
if [ -z "$TOTAL_COUNT" ] || [ "$TOTAL_COUNT" = "0" ]; then
echo -e "${RED}No metrics found for endpoint: $ENDPOINT${NC}"
echo ""
echo "Available endpoints with metrics:"
echo "$METRICS" | grep -E "request_latency_total.*endpoint=" | sed -n 's/.*endpoint="\([^"]*\)".*/\1/p' | sort | uniq | head -20
exit 1
fi
# Display summary
echo -e "${GREEN}📊 REQUEST SUMMARY${NC}"
echo -e "├─ Total Requests: ${BLUE}${TOTAL_COUNT}${NC}"
echo -e "├─ Total Time: ${BLUE}$(format_time "$TOTAL_SUM")${NC}"
echo -e "└─ Average Time: ${BLUE}$(format_time "$TOTAL_MEAN")${NC}"
echo ""
# Display component breakdown
echo -e "${GREEN}⏱️ LATENCY BREAKDOWN${NC}"
echo -e "├─ ${YELLOW}Total Request Time:${NC}"
echo -e "│ ├─ Mean: ${BLUE}$(format_time "$TOTAL_MEAN")${NC}"
echo -e "│ ├─ P50: ${BLUE}$(get_percentile "request_latency_total_seconds" 50)${NC}"
echo -e "│ ├─ P95: ${BLUE}$(get_percentile "request_latency_total_seconds" 95)${NC}"
echo -e "│ └─ P99: ${BLUE}$(get_percentile "request_latency_total_seconds" 99)${NC}"
echo -e "│"
echo -e "├─ ${YELLOW}Database Operations:${NC}"
echo -e "│ ├─ Total Operations: ${BLUE}${DB_COUNT:-0}${NC} (across ${BLUE}${TOTAL_COUNT}${NC} requests)"
echo -e "│ ├─ Avg Operations/Request: ${BLUE}$(echo "scale=1; ${DB_COUNT:-0} / ${TOTAL_COUNT}" | bc)${NC}"
echo -e "│ ├─ Avg Time/Operation: ${BLUE}$(format_time "$DB_MEAN")${NC}"
echo -e "│ ├─ Avg Time/Request: ${BLUE}$(format_time "$DB_TIME_PER_REQUEST")${NC}"
echo -e "│ └─ Percentage of Request: ${BLUE}${DB_PERCENTAGE:-0}%${NC}"
echo -e "│"
echo -e "├─ ${YELLOW}Search Operations:${NC}"
echo -e "│ ├─ Total Operations: ${BLUE}${SEARCH_COUNT:-0}${NC} (across ${BLUE}${TOTAL_COUNT}${NC} requests)"
echo -e "│ ├─ Avg Operations/Request: ${BLUE}$(echo "scale=1; ${SEARCH_COUNT:-0} / ${TOTAL_COUNT}" | bc)${NC}"
echo -e "│ ├─ Avg Time/Operation: ${BLUE}$(format_time "$SEARCH_MEAN")${NC}"
echo -e "│ ├─ Avg Time/Request: ${BLUE}$(format_time "$SEARCH_TIME_PER_REQUEST")${NC}"
echo -e "│ └─ Percentage of Request: ${BLUE}${SEARCH_PERCENTAGE:-0}%${NC}"
echo -e "│"
echo -e "└─ ${YELLOW}Internal Processing:${NC}"
echo -e " ├─ Avg Time/Request: ${BLUE}$(format_time "$INTERNAL_TIME_PER_REQUEST")${NC}"
echo -e " └─ Percentage of Request: ${BLUE}${INTERNAL_PERCENTAGE:-0}%${NC}"
echo ""
# Calculate and display percentage summary
if [ -n "$DB_PERCENTAGE" ] || [ -n "$SEARCH_PERCENTAGE" ] || [ -n "$INTERNAL_PERCENTAGE" ]; then
echo -e "${GREEN}📈 TIME DISTRIBUTION${NC}"
# Create a simple bar chart
DB_PCT=${DB_PERCENTAGE:-0}
SEARCH_PCT=${SEARCH_PERCENTAGE:-0}
INTERNAL_PCT=${INTERNAL_PERCENTAGE:-0}
# Function to create bar
create_bar() {
local pct=$1
local width=$(echo "scale=0; $pct / 2" | bc)
printf '█%.0s' $(seq 1 $width)
}
echo -e "Database: $(create_bar $DB_PCT) ${DB_PCT}%"
echo -e "Search: $(create_bar $SEARCH_PCT) ${SEARCH_PCT}%"
echo -e "Internal: $(create_bar $INTERNAL_PCT) ${INTERNAL_PCT}%"
echo ""
fi
# Show raw metrics if requested
if [ "$SHOW_RAW" = true ]; then
echo -e "${GREEN}📋 RAW METRICS${NC}"
echo "$METRICS" | grep -E "(request_latency|request_percentage|request_operations).*endpoint=\"${ENDPOINT_ESCAPED}\"" | sort
echo ""
fi
echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════${NC}"
bin/distributed-test/CLUSTER-SIZING-RUNBOOK.md
+469
View File
@@ -0,0 +1,469 @@
# OpenMetadata Cluster Sizing Runbook
This runbook walks through benchmarking an OpenMetadata cluster from 10K to 5M entities, identifying where performance breaks, and applying the right configuration for your target scale.
## Prerequisites
- OpenMetadata server running and accessible via HTTP
- Admin port exposed (optional, recommended for diagnostics — typically 8586)
- Python 3 installed on the benchmark host (for JSON parsing)
- `curl` and `bash` available
- Sufficient disk space for test data (each run generates JSON reports ~1-5MB)
- Network connectivity from benchmark host to server with low latency
## Quick Start
Run a full progressive benchmark with a single command:
```bash
cd bin/distributed-test
./scripts/benchmark-sizing.sh \
--server http://your-server:8585 \
--workers 30 \
--ramp \
--mixed \
--output-dir ./sizing-results
```
This will:
1. Run a ramp test to find optimal concurrency
2. Loop through scales: 10k → 50k → 100k → 200k → 500k → ~2M → ~5M
3. At each scale, run both sequential and realistic modes
4. Stop automatically when a break-point is detected
5. Generate `SIZING-SUMMARY.md` with results and recommendations
## Step-by-Step Guide
### Step 1: Environment Setup
#### Option A: Point to existing server
```bash
# Verify connectivity
curl -s http://your-server:8585/api/v1/system/version | python3 -m json.tool
```
#### Option B: Start local distributed cluster
```bash
cd bin/distributed-test
./scripts/start.sh
# Servers will be available at:
# Server 1: http://localhost:8585 (admin: 8586)
# Server 2: http://localhost:8587 (admin: 8588)
# Server 3: http://localhost:8589 (admin: 8590)
```
#### Note current configuration
Before benchmarking, record your current settings so you can compare before/after:
```bash
# Check server version
curl -s http://localhost:8585/api/v1/system/version
# If admin port is exposed, check diagnostics
curl -s http://localhost:8586/metrics | grep -E "jvm_memory|jetty_threads|hikari"
```
Key settings to note:
- JVM heap size (`-Xmx`)
- `SERVER_MAX_THREADS`
- `DB_CONNECTION_POOL_MAX_SIZE`
- `SERVER_ENABLE_VIRTUAL_THREAD`
- `BULK_OPERATION_QUEUE_SIZE`
### Step 2: Find Optimal Concurrency (Optional)
The ramp test progressively increases worker count to find the sweet spot between throughput and latency:
```bash
./scripts/perf-test.sh --ramp --scale 10k --workers 64 --admin-port 8586
```
**What to look for in ramp results:**
| Workers | RPS | p95ms | Interpretation |
|---------|------|-------|----------------|
| 1-4 | Low | Low | Underutilizing server capacity |
| 8-16 | Peak | Low | Sweet spot — use this worker count |
| 32+ | Flat | Rising | Server saturated — more workers hurt |
The ramp test output will suggest an `optimal_workers` value. Use it for subsequent runs.
### Step 3: Run Progressive Benchmark
#### Full run (recommended)
```bash
./scripts/benchmark-sizing.sh \
--server http://localhost:8585 \
--admin-port 8586 \
--workers 30 \
--ramp \
--mixed \
--mixed-duration 60 \
--output-dir ./sizing-results
```
#### Quick 2-tier smoke test
```bash
./scripts/benchmark-sizing.sh \
--server http://localhost:8585 \
--start-scale 10k \
--end-scale 50k \
--output-dir /tmp/sizing-test
```
#### Sequential mode only (simpler, less contention)
```bash
./scripts/benchmark-sizing.sh \
--server http://localhost:8585 \
--modes seq \
--end-scale 500k
```
#### Resume after failure
```bash
./scripts/benchmark-sizing.sh \
--server http://localhost:8585 \
--skip-existing \
--output-dir ./sizing-results
```
#### What to expect at each scale tier
| Scale | Entities | Typical Duration | What It Tests |
|-------|----------|-----------------|---------------|
| 10k | ~10,000 | 2-5 min | Baseline — should pass easily |
| 50k | ~50,000 | 5-15 min | Small production workload |
| 100k | ~100,000 | 10-30 min | Medium production workload |
| 200k | ~200,000 | 20-60 min | Large production workload |
| 500k | ~500,000 | 1-3 hours | Enterprise scale — DB pool pressure |
| large | ~2,000,000 | 3-8 hours | Large enterprise — thread/memory limits |
| xlarge | ~5,000,000 | 8-24 hours | Extreme scale — full cluster stress |
#### Monitoring during benchmark
While the benchmark runs, monitor the server:
```bash
# Watch server logs for errors
docker logs -f openmetadata-server-1 2>&1 | grep -E "ERROR|WARN|OOM"
# Watch DB connection pool (if admin port exposed)
watch -n 5 'curl -s http://localhost:8586/metrics | grep hikari'
# Watch JVM heap
watch -n 5 'curl -s http://localhost:8586/metrics | grep "jvm_memory_bytes_used.*heap"'
```
### Step 4: Analyze Results
After the benchmark completes, open the generated summary:
```bash
cat ./sizing-results/SIZING-SUMMARY.md
```
#### Reading the Progressive Results table
```
| Scale | Mode | Entities | RPS | p95 (ms) | Errors % | Assessment |
|-------|------------|----------|-------|----------|----------|------------|
| 10k | Sequential | 10234 | 456.2 | 312 | 0.05 | adequate |
| 10k | Realistic | 10234 | 389.1 | 445 | 0.12 | adequate |
| 50k | Sequential | 50120 | 412.3 | 456 | 0.23 | adequate |
| 50k | Realistic | 50120 | 312.4 | 890 | 1.45 | marginal |
| 100k | Realistic | 98450 | 189.2 | 2345 | 12.3 | undersized |
```
Key columns:
- **RPS**: Higher is better. Watch for drops between tiers.
- **p95**: 95th percentile latency. Under 500ms is good, over 2000ms is concerning.
- **Errors %**: Under 1% is acceptable. Over 5% signals resource exhaustion.
- **Assessment**: `adequate` (good), `marginal` (tuning needed), `undersized` (upgrade needed).
#### Interpreting sequential vs realistic differences
- **Sequential** runs entity types one at a time — measures raw throughput per entity type
- **Realistic** runs all types concurrently — exposes cross-entity contention (DB locks, thread pool pressure)
- A large gap (>30% RPS drop or >2x p95 increase) in realistic mode indicates contention issues
- If sequential is fine but realistic breaks, focus on: DB pool size, thread count, bulk executor tuning
#### Understanding the break-point
The benchmark stops when it detects:
- Assessment = `undersized`
- Error rate > 10%
- p95 latency > 10 seconds
- Throughput per entity degraded >50% from previous tier
The tier just before the break-point is your current cluster's capacity ceiling.
#### Reading cluster_sizing recommendations
Each JSON report contains a `cluster_sizing` section with specific recommendations:
```bash
python3 -c "
import json
with open('./sizing-results/sizing-50k-realistic.json') as f:
data = json.load(f)
for k, v in data['cluster_sizing']['recommendations'].items():
if isinstance(v, dict) and 'recommended_env' in v:
print(f\"{k}: {v['recommended_env']} (was: {v.get('current_env', 'unknown')})\")
"
```
### Step 5: Apply Recommendations
#### Configuration Matrix
| Parameter | Small (<50K) | Medium (50-200K) | Large (200K-2M) | XLarge (2-5M) |
|-----------|-------------|-------------------|-----------------|---------------|
| `SERVER_MAX_THREADS` | 150 | 300 | 500 | 750 |
| `SERVER_ENABLE_VIRTUAL_THREAD` | false | true | true | true |
| `DB_CONNECTION_POOL_MAX_SIZE` | 50 | 100 | 200 | 300 |
| `DB_CONNECTION_TIMEOUT` | 30000 | 30000 | 60000 | 60000 |
| `ELASTICSEARCH_MAX_CONN_TOTAL` | 50 | 100 | 200 | 300 |
| `BULK_OPERATION_QUEUE_SIZE` | 1000 | 2000 | 5000 | 10000 |
| `BULK_OPERATION_MAX_THREADS` | 10 | 20 | 30 | 50 |
| `SERVER_ACCEPT_QUEUE_SIZE` | 50 | 100 | 200 | 500 |
| JVM Heap (`-Xmx`) | 2G | 4G | 8G | 16G |
#### Applying changes
**Docker Compose:**
Add environment variables to your server service:
```yaml
environment:
SERVER_MAX_THREADS: "300"
SERVER_ENABLE_VIRTUAL_THREAD: "true"
DB_CONNECTION_POOL_MAX_SIZE: "100"
ELASTICSEARCH_MAX_CONN_TOTAL: "100"
BULK_OPERATION_QUEUE_SIZE: "2000"
BULK_OPERATION_MAX_THREADS: "20"
JAVA_OPTS: "-Xmx4g -Xms4g"
```
**Kubernetes / Helm:**
```yaml
# values.yaml
openmetadata:
config:
serverMaxThreads: 300
enableVirtualThread: true
database:
connectionPoolMaxSize: 100
connectionTimeout: 30000
elasticsearch:
maxConnectionsTotal: 100
bulkOperation:
queueSize: 2000
maxThreads: 20
jvmOpts: "-Xmx4g -Xms4g"
```
**Bare metal / `openmetadata.yaml`:**
```yaml
server:
applicationConnectors:
- type: http
port: 8585
maxThreads: 300
enableVirtualThread: true
database:
hikariConfig:
maximumPoolSize: 100
connectionTimeout: 30000
elasticsearch:
maxConnectionsTotal: 100
```
#### Verifying changes took effect
After restarting the server with new settings:
```bash
# Check via diagnostics endpoint
curl -s http://localhost:8585/api/v1/system/diagnostics | python3 -c "
import json, sys
d = json.load(sys.stdin)
j = d.get('jetty', {})
db = d.get('database', {})
print(f\"Jetty max threads: {j.get('threads_max', 'N/A')}\")
print(f\"DB pool max: {db.get('pool_max', 'N/A')}\")
print(f\"Virtual threads: {j.get('virtual_threads_enabled', 'N/A')}\")
"
# Check via Prometheus metrics (admin port)
curl -s http://localhost:8586/metrics | grep -E "jetty_threads_max|hikari.*max"
```
### Step 6: Re-validate
After applying configuration changes, re-run the benchmark at your target scale:
```bash
# Re-run just the tier that previously broke
./scripts/perf-test.sh \
--scale 100k \
--realistic \
--mixed \
--workers 30 \
--admin-port 8586 \
--output ./sizing-results/sizing-100k-realistic-tuned.json
# Or re-run the full progressive suite
./scripts/benchmark-sizing.sh \
--server http://localhost:8585 \
--admin-port 8586 \
--output-dir ./sizing-results-tuned
```
Compare before/after:
```bash
# Side-by-side comparison
python3 -c "
import json
with open('./sizing-results/sizing-100k-realistic.json') as f:
before = json.load(f)
with open('./sizing-results/sizing-100k-realistic-tuned.json') as f:
after = json.load(f)
b = before['overall']
a = after['overall']
print(f\"{'Metric':<20} {'Before':>12} {'After':>12} {'Change':>12}\")
print('-' * 56)
for key in ['overall_throughput_rps', 'overall_error_rate_pct']:
bv = b.get(key, 0)
av = a.get(key, 0)
diff = ((av - bv) / bv * 100) if bv > 0 else 0
print(f\"{key:<20} {bv:>12.1f} {av:>12.1f} {diff:>+11.1f}%\")
print(f\"\nBefore: {before['cluster_sizing']['assessment']}\")
print(f\"After: {after['cluster_sizing']['assessment']}\")
"
```
## Configuration Reference
| Parameter | Default | Description | When to Increase |
|-----------|---------|-------------|-----------------|
| `SERVER_MAX_THREADS` | 150 | Jetty HTTP thread pool | Jetty utilization >80% |
| `SERVER_ENABLE_VIRTUAL_THREAD` | false | Use JDK21 virtual threads | Always enable at >50K entities |
| `DB_CONNECTION_POOL_MAX_SIZE` | 50 | HikariCP max connections | DB pool utilization >70% or pending >0 |
| `DB_CONNECTION_TIMEOUT` | 30000 | Max wait for DB connection (ms) | Connection acquire timeouts |
| `ELASTICSEARCH_MAX_CONN_TOTAL` | 50 | Max HTTP connections to ES/OS | Search latency spikes |
| `BULK_OPERATION_QUEUE_SIZE` | 1000 | Async bulk operation queue depth | 503 "queue full" errors |
| `BULK_OPERATION_MAX_THREADS` | 10 | Worker threads for bulk ops | Queue filling up, low throughput |
| `SERVER_ACCEPT_QUEUE_SIZE` | 50 | TCP accept backlog | Connection refused under load |
## Interpreting Results
### Assessment Meanings
| Assessment | Meaning | Action |
|------------|---------|--------|
| `adequate` | Server handles this scale with acceptable latency and error rates | No changes needed |
| `marginal` | Performance is degrading — latency rising, some errors appearing | Tune configuration (Step 5) |
| `undersized` | Server cannot handle this scale — high errors, extreme latency | Upgrade resources or reduce scale target |
### Common Patterns and Fixes
| Pattern | Symptom | Fix |
|---------|---------|-----|
| DB pool exhaustion | p95 spikes, bimodal latency, pending connections >0 | Increase `DB_CONNECTION_POOL_MAX_SIZE` |
| Thread pool saturation | Jetty utilization >90%, queue depth growing | Increase `SERVER_MAX_THREADS`, enable virtual threads |
| Bulk executor overflow | 503 errors with "queue full" | Increase `BULK_OPERATION_QUEUE_SIZE` and `BULK_OPERATION_MAX_THREADS` |
| GC pressure | p99/p95 ratio >3x, periodic latency spikes | Increase heap, tune GC settings |
| Search bottleneck | High search latency in breakdown, low ES connection usage | Increase `ELASTICSEARCH_MAX_CONN_TOTAL` |
| Sequential OK, realistic broken | Performance fine per-type but breaks under mixed load | All of the above — concurrent load compounds contention |
### Sequential vs Realistic Differences
- **<10% RPS drop**: Excellent — server handles contention well
- **10-30% RPS drop**: Normal — some contention expected
- **30-50% RPS drop**: Concerning — review DB pool and thread settings
- **>50% RPS drop**: Critical contention — likely needs resource upgrades
## Troubleshooting
### Server OOM during benchmark
Symptoms: Server process killed, connection refused mid-run.
```bash
# Check if server was OOM-killed
docker logs openmetadata-server-1 2>&1 | tail -50
dmesg | grep -i "oom\|killed"
# Fix: Increase heap
# In docker-compose.yaml:
# JAVA_OPTS: "-Xmx8g -Xms8g"
```
### Connection refused errors
Symptoms: Benchmark shows many `connection_error` entries.
```bash
# Check server is running
curl -s http://localhost:8585/api/v1/system/version
# Check accept queue
curl -s http://localhost:8586/metrics | grep "jetty_queued"
# Fix: Increase SERVER_ACCEPT_QUEUE_SIZE, check firewall rules
```
### Benchmark script hangs
Symptoms: No progress for >5 minutes.
```bash
# Check server health
curl -s http://localhost:8585/api/v1/system/version
# Check server thread dumps for deadlocks
curl -s http://localhost:8586/threads
# The script has a 60s per-request timeout; if the server is extremely slow
# but responsive, individual requests may be timing out and retrying
```
### How to resume after failure
The `--skip-existing` flag lets you resume:
```bash
# First, check what's already completed
ls -la ./sizing-results/sizing-*.json
# Resume from where it left off
./scripts/benchmark-sizing.sh \
--skip-existing \
--output-dir ./sizing-results
```
Individual tiers can also be re-run manually:
```bash
./scripts/perf-test.sh \
--scale 100k \
--realistic \
--workers 30 \
--output ./sizing-results/sizing-100k-realistic.json
```
bin/distributed-test/DIAGNOSTICS.md
+318
View File
@@ -0,0 +1,318 @@
# Server-Side Diagnostics & Load Test Correlation
The diagnostics endpoint (`GET /api/v1/system/diagnostics`) provides a single-call performance snapshot of the OpenMetadata server. Combined with the load test script, it enables pinpointing **where** time is spent during high-load scenarios and produces actionable tuning recommendations.
## The Diagnostics Endpoint
### Basic Usage
```bash
curl -H "Authorization: Bearer $TOKEN" \
http://localhost:8585/api/v1/system/diagnostics | python3 -m json.tool
```
### Response Structure
```json
{
"timestamp": "2026-03-02T19:00:00Z",
"jvm": { ... },
"jetty": { ... },
"database": { ... },
"database_queries": { ... },
"bulk_executor": { ... },
"request_latency": { ... }
}
```
Each section is explained below.
---
## Understanding Each Section
### JVM
| Field | What It Tells You |
|-------|-------------------|
| `heap_used_bytes` / `heap_max_bytes` | Current heap consumption vs maximum. |
| `heap_usage_pct` | If >85% after load, GC pressure is likely adding tail latency. |
| `gc_pause_total_ms` | Cumulative GC pause time since JVM start. Compare before/after load to see how much GC occurred during the test. |
| `gc_count` | Total GC collections. A large delta during load means frequent stop-the-world pauses. |
| `thread_count` / `thread_peak` | Active JVM threads. Correlate with Jetty thread pool. |
| `cpu_process_pct` | Process CPU utilization (0-100). If pinned at 100%, the server is CPU-bound. |
| `uptime_seconds` | Useful to confirm the server wasn't restarted mid-test. |
### Jetty (Thread Pool)
| Field | What It Tells You |
|-------|-------------------|
| `threads_busy` / `threads_max` | How many request-handling threads are in use. |
| `utilization_pct` | **Key metric.** If >90% with `queue_size > 0`, the thread pool is saturated and requests are queuing. |
| `queue_size` | Requests waiting for a free thread. Non-zero means latency is being added by queuing. |
| `queue_time_avg_ms` | Average time a request waits in the queue before getting a thread. |
| `virtual_threads_enabled` | Whether Java 21 virtual threads are active (eliminates thread pool as a bottleneck). |
### Database (HikariCP Pool)
| Field | What It Tells You |
|-------|-------------------|
| `pool_active` / `pool_max` | Active DB connections vs maximum pool size. |
| `pool_usage_pct` | **Key metric.** If >80%, connection contention is likely. Requests wait for a free connection. |
| `pool_pending` | Threads waiting for a DB connection. If >0 during load, the pool is undersized. |
| `pool_idle` | Spare connections. If 0 during load, the pool is fully utilized. |
| `connection_acquire_avg_ms` | Average time to acquire a connection from the pool. High values (>50ms) indicate pool contention. |
| `connection_acquire_max_ms` | Maximum connection acquire time observed. |
| `connection_acquire_count` | Total number of connection acquire operations. |
### Database Queries (Per-Type Profiling)
Breaks down DB query timing by operation type (select, insert, update, delete):
| Field | What It Tells You |
|-------|-------------------|
| `total_operations` | Sum of all DB operations across all types. |
| `{type}.count` | Number of queries of this type. |
| `{type}.mean_ms` | Average query duration. |
| `{type}.max_ms` | Maximum query duration. Spikes indicate lock contention or full table scans. |
| `{type}.p95_ms` | 95th percentile query duration. If >100ms, investigate slow queries. |
| `{type}.total_ms` | Total cumulative time spent in queries of this type. |
**Reading the profile:** If `select.p95_ms` is 200ms while `insert.p95_ms` is only 20ms, read queries are the bottleneck. This often indicates missing indexes or N+1 query patterns.
### Bulk Executor
| Field | What It Tells You |
|-------|-------------------|
| `queue_depth` / `queue_capacity` | Items in the async processing queue. |
| `queue_usage_pct` | If >70%, approaching the rejection threshold (HTTP 503 errors). |
| `active_threads` / `max_threads` | Worker threads actively processing bulk operations. |
| `has_capacity` | `false` means the next bulk submission will be rejected with 503. |
### Request Latency (Per-Endpoint Breakdown)
This is the most actionable section. For each `METHOD /endpoint` combination:
| Field | What It Tells You |
|-------|-------------------|
| `count` | Total requests processed for this endpoint. |
| `avg_total_ms` | Average end-to-end latency. |
| `avg_db_ms` / `db_pct` | Time spent in database queries and its percentage of total. |
| `avg_search_ms` / `search_pct` | Time spent in search/Elasticsearch operations. |
| `avg_internal_ms` / `internal_pct` | Time in Java code (serialization, validation, business logic). |
| `avg_db_ops` / `avg_search_ops` | Average number of DB/search round-trips per request. |
**Reading the breakdown:** If `PUT /v1/tables` shows `db_pct: 56%`, then 56% of the request time is spent waiting for database queries. Combined with `database.pool_usage_pct: 85%`, this tells you the DB connection pool is the bottleneck.
---
## Load Test Integration
The load test script automatically queries the diagnostics endpoint at three points:
1. **Before load** — baseline snapshot
2. **During load** — sampled every 10 seconds by the health monitor
3. **After load** — final snapshot for comparison
### Running a Load Test with Diagnostics
```bash
# Basic: diagnostics are collected automatically
./perf-test.sh --scale small --server http://localhost:8585 --admin-port 8586
# With explicit token
./perf-test.sh --scale medium --server http://localhost:8585 \
--admin-port 8586 --token "$MY_TOKEN" --output /tmp/bench.json
```
The `--admin-port` flag enables both Prometheus scraping and diagnostics collection. Diagnostics work without it too (they use the main API port).
### Console Output
After the benchmark table, you'll see a `SERVER-SIDE BREAKDOWN` section:
```
SERVER-SIDE BREAKDOWN (from /api/v1/system/diagnostics):
JVM: heap 1.2GB/2GB (60%), GC pauses +450ms during load
Jetty: 142/150 threads busy (95%), queue depth: 23
DB Pool: 85/100 active (85%), 12 pending connections
Bulk Executor: queue 450/1000 (45%)
Latency Breakdown (PUT endpoints):
Endpoint Total DB% Search% Internal%
/v1/tables 320ms 56.2% 14.1% 29.7%
/v1/topics 180ms 48.0% 22.0% 30.0%
/v1/dashboards 250ms 52.0% 18.0% 30.0%
BOTTLENECK: DB bottleneck on PUT /v1/tables: 56.2% of request time in DB, pool at 85.0% utilization
```
### JSON Report
The report includes top-level `diagnostics_before` and `diagnostics_after` objects, plus `cluster_sizing.server_side_analysis`:
```bash
cat /tmp/bench.json | python3 -c "
import json, sys
r = json.load(sys.stdin)
# Check if diagnostics were available
diag = r.get('diagnostics_after', {})
if diag:
jvm = diag['jvm']
print(f'Heap: {jvm[\"heap_usage_pct\"]}%')
print(f'GC pauses: {jvm[\"gc_pause_total_ms\"]}ms')
jetty = diag['jetty']
print(f'Jetty: {jetty[\"threads_busy\"]}/{jetty[\"threads_max\"]} ({jetty[\"utilization_pct\"]}%)')
db = diag['database']
print(f'DB pool: {db[\"pool_active\"]}/{db[\"pool_max\"]} ({db[\"pool_usage_pct\"]}%)')
for ep, data in diag.get('request_latency', {}).items():
print(f'{ep}: total={data[\"avg_total_ms\"]}ms '
f'DB={data[\"db_pct\"]}% Search={data[\"search_pct\"]}% '
f'Internal={data[\"internal_pct\"]}%')
else:
print('Diagnostics not available (server may be older version)')
"
```
---
## Bottleneck Detection Rules
The load test applies these rules automatically and surfaces them in findings:
| Condition | Diagnosis | Recommended Fix |
|-----------|-----------|-----------------|
| `db_pct > 60%` AND `pool_usage_pct > 80%` | DB is the bottleneck | `export DB_CONNECTION_POOL_MAX_SIZE=150` |
| `jetty.utilization_pct > 90%` AND `queue_size > 0` | Thread pool saturated | `export SERVER_MAX_THREADS=300` or enable virtual threads |
| `search_pct > 30%` for any endpoint | Search indexing consuming latency | `export ELASTICSEARCH_MAX_CONN_TOTAL=50` |
| `bulk_executor.queue_usage_pct > 70%` | Near bulk rejection threshold | `export BULK_OPERATION_QUEUE_SIZE=2000` |
| `jvm.heap_usage_pct > 85%` after load | Memory pressure / GC tail latency | Increase JVM heap (`-Xmx`) |
| `database_queries.{type}.p95_ms > 100ms` | Slow DB queries of that type | Add indexes, optimize queries |
| `connection_acquire_avg_ms > 50ms` | Connection pool contention | `export DB_CONNECTION_POOL_MAX_SIZE=150` |
---
## Common Scenarios
### Scenario 1: High Latency, DB is the Bottleneck
**Symptoms:** p95 latency >2s, `db_pct` >60%, `pool_usage_pct` >80%.
```
Latency Breakdown:
/v1/tables 320ms DB=62% Search=12% Internal=26%
DB Pool: 95/100 active (95%), 8 pending
```
**What's happening:** Every PUT requires multiple DB round-trips. At 95% pool utilization with 8 pending connections, requests are waiting for a free connection.
**Fix:**
```bash
export DB_CONNECTION_POOL_MAX_SIZE=150
export DB_CONNECTION_TIMEOUT=10000 # Fail fast instead of waiting 30s
```
### Scenario 2: Thread Pool Exhaustion
**Symptoms:** Connection refused errors, `utilization_pct` >95%, `queue_size` growing.
```
Jetty: 148/150 threads busy (99%), queue depth: 45
```
**What's happening:** All Jetty threads are busy. New requests queue up, adding latency. If the queue fills, connections get refused.
**Fix:**
```bash
export SERVER_MAX_THREADS=300
# OR enable virtual threads (preferred for I/O-bound workloads):
export SERVER_ENABLE_VIRTUAL_THREAD=true
```
### Scenario 3: GC Pressure
**Symptoms:** Periodic latency spikes, `heap_usage_pct` >85%, large GC pause delta.
```
JVM: heap 1.8GB/2GB (90%), GC pauses +2300ms during load
```
**What's happening:** The JVM is spending significant time in garbage collection. This manifests as periodic latency spikes and throughput drops.
**Fix:**
```bash
# Increase heap
export OPENMETADATA_HEAP_OPTS="-Xmx4g -Xms4g"
```
### Scenario 4: Bulk Executor Queue Filling
**Symptoms:** HTTP 503 errors on PUT endpoints.
```
Bulk Executor: queue 980/1000 (98%)
has_capacity: false
```
**What's happening:** The async processing queue is full. New requests that need bulk processing are rejected with 503.
**Fix:**
```bash
export BULK_OPERATION_QUEUE_SIZE=2000
export BULK_OPERATION_MAX_THREADS=20
```
### Scenario 5: Slow DB Queries
**Symptoms:** High p95 latency, `database_queries.select.p95_ms` >100ms, `db_pct` >60%.
```
DB Query Profile (total operations: 125,000):
Type Count Mean Max
select 80,000 12.3ms 450.0ms
insert 40,000 25.1ms 890.0ms
Connection acquire avg: 85.2ms
```
**What's happening:** Individual DB queries are slow (p95 >100ms) and connection acquire time is elevated, indicating both query performance issues and pool contention.
**Fix:**
```bash
# Increase pool size to reduce acquire contention
export DB_CONNECTION_POOL_MAX_SIZE=150
# Reduce connection timeout to fail fast
export DB_CONNECTION_TIMEOUT=10000
# Consider adding database indexes for slow SELECT queries
```
---
## Comparing Before/After Snapshots
The most valuable analysis comes from comparing diagnostics before and after load:
| Metric | Before | After | Interpretation |
|--------|--------|-------|----------------|
| `heap_usage_pct` | 25% | 85% | Significant memory allocation during load |
| `gc_pause_total_ms` | 200 | 2500 | 2.3s of GC pauses during the test |
| `pool_active` | 2 | 95 | Pool went from idle to near-max |
| `pool_pending` | 0 | 8 | Connection contention appeared |
| `queue_depth` | 0 | 450 | Bulk queue built up under load |
If the `diagnostics_during` samples are available in the health monitor data, you can plot these metrics over time to see exactly when bottlenecks emerged.
---
## Graceful Fallback
If the server doesn't have the diagnostics endpoint (older version), the load test:
- Prints a notice: `Diagnostics endpoint returned status=404 (may not be available)`
- Falls back to Prometheus scraping (if `--admin-port` is set)
- Skips the `SERVER-SIDE BREAKDOWN` section in the console output
- Omits `diagnostics_before`/`diagnostics_after` from the JSON report
No hard dependency — the load test works with or without it.
bin/distributed-test/USAGE.md
+246
View File
@@ -0,0 +1,246 @@
# Distributed Indexing Load Test Scripts
Scripts for generating test data and triggering reindexing to load-test the OpenMetadata search indexing pipeline.
## Quick Start
```bash
# 1. Start the environment
./scripts/start.sh
# 2. Load test data (~50K entities)
./scripts/perf-test.sh --scale small --server http://localhost:8585
# 3. Trigger reindex
./scripts/trigger-reindex.sh
# 4. Monitor logs
./scripts/logs.sh
# 5. Stop the environment
./scripts/stop.sh
```
## perf-test.sh
Generates entities across 30+ entity types, including time-series data, lineage edges, and data quality entities. Uses concurrent workers for high throughput.
### Scale Presets
Use `--scale` to pick a preset:
| Preset | Approximate Total | Use Case |
|--------|-------------------|----------|
| `small` | ~50K | Quick smoke tests, CI |
| `medium` | ~500K | Integration testing |
| `large` | ~2M | Performance validation |
| `xlarge` | ~5M | Full-scale load testing |
```bash
# Small smoke test
./perf-test.sh --scale small --server http://localhost:8585
# Full 5M load test
./perf-test.sh --scale xlarge --server http://localhost:8585
# Quick mode (~10K, fastest)
./perf-test.sh --quick --server http://localhost:8585
```
Default (no `--scale` or `--quick`) produces ~46K entities for backward compatibility.
### Overriding Individual Counts
Any `--entity-type NUM` flag overrides the preset for that entity type:
```bash
# Small preset but with 100K tables
./perf-test.sh --scale small --tables 100000
# Only create tables and dashboards (everything else stays at preset counts)
./perf-test.sh --scale small --tables 50000 --dashboards 10000
```
### All Flags
#### Entity counts
| Flag | Default | Description |
|------|---------|-------------|
| `--tables NUM` | 20000 | Database tables |
| `--topics NUM` | 3000 | Kafka/messaging topics |
| `--dashboards NUM` | 5000 | Looker dashboards |
| `--charts NUM` | 10000 | Dashboard charts |
| `--pipelines NUM` | 3000 | Airflow pipelines |
| `--stored-procedures NUM` | 0 | Stored procedures |
| `--containers NUM` | 2000 | S3 containers |
| `--search-indexes NUM` | 1000 | Elasticsearch indexes |
| `--mlmodels NUM` | 2000 | ML models |
| `--queries NUM` | 0 | SQL queries |
| `--data-models NUM` | 0 | Dashboard data models |
| `--test-suites NUM` | 0 | Test suites |
| `--test-cases NUM` | 0 | Test cases (linked to tables) |
| `--glossaries NUM` | 50 | Glossaries |
| `--glossary-terms NUM` | 5000 | Glossary terms |
| `--classifications NUM` | 20 | Tag classifications |
| `--tags NUM` | 1000 | Tags |
| `--users NUM` | 0 | Users |
| `--teams NUM` | 0 | Teams |
| `--domains NUM` | 0 | Domains |
| `--data-products NUM` | 0 | Data products (need domains) |
| `--api-collections NUM` | 0 | API collections |
| `--api-endpoints NUM` | 0 | API endpoints (need collections) |
| `--lineage-edges NUM` | 0 | Lineage edges between entities |
#### Time-series entity counts
| Flag | Default | Description |
|------|---------|-------------|
| `--test-case-results NUM` | 0 | Test case results (need test cases) |
| `--entity-report-data NUM` | 0 | Entity report data insights |
| `--web-analytic-views NUM` | 0 | Web analytic entity view reports |
| `--web-analytic-activity NUM` | 0 | Web analytic user activity reports |
| `--raw-cost-analysis NUM` | 0 | Raw cost analysis reports |
| `--aggregated-cost-analysis NUM` | 0 | Aggregated cost analysis reports |
#### Other options
| Flag | Default | Description |
|------|---------|-------------|
| `--server URL` | `http://localhost:8585` | Target OpenMetadata server |
| `--workers NUM` | 20 | Concurrent HTTP workers |
| `--quick` | - | Quick mode preset (~10K entities) |
| `--scale PRESET` | - | Scale preset (small/medium/large/xlarge) |
| `--skip-reads` | - | Skip read benchmarking phase (Phase 8) |
| `--only-reads` | - | Skip write phases; discover existing entities and run reads only |
| `--mixed` | - | Run mixed read/write workload (Phase 9) |
| `--mixed-duration SECS` | 60 | Duration of mixed workload in seconds |
| `--read-ratio PCT` | 80 | Read percentage in mixed workload (0-100) |
| `--realistic` | - | Run Phase 4 entity creation concurrently across entity types using a shared worker pool |
### Entity Creation Order
The script creates entities in dependency order across up to 9 phases:
```
Phase 1 Metadata domains, classifications, tags, glossaries, terms, users, teams
Phase 2 Services database, dashboard, pipeline, messaging, ML, storage, search, API
Phase 3 Infrastructure databases, schemas, API collections
Phase 4 Core entities tables, dashboards, charts, topics, pipelines, storedProcedures,
containers, searchIndexes, mlmodels, queries, dataModels,
apiEndpoints, dataProducts
Phase 5 Data Quality testSuites, testCases
Phase 6 Lineage table->table (60%), table->dashboard (25%), pipeline->table (15%)
Phase 7 Time-Series testCaseResults, entityReportData, webAnalyticViews,
webAnalyticActivity, rawCostAnalysis, aggCostAnalysis
Phase 8 Read Benchmarks entity fetch, paginated list, search queries, lineage traversal
Phase 9 Mixed Workload concurrent reads + writes for configurable duration (--mixed)
```
Phases 8 and 9 are optional:
- Phase 8 runs automatically unless `--skip-reads` is passed
- Phase 9 only runs when `--mixed` is passed
- `--only-reads` skips phases 1-7, discovers existing entities, and runs Phase 8
### Entity Linking
- **Tables, dashboards, pipelines**: IDs collected during Phase 4 for use in lineage (Phase 6)
- **Test cases**: FQNs collected for testCaseResult creation (Phase 7)
- **Lineage edges**: Use collected UUIDs via `PUT /api/v1/lineage`
- Collections are capped at `max(lineage_edges * 2, test_case_results)` to bound memory
### Auto-Scaling Infrastructure
Databases and schemas scale automatically with table count:
- `NUM_DATABASES = max(1, tables / 50000)`
- `SCHEMAS_PER_DB = min(20, tables / (databases * 5000))`
- This keeps ~5000 tables per schema at any scale
### Retry Logic
HTTP requests retry up to 3 times with exponential backoff (1s, 2s, 4s) on:
- 5xx server errors
- Connection errors / timeouts
### Realistic Concurrent Workload (`--realistic`)
By default, `--workers N` creates N concurrent workers **per entity type**, but entity types run
sequentially (all tables first, then all dashboards, etc.). With `--realistic`, all Phase 4 entity
types are created concurrently through a **single shared worker pool**, simulating real-world traffic
where tables, dashboards, topics, and pipelines all hit the server at the same time.
This exposes contention patterns not visible in sequential mode:
- Cross-entity DB lock contention
- Shared thread pool pressure
- Connection pool exhaustion under mixed workloads
```bash
# Realistic mode: all entity types hit the server concurrently
./perf-test.sh --scale 10k --realistic --server http://localhost:8585
# Compare with sequential mode (default)
./perf-test.sh --scale 10k --server http://localhost:8585
```
The report includes a `realistic_combined` entry showing combined RPS and latency distribution
across all entity types, in addition to individual per-entity-type metrics.
### Performance Tips
- Use `--workers 30` or higher if the server can handle it
- Time-series and lineage phases use `min(10, workers)` to avoid overwhelming the server
- At `xlarge` scale, expect the script to run for several hours depending on server capacity
- Monitor server logs for 429/503 errors and reduce workers if needed
### Multi-Scale Benchmarking
Run benchmarks across multiple asset counts to compare performance at different scales:
```bash
# Benchmark at 10k, 50k, 100k, and 200k entities
for scale in 10k 50k 100k 200k; do
./scripts/perf-test.sh --scale "$scale" --server http://localhost:8585 \
--output "/tmp/bench-${scale}.json" --workers 20 2>&1 | tee "/tmp/bench-${scale}.log"
done
```
With read and mixed workload benchmarks included:
```bash
for scale in 10k 50k 100k; do
./scripts/perf-test.sh --scale "$scale" --server http://localhost:8585 \
--mixed --mixed-duration 30 \
--output "/tmp/bench-${scale}.json" 2>&1 | tee "/tmp/bench-${scale}.log"
done
```
Compare results across scales:
```bash
for f in /tmp/bench-*.json; do
echo "=== $(basename $f) ==="
python3 -c "
import json
r = json.load(open('$f'))
o = r['overall']
print(f\" Entities: {o['total_entities_created']:,} RPS: {o['overall_throughput_rps']:.1f}\"
f\" Errors: {o['overall_error_rate_pct']:.1f}% Time: {o['total_wall_clock_s']:.0f}s\")
"
done
```
## Verification After Loading
```bash
# 1. Trigger reindex
./scripts/trigger-reindex.sh
# 2. Check partition table for all entity types
mysql -e "SELECT DISTINCT entityType FROM search_index_partition ORDER BY entityType;"
# 3. Verify counts in UI
# - Data Assets: tables, topics, dashboards, pipelines, etc.
# - Data Quality: test suites and test cases
# - Lineage: visible edges between tables/dashboards/pipelines
# - Data Insights: time-series charts for entity reports, web analytics, cost analysis
```
bin/distributed-test/scripts/benchmark-sizing.sh
Executable
+714
View File
@@ -0,0 +1,714 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PERF_TEST="$SCRIPT_DIR/perf-test.sh"
# ─── Defaults ───────────────────────────────────────────────────────────────────
SERVER="http://localhost:8585"
WORKERS=30
OUTPUT_DIR="./sizing-results"
START_SCALE="10k"
END_SCALE="xlarge"
MODES="seq,realistic"
ADMIN_PORT=""
TOKEN=""
RAMP=false
SKIP_EXISTING=false
MIXED=false
MIXED_DURATION=60
NO_BREAK=false
# ─── Scale Ladder (ordered) ────────────────────────────────────────────────────
ALL_SCALES=(10k 50k 100k 200k 500k large xlarge)
# ─── Colors ─────────────────────────────────────────────────────────────────────
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m'
# ─── Usage ──────────────────────────────────────────────────────────────────────
usage() {
cat <<'EOF'
Usage: benchmark-sizing.sh [OPTIONS]
Progressive benchmark runner that loops through scale tiers, runs perf-test.sh
at each tier, captures JSON outputs, and generates a final comparison report.
Scale Ladder: 10k → 50k → 100k → 200k → 500k → large (~2M) → xlarge (~5M)
OPTIONS:
--server URL Target server (default: http://localhost:8585)
--workers NUM Worker count for all runs (default: 30)
--output-dir DIR Directory for JSON reports + final summary (default: ./sizing-results)
--start-scale PRESET First scale tier to run (default: 10k)
--end-scale PRESET Last scale tier to run (default: xlarge)
--modes MODES Comma-separated: seq, realistic, or both (default: seq,realistic)
--admin-port PORT Pass through to perf-test.sh for server diagnostics
--token TOKEN Auth token pass-through
--ramp Run ramp test at first tier to find optimal workers
--skip-existing Skip tiers that already have JSON output in output-dir
--mixed Also run mixed read/write workload at each tier
--mixed-duration SECS Duration for mixed workload (default: 60)
--no-break Don't stop at break-points, run all tiers regardless
-h, --help Show this help message
EXAMPLES:
# Quick 2-tier test
benchmark-sizing.sh --start-scale 10k --end-scale 50k
# Full progressive benchmark with ramp and mixed workloads
benchmark-sizing.sh --ramp --mixed --output-dir /data/sizing
# Resume after failure (skips completed tiers)
benchmark-sizing.sh --skip-existing --output-dir ./sizing-results
# Single mode only
benchmark-sizing.sh --modes realistic --end-scale 500k
EOF
exit 0
}
# ─── Parse Arguments ────────────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
case "$1" in
--server) SERVER="$2"; shift 2 ;;
--workers) WORKERS="$2"; shift 2 ;;
--output-dir) OUTPUT_DIR="$2"; shift 2 ;;
--start-scale) START_SCALE="$2"; shift 2 ;;
--end-scale) END_SCALE="$2"; shift 2 ;;
--modes) MODES="$2"; shift 2 ;;
--admin-port) ADMIN_PORT="$2"; shift 2 ;;
--token) TOKEN="$2"; shift 2 ;;
--ramp) RAMP=true; shift ;;
--skip-existing) SKIP_EXISTING=true; shift ;;
--mixed) MIXED=true; shift ;;
--mixed-duration) MIXED_DURATION="$2"; shift 2 ;;
--no-break) NO_BREAK=true; shift ;;
-h|--help) usage ;;
*) echo "Unknown option: $1"; usage ;;
esac
done
# ─── Validate ───────────────────────────────────────────────────────────────────
if [[ ! -f "$PERF_TEST" ]]; then
echo -e "${RED}Error: perf-test.sh not found at $PERF_TEST${NC}"
exit 1
fi
validate_scale() {
local scale="$1"
for s in "${ALL_SCALES[@]}"; do
[[ "$s" == "$scale" ]] && return 0
done
echo -e "${RED}Error: Invalid scale '$scale'. Valid: ${ALL_SCALES[*]}${NC}"
exit 1
}
validate_scale "$START_SCALE"
validate_scale "$END_SCALE"
# ─── Resolve Scale Range ───────────────────────────────────────────────────────
resolve_scales() {
local start="$1" end="$2"
local collecting=false
SCALES=()
for s in "${ALL_SCALES[@]}"; do
[[ "$s" == "$start" ]] && collecting=true
if $collecting; then
SCALES+=("$s")
fi
[[ "$s" == "$end" ]] && break
done
if [[ ${#SCALES[@]} -eq 0 ]]; then
echo -e "${RED}Error: No scales resolved between $start and $end${NC}"
exit 1
fi
}
resolve_scales "$START_SCALE" "$END_SCALE"
# ─── Parse Modes ────────────────────────────────────────────────────────────────
IFS=',' read -ra MODE_LIST <<< "$MODES"
for m in "${MODE_LIST[@]}"; do
if [[ "$m" != "seq" && "$m" != "realistic" ]]; then
echo -e "${RED}Error: Invalid mode '$m'. Valid: seq, realistic${NC}"
exit 1
fi
done
# ─── Setup Output Directory ────────────────────────────────────────────────────
mkdir -p "$OUTPUT_DIR"
SUMMARY_CSV="$OUTPUT_DIR/.sizing-progress.csv"
SUMMARY_MD="$OUTPUT_DIR/SIZING-SUMMARY.md"
LOG_FILE="$OUTPUT_DIR/benchmark-sizing.log"
log() {
local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
echo "$msg" >> "$LOG_FILE"
echo -e "$msg"
}
# ─── Print Banner ──────────────────────────────────────────────────────────────
echo ""
echo -e "${BOLD}══════════════════════════════════════════════════════════════════${NC}"
echo -e "${BOLD} OpenMetadata Progressive Cluster Sizing Benchmark${NC}"
echo -e "${BOLD}══════════════════════════════════════════════════════════════════${NC}"
echo ""
echo -e " Server: ${CYAN}$SERVER${NC}"
echo -e " Workers: ${CYAN}$WORKERS${NC}"
echo -e " Scale range: ${CYAN}${SCALES[*]}${NC}"
echo -e " Modes: ${CYAN}${MODE_LIST[*]}${NC}"
echo -e " Output: ${CYAN}$OUTPUT_DIR${NC}"
echo -e " Ramp test: ${CYAN}$RAMP${NC}"
echo -e " Mixed: ${CYAN}$MIXED${NC}"
echo -e " No-break: ${CYAN}$NO_BREAK${NC}"
[[ -n "$ADMIN_PORT" ]] && echo -e " Admin port: ${CYAN}$ADMIN_PORT${NC}"
echo ""
# ─── Connectivity Check ────────────────────────────────────────────────────────
log "Checking server connectivity..."
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$SERVER/api/v1/system/version" 2>/dev/null || echo "000")
if [[ "$HTTP_CODE" == "000" ]]; then
echo -e "${RED}Error: Cannot reach $SERVER (connection refused or timeout)${NC}"
exit 1
elif [[ "$HTTP_CODE" -ge 400 ]]; then
echo -e "${YELLOW}Warning: Server returned HTTP $HTTP_CODE (may need auth token)${NC}"
fi
log "Server reachable (HTTP $HTTP_CODE)"
# ─── Build perf-test.sh Base Args ──────────────────────────────────────────────
build_perf_args() {
local scale="$1" mode="$2" output_file="$3"
local args=()
args+=(--server "$SERVER")
args+=(--workers "$WORKERS")
args+=(--scale "$scale")
args+=(--output "$output_file")
[[ -n "$ADMIN_PORT" ]] && args+=(--admin-port "$ADMIN_PORT")
[[ -n "$TOKEN" ]] && args+=(--token "$TOKEN")
[[ "$mode" == "realistic" ]] && args+=(--realistic)
if $MIXED; then
args+=(--mixed)
args+=(--mixed-duration "$MIXED_DURATION")
fi
echo "${args[@]}"
}
# ─── Extract Metrics from JSON ──────────────────────────────────────────────────
extract_metrics() {
local json_file="$1"
if [[ ! -f "$json_file" ]]; then
echo "N/A,0,0,0,0,0,N/A"
return
fi
python3 -c "
import json, sys
with open('$json_file') as f:
data = json.load(f)
overall = data.get('overall', {})
sizing = data.get('cluster_sizing', {})
server = data.get('server_info', {})
total = overall.get('total_entities_created', 0)
rps = overall.get('overall_throughput_rps', 0)
error_rate = overall.get('overall_error_rate_pct', 0)
wall_clock = overall.get('total_wall_clock_s', 0)
assessment = sizing.get('assessment', 'unknown')
# Find max p95 and p99 across write entities
max_p95 = 0
max_p99 = 0
entities = data.get('entities', {})
for name, ent in entities.items():
if name.startswith('read_') or name.startswith('mixed_'):
continue
lat = ent.get('latency_ms', {})
p95 = lat.get('p95', 0)
p99 = lat.get('p99', 0)
if p95 > max_p95:
max_p95 = p95
if p99 > max_p99:
max_p99 = p99
print(f'{total},{rps:.1f},{max_p95:.0f},{max_p99:.0f},{error_rate:.2f},{wall_clock:.0f},{assessment}')
" 2>/dev/null || echo "N/A,0,0,0,0,0,error"
}
# ─── Extract Config Summary from JSON ───────────────────────────────────────────
extract_config() {
local json_file="$1"
if [[ ! -f "$json_file" ]]; then
echo "N/A"
return
fi
python3 -c "
import json
with open('$json_file') as f:
data = json.load(f)
cs = data.get('cluster_sizing', {}).get('config_summary', [])
print('; '.join(cs) if cs else 'N/A')
" 2>/dev/null || echo "N/A"
}
# ─── Extract Server Info ────────────────────────────────────────────────────────
extract_server_info() {
local json_file="$1"
python3 -c "
import json
with open('$json_file') as f:
data = json.load(f)
si = data.get('server_info', {})
version = si.get('version', 'unknown')
diag = data.get('diagnostics_before', {})
jvm = diag.get('jvm', {})
heap_max = jvm.get('heap_max_bytes', 0)
heap_gb = heap_max / (1024**3) if heap_max else 0
jetty = diag.get('jetty', {})
max_threads = jetty.get('threads_max', 'unknown')
db = diag.get('database', {})
db_pool = db.get('pool_max', 'unknown')
print(f'Version: {version}')
print(f'Heap: {heap_gb:.1f}GB')
print(f'Threads: {max_threads}')
print(f'DB Pool: {db_pool}')
" 2>/dev/null || echo "Version: unknown"
}
# ─── Break-point Detection ──────────────────────────────────────────────────────
PREV_RPS_PER_ENTITY=""
is_broken() {
local metrics="$1"
IFS=',' read -r total rps p95 p99 error_rate wall_clock assessment <<< "$metrics"
[[ "$assessment" == "undersized" ]] && return 0
if python3 -c "exit(0 if float('$error_rate') > 10 else 1)" 2>/dev/null; then
return 0
fi
if python3 -c "exit(0 if float('$p95') > 10000 else 1)" 2>/dev/null; then
return 0
fi
if [[ -n "$PREV_RPS_PER_ENTITY" && "$total" != "0" && "$total" != "N/A" ]]; then
if python3 -c "
curr_rpe = float('$rps') / max(float('$total'), 1)
prev_rpe = float('$PREV_RPS_PER_ENTITY')
exit(0 if prev_rpe > 0 and curr_rpe / prev_rpe < 0.5 else 1)
" 2>/dev/null; then
return 0
fi
fi
return 1
}
# ─── Results Tracking ──────────────────────────────────────────────────────────
declare -a RESULT_LINES=()
BREAK_SCALE=""
BREAK_MODE=""
BREAK_REASON=""
add_result() {
local scale="$1" mode="$2" metrics="$3" config="$4"
RESULT_LINES+=("$scale,$mode,$metrics,$config")
}
# ─── Optional Ramp Test ────────────────────────────────────────────────────────
if $RAMP; then
FIRST_SCALE="${SCALES[0]}"
log "${BOLD}Running ramp test at scale $FIRST_SCALE to find optimal workers...${NC}"
RAMP_OUTPUT="$OUTPUT_DIR/sizing-${FIRST_SCALE}-ramp.json"
RAMP_ARGS=(--server "$SERVER" --workers "$WORKERS" --scale "$FIRST_SCALE" --ramp --output "$RAMP_OUTPUT")
[[ -n "$ADMIN_PORT" ]] && RAMP_ARGS+=(--admin-port "$ADMIN_PORT")
[[ -n "$TOKEN" ]] && RAMP_ARGS+=(--token "$TOKEN")
if $SKIP_EXISTING && [[ -f "$RAMP_OUTPUT" ]]; then
log "Ramp output exists, skipping (--skip-existing)"
else
log "Running: perf-test.sh ${RAMP_ARGS[*]}"
if "$PERF_TEST" "${RAMP_ARGS[@]}" 2>&1 | tee -a "$LOG_FILE"; then
OPTIMAL=$(python3 -c "
import json
with open('$RAMP_OUTPUT') as f:
data = json.load(f)
rt = data.get('ramp_test', {})
print(rt.get('optimal_workers', $WORKERS))
" 2>/dev/null || echo "$WORKERS")
log "${GREEN}Ramp test complete. Optimal workers: $OPTIMAL${NC}"
echo -e "\n${YELLOW}Ramp test suggests $OPTIMAL workers. Current setting: $WORKERS${NC}"
echo -e "${YELLOW}To use the suggested value, re-run with --workers $OPTIMAL${NC}\n"
else
log "${YELLOW}Ramp test failed, continuing with $WORKERS workers${NC}"
fi
fi
fi
# ─── Main Benchmark Loop ──────────────────────────────────────────────────────
BENCHMARK_START=$(date +%s)
TOTAL_TIERS=${#SCALES[@]}
TIER_NUM=0
BROKEN=false
for scale in "${SCALES[@]}"; do
TIER_NUM=$((TIER_NUM + 1))
echo ""
echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${BOLD} Tier $TIER_NUM/$TOTAL_TIERS: scale=$scale${NC}"
echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
for mode in "${MODE_LIST[@]}"; do
OUTPUT_FILE="$OUTPUT_DIR/sizing-${scale}-${mode}.json"
MODE_LABEL=$([[ "$mode" == "seq" ]] && echo "sequential" || echo "realistic")
log " [$scale / $MODE_LABEL] Starting..."
if $SKIP_EXISTING && [[ -f "$OUTPUT_FILE" ]]; then
log " [$scale / $MODE_LABEL] Output exists, skipping (--skip-existing)"
METRICS=$(extract_metrics "$OUTPUT_FILE")
CONFIG=$(extract_config "$OUTPUT_FILE")
add_result "$scale" "$mode" "$METRICS" "$CONFIG"
IFS=',' read -r _total _rps _p95 _p99 _err _wall _assess <<< "$METRICS"
echo -e " ${GREEN}[SKIP]${NC} $scale/$MODE_LABEL: ${_total} entities, ${_rps} RPS, p95=${_p95}ms, err=${_err}%, ${_assess}"
continue
fi
PERF_ARGS=$(build_perf_args "$scale" "$mode" "$OUTPUT_FILE")
log " Running: perf-test.sh $PERF_ARGS"
RUN_START=$(date +%s)
EXIT_CODE=0
# shellcheck disable=SC2086
"$PERF_TEST" $PERF_ARGS 2>&1 | tee -a "$LOG_FILE" || EXIT_CODE=$?
RUN_END=$(date +%s)
RUN_DURATION=$((RUN_END - RUN_START))
if [[ $EXIT_CODE -ne 0 ]]; then
log " ${RED}[$scale / $MODE_LABEL] FAILED (exit code $EXIT_CODE) after ${RUN_DURATION}s${NC}"
add_result "$scale" "$mode" "FAILED,0,0,0,100,${RUN_DURATION},failed" "N/A"
BREAK_SCALE="$scale"
BREAK_MODE="$mode"
BREAK_REASON="perf-test.sh exited with code $EXIT_CODE"
BROKEN=true
break
fi
METRICS=$(extract_metrics "$OUTPUT_FILE")
CONFIG=$(extract_config "$OUTPUT_FILE")
add_result "$scale" "$mode" "$METRICS" "$CONFIG"
IFS=',' read -r _total _rps _p95 _p99 _err _wall _assess <<< "$METRICS"
log " ${GREEN}[$scale / $MODE_LABEL] Done:${NC} ${_total} entities, ${_rps} RPS, p95=${_p95}ms, err=${_err}%, ${_assess} (${RUN_DURATION}s)"
if is_broken "$METRICS"; then
if [[ "$_assess" == "undersized" ]]; then
reason="Cluster assessed as undersized"
elif python3 -c "exit(0 if float('$_err') > 10 else 1)" 2>/dev/null; then
reason="Error rate ${_err}% exceeds 10% threshold"
elif python3 -c "exit(0 if float('$_p95') > 10000 else 1)" 2>/dev/null; then
reason="p95 latency ${_p95}ms exceeds 10s threshold"
else
reason="Throughput degraded >50% from previous tier"
fi
if $NO_BREAK; then
echo -e "\n ${YELLOW}${BOLD}BREAK-POINT WOULD FIRE at $scale/$MODE_LABEL: $reason (--no-break, continuing)${NC}\n"
log "BREAK-POINT SUPPRESSED at $scale/$MODE_LABEL: $reason (--no-break)"
if [[ -z "$BREAK_SCALE" ]]; then
BREAK_SCALE="$scale"
BREAK_MODE="$mode"
BREAK_REASON="$reason (continued with --no-break)"
fi
else
BREAK_SCALE="$scale"
BREAK_MODE="$mode"
BREAK_REASON="$reason"
echo -e "\n ${RED}${BOLD}BREAK-POINT DETECTED at $scale/$MODE_LABEL: $BREAK_REASON${NC}\n"
log "BREAK-POINT DETECTED at $scale/$MODE_LABEL: $BREAK_REASON"
BROKEN=true
break
fi
fi
if [[ "$_total" != "0" && "$_total" != "N/A" ]]; then
PREV_RPS_PER_ENTITY=$(python3 -c "print(float('$_rps') / max(float('$_total'), 1))" 2>/dev/null || echo "")
fi
done
if $BROKEN; then
break
fi
# Tier comparison
if [[ ${#MODE_LIST[@]} -gt 1 ]]; then
SEQ_FILE="$OUTPUT_DIR/sizing-${scale}-seq.json"
REAL_FILE="$OUTPUT_DIR/sizing-${scale}-realistic.json"
if [[ -f "$SEQ_FILE" && -f "$REAL_FILE" ]]; then
echo ""
echo -e " ${CYAN}Tier $scale comparison:${NC}"
SEQ_M=$(extract_metrics "$SEQ_FILE")
REAL_M=$(extract_metrics "$REAL_FILE")
IFS=',' read -r s_total s_rps s_p95 s_p99 s_err s_wall s_assess <<< "$SEQ_M"
IFS=',' read -r r_total r_rps r_p95 r_p99 r_err r_wall r_assess <<< "$REAL_M"
printf " %-12s %8s %8s %8s %8s %10s\n" "" "Entities" "RPS" "p95ms" "Errors%" "Assessment"
printf " %-12s %8s %8s %8s %8s %10s\n" "Sequential" "$s_total" "$s_rps" "$s_p95" "${s_err}%" "$s_assess"
printf " %-12s %8s %8s %8s %8s %10s\n" "Realistic" "$r_total" "$r_rps" "$r_p95" "${r_err}%" "$r_assess"
echo ""
fi
fi
done
BENCHMARK_END=$(date +%s)
BENCHMARK_DURATION=$((BENCHMARK_END - BENCHMARK_START))
# ─── Generate SIZING-SUMMARY.md ────────────────────────────────────────────────
log "Generating $SUMMARY_MD..."
# Try to get server info from the first available JSON
SERVER_INFO=""
for scale in "${SCALES[@]}"; do
for mode in "${MODE_LIST[@]}"; do
F="$OUTPUT_DIR/sizing-${scale}-${mode}.json"
if [[ -f "$F" ]]; then
SERVER_INFO=$(extract_server_info "$F")
break 2
fi
done
done
{
echo "# OpenMetadata Cluster Sizing Summary"
echo ""
echo "Generated: $(date '+%Y-%m-%d %H:%M:%S')"
echo "Total benchmark time: ${BENCHMARK_DURATION}s ($((BENCHMARK_DURATION / 60))m $((BENCHMARK_DURATION % 60))s)"
echo ""
echo "## Server Configuration"
echo ""
echo '```'
echo "Server: $SERVER"
echo "Workers: $WORKERS"
if [[ -n "$SERVER_INFO" ]]; then
echo "$SERVER_INFO"
fi
echo '```'
echo ""
echo "## Progressive Results"
echo ""
echo "| Scale | Mode | Entities | RPS | p95 (ms) | p99 (ms) | Errors % | Assessment | Duration |"
echo "|-------|------|----------|-----|----------|----------|----------|------------|----------|"
LAST_ADEQUATE_SCALE=""
LAST_ADEQUATE_CONFIG=""
for line in "${RESULT_LINES[@]}"; do
IFS=',' read -r scale mode total rps p95 p99 err wall assess config <<< "$line"
mode_label=$([[ "$mode" == "seq" ]] && echo "Sequential" || echo "Realistic")
# Format duration
if [[ "$wall" =~ ^[0-9]+$ ]]; then
duration="${wall}s"
else
duration="$wall"
fi
# Assessment emoji
case "$assess" in
adequate) assess_fmt="$assess" ;;
marginal) assess_fmt="$assess" ;;
undersized) assess_fmt="$assess" ;;
failed) assess_fmt="FAILED" ;;
*) assess_fmt="$assess" ;;
esac
echo "| $scale | $mode_label | $total | $rps | $p95 | $p99 | $err | $assess_fmt | $duration |"
if [[ "$assess" == "adequate" || "$assess" == "marginal" ]]; then
LAST_ADEQUATE_SCALE="$scale"
LAST_ADEQUATE_CONFIG="$config"
fi
done
echo ""
# Break-point section
if [[ -n "$BREAK_SCALE" ]]; then
echo "## Break-Point Detected"
echo ""
echo "**Scale:** $BREAK_SCALE "
echo "**Mode:** $([[ "$BREAK_MODE" == "seq" ]] && echo "Sequential" || echo "Realistic") "
echo "**Reason:** $BREAK_REASON"
echo ""
if [[ -n "$LAST_ADEQUATE_SCALE" ]]; then
echo "The cluster handled **$LAST_ADEQUATE_SCALE** scale adequately but broke at **$BREAK_SCALE**."
echo ""
fi
else
echo "## Result"
echo ""
echo "No break-point detected. The cluster handled all tested scales up to **${SCALES[${#SCALES[@]}-1]}**."
echo ""
fi
# Recommended configuration
if [[ -n "$LAST_ADEQUATE_CONFIG" && "$LAST_ADEQUATE_CONFIG" != "N/A" ]]; then
echo "## Recommended Configuration"
echo ""
echo "Based on the last adequate tier ($LAST_ADEQUATE_SCALE):"
echo ""
echo '```bash'
IFS=';' read -ra CONFIGS <<< "$LAST_ADEQUATE_CONFIG"
for cfg in "${CONFIGS[@]}"; do
cfg_trimmed=$(echo "$cfg" | xargs)
[[ -n "$cfg_trimmed" ]] && echo "export $cfg_trimmed"
done
echo '```'
echo ""
fi
# Sequential vs Realistic comparison
if [[ ${#MODE_LIST[@]} -gt 1 ]]; then
echo "## Sequential vs Realistic Comparison"
echo ""
echo "| Scale | Seq RPS | Real RPS | RPS Diff | Seq p95 | Real p95 | p95 Diff |"
echo "|-------|---------|----------|----------|---------|----------|----------|"
for scale in "${SCALES[@]}"; do
SEQ_FILE="$OUTPUT_DIR/sizing-${scale}-seq.json"
REAL_FILE="$OUTPUT_DIR/sizing-${scale}-realistic.json"
if [[ -f "$SEQ_FILE" && -f "$REAL_FILE" ]]; then
python3 -c "
import json
with open('$SEQ_FILE') as f:
seq = json.load(f)
with open('$REAL_FILE') as f:
real = json.load(f)
s_rps = seq.get('overall', {}).get('overall_throughput_rps', 0)
r_rps = real.get('overall', {}).get('overall_throughput_rps', 0)
rps_diff = ((r_rps - s_rps) / s_rps * 100) if s_rps > 0 else 0
# Max p95
def max_p95(data):
mx = 0
for name, ent in data.get('entities', {}).items():
if name.startswith('read_') or name.startswith('mixed_'):
continue
p = ent.get('latency_ms', {}).get('p95', 0)
if p > mx:
mx = p
return mx
s_p95 = max_p95(seq)
r_p95 = max_p95(real)
p95_diff = ((r_p95 - s_p95) / s_p95 * 100) if s_p95 > 0 else 0
print(f'| $scale | {s_rps:.1f} | {r_rps:.1f} | {rps_diff:+.1f}% | {s_p95:.0f} | {r_p95:.0f} | {p95_diff:+.1f}% |')
" 2>/dev/null || true
fi
done
echo ""
echo "> **Sequential** runs entity types one at a time. **Realistic** runs all concurrently through a shared worker pool, exposing cross-entity contention."
echo ""
fi
# Individual tier details
echo "## Tier Details"
echo ""
for scale in "${SCALES[@]}"; do
for mode in "${MODE_LIST[@]}"; do
F="$OUTPUT_DIR/sizing-${scale}-${mode}.json"
[[ ! -f "$F" ]] && continue
mode_label=$([[ "$mode" == "seq" ]] && echo "Sequential" || echo "Realistic")
echo "### $scale ($mode_label)"
echo ""
python3 -c "
import json
with open('$F') as f:
data = json.load(f)
entities = data.get('entities', {})
print('| Entity | Count | RPS | p50 | p95 | p99 | Errors % |')
print('|--------|-------|-----|-----|-----|-----|----------|')
for name, ent in sorted(entities.items()):
count = ent.get('created', ent.get('total_requests', 0))
rps = ent.get('throughput_rps', 0)
lat = ent.get('latency_ms', {})
p50 = lat.get('p50', 0)
p95 = lat.get('p95', 0)
p99 = lat.get('p99', 0)
err = ent.get('error_rate_pct', 0)
print(f'| {name} | {count} | {rps:.1f} | {p50:.0f} | {p95:.0f} | {p99:.0f} | {err:.2f} |')
# Sizing findings
sizing = data.get('cluster_sizing', {})
findings = sizing.get('findings', [])
if findings:
print()
print('**Findings:**')
for f in findings:
print(f'- {f}')
" 2>/dev/null || echo "_Failed to parse results_"
echo ""
done
done
echo "---"
echo ""
echo "Generated by \`benchmark-sizing.sh\` on $(date '+%Y-%m-%d %H:%M:%S')"
} > "$SUMMARY_MD"
# ─── Final Console Summary ─────────────────────────────────────────────────────
echo ""
echo -e "${BOLD}══════════════════════════════════════════════════════════════════${NC}"
echo -e "${BOLD} SIZING BENCHMARK COMPLETE${NC}"
echo -e "${BOLD}══════════════════════════════════════════════════════════════════${NC}"
echo ""
printf " %-8s %-12s %8s %8s %8s %8s %10s\n" "Scale" "Mode" "Entities" "RPS" "p95ms" "Errors%" "Assessment"
printf " %-8s %-12s %8s %8s %8s %8s %10s\n" "────────" "────────────" "────────" "────────" "────────" "────────" "──────────"
for line in "${RESULT_LINES[@]}"; do
IFS=',' read -r scale mode total rps p95 p99 err wall assess config <<< "$line"
mode_label=$([[ "$mode" == "seq" ]] && echo "Sequential" || echo "Realistic")
case "$assess" in
adequate) color="$GREEN" ;;
marginal) color="$YELLOW" ;;
undersized) color="$RED" ;;
failed) color="$RED" ;;
*) color="$NC" ;;
esac
printf " %-8s %-12s %8s %8s %8s %8s ${color}%10s${NC}\n" \
"$scale" "$mode_label" "$total" "$rps" "$p95" "${err}%" "$assess"
done
echo ""
if [[ -n "$BREAK_SCALE" ]]; then
echo -e " ${RED}${BOLD}Break-point: $BREAK_SCALE ($BREAK_MODE) — $BREAK_REASON${NC}"
if [[ -n "$LAST_ADEQUATE_SCALE" ]]; then
echo -e " ${GREEN}Last adequate scale: $LAST_ADEQUATE_SCALE${NC}"
fi
else
echo -e " ${GREEN}No break-point detected — cluster handled all scales through ${SCALES[${#SCALES[@]}-1]}${NC}"
fi
echo ""
echo -e " Total time: ${BENCHMARK_DURATION}s ($((BENCHMARK_DURATION / 60))m $((BENCHMARK_DURATION % 60))s)"
echo -e " Summary: ${CYAN}$SUMMARY_MD${NC}"
echo -e " Log: ${CYAN}$LOG_FILE${NC}"
echo ""
bin/distributed-test/scripts/gc-reindex-report.sh
Executable
+370
View File
@@ -0,0 +1,370 @@
#!/bin/bash
# Captures GC pause statistics during a reindex run and compares them to a saved baseline.
#
# Usage:
# ./gc-reindex-report.sh --token TOKEN [--server URL] [--gclog PATH] [--save-baseline]
#
# What it does:
# 1. Reads the server's GC log (or polls /api/v1/system/metrics for JVM stats if no GC log)
# 2. Triggers a full reindex
# 3. Polls until the reindex finishes
# 4. Summarises: total STW pause time, max pause, p99 pause, probe failures, throughput
# 5. Optionally saves the result as a baseline for future comparison
#
# GC log note:
# Start the server with: -Xlog:gc*:file=/tmp/om-gc.log:time,uptime,tags:filecount=5,filesize=20m
# Then pass --gclog /tmp/om-gc.log to this script.
#
# Without --gclog the script uses the Prometheus /metrics endpoint to estimate GC pressure.
set -euo pipefail
SERVER_URL="http://localhost:8585"
TOKEN=""
GC_LOG=""
SAVE_BASELINE=false
BASELINE_FILE="$(dirname "$0")/../gc-baseline.json"
POLL_INTERVAL=10 # seconds between status polls
while [[ $# -gt 0 ]]; do
case $1 in
--server) SERVER_URL="$2"; shift 2 ;;
--token) TOKEN="$2"; shift 2 ;;
--gclog) GC_LOG="$2"; shift 2 ;;
--save-baseline) SAVE_BASELINE=true; shift ;;
--baseline) BASELINE_FILE="$2"; shift 2 ;;
-h|--help)
sed -n '2,20p' "$0" | sed 's/^# //;s/^#//'
exit 0
;;
*) echo "Unknown arg: $1"; exit 1 ;;
esac
done
if [[ -z "$TOKEN" ]]; then
echo "ERROR: --token is required"; exit 1
fi
AUTH_HEADER="Authorization: Bearer $TOKEN"
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
# ── helpers ───────────────────────────────────────────────────────────────────
jq_required() {
if ! command -v jq &>/dev/null; then
echo "ERROR: jq is required. Install with: brew install jq"; exit 1
fi
}
curl_json() {
curl -sf -H "$AUTH_HEADER" -H "Content-Type: application/json" "$@"
}
# ── capture GC snapshot via /metrics ─────────────────────────────────────────
gc_metrics_snapshot() {
local label="$1"
local result
# Try Prometheus endpoint first (if enabled)
if result=$(curl -sf -H "$AUTH_HEADER" "${SERVER_URL}/api/v1/system/metrics" 2>/dev/null); then
local gc_count gc_time
gc_count=$(echo "$result" | grep '^jvm_gc_pause_seconds_count' | awk '{s+=$NF} END{print int(s)}' || echo 0)
gc_time=$(echo "$result" | grep '^jvm_gc_pause_seconds_sum' | awk '{s+=$NF} END{printf "%.3f", s}' || echo "0.000")
echo "${label}:gc_count=${gc_count},gc_time_s=${gc_time}"
else
echo "${label}:gc_count=N/A,gc_time_s=N/A"
fi
}
# ── parse GC log (G1GC format) ────────────────────────────────────────────────
parse_gc_log() {
local logfile="$1"
if [[ ! -f "$logfile" ]]; then
echo "GC_LOG_NOT_FOUND"; return
fi
python3 - "$logfile" <<'PYEOF'
import re, sys, statistics
log_file = sys.argv[1]
pauses = []
# Match G1GC pause lines: [0.123s][info][gc] GC(0) Pause ... 12.345ms
pause_re = re.compile(r'Pause (?:Young|Full|Initial Mark|Remark|Cleanup)[^0-9]*(\d+\.\d+)ms', re.IGNORECASE)
# Also match: GC(N) Pause ... Nms
generic_re = re.compile(r'\bPause\b.*?(\d+\.\d+)ms')
with open(log_file) as f:
for line in f:
m = pause_re.search(line) or generic_re.search(line)
if m:
pauses.append(float(m.group(1)))
if not pauses:
print("no_pauses_found")
sys.exit(0)
pauses.sort()
p99_idx = max(0, int(len(pauses) * 0.99) - 1)
print(f"count={len(pauses)}")
print(f"total_ms={sum(pauses):.1f}")
print(f"max_ms={max(pauses):.1f}")
print(f"p99_ms={pauses[p99_idx]:.1f}")
print(f"mean_ms={statistics.mean(pauses):.1f}")
PYEOF
}
# ── probe health once ─────────────────────────────────────────────────────────
probe_health() {
local http_code
http_code=$(curl -o /dev/null -sf -w "%{http_code}" \
--max-time 1 \
-H "$AUTH_HEADER" \
"${SERVER_URL}/api/v1/system/health" 2>/dev/null || echo "000")
echo "$http_code"
}
# ── trigger reindex ───────────────────────────────────────────────────────────
trigger_reindex() {
local payload='{"recreateIndex":false}'
curl_json -X POST \
"${SERVER_URL}/api/v1/apps/trigger/SearchIndexingApplication" \
-d "$payload" > /dev/null 2>&1 || true
echo "Reindex triggered"
}
# ── poll reindex status ───────────────────────────────────────────────────────
STATS_FILE="/tmp/om-reindex-stats-$$.txt"
poll_until_done() {
local start_ts
start_ts=$(date +%s)
local probe_failures=0
local probe_total=0
local max_latency_ms=0
local probes_over_1000=0
local consecutive_slow=0
local max_consecutive_slow=0
local last_probe_ts=0
echo ""
echo "Polling reindex status every ${POLL_INTERVAL}s ..."
echo " (probing /system/health every 10s — k8s periodSeconds=10, failureThreshold=3)"
echo ""
while true; do
sleep "$POLL_INTERVAL"
local now
now=$(date +%s)
# Probe health every 10s (k8s default periodSeconds)
if (( now - last_probe_ts >= 10 )); then
local t0 code latency
t0=$(python3 -c "import time; print(int(time.time()*1000))")
code=$(probe_health)
latency=$(( $(python3 -c "import time; print(int(time.time()*1000))") - t0 ))
probe_total=$(( probe_total + 1 ))
last_probe_ts=$now
if (( latency > max_latency_ms )); then max_latency_ms=$latency; fi
if (( latency > 1000 )); then
probes_over_1000=$(( probes_over_1000 + 1 ))
consecutive_slow=$(( consecutive_slow + 1 ))
if (( consecutive_slow > max_consecutive_slow )); then
max_consecutive_slow=$consecutive_slow
fi
else
consecutive_slow=0
fi
if [[ "$code" != "200" ]]; then
probe_failures=$(( probe_failures + 1 ))
echo " [$(date +%H:%M:%S)] PROBE FAIL code=$code latency=${latency}ms consecutive=${consecutive_slow}" >&2
elif (( latency > 1000 )); then
echo " [$(date +%H:%M:%S)] SLOW PROBE latency=${latency}ms *** >1000ms (k8s timeout risk) consecutive=${consecutive_slow}" >&2
else
echo " [$(date +%H:%M:%S)] probe ok latency=${latency}ms" >&2
fi
if (( max_consecutive_slow >= 3 )); then
echo " *** WARNING: ${max_consecutive_slow} consecutive slow probes — k8s WOULD RESTART POD ***" >&2
fi
fi
# Check app status
local status_json
status_json=$(curl_json "${SERVER_URL}/api/v1/apps/name/SearchIndexingApplication/status?offset=0&limit=1" 2>/dev/null || echo '{}')
local app_status
app_status=$(echo "$status_json" | python3 -c "
import json,sys
try:
d=json.load(sys.stdin)
runs=d.get('data',[])
if runs: print(runs[0].get('status','unknown'))
else: print('pending')
except: print('unknown')
" 2>/dev/null || echo "unknown")
if [[ "$app_status" == "success" || "$app_status" == "failed" ]]; then
local elapsed=$(( now - start_ts ))
echo "" >&2
echo " Reindex finished: status=$app_status elapsed=${elapsed}s" >&2
# Write structured results to stats file (not stdout)
{
echo "PROBE_FAILURES=$probe_failures"
echo "PROBE_TOTAL=$probe_total"
echo "MAX_LATENCY_MS=$max_latency_ms"
echo "PROBES_OVER_1000=$probes_over_1000"
echo "MAX_CONSECUTIVE_SLOW=$max_consecutive_slow"
echo "ELAPSED_S=$elapsed"
} > "$STATS_FILE"
return
fi
done
}
# ── main ──────────────────────────────────────────────────────────────────────
jq_required
echo "═══════════════════════════════════════════════════════════════"
echo " GC / Reindex Report — $(date)"
echo " Server: $SERVER_URL"
echo "═══════════════════════════════════════════════════════════════"
# Snapshot before
METRICS_BEFORE=$(gc_metrics_snapshot "before")
GC_LOG_LINES_BEFORE=0
if [[ -n "$GC_LOG" && -f "$GC_LOG" ]]; then
GC_LOG_LINES_BEFORE=$(wc -l < "$GC_LOG")
echo "GC log: $GC_LOG (current lines: $GC_LOG_LINES_BEFORE)"
fi
trigger_reindex
# Poll — probe lines print live to stderr, structured results written to STATS_FILE
poll_until_done
PROBE_FAILURES=$(grep '^PROBE_FAILURES=' "$STATS_FILE" | cut -d= -f2)
PROBE_TOTAL=$(grep '^PROBE_TOTAL=' "$STATS_FILE" | cut -d= -f2)
MAX_LATENCY=$(grep '^MAX_LATENCY_MS=' "$STATS_FILE" | cut -d= -f2)
PROBES_OVER_1000=$(grep '^PROBES_OVER_1000=' "$STATS_FILE" | cut -d= -f2)
MAX_CONSECUTIVE=$(grep '^MAX_CONSECUTIVE_SLOW=' "$STATS_FILE" | cut -d= -f2)
ELAPSED_S=$(grep '^ELAPSED_S=' "$STATS_FILE" | cut -d= -f2)
rm -f "$STATS_FILE"
# Snapshot after
METRICS_AFTER=$(gc_metrics_snapshot "after")
# GC log delta
GC_LOG_STATS=""
if [[ -n "$GC_LOG" && -f "$GC_LOG" ]]; then
# Extract only lines added since reindex started
TMPLOG="/tmp/om-gc-run-${TIMESTAMP}.log"
tail -n "+${GC_LOG_LINES_BEFORE}" "$GC_LOG" > "$TMPLOG"
GC_LOG_STATS=$(parse_gc_log "$TMPLOG")
rm -f "$TMPLOG"
fi
# Parse metrics delta
GC_COUNT_BEFORE=$(echo "$METRICS_BEFORE" | sed 's/.*gc_count=\([^,]*\).*/\1/')
GC_TIME_BEFORE=$(echo "$METRICS_BEFORE" | sed 's/.*gc_time_s=\([^,]*\).*/\1/')
GC_COUNT_AFTER=$(echo "$METRICS_AFTER" | sed 's/.*gc_count=\([^,]*\).*/\1/')
GC_TIME_AFTER=$(echo "$METRICS_AFTER" | sed 's/.*gc_time_s=\([^,]*\).*/\1/')
# Compute deltas if numeric
GC_COUNT_DELTA="N/A"
GC_TIME_DELTA="N/A"
if [[ "$GC_COUNT_BEFORE" =~ ^[0-9]+$ && "$GC_COUNT_AFTER" =~ ^[0-9]+$ ]]; then
GC_COUNT_DELTA=$(( GC_COUNT_AFTER - GC_COUNT_BEFORE ))
fi
if [[ "$GC_TIME_BEFORE" =~ ^[0-9.]+$ && "$GC_TIME_AFTER" =~ ^[0-9.]+$ ]]; then
GC_TIME_DELTA=$(python3 -c "print(f'{float('$GC_TIME_AFTER') - float('$GC_TIME_BEFORE'):.3f}')")
fi
echo ""
echo "══════════════════ RESULTS ════════════════════════════════════"
echo ""
printf " %-30s %s\n" "Reindex duration" "${ELAPSED_S}s"
printf " %-30s %s\n" "Health probe total" "${PROBE_TOTAL}"
printf " %-30s %s\n" "Probe failures (non-200)" "${PROBE_FAILURES:-0} $([ "${PROBE_FAILURES:-0}" -eq 0 ] && echo '✓ none' || echo '⚠ k8s would have restarted pod')"
printf " %-30s %s\n" "Probes >1000ms" "${PROBES_OVER_1000:-0} $([ "${PROBES_OVER_1000:-0}" -eq 0 ] && echo '✓ none' || echo '⚠ GC pressure detected')"
printf " %-30s %s\n" "Max consecutive >1000ms" "${MAX_CONSECUTIVE:-0} $([ "${MAX_CONSECUTIVE:-0}" -lt 3 ] && echo '✓ ok' || echo '*** k8s WOULD RESTART POD ***')"
printf " %-30s %s\n" "Max probe latency" "${MAX_LATENCY:-0}ms $([ "${MAX_LATENCY:-0}" -lt 1000 ] && echo '✓ ok' || echo '⚠ high')"
echo ""
printf " %-30s %s\n" "GC collections (delta)" "$GC_COUNT_DELTA"
printf " %-30s %s\n" "GC pause time (delta)" "${GC_TIME_DELTA}s"
echo ""
if [[ -n "$GC_LOG_STATS" && "$GC_LOG_STATS" != "no_pauses_found" && "$GC_LOG_STATS" != "GC_LOG_NOT_FOUND" ]]; then
echo " GC log pause breakdown:"
echo "$GC_LOG_STATS" | sed 's/^/ /'
echo ""
fi
# Build JSON result
RESULT_JSON=$(python3 -c "
import json, sys
result = {
'timestamp': '$(date -u +%Y-%m-%dT%H:%M:%SZ)',
'server': '$SERVER_URL',
'elapsed_s': int('${ELAPSED_S:-0}'),
'probe_total': int('${PROBE_TOTAL:-0}'),
'probe_failures': int('${PROBE_FAILURES:-0}'),
'probes_over_1000ms': int('${PROBES_OVER_1000:-0}'),
'max_consecutive_slow': int('${MAX_CONSECUTIVE:-0}'),
'max_probe_latency_ms': int('${MAX_LATENCY:-0}'),
'gc_collections_delta': '${GC_COUNT_DELTA}',
'gc_pause_time_delta_s': '${GC_TIME_DELTA}',
'gc_log_stats': '${GC_LOG_STATS:-none}'
}
print(json.dumps(result, indent=2))
")
# Compare to baseline
if [[ -f "$BASELINE_FILE" ]]; then
echo " Comparison to baseline ($BASELINE_FILE):"
python3 - "$BASELINE_FILE" <<PYEOF
import json, sys
with open(sys.argv[1]) as f:
baseline = json.load(f)
current = $RESULT_JSON
def pct(new, old):
if old == 0: return '+∞%'
delta = new - old
return f'{delta/old*100:+.1f}%'
print(f" {'Metric':<30} {'Baseline':>12} {'Current':>12} {'Change':>10}")
print(f" {'-'*30} {'-'*12} {'-'*12} {'-'*10}")
for k in ['elapsed_s','probe_failures','probes_over_1000ms','max_consecutive_slow','max_probe_latency_ms']:
b = baseline.get(k, 'N/A')
c = current.get(k, 'N/A')
change = pct(c, b) if isinstance(b, (int,float)) and isinstance(c, (int,float)) else 'N/A'
print(f" {k:<30} {str(b):>12} {str(c):>12} {change:>10}")
PYEOF
echo ""
fi
# Save result
OUT_FILE="gc-report-${TIMESTAMP}.json"
echo "$RESULT_JSON" > "$OUT_FILE"
echo " Report saved to: $OUT_FILE"
if [[ "$SAVE_BASELINE" == "true" ]]; then
cp "$OUT_FILE" "$BASELINE_FILE"
echo " Saved as baseline: $BASELINE_FILE"
fi
echo ""
echo "══════════════════════════════════════════════════════════════"
echo ""
echo " Tip: Run once with --save-baseline to record a baseline, then"
echo " run again after code changes to see the delta."
echo ""
bin/distributed-test/scripts/logs.sh
Executable
+130
View File
@@ -0,0 +1,130 @@
#!/bin/bash
# View aggregated logs from all OM servers with color coding
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_DIR"
# Colors for different servers
COLOR_SERVER1="\033[1;32m" # Green
COLOR_SERVER2="\033[1;34m" # Blue
COLOR_SERVER3="\033[1;35m" # Magenta
COLOR_MYSQL="\033[1;33m" # Yellow
COLOR_OPENSEARCH="\033[1;36m" # Cyan
COLOR_RESET="\033[0m"
# Default values
FOLLOW=false
SERVER_FILTER=""
GREP_PATTERN=""
TAIL_LINES=100
# Parse arguments
while [[ $# -gt 0 ]]; do
case $1 in
-f|--follow)
FOLLOW=true
shift
;;
--server)
SERVER_FILTER="$2"
shift 2
;;
--grep)
GREP_PATTERN="$2"
shift 2
;;
--tail)
TAIL_LINES="$2"
shift 2
;;
-h|--help)
echo "Usage: $0 [OPTIONS]"
echo ""
echo "Options:"
echo " -f, --follow Follow log output (like tail -f)"
echo " --server NUM Filter to specific server (1, 2, or 3)"
echo " --grep PATTERN Filter logs by pattern"
echo " --tail NUM Number of lines to show (default: 100)"
echo " -h, --help Show this help message"
echo ""
echo "Examples:"
echo " $0 -f # Follow all server logs"
echo " $0 --server 1 -f # Follow only server 1 logs"
echo " $0 --grep 'partition' -f # Follow logs containing 'partition'"
echo " $0 --grep 'SearchIndex' --tail 500"
exit 0
;;
*)
echo "Unknown option: $1"
exit 1
;;
esac
done
# Build the docker compose logs command
COMPOSE_ARGS=""
if [ "$FOLLOW" == "true" ]; then
COMPOSE_ARGS="$COMPOSE_ARGS -f"
fi
COMPOSE_ARGS="$COMPOSE_ARGS --tail=$TAIL_LINES"
# Determine which services to show
SERVICES=""
if [ -n "$SERVER_FILTER" ]; then
case $SERVER_FILTER in
1) SERVICES="openmetadata-server-1" ;;
2) SERVICES="openmetadata-server-2" ;;
3) SERVICES="openmetadata-server-3" ;;
mysql) SERVICES="mysql" ;;
opensearch) SERVICES="opensearch" ;;
*)
echo "Invalid server filter: $SERVER_FILTER"
echo "Use 1, 2, 3, mysql, or opensearch"
exit 1
;;
esac
else
SERVICES="openmetadata-server-1 openmetadata-server-2 openmetadata-server-3"
fi
# Function to colorize output
colorize_logs() {
while IFS= read -r line; do
if [[ $line == *"openmetadata-server-1"* ]] || [[ $line == *"om-server-1"* ]]; then
echo -e "${COLOR_SERVER1}[SERVER-1]${COLOR_RESET} $line"
elif [[ $line == *"openmetadata-server-2"* ]] || [[ $line == *"om-server-2"* ]]; then
echo -e "${COLOR_SERVER2}[SERVER-2]${COLOR_RESET} $line"
elif [[ $line == *"openmetadata-server-3"* ]] || [[ $line == *"om-server-3"* ]]; then
echo -e "${COLOR_SERVER3}[SERVER-3]${COLOR_RESET} $line"
elif [[ $line == *"mysql"* ]]; then
echo -e "${COLOR_MYSQL}[MYSQL]${COLOR_RESET} $line"
elif [[ $line == *"opensearch"* ]]; then
echo -e "${COLOR_OPENSEARCH}[OPENSEARCH]${COLOR_RESET} $line"
else
echo "$line"
fi
done
}
echo "======================================"
echo "Distributed Test Logs"
echo "======================================"
echo "Services: $SERVICES"
if [ -n "$GREP_PATTERN" ]; then
echo "Filter: $GREP_PATTERN"
fi
echo "--------------------------------------"
echo ""
# Run the logs command with optional grep filter
if [ -n "$GREP_PATTERN" ]; then
docker compose logs $COMPOSE_ARGS $SERVICES 2>&1 | grep --line-buffered -i "$GREP_PATTERN" | colorize_logs
else
docker compose logs $COMPOSE_ARGS $SERVICES 2>&1 | colorize_logs
fi
bin/distributed-test/scripts/perf-test.sh
Executable
+3285
View File
File diff suppressed because it is too large Load Diff
bin/distributed-test/scripts/start.sh
Executable
+126
View File
@@ -0,0 +1,126 @@
#!/bin/bash
# Start the distributed test environment
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
ROOT_DIR="$(cd "$PROJECT_DIR/../../.." && pwd)"
cd "$PROJECT_DIR"
# Load environment variables (handle values with spaces properly)
if [ -f .env ]; then
set -a
source .env
set +a
fi
echo "======================================"
echo "Distributed Search Indexing Test Setup"
echo "======================================"
echo ""
echo "Configuration:"
echo " - MySQL Port: ${MYSQL_PORT:-3306}"
echo " - OpenSearch Port: ${OPENSEARCH_PORT:-9200}"
echo " - Server 1: http://localhost:8585"
echo " - Server 2: http://localhost:8587"
echo " - Server 3: http://localhost:8589"
echo ""
# Parse arguments
BUILD_FLAG=""
SKIP_MVN=false
for arg in "$@"; do
case $arg in
--build|-b)
BUILD_FLAG="--build"
;;
--skip-mvn|-s)
SKIP_MVN=true
;;
esac
done
# Check if distribution exists, if not build with Maven
DIST_TAR=$(find "$ROOT_DIR/openmetadata-dist/target" -name "openmetadata-*.tar.gz" 2>/dev/null | head -1)
if [ -z "$DIST_TAR" ] && [ "$SKIP_MVN" != "true" ]; then
echo "OpenMetadata distribution not found. Building with Maven..."
echo "This may take several minutes on first run."
echo ""
cd "$ROOT_DIR"
mvn clean install -DskipTests -Pquickstart -pl '!openmetadata-ui' -am
cd "$PROJECT_DIR"
BUILD_FLAG="--build"
echo ""
echo "Maven build complete."
elif [ "$SKIP_MVN" == "true" ]; then
echo "Skipping Maven build (--skip-mvn flag)"
fi
if [ -n "$BUILD_FLAG" ]; then
echo "Building Docker images..."
fi
# Start the services
echo "Starting services..."
docker compose up -d $BUILD_FLAG
echo ""
echo "Waiting for services to be healthy..."
# Wait for MySQL
echo -n " MySQL: "
until docker compose exec -T mysql mysqladmin ping -h localhost -uroot -p${MYSQL_ROOT_PASSWORD:-password} --silent 2>/dev/null; do
echo -n "."
sleep 2
done
echo " Ready"
# Wait for OpenSearch
echo -n " OpenSearch: "
until curl -s http://localhost:${OPENSEARCH_PORT:-9200}/_cluster/health 2>/dev/null | grep -qE '"status":"(green|yellow)"'; do
echo -n "."
sleep 2
done
echo " Ready"
# Wait for each OM server
for server_num in 1 2 3; do
case $server_num in
1) port=8586 ;;
2) port=8588 ;;
3) port=8590 ;;
esac
echo -n " Server $server_num (admin port $port): "
timeout=120
elapsed=0
until curl -s "http://localhost:$port/healthcheck" >/dev/null 2>&1; do
echo -n "."
sleep 3
elapsed=$((elapsed + 3))
if [ $elapsed -ge $timeout ]; then
echo " Timeout waiting for server $server_num"
echo "Check logs with: ./scripts/logs.sh -f --server $server_num"
exit 1
fi
done
echo " Ready"
done
echo ""
echo "======================================"
echo "All services are up and running!"
echo "======================================"
echo ""
echo "Next steps:"
echo " 1. Load test data: ./scripts/perf-test.sh --tables 10000"
echo " 2. Trigger reindexing: ./scripts/trigger-reindex.sh"
echo " 3. Watch logs: ./scripts/logs.sh -f"
echo ""
echo "Server endpoints:"
echo " - Server 1: http://localhost:8585 (trigger reindex here)"
echo " - Server 2: http://localhost:8587"
echo " - Server 3: http://localhost:8589"
echo ""
bin/distributed-test/scripts/stop.sh
Executable
+24
View File
@@ -0,0 +1,24 @@
#!/bin/bash
# Stop the distributed test environment
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_DIR"
echo "Stopping distributed test environment..."
# Check for cleanup flag
if [ "$1" == "--clean" ] || [ "$1" == "-c" ]; then
echo "Removing containers and volumes..."
docker compose down -v --remove-orphans
echo "Cleaned up all containers and volumes."
else
docker compose down --remove-orphans
echo "Containers stopped. Data volumes preserved."
echo "Use --clean or -c to also remove volumes."
fi
echo "Done."
bin/distributed-test/scripts/trigger-reindex.sh
Executable
+145
View File
@@ -0,0 +1,145 @@
#!/bin/bash
# Trigger reindexing via REST API
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
# Default values
SERVER_URL="http://localhost:8585"
ENTITY_TYPES=""
BATCH_SIZE=100
PARTITION_SIZE=10000
# Parse arguments
while [[ $# -gt 0 ]]; do
case $1 in
--server)
SERVER_URL="$2"
shift 2
;;
--entities)
ENTITY_TYPES="$2"
shift 2
;;
--batch-size)
BATCH_SIZE="$2"
shift 2
;;
--partition-size)
PARTITION_SIZE="$2"
shift 2
;;
-h|--help)
echo "Usage: $0 [OPTIONS]"
echo ""
echo "Options:"
echo " --server URL Target server URL (default: http://localhost:8585)"
echo " --entities TYPES Comma-separated entity types to reindex (default: all)"
echo " --batch-size NUM Batch size for indexing (default: 100)"
echo " --partition-size NUM Partition size for distributed indexing (default: 10000, range: 1000-50000)"
echo " Smaller values = more partitions = better distribution across servers"
echo " -h, --help Show this help message"
echo ""
echo "Examples:"
echo " $0 # Reindex all on server 1"
echo " $0 --server http://localhost:8587 # Trigger on server 2"
echo " $0 --entities table,dashboard # Reindex only tables and dashboards"
echo " $0 --partition-size 2000 # Use smaller partitions for better distribution"
exit 0
;;
*)
echo "Unknown option: $1"
exit 1
;;
esac
done
echo "======================================"
echo "Triggering Search Reindexing"
echo "======================================"
echo "Server: $SERVER_URL"
echo "Indexing mode: staged indexes with alias promotion"
echo "Batch size: $BATCH_SIZE"
echo "Partition size: $PARTITION_SIZE"
if [ -n "$ENTITY_TYPES" ]; then
echo "Entity types: $ENTITY_TYPES"
else
echo "Entity types: all"
fi
echo ""
# First, get a JWT token (using admin user)
echo "Authenticating..."
TOKEN_RESPONSE=$(curl -s -X POST "${SERVER_URL}/api/v1/users/login" \
-H "Content-Type: application/json" \
-d '{"email": "admin@open-metadata.org", "password": "admin"}')
ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | grep -o '"accessToken":"[^"]*"' | cut -d'"' -f4)
if [ -z "$ACCESS_TOKEN" ]; then
echo "Failed to authenticate. Response: $TOKEN_RESPONSE"
echo ""
echo "Trying with basic auth token..."
# Try to get the bot token instead
ACCESS_TOKEN="eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJvcGVuLW1ldGFkYXRhLm9yZyIsInN1YiI6ImFkbWluIiwiZW1haWwiOiJhZG1pbkBvcGVuLW1ldGFkYXRhLm9yZyIsImlzQm90IjpmYWxzZSwiaXNBZG1pbiI6dHJ1ZSwidG9rZW5UeXBlIjoiUEVSU09OQUxfQUNDRVNTIiwiaWF0IjoxNjk1MjM3MzY2LCJleHAiOjE2OTc4MjkzNjZ9.placeholder"
fi
echo "Authenticated successfully."
echo ""
# Build entities array
if [ -n "$ENTITY_TYPES" ]; then
# Convert comma-separated to JSON array
ENTITIES_JSON=$(echo "$ENTITY_TYPES" | sed 's/,/","/g' | sed 's/^/["/' | sed 's/$/"]/')
else
ENTITIES_JSON='["all"]'
fi
REQUEST_BODY=$(cat <<EOF
{
"entities": $ENTITIES_JSON,
"batchSize": $BATCH_SIZE,
"partitionSize": $PARTITION_SIZE,
"runMode": "BATCH"
}
EOF
)
echo "Request body:"
echo "$REQUEST_BODY"
echo ""
# Trigger the reindex
echo "Triggering reindex..."
RESPONSE=$(curl -s -X POST "${SERVER_URL}/api/v1/apps/name/SearchIndexingApplication/trigger" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $ACCESS_TOKEN" \
-d "$REQUEST_BODY")
echo "Response:"
echo "$RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$RESPONSE"
echo ""
# Check job status
echo "Checking job status..."
sleep 2
STATUS_RESPONSE=$(curl -s -X GET "${SERVER_URL}/api/v1/apps/name/SearchIndexingApplication/status" \
-H "Authorization: Bearer $ACCESS_TOKEN")
echo "Job status:"
echo "$STATUS_RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$STATUS_RESPONSE"
echo ""
echo "======================================"
echo "Reindex triggered!"
echo "======================================"
echo ""
echo "Monitor progress with:"
echo " ./scripts/logs.sh -f --grep 'partition\\|SearchIndex\\|Distributed'"
echo ""
echo "Check job status:"
echo " curl -s '${SERVER_URL}/api/v1/apps/name/SearchIndexingApplication/status'"
echo ""
bin/openmetadata-server-start.sh
+128
View File
@@ -0,0 +1,128 @@
#!/bin/bash
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ];
then
echo "USAGE: $0 [-daemon] openmetadata.yaml"
exit 1
fi
base_dir=$(dirname $0)/..
OPENMETADATA_HOME=$base_dir
# OpenMetadata env script
. $OPENMETADATA_HOME/conf/openmetadata-env.sh
if [ "x$OPENMETADATA_HEAP_OPTS" = "x" ]; then
export OPENMETADATA_HEAP_OPTS="-Xmx1G -Xms1G"
fi
EXTRA_ARGS="-name OpenMetadataServer"
# create logs directory
if [ "x$LOG_DIR" = "x" ]; then
LOG_DIR="$base_dir/logs"
fi
if [ ! -d "$LOG_DIR" ]; then
mkdir -p "$LOG_DIR"
fi
# classpath addition for release
echo $CLASSPATH
for file in $base_dir/libs/*.jar;
do
CLASSPATH=$CLASSPATH:$file
done
if [ ! "x$EXT_CLASSPATH" = "x" ]; then
CLASSPATH=$CLASSPATH:$EXT_CLASSPATH;
fi
COMMAND=$1
while [ "$COMMAND" != "" ]; do
case $COMMAND in
-name)
DAEMON_NAME=$2
shift 2
;;
-daemon)
DAEMON_NAME=OpenMetadataServer
DAEMON_MODE=true
shift
;;
*)
break
;;
esac
COMMAND=$1
done
CONSOLE_OUTPUT_FILE=$LOG_DIR/$DAEMON_NAME.out
# Which java to use
if [ -z "$JAVA_HOME" ]; then
JAVA="java"
else
JAVA="$JAVA_HOME/bin/java"
fi
# Set Debug options if enabled
if [ "x$OPENMETADATA_DEBUG" != "x" ]; then
# Use default ports
DEFAULT_JAVA_DEBUG_PORT="0.0.0.0:5005"
if [ -z "$JAVA_DEBUG_PORT" ]; then
JAVA_DEBUG_PORT="$DEFAULT_JAVA_DEBUG_PORT"
fi
# Use the defaults if JAVA_DEBUG_OPTS was not set
DEFAULT_JAVA_DEBUG_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=${DEBUG_SUSPEND_FLAG:-n},address=$JAVA_DEBUG_PORT"
if [ -z "$JAVA_DEBUG_OPTS" ]; then
JAVA_DEBUG_OPTS="$DEFAULT_JAVA_DEBUG_OPTS"
fi
echo "Enabling Java debug options: $JAVA_DEBUG_OPTS"
OPENMETADATA_OPTS="$JAVA_DEBUG_OPTS $OPENMETADATA_OPTS"
fi
# GC options
if [ -z "$OPENMETADATA_GC_LOG_OPTS" ]; then
OPENMETADATA_GC_LOG_OPTS="-Xlog:gc*:file=$LOG_DIR/openmetadata-gc.log:time,tags:filecount=10,filesize=102400"
fi
# JVM performance options
if [ -z "$OPENMETADATA_JVM_PERFORMANCE_OPTS" ]; then
export OPENMETADATA_JVM_PERFORMANCE_OPTS="\
-server -XX:+UseG1GC \
-XX:+UnlockExperimentalVMOptions \
-XX:MaxGCPauseMillis=200 \
-XX:G1NewSizePercent=5 \
-XX:G1MaxNewSizePercent=20 \
-XX:InitiatingHeapOccupancyPercent=45 \
-XX:+ExplicitGCInvokesConcurrent \
-XX:+UseStringDeduplication \
-XX:-UseLargePages \
-XX:+UseCompressedOops -XX:+UseCompressedClassPointers \
-Djava.awt.headless=true"
fi
#Application classname
APP_CLASS="org.openmetadata.service.OpenMetadataApplication"
# Launch mode
if [ "x$DAEMON_MODE" = "xtrue" ]; then
nohup $JAVA $OPENMETADATA_HEAP_OPTS $OPENMETADATA_JVM_PERFORMANCE_OPTS $OPENMETADATA_GC_LOG_OPTS -cp $CLASSPATH $OPENMETADATA_OPTS "$APP_CLASS" "server" "$@" > "$CONSOLE_OUTPUT_FILE" 2>&1 < /dev/null &
else
exec $JAVA $OPENMETADATA_HEAP_OPTS $OPENMETADATA_JVM_PERFORMANCE_OPTS $OPENMETADATA_GC_LOG_OPTS -cp $CLASSPATH $OPENMETADATA_OPTS "$APP_CLASS" "server" "$@"
fi
bin/openmetadata.sh
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env bash
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Home Dir
base_dir=$(dirname $0)/..
CATALOG_HOME=$base_dirbase_dir=$(dirname $0)/..
CATALOG_HOME=$base_dir
PID_DIR=$base_dir/logs
LOG_DIR=$base_dir/logs
mkdir -p $LOG_DIR
[ -z $MAX_WAIT_TIME ] && MAX_WAIT_TIME=120
# OpenMetadata env script
. $CATALOG_HOME/conf/openmetadata-env.sh
function catalogStart {
catalogStatus -q
if [[ $? -eq 0 ]]; then
rm -f ${PID_FILE}
echo "Starting OpenMetadata"
APP_CLASS="org.openmetadata.service.OpenMetadataApplication"
cd ${CATALOG_HOME}
nohup ${JAVA} ${CATALOG_HEAP_OPTS} ${CATALOG_JVM_PERF_OPTS} ${CATALOG_DEBUG_OPTS} ${CATALOG_GC_LOG_OPTS} ${CATALOG_JMX_OPTS} -cp ${CLASSPATH} "${APP_CLASS}" "server" "$@" 2>>"${ERR_FILE}" 1>>"${OUT_FILE}" &
cd - &> /dev/null
echo $! > ${PID_FILE}
echo $(catalogStatus)
else
echo "OpenMetadata already running with PID: ${PID}"
fi
}
function catalogStop {
catalogStatus -q
if [[ $? -eq 1 ]]; then
echo "Stopping OpenMetadata [${PID}] "
kill -s KILL ${PID} 1>>"${OUT_FILE}" 2>>"${ERR_FILE}"
else
echo "OpenMetadata not running"
fi
}
function catalogStatus {
local verbose=1
while true; do
case ${1} in
-q) verbose=0; shift;;
*) break;;
esac
done
getPID
if [[ $? -eq 1 ]] || [[ ${PID} -eq 0 ]]; then
[[ "${verbose}" -eq 1 ]] && echo "OpenMetadata not running."
return 0
fi
ps -p ${PID} > /dev/null
if [[ $? -eq 0 ]]; then
[[ "${verbose}" -eq 1 ]] && echo "OpenMetadata running with PID=${PID}."
return 1
else
[[ "${verbose}" -eq 1 ]] && echo "OpenMetadata not running."
return 0
fi
}
# Removes the PID file if OpenMetadata is not running
function catalogClean {
catalogStatus -q
if [[ $? -eq 0 ]]; then
rm -f ${PID_FILE} ${OUT_FILE} ${ERR_FILE}
echo "Removed the ${PID_FILE}, ${OUT_FILE} and ${ERR_FILE} files."
else
echo "Can't clean files. OpenMetadata running with PID=${PID}."
fi
}
# Returns 0 if the OpenMetadata is running and sets the $PID variable.
function getPID {
if [ ! -d $PID_DIR ]; then
printf "Can't find pid dir.\n"
exit 1
fi
if [ ! -f $PID_FILE ]; then
PID=0
return 1
fi
PID="$(<$PID_FILE)"
return 0
}
# Home Dir
base_dir=$(dirname $0)/..
cd $base_dir && base_dir=`pwd` && cd - &> /dev/null
# App name
APP_NAME=catalog
#if CATALOG_DIR is not set its a dev env.
if [ "x$CATALOG_DIR" != "x" ]; then
CATALOG_HOME=$CATALOG_DIR/$APP_NAME
else
CATALOG_HOME=$base_dir
PID_DIR=$CATALOG_HOME/logs
LOG_DIR=$CATALOG_HOME/logs
mkdir -p $LOG_DIR
fi
# CATALOG env script
. ${CATALOG_HOME}/conf/openmetadata-env.sh
PID=0
[ -z $PID_DIR ] && PID_DIR="/var/run/$APP_NAME"
[ -z $LOG_DIR ] && LOG_DIR="/var/log/$APP_NAME"
# Name of PID, OUT and ERR files
PID_FILE="${PID_DIR}/${APP_NAME}.pid"
OUT_FILE="${LOG_DIR}/${APP_NAME}.log"
ERR_FILE="${LOG_DIR}/${APP_NAME}.err"
CLASSPATH=${CLASSPATH}:/etc/catalog/conf
# classpath addition for release
for file in ${CATALOG_HOME}/libs/*.jar;
do
CLASSPATH=${CLASSPATH}:${file}
done
# Which java to use
if [[ -z "${JAVA_HOME}" ]]; then
JAVA="java"
else
JAVA="${JAVA_HOME}/bin/java"
fi
if [[ "x${CATALOG_HEAP_OPTS}" = "x" ]]; then
export CATALOG_HEAP_OPTS="-Xmx2G -Xms2G"
fi
# JVM performance options
if [[ -z "${CATALOG_JVM_PERF_OPTS}" ]]; then
CATALOG_JVM_PERF_OPTS="-server -XX:+UseG1GC -XX:+UnlockExperimentalVMOptions -XX:MaxGCPauseMillis=200 -XX:G1NewSizePercent=5 -XX:G1MaxNewSizePercent=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -Djava.awt.headless=true"
fi
# JMX settings
if [ -z "$CATALOG_JMX_OPTS" ]; then
CATALOG_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false "
fi
# JMX port to use
if [ $JMX_PORT ]; then
CATALOG_JMX_OPTS="$CATALOG_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
fi
# GC options
GC_LOG_FILE_NAME='CATALOG-gc.log'
if [ -z "$CATALOG_GC_LOG_OPTS" ]; then
JAVA_MAJOR_VERSION=$($JAVA -version 2>&1 | sed -E -n 's/.* version "([0-9]*).*$/\1/p')
if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then
CATALOG_GC_LOG_OPTS="-Xlog:gc*:file=$LOG_DIR/$GC_LOG_FILE_NAME:time,tags:filecount=10,filesize=102400"
else
CATALOG_GC_LOG_OPTS="-Xloggc:$LOG_DIR/$GC_LOG_FILE_NAME -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M"
fi
fi
option="$1"
shift
case "${option}" in
start)
conf="$CATALOG_HOME/conf/openmetadata.yaml"
if [[ $# -eq 1 ]]; then conf="${1}"; fi
catalogStart "${conf}"
;;
stop)
catalogStop
;;
status)
catalogStatus
;;
clean)
catalogClean
;;
*)
echo "Usage: $0 {start|stop|status|clean}"
;;
esac
bin/thread-monitor.sh
Executable
+115
View File
@@ -0,0 +1,115 @@
#!/bin/bash
# Thread and CPU monitor for OpenMetadata reindexing
# Usage: ./bin/thread-monitor.sh <PID> [interval_seconds]
#
# Monitors:
# - Thread count by name prefix (reindex-*, om-*, etc.)
# - Total thread count over time
# - CPU usage per thread group
# - Ever-increasing thread detection
PID=${1:?Usage: $0 <PID> [interval_seconds]}
INTERVAL=${2:-5}
LOG_DIR="/tmp/om-thread-monitor"
mkdir -p "$LOG_DIR"
THREAD_LOG="$LOG_DIR/threads.csv"
SUMMARY_LOG="$LOG_DIR/summary.log"
PREV_COUNTS_FILE="$LOG_DIR/.prev_counts"
echo "timestamp,total_threads,reindex_threads,om_threads,virtual_threads,pool_threads,fjp_threads" > "$THREAD_LOG"
echo "" > "$SUMMARY_LOG"
echo "=== OpenMetadata Thread Monitor ==="
echo "PID: $PID"
echo "Interval: ${INTERVAL}s"
echo "Logs: $LOG_DIR"
echo "Press Ctrl+C to stop and see summary"
echo ""
iteration=0
monitor() {
local ts=$(date '+%H:%M:%S')
# Get all threads for this process
local thread_dump=$(jstack "$PID" 2>/dev/null)
if [ $? -ne 0 ]; then
echo "[$ts] ERROR: Cannot get thread dump for PID $PID (process may have exited)"
return 1
fi
# Count threads by category
local total=$(echo "$thread_dump" | grep -c '^"')
local reindex=$(echo "$thread_dump" | grep -c '^"reindex-')
local om=$(echo "$thread_dump" | grep -c '^"om-')
local virtual=$(echo "$thread_dump" | grep -c 'VirtualThread')
local pool=$(echo "$thread_dump" | grep -c '^"pool-')
local fjp=$(echo "$thread_dump" | grep -c '^"ForkJoinPool')
echo "$ts,$total,$reindex,$om,$virtual,$pool,$fjp" >> "$THREAD_LOG"
# Detailed breakdown
echo "[$ts] total=$total | reindex=$reindex | om=$om | virtual=$virtual | pool=$pool | fjp=$fjp"
# Every 3rd iteration, show detailed reindex thread breakdown
if [ $((iteration % 3)) -eq 0 ]; then
echo " Reindex breakdown:"
echo "$thread_dump" | grep '^"reindex-' | sed 's/".*//' | sed 's/"//g' | sed 's/-[0-9]*$//' | sort | uniq -c | sort -rn | head -15 | while read count name; do
printf " %-45s %d\n" "$name" "$count"
done
# Show unnamed/pool threads (potential leaks)
local unnamed=$(echo "$thread_dump" | grep '^"pool-' | sed 's/".*//' | sed 's/"//g' | sort | uniq -c | sort -rn)
if [ -n "$unnamed" ]; then
echo " Unnamed pool threads (potential leaks):"
echo "$unnamed" | head -10 | while read count name; do
printf " %-45s %d\n" "$name" "$count"
done
fi
fi
# Detect growth: compare with previous counts
if [ -f "$PREV_COUNTS_FILE" ]; then
local prev_total=$(cat "$PREV_COUNTS_FILE" | head -1)
local delta=$((total - prev_total))
if [ $delta -gt 10 ]; then
echo " ⚠ THREAD GROWTH: +$delta threads since last check"
fi
fi
echo "$total" > "$PREV_COUNTS_FILE"
# Check for RUNNABLE threads (CPU consumers)
if [ $((iteration % 6)) -eq 0 ]; then
echo " Top RUNNABLE threads (CPU consumers):"
echo "$thread_dump" | grep -B1 'java.lang.Thread.State: RUNNABLE' | grep '^"' | sed 's/".*//' | sed 's/"//g' | sed 's/-[0-9]*$//' | sort | uniq -c | sort -rn | head -10 | while read count name; do
printf " %-45s %d RUNNABLE\n" "$name" "$count"
done
fi
# Every 12th iteration, save a full thread dump
if [ $((iteration % 12)) -eq 0 ] && [ $iteration -gt 0 ]; then
local dump_file="$LOG_DIR/thread-dump-$(date '+%H%M%S').txt"
echo "$thread_dump" > "$dump_file"
echo " [Saved full thread dump to $dump_file]"
fi
iteration=$((iteration + 1))
}
cleanup() {
echo ""
echo "=== SUMMARY ==="
echo "Thread count over time:"
cat "$THREAD_LOG"
echo ""
echo "Check $LOG_DIR for detailed logs and thread dumps"
exit 0
}
trap cleanup INT TERM
while true; do
monitor || exit 1
sleep "$INTERVAL"
done