wehub-resource-sync
9e8f1bbeed
Dashboard / frontend (push) Failing after 0s
Dashboard / api (push) Failing after 0s
Lint PowerShell / powershell-lint (ubuntu-latest) (push) Failing after 1s
Python Lint / Lint Python with Ruff (push) Failing after 1s
ShellCheck / Lint shell scripts (push) Failing after 1s
Matrix Smoke / linux-smoke (push) Failing after 1s
Matrix Smoke / distro: cachyos (push) Failing after 15s
Matrix Smoke / distro: linux-mint-21.3 (push) Failing after 15s
Matrix Smoke / distro: debian-12 (push) Failing after 5m21s
Matrix Smoke / distro: fedora-41 (push) Failing after 4m56s
Matrix Smoke / distro: ubuntu-24.04 (push) Failing after 2m13s
Matrix Smoke / distro: rocky-9 (push) Failing after 10m39s
Matrix Smoke / distro: manjaro (push) Failing after 12m11s
Matrix Smoke / distro: opensuse-tw (push) Failing after 11m53s
Matrix Smoke / distro: archlinux (push) Failing after 20m3s
Matrix Smoke / distro: ubuntu-22.04 (push) Failing after 13m49s
Validate .env Schema / tier-1-env-validation (push) Successful in 52s
Validate .env Schema / tier-2-env-validation (push) Successful in 44s
Validate .env Schema / tier-3-env-validation (push) Successful in 52s
Validate .env Schema / tier-4-env-validation (push) Successful in 51s
Validate Extensions Catalog / Check catalog is up-to-date (push) Failing after 9m47s
Secret Scan / Scan for secrets (push) Failing after 21m4s
Validate Docker Compose / Validate Docker Compose files (push) Has been cancelled
Python Type Check / Type check with mypy (push) Has been cancelled
Validate .env Schema / tier-0-env-validation (push) Has been cancelled
Test Linux / integration-smoke (push) Has been cancelled
Lint PowerShell / powershell-lint (windows-latest) (push) Has been cancelled
Matrix Smoke / macos-smoke (push) Has been cancelled
752 lines
23 KiB
Bash
Executable File
752 lines
23 KiB
Bash
Executable File
#!/bin/bash
|
|
#=============================================================================
|
|
# ods-test.sh - ODS Validation Suite (M8)
|
|
#
|
|
# Mission M8: Agent Bench Testing Systems
|
|
# Validates all critical paths: LLM, STT, TTS, embeddings, tool calling,
|
|
# voice round-trip. Clear pass/fail with actionable errors.
|
|
#
|
|
# Target: <2 minutes runtime
|
|
# Pass criteria: All critical tests pass
|
|
#
|
|
# Usage:
|
|
# ./ods-test.sh # Run all tests
|
|
# ./ods-test.sh --quick # Fast mode (~30s, no inference)
|
|
# ./ods-test.sh --json # JSON output for automation
|
|
# ./ods-test.sh --service llm # Test specific service
|
|
#
|
|
# Exit codes:
|
|
# 0 - All critical tests passed
|
|
# 1 - One or more tests failed
|
|
# 2 - Configuration error
|
|
#=============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ODS_DIR="${ODS_DIR:-$HOME/.ods}"
|
|
ENV_FILE="${ENV_FILE:-$ODS_DIR/.env}"
|
|
TIMEOUT=15
|
|
QUICK_TIMEOUT=5
|
|
|
|
# Source service registry for port resolution
|
|
_DT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
if [[ -f "$_DT_DIR/lib/service-registry.sh" ]]; then
|
|
export SCRIPT_DIR="$_DT_DIR"
|
|
. "$_DT_DIR/lib/service-registry.sh"
|
|
sr_load
|
|
[[ -f "$_DT_DIR/lib/safe-env.sh" ]] && . "$_DT_DIR/lib/safe-env.sh"
|
|
load_env_file "$_DT_DIR/.env"
|
|
sr_resolve_ports
|
|
fi
|
|
|
|
# Service endpoints — resolved from registry
|
|
LLM_HOST="${LLM_HOST:-localhost}"
|
|
LLM_PORT="${LLM_PORT:-${SERVICE_PORTS[llama-server]:-11434}}"
|
|
LLM_URL="http://${LLM_HOST}:${LLM_PORT}"
|
|
WHISPER_HOST="${WHISPER_HOST:-localhost}"
|
|
WHISPER_PORT="${WHISPER_PORT:-${SERVICE_PORTS[whisper]:-9000}}"
|
|
TTS_HOST="${TTS_HOST:-localhost}"
|
|
TTS_PORT="${TTS_PORT:-${SERVICE_PORTS[tts]:-8880}}"
|
|
EMBEDDING_HOST="${EMBEDDING_HOST:-localhost}"
|
|
EMBEDDING_PORT="${EMBEDDING_PORT:-${SERVICE_PORTS[embeddings]:-9103}}"
|
|
LIVEKIT_HOST="${LIVEKIT_HOST:-localhost}"
|
|
LIVEKIT_PORT="${LIVEKIT_PORT:-7880}"
|
|
PRIVACY_SHIELD_PORT="${PRIVACY_SHIELD_PORT:-${SERVICE_PORTS[privacy-shield]:-8085}}"
|
|
|
|
# Colors (ANSI escape sequences)
|
|
RED='\e[0;31m'
|
|
GREEN='\e[0;32m'
|
|
YELLOW='\e[1;33m'
|
|
BLUE='\e[0;34m'
|
|
CYAN='\e[0;36m'
|
|
NC='\e[0m'
|
|
BOLD='\e[1m'
|
|
|
|
# Test tracking
|
|
TOTAL_TESTS=0
|
|
PASSED_TESTS=0
|
|
FAILED_TESTS=0
|
|
SKIPPED_TESTS=0
|
|
START_TIME=0
|
|
|
|
# Mode flags
|
|
JSON_OUTPUT=false
|
|
QUICK_MODE=false
|
|
SPECIFIC_SERVICE=""
|
|
VERBOSE=false
|
|
|
|
# Results storage
|
|
RESULTS_STATUS=()
|
|
RESULTS_NAMES=()
|
|
RESULTS_DETAILS=()
|
|
|
|
#--------------------------------------------------------------------------
|
|
# Utility Functions
|
|
#--------------------------------------------------------------------------
|
|
|
|
load_env() {
|
|
[[ -f "$_DT_DIR/lib/safe-env.sh" ]] && . "$_DT_DIR/lib/safe-env.sh"
|
|
load_env_file "$ENV_FILE"
|
|
}
|
|
|
|
log() {
|
|
[[ "$VERBOSE" == "true" ]] && echo "$@" >&2
|
|
}
|
|
|
|
# Portable millisecond timestamp (macOS BSD date lacks %N)
|
|
_now_ms() {
|
|
python3 -c 'import time; print(int(time.time() * 1000))' 2>/dev/null || echo "$(date +%s)000"
|
|
}
|
|
|
|
print_header() {
|
|
if [[ "$JSON_OUTPUT" != "true" ]]; then
|
|
echo ""
|
|
echo "========================================"
|
|
echo " ODS - VALIDATION SUITE"
|
|
echo " Mission M8: Critical Path Testing"
|
|
echo "========================================"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
record_result() {
|
|
local name="$1"
|
|
local status="$2"
|
|
local details="${3:-}"
|
|
|
|
RESULTS_NAMES+=("$name")
|
|
RESULTS_STATUS+=("$status")
|
|
RESULTS_DETAILS+=("$details")
|
|
TOTAL_TESTS=$((TOTAL_TESTS + 1))
|
|
|
|
case "$status" in
|
|
pass) PASSED_TESTS=$((PASSED_TESTS + 1)) ;;
|
|
fail) FAILED_TESTS=$((FAILED_TESTS + 1)) ;;
|
|
skip) SKIPPED_TESTS=$((SKIPPED_TESTS + 1)) ;;
|
|
esac
|
|
}
|
|
|
|
print_test() {
|
|
local name="$1"
|
|
local status="$2"
|
|
local details="${3:-}"
|
|
|
|
if [[ "$JSON_OUTPUT" != "true" ]]; then
|
|
local icon
|
|
case "$status" in
|
|
pass) icon="OK" ;;
|
|
fail) icon="FAIL" ;;
|
|
skip) icon="SKIP" ;;
|
|
esac
|
|
|
|
printf " %-40s [%s]" "$name" "$icon"
|
|
[[ -n "$details" ]] && printf " %s" "$details"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
#--------------------------------------------------------------------------
|
|
# Core Test Functions
|
|
#--------------------------------------------------------------------------
|
|
|
|
test_http() {
|
|
local name="$1"
|
|
local url="$2"
|
|
local expected="${3:-200}"
|
|
local method="${4:-GET}"
|
|
local payload="${5:-}"
|
|
local custom_timeout="${6:-$TIMEOUT}"
|
|
|
|
local response_code
|
|
local start_time end_time duration_ms
|
|
|
|
start_time=$(_now_ms)
|
|
|
|
if [[ -n "$payload" && "$method" == "POST" ]]; then
|
|
response_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
|
--max-time "$custom_timeout" \
|
|
-X POST "$url" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$payload" 2>/dev/null || echo "000")
|
|
else
|
|
response_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
|
--max-time "$custom_timeout" \
|
|
"$url" 2>/dev/null || echo "000")
|
|
fi
|
|
|
|
end_time=$(_now_ms)
|
|
duration_ms=$(( end_time - start_time ))
|
|
|
|
if [[ "$response_code" == "$expected" ]]; then
|
|
record_result "$name" "pass" "${duration_ms}ms"
|
|
print_test "$name" "pass" "${duration_ms}ms"
|
|
return 0
|
|
else
|
|
record_result "$name" "fail" "HTTP $response_code"
|
|
print_test "$name" "fail" "HTTP $response_code"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
test_tcp() {
|
|
local name="$1"
|
|
local host="$2"
|
|
local port="$3"
|
|
|
|
if timeout "$TIMEOUT" bash -c "cat < /dev/null > /dev/tcp/$host/$port" 2>/dev/null; then
|
|
record_result "$name" "pass"
|
|
print_test "$name" "pass"
|
|
return 0
|
|
else
|
|
record_result "$name" "fail" "port $port unreachable"
|
|
print_test "$name" "fail" "port closed"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
#--------------------------------------------------------------------------
|
|
# Service Test Suites
|
|
#--------------------------------------------------------------------------
|
|
|
|
test_docker() {
|
|
echo ""
|
|
echo "> Docker Infrastructure"
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
record_result "Docker Available" "fail" "docker not installed"
|
|
print_test "Docker Available" "fail" "not installed"
|
|
return 0
|
|
fi
|
|
|
|
record_result "Docker Available" "pass"
|
|
print_test "Docker Available" "pass"
|
|
|
|
if ! timeout 10 docker info &>/dev/null; then
|
|
record_result "Docker Daemon" "fail" "daemon not running"
|
|
print_test "Docker Daemon" "fail" "not running"
|
|
return 0
|
|
fi
|
|
|
|
record_result "Docker Daemon" "pass"
|
|
print_test "Docker Daemon" "pass"
|
|
|
|
local running_count
|
|
running_count=$(timeout 10 docker ps --format '{{.Names}}' 2>/dev/null | wc -l)
|
|
record_result "Running Containers" "pass" "$running_count containers"
|
|
print_test "Running Containers" "pass" "$running_count containers"
|
|
}
|
|
|
|
test_gpu() {
|
|
echo ""
|
|
echo "> GPU Resources"
|
|
|
|
if ! command -v nvidia-smi &>/dev/null; then
|
|
record_result "NVIDIA GPU" "skip" "nvidia-smi not found"
|
|
print_test "NVIDIA GPU" "skip"
|
|
return 0
|
|
fi
|
|
|
|
local gpu_info
|
|
gpu_info=$(timeout 10 nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu --format=csv,noheader 2>/dev/null | head -1)
|
|
|
|
if [[ -n "$gpu_info" ]]; then
|
|
local name mem_used mem_total util mem_pct
|
|
name=$(echo "$gpu_info" | cut -d',' -f1 | xargs)
|
|
mem_used=$(echo "$gpu_info" | cut -d',' -f2 | xargs | cut -d' ' -f1)
|
|
mem_total=$(echo "$gpu_info" | cut -d',' -f3 | xargs | cut -d' ' -f1)
|
|
|
|
mem_pct=$(( mem_used * 100 / mem_total ))
|
|
|
|
record_result "GPU Available" "pass" "$name"
|
|
print_test "GPU Available" "pass" "$name"
|
|
|
|
if [[ $mem_pct -gt 90 ]]; then
|
|
record_result "GPU Memory" "fail" "${mem_pct}% used - critical"
|
|
print_test "GPU Memory" "fail" "${mem_pct}% used"
|
|
else
|
|
record_result "GPU Memory" "pass" "${mem_pct}% used"
|
|
print_test "GPU Memory" "pass" "${mem_pct}% used"
|
|
fi
|
|
else
|
|
record_result "NVIDIA GPU" "fail" "no GPU detected"
|
|
print_test "NVIDIA GPU" "fail" "not detected"
|
|
fi
|
|
}
|
|
|
|
test_llm() {
|
|
echo ""
|
|
echo "> LLM Inference (llama-server)"
|
|
|
|
test_http "LLM Health" "$LLM_URL/health" "200" || return 1
|
|
test_http "LLM Models API" "$LLM_URL/v1/models" "200"
|
|
|
|
if [[ "$QUICK_MODE" == "true" ]]; then
|
|
record_result "LLM Inference" "skip" "quick mode"
|
|
print_test "LLM Inference" "skip"
|
|
return 0
|
|
fi
|
|
|
|
local model_id
|
|
model_id=$(curl -s --max-time 10 "$LLM_URL/v1/models" 2>/dev/null | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4)
|
|
model_id="${model_id:-local}"
|
|
|
|
local payload="{\"model\": \"$model_id\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello\"}], \"max_tokens\": 10}"
|
|
local response
|
|
|
|
response=$(curl -s --max-time 30 \
|
|
-X POST "$LLM_URL/v1/chat/completions" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$payload" 2>/dev/null)
|
|
|
|
if echo "$response" | grep -q '"content"'; then
|
|
local tokens_used
|
|
tokens_used=$(echo "$response" | grep -o '"total_tokens":[0-9]*' | cut -d: -f2)
|
|
record_result "LLM Inference" "pass" "${tokens_used} tokens"
|
|
print_test "LLM Inference" "pass" "${tokens_used} tokens"
|
|
else
|
|
record_result "LLM Inference" "fail" "no content in response"
|
|
print_test "LLM Inference" "fail"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
test_tool_calling() {
|
|
echo ""
|
|
echo "> Tool Calling M8 Critical"
|
|
|
|
if [[ "$QUICK_MODE" == "true" ]]; then
|
|
record_result "Tool Calling" "skip" "quick mode"
|
|
print_test "Tool Calling" "skip"
|
|
return 0
|
|
fi
|
|
|
|
local tools='[{"type":"function","function":{"name":"get_weather","description":"Get weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}'
|
|
local payload="{\"model\": \"Qwen/Qwen2.5-32B-Instruct-AWQ\", \"messages\": [{\"role\": \"user\", \"content\": \"What is the weather in Tokyo?\"}], \"tools\": $tools, \"tool_choice\": \"auto\", \"max_tokens\": 100}"
|
|
|
|
local response
|
|
response=$(curl -s --max-time 30 \
|
|
-X POST "$LLM_URL/v1/chat/completions" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$payload" 2>/dev/null)
|
|
|
|
if echo "$response" | grep -q '"tool_calls"'; then
|
|
record_result "Tool Calling" "pass" "function called"
|
|
print_test "Tool Calling" "pass" "function called"
|
|
else
|
|
record_result "Tool Calling" "fail" "no tool_calls in response"
|
|
print_test "Tool Calling" "fail" "no tool call"
|
|
fi
|
|
}
|
|
|
|
test_whisper() {
|
|
echo ""
|
|
echo "> Whisper Speech-to-Text"
|
|
|
|
test_tcp "Whisper Port" "$WHISPER_HOST" "$WHISPER_PORT"
|
|
|
|
local health_url="http://${WHISPER_HOST}:${WHISPER_PORT}/health"
|
|
local response
|
|
response=$(curl -s --max-time "$TIMEOUT" "$health_url" 2>/dev/null || echo "")
|
|
|
|
if [[ -n "$response" ]]; then
|
|
if echo "$response" | grep -qiE "ok|healthy|ready"; then
|
|
record_result "Whisper Health" "pass"
|
|
print_test "Whisper Health" "pass"
|
|
else
|
|
record_result "Whisper Health" "pass" "responding"
|
|
print_test "Whisper Health" "pass" "responding"
|
|
fi
|
|
else
|
|
whisper_http_exit=0
|
|
test_http "Whisper HTTP" "http://${WHISPER_HOST}:${WHISPER_PORT}/" "200" || whisper_http_exit=$?
|
|
[[ $whisper_http_exit -ne 0 ]] && log "Whisper HTTP check failed (exit $whisper_http_exit)"
|
|
fi
|
|
}
|
|
|
|
test_tts() {
|
|
echo ""
|
|
echo "> Kokoro TTS Text-to-Speech"
|
|
|
|
test_tcp "TTS Port" "$TTS_HOST" "$TTS_PORT"
|
|
|
|
local voices_url="http://${TTS_HOST}:${TTS_PORT}/v1/audio/voices"
|
|
local response
|
|
response=$(curl -s --max-time "$TIMEOUT" "$voices_url" 2>/dev/null || echo "")
|
|
|
|
if echo "$response" | grep -q '"voices"'; then
|
|
local voice_count
|
|
voice_count=$(echo "$response" | grep -o '"voice_id"' | wc -l)
|
|
record_result "TTS Voices" "pass" "$voice_count voices"
|
|
print_test "TTS Voices" "pass" "$voice_count voices"
|
|
else
|
|
tts_api_exit=0
|
|
test_http "TTS API" "http://${TTS_HOST}:${TTS_PORT}/" "200" || tts_api_exit=$?
|
|
[[ $tts_api_exit -ne 0 ]] && log "TTS API check failed (exit $tts_api_exit)"
|
|
fi
|
|
}
|
|
|
|
test_embeddings() {
|
|
echo ""
|
|
echo "> Embeddings TEI"
|
|
|
|
test_tcp "Embeddings Port" "$EMBEDDING_HOST" "$EMBEDDING_PORT"
|
|
|
|
local health_url="http://${EMBEDDING_HOST}:${EMBEDDING_PORT}/health"
|
|
local response
|
|
response=$(curl -s --max-time "$TIMEOUT" "$health_url" 2>/dev/null || echo "")
|
|
|
|
if echo "$response" | grep -q "ok"; then
|
|
record_result "Embeddings Health" "pass"
|
|
print_test "Embeddings Health" "pass"
|
|
else
|
|
local payload='{"inputs": "test sentence"}'
|
|
embeddings_api_exit=0
|
|
test_http "Embeddings API" "http://${EMBEDDING_HOST}:${EMBEDDING_PORT}/embed" "200" "POST" "$payload" || embeddings_api_exit=$?
|
|
[[ $embeddings_api_exit -ne 0 ]] && log "Embeddings API check failed (exit $embeddings_api_exit)"
|
|
fi
|
|
}
|
|
|
|
test_voice_roundtrip() {
|
|
echo ""
|
|
echo "> Voice Round-Trip M8 Critical"
|
|
|
|
if [[ "$QUICK_MODE" == "true" ]]; then
|
|
record_result "Voice Round-Trip" "skip" "quick mode"
|
|
print_test "Voice Round-Trip" "skip"
|
|
return 0
|
|
fi
|
|
|
|
local whisper_ready=false
|
|
local tts_ready=false
|
|
local llm_ready=false
|
|
|
|
if curl -s --max-time 5 "http://${WHISPER_HOST}:${WHISPER_PORT}/health" &>/dev/null; then
|
|
whisper_ready=true
|
|
fi
|
|
if curl -s --max-time 5 "http://${TTS_HOST}:${TTS_PORT}/v1/audio/voices" &>/dev/null; then
|
|
tts_ready=true
|
|
fi
|
|
if curl -s --max-time 5 "$LLM_URL/health" &>/dev/null; then
|
|
llm_ready=true
|
|
fi
|
|
|
|
if [[ "$whisper_ready" != "true" ]]; then
|
|
record_result "Voice Round-Trip" "skip" "Whisper unavailable"
|
|
print_test "Voice Round-Trip" "skip" "Whisper unavailable"
|
|
return 0
|
|
fi
|
|
if [[ "$tts_ready" != "true" ]]; then
|
|
record_result "Voice Round-Trip" "skip" "TTS unavailable"
|
|
print_test "Voice Round-Trip" "skip" "TTS unavailable"
|
|
return 0
|
|
fi
|
|
if [[ "$llm_ready" != "true" ]]; then
|
|
record_result "Voice Round-Trip" "skip" "LLM unavailable"
|
|
print_test "Voice Round-Trip" "skip" "LLM unavailable"
|
|
return 0
|
|
fi
|
|
|
|
local start_time end_time duration_ms
|
|
start_time=$(_now_ms)
|
|
|
|
local llm_payload='{"model": "Qwen/Qwen2.5-32B-Instruct-AWQ", "messages": [{"role": "user", "content": "What is the weather today?"}], "max_tokens": 50}'
|
|
local llm_response
|
|
llm_response=$(curl -s --max-time 15 \
|
|
-X POST "$LLM_URL/v1/chat/completions" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$llm_payload" 2>/dev/null)
|
|
|
|
if ! echo "$llm_response" | grep -q '"content"'; then
|
|
record_result "Voice Round-Trip" "fail" "LLM step failed"
|
|
print_test "Voice Round-Trip" "fail" "LLM failed"
|
|
return 1
|
|
fi
|
|
|
|
local tts_text="The weather today is sunny and 75 degrees."
|
|
local tts_payload="{\"model\": \"kokoro\", \"input\": \"$tts_text\", \"voice\": \"af_bella\"}"
|
|
local tts_response
|
|
tts_response=$(curl -s --max-time 15 \
|
|
-X POST "http://${TTS_HOST}:${TTS_PORT}/v1/audio/speech" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$tts_payload" 2>/dev/null)
|
|
|
|
end_time=$(_now_ms)
|
|
duration_ms=$(( end_time - start_time ))
|
|
|
|
if [[ -n "$tts_response" ]] && [[ ${#tts_response} -gt 100 ]]; then
|
|
record_result "Voice Round-Trip" "pass" "${duration_ms}ms"
|
|
print_test "Voice Round-Trip" "pass" "${duration_ms}ms"
|
|
else
|
|
record_result "Voice Round-Trip" "fail" "TTS step failed"
|
|
print_test "Voice Round-Trip" "fail" "TTS failed"
|
|
fi
|
|
}
|
|
|
|
test_privacy_shield() {
|
|
echo ""
|
|
echo "> Privacy Shield M3"
|
|
|
|
local shield_url="http://localhost:${PRIVACY_SHIELD_PORT}"
|
|
|
|
test_http "Privacy Shield Health" "$shield_url/health" "200" || return 0
|
|
|
|
if [[ "$QUICK_MODE" == "true" ]]; then
|
|
record_result "Privacy Shield Proxy" "skip" "quick mode"
|
|
print_test "Privacy Shield Proxy" "skip"
|
|
return 0
|
|
fi
|
|
|
|
local payload='{"model": "test", "messages": [{"role": "user", "content": "My email is john@example.com"}], "max_tokens": 10}'
|
|
local response
|
|
response=$(curl -s --max-time 10 \
|
|
-X POST "$shield_url/v1/chat/completions" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$payload" 2>/dev/null || echo "")
|
|
|
|
if [[ -n "$response" ]]; then
|
|
record_result "Privacy Shield Proxy" "pass" "PII scrubbing active"
|
|
print_test "Privacy Shield Proxy" "pass" "PII scrubbing"
|
|
else
|
|
record_result "Privacy Shield Proxy" "fail" "no response"
|
|
print_test "Privacy Shield Proxy" "fail"
|
|
fi
|
|
}
|
|
|
|
test_livekit() {
|
|
echo ""
|
|
echo "> LiveKit Voice Infrastructure"
|
|
|
|
test_tcp "LiveKit Port" "$LIVEKIT_HOST" "$LIVEKIT_PORT"
|
|
livekit_health_exit=0
|
|
test_http "LiveKit Health" "http://${LIVEKIT_HOST}:${LIVEKIT_PORT}/" "200" || livekit_health_exit=$?
|
|
[[ $livekit_health_exit -ne 0 ]] && log "LiveKit health check failed (exit $livekit_health_exit)"
|
|
}
|
|
|
|
#--------------------------------------------------------------------------
|
|
# Summary and Output
|
|
#--------------------------------------------------------------------------
|
|
|
|
print_summary() {
|
|
local end_time elapsed_secs
|
|
end_time=$(date +%s)
|
|
elapsed_secs=$((end_time - START_TIME))
|
|
|
|
if [[ "$JSON_OUTPUT" == "true" ]]; then
|
|
_print_json_summary "$elapsed_secs"
|
|
else
|
|
_print_text_summary "$elapsed_secs"
|
|
fi
|
|
}
|
|
|
|
_print_json_summary() {
|
|
local elapsed="$1"
|
|
|
|
echo "{"
|
|
echo " \"timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\","
|
|
echo " \"runtime_seconds\": $elapsed,"
|
|
echo " \"summary\": {"
|
|
echo " \"total\": $TOTAL_TESTS,"
|
|
echo " \"passed\": $PASSED_TESTS,"
|
|
echo " \"failed\": $FAILED_TESTS,"
|
|
echo " \"skipped\": $SKIPPED_TESTS,"
|
|
echo " \"success\": $([ $FAILED_TESTS -eq 0 ] && echo 'true' || echo 'false')"
|
|
echo " },"
|
|
echo " \"results\": ["
|
|
|
|
local first=true
|
|
local i
|
|
for i in "${!RESULTS_NAMES[@]}"; do
|
|
[[ "$first" == "true" ]] || echo ","
|
|
printf ' {"name": "%s", "status": "%s", "details": "%s"}' \
|
|
"${RESULTS_NAMES[$i]}" "${RESULTS_STATUS[$i]}" "${RESULTS_DETAILS[$i]:-}"
|
|
first=false
|
|
done
|
|
echo ""
|
|
echo " ]"
|
|
echo "}"
|
|
}
|
|
|
|
_print_text_summary() {
|
|
local elapsed="$1"
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo ""
|
|
|
|
if [[ $FAILED_TESTS -eq 0 ]]; then
|
|
echo -e " ${GREEN}ALL TESTS PASSED${NC}"
|
|
echo ""
|
|
echo " Passed: $PASSED_TESTS | Skipped: $SKIPPED_TESTS"
|
|
echo " Runtime: ${elapsed}s"
|
|
echo ""
|
|
echo -e " ${BOLD}ODS is ready!${NC}"
|
|
else
|
|
echo -e " ${RED}SOME TESTS FAILED${NC}"
|
|
echo ""
|
|
echo -e " Passed: ${GREEN}${PASSED_TESTS}${NC} | Failed: ${RED}${FAILED_TESTS}${NC} | Skipped: ${YELLOW}${SKIPPED_TESTS}${NC}"
|
|
echo " Runtime: ${elapsed}s"
|
|
echo ""
|
|
echo " Failed tests:"
|
|
local i
|
|
for i in "${!RESULTS_NAMES[@]}"; do
|
|
if [[ "${RESULTS_STATUS[$i]}" == "fail" ]]; then
|
|
echo " - ${RESULTS_NAMES[$i]}"
|
|
fi
|
|
done
|
|
echo ""
|
|
echo "Actionable fixes:"
|
|
|
|
if [[ "${RESULTS_STATUS[0]:-}" == "fail" ]] && [[ "${RESULTS_NAMES[0]:-}" == *"LLM"* ]]; then
|
|
echo " - LLM not responding - check: docker logs ods-llama-server"
|
|
fi
|
|
|
|
local i
|
|
for i in "${!RESULTS_NAMES[@]}"; do
|
|
if [[ "${RESULTS_STATUS[$i]}" == "fail" ]]; then
|
|
case "${RESULTS_NAMES[$i]}" in
|
|
"Tool Calling") echo " - Tool calling failed - check llama-server tool support" ;;
|
|
"Whisper Port") echo " - Whisper not running - start: docker compose up whisper" ;;
|
|
"TTS Port") echo " - TTS not running - start: docker compose up kokoro-tts" ;;
|
|
esac
|
|
fi
|
|
done
|
|
fi
|
|
|
|
echo ""
|
|
}
|
|
|
|
#--------------------------------------------------------------------------
|
|
# Main
|
|
#--------------------------------------------------------------------------
|
|
|
|
show_help() {
|
|
cat << 'EOF'
|
|
ODS Validation Suite (M8)
|
|
|
|
USAGE:
|
|
ods-test.sh [OPTIONS]
|
|
|
|
OPTIONS:
|
|
--quick, -q Fast mode (~30s, no inference tests)
|
|
--json, -j Output results as JSON
|
|
--service, -s Test specific service only
|
|
--verbose, -v Show detailed output
|
|
--help, -h Show this help
|
|
|
|
SERVICES:
|
|
docker, gpu, llm, tool-calling, whisper, tts,
|
|
embeddings, voice-roundtrip, privacy-shield, livekit
|
|
|
|
EXAMPLES:
|
|
ods-test.sh # Run all tests
|
|
ods-test.sh --quick # Fast health check
|
|
ods-test.sh --json > results.json
|
|
ods-test.sh --service llm # Test LLM only
|
|
|
|
EXIT CODES:
|
|
0 - All tests passed
|
|
1 - One or more tests failed
|
|
2 - Configuration error
|
|
|
|
EOF
|
|
}
|
|
|
|
run_all_tests() {
|
|
print_header
|
|
|
|
test_docker
|
|
test_gpu
|
|
test_llm
|
|
test_tool_calling
|
|
test_whisper
|
|
test_tts
|
|
test_embeddings
|
|
test_voice_roundtrip
|
|
test_privacy_shield
|
|
test_livekit
|
|
}
|
|
|
|
run_specific_service() {
|
|
local service="$1"
|
|
|
|
print_header
|
|
|
|
case "$service" in
|
|
docker) test_docker ;;
|
|
gpu) test_gpu ;;
|
|
llm) test_llm ;;
|
|
tool-calling) test_tool_calling ;;
|
|
whisper) test_whisper ;;
|
|
tts) test_tts ;;
|
|
embeddings) test_embeddings ;;
|
|
voice-roundtrip) test_voice_roundtrip ;;
|
|
privacy-shield) test_privacy_shield ;;
|
|
livekit) test_livekit ;;
|
|
*)
|
|
echo "Unknown service: $service" >&2
|
|
echo "Available: docker, gpu, llm, tool-calling, whisper, tts, embeddings, voice-roundtrip, privacy-shield, livekit" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main() {
|
|
START_TIME=$(date +%s)
|
|
|
|
# Parse arguments
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--json|-j)
|
|
JSON_OUTPUT=true
|
|
shift
|
|
;;
|
|
--quick|-q)
|
|
QUICK_MODE=true
|
|
TIMEOUT=$QUICK_TIMEOUT
|
|
shift
|
|
;;
|
|
--service|-s)
|
|
SPECIFIC_SERVICE="$2"
|
|
shift 2
|
|
;;
|
|
--verbose|-v)
|
|
VERBOSE=true
|
|
shift
|
|
;;
|
|
--help|-h)
|
|
show_help
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unknown option: $1" >&2
|
|
show_help
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Load environment
|
|
load_env
|
|
|
|
# Run tests
|
|
if [[ -n "$SPECIFIC_SERVICE" ]]; then
|
|
run_specific_service "$SPECIFIC_SERVICE"
|
|
else
|
|
run_all_tests
|
|
fi
|
|
|
|
# Print summary
|
|
print_summary
|
|
|
|
# Exit with appropriate code
|
|
if [[ $FAILED_TESTS -eq 0 ]]; then
|
|
exit 0
|
|
else
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
main "$@"
|