wehub-resource-sync
4b6817381b
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
407 lines
12 KiB
YAML
407 lines
12 KiB
YAML
- id: slash_help_prefix
|
|
input: "/help"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "slash"
|
|
content: "/help"
|
|
expected_executed_actions:
|
|
- kind: "slash"
|
|
content: "/help"
|
|
expected_terminal_response_contains:
|
|
- "ran /help"
|
|
expected_session_history_delta:
|
|
- type: "slash"
|
|
text: "/help"
|
|
ok: true
|
|
|
|
- id: bare_help_alias
|
|
input: "help"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "slash"
|
|
content: "/help"
|
|
expected_executed_actions:
|
|
- kind: "slash"
|
|
content: "/help"
|
|
expected_terminal_response_contains:
|
|
- "ran /help"
|
|
expected_session_history_delta:
|
|
- type: "slash"
|
|
text: "/help"
|
|
ok: true
|
|
|
|
- id: docs_run_investigation
|
|
input: "how do I run an investigation?"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "docs:run_investigation"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "opensre investigate"
|
|
- "alert"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "how do I run an investigation?"
|
|
ok: true
|
|
|
|
- id: docs_datadog_setup
|
|
input: "how do I configure datadog?"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "docs:datadog_setup"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "integrations"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "how do I configure datadog?"
|
|
ok: true
|
|
|
|
- id: docs_supported_integrations
|
|
input: "what are the supported integrations?"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "docs:supported_integrations"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "integrations"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "what are the supported integrations?"
|
|
ok: true
|
|
|
|
- id: docs_railway_deploy
|
|
input: "how to deploy OpenSRE on Railway?"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "docs:railway_deploy"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "railway"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "how to deploy OpenSRE on Railway?"
|
|
ok: true
|
|
|
|
- id: docs_command_question
|
|
input: "what command do I use?"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "docs:command_selection"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "investigate"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "what command do I use?"
|
|
ok: true
|
|
|
|
- id: cli_agent_run_sample_alert
|
|
input: "run a sample alert"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "sample_alert"
|
|
content: "generic"
|
|
expected_executed_actions:
|
|
- kind: "alert"
|
|
content: "sample:generic"
|
|
expected_terminal_response_contains:
|
|
- "sample alert generic"
|
|
expected_session_history_delta:
|
|
- type: "alert"
|
|
text: "sample:generic"
|
|
ok: true
|
|
|
|
- id: cli_agent_try_sample_alert
|
|
input: "try a sample alert"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "sample_alert"
|
|
content: "generic"
|
|
expected_executed_actions:
|
|
- kind: "alert"
|
|
content: "sample:generic"
|
|
expected_terminal_response_contains:
|
|
- "sample alert generic"
|
|
expected_session_history_delta:
|
|
- type: "alert"
|
|
text: "sample:generic"
|
|
ok: true
|
|
|
|
- id: cli_agent_synthetic_connection_exhaustion
|
|
input: "run synthetic test 002-connection-exhaustion"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "synthetic_test"
|
|
content: "rds_postgres:002-connection-exhaustion"
|
|
expected_executed_actions:
|
|
- kind: "synthetic_test"
|
|
content: "rds_postgres:002-connection-exhaustion"
|
|
expected_terminal_response_contains:
|
|
- "synthetic test rds_postgres:002-connection-exhaustion"
|
|
expected_session_history_delta:
|
|
- type: "synthetic_test"
|
|
text: "rds_postgres:002-connection-exhaustion"
|
|
ok: true
|
|
|
|
- id: cli_agent_synthetic_storage_full
|
|
input: "run synthetic test 003-storage-full"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "synthetic_test"
|
|
content: "rds_postgres:003-storage-full"
|
|
expected_executed_actions:
|
|
- kind: "synthetic_test"
|
|
content: "rds_postgres:003-storage-full"
|
|
expected_terminal_response_contains:
|
|
- "synthetic test rds_postgres:003-storage-full"
|
|
expected_session_history_delta:
|
|
- type: "synthetic_test"
|
|
text: "rds_postgres:003-storage-full"
|
|
ok: true
|
|
|
|
- id: cli_agent_synthetic_bare_numeric_id
|
|
input: "run synthetic test 005 now"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "synthetic_test"
|
|
content: "rds_postgres:005-failover"
|
|
expected_executed_actions:
|
|
- kind: "synthetic_test"
|
|
content: "rds_postgres:005-failover"
|
|
expected_terminal_response_contains:
|
|
- "synthetic test rds_postgres:005-failover"
|
|
expected_session_history_delta:
|
|
- type: "synthetic_test"
|
|
text: "rds_postgres:005-failover"
|
|
ok: true
|
|
|
|
- id: cli_agent_synthetic_cpu_saturation
|
|
input: "launch synthetic test 004-cpu-saturation-bad-query"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "synthetic_test"
|
|
content: "rds_postgres:004-cpu-saturation-bad-query"
|
|
expected_executed_actions:
|
|
- kind: "synthetic_test"
|
|
content: "rds_postgres:004-cpu-saturation-bad-query"
|
|
expected_terminal_response_contains:
|
|
- "synthetic test rds_postgres:004-cpu-saturation-bad-query"
|
|
expected_session_history_delta:
|
|
- type: "synthetic_test"
|
|
text: "rds_postgres:004-cpu-saturation-bad-query"
|
|
ok: true
|
|
|
|
- id: cli_agent_show_connected_services
|
|
input: "show me connected services"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "slash"
|
|
content: "/integrations list"
|
|
expected_executed_actions:
|
|
- kind: "slash"
|
|
content: "/integrations list"
|
|
expected_terminal_response_contains:
|
|
- "ran /integrations list"
|
|
expected_session_history_delta:
|
|
- type: "slash"
|
|
text: "/integrations list"
|
|
ok: true
|
|
|
|
- id: cli_agent_health_then_connected_services
|
|
input: "check the health of my opensre and then show me all connected services"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "slash"
|
|
content: "/health"
|
|
- kind: "slash"
|
|
content: "/integrations list"
|
|
expected_executed_actions:
|
|
- kind: "slash"
|
|
content: "/health"
|
|
- kind: "slash"
|
|
content: "/integrations list"
|
|
expected_terminal_response_contains:
|
|
- "ran /health"
|
|
- "ran /integrations list"
|
|
expected_session_history_delta:
|
|
- type: "slash"
|
|
text: "/health"
|
|
ok: true
|
|
- type: "slash"
|
|
text: "/integrations list"
|
|
ok: true
|
|
|
|
- id: cli_agent_connect_local_llama
|
|
input: "please connect to local llama"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "provider:local_llama_connect"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "ollama"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "please connect to local llama"
|
|
ok: true
|
|
|
|
- id: cli_agent_low_signal_greeting
|
|
input: "hello"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "chat:greeting"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "OpenSRE"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "hello"
|
|
ok: true
|
|
|
|
- id: cli_agent_freeform_operational
|
|
input: "why is the database slow?"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "diagnostic_question:database_slow"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "incident"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "why is the database slow?"
|
|
ok: true
|
|
|
|
- id: cli_agent_alert_json_blob
|
|
input: '{"alertname": "HighCPU", "severity": "critical"}'
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "alert_payload:json_blob"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "HighCPU"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: '{"alertname": "HighCPU", "severity": "critical"}'
|
|
ok: true
|
|
|
|
- id: new_alert_checkout_502_rate
|
|
input: "the checkout service is returning 502 errors for 30% of requests"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "incident_description:checkout_502_rate"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "checkout"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "the checkout service is returning 502 errors for 30% of requests"
|
|
ok: true
|
|
|
|
# A bare incident description (no explicit "investigate") is an assistant_handoff,
|
|
# consistent with new_alert_checkout_502_rate. The action agent is the sole tool
|
|
# selector and the system prompt sends incident descriptions to the assistant
|
|
# unless the user explicitly asks to investigate; there is no regex override that
|
|
# upgrades cpu/spiking pastes to an investigation.
|
|
- id: new_alert_orders_cpu_99
|
|
input: "CPU is spiking to 99% on the orders-api pods"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "incident_description:orders_cpu_99"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "orders"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "CPU is spiking to 99% on the orders-api pods"
|
|
ok: true
|
|
|
|
- id: follow_up_why_failed_with_state
|
|
input: "why did it fail?"
|
|
expected_turn_kind: "agent"
|
|
with_prior_state: true
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "follow_up:last_failure"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "failed"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "why did it fail?"
|
|
ok: true
|
|
|
|
- id: follow_up_spike_cause_with_state
|
|
input: "what caused the spike?"
|
|
expected_turn_kind: "agent"
|
|
with_prior_state: true
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "follow_up:spike_cause"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "incident"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "what caused the spike?"
|
|
ok: true
|
|
|
|
- id: follow_up_what_happened_with_state
|
|
input: "what happened?"
|
|
expected_turn_kind: "agent"
|
|
with_prior_state: true
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "follow_up:last_investigation_summary"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "incident"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "what happened?"
|
|
ok: true
|
|
|
|
- id: follow_up_last_investigation_wording_with_state
|
|
input: "No, during the last investigation"
|
|
expected_turn_kind: "agent"
|
|
with_prior_state: true
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "follow_up:last_investigation_summary"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "investigation"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "No, during the last investigation"
|
|
ok: true
|
|
|
|
- id: how_to_latency_measurement
|
|
input: "how do I measure latency with opensre?"
|
|
expected_turn_kind: "agent"
|
|
expected_planned_actions:
|
|
- kind: "assistant_handoff"
|
|
content: "docs:latency_measurement"
|
|
expected_executed_actions: []
|
|
expected_terminal_response_contains:
|
|
- "latency"
|
|
expected_session_history_delta:
|
|
- type: "cli_agent"
|
|
text: "how do I measure latency with opensre?"
|
|
ok: true
|
|
|
|
# Priority prompt #12 ("sniffdog investigate --service myportfolio") is intentionally
|
|
# omitted from turn scenarios: live planning may return cli_command while the
|
|
# execution oracle usually handoffs with guidance, so no stable planned/executed
|
|
# contract survives test_live_action_planning + test_live_turn_execution_oracle together.
|