4b6817381b
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
220 lines
8.3 KiB
Python
220 lines
8.3 KiB
Python
"""Per-kind message builders for scheduled tasks.
|
|
|
|
Each task kind maps to a function that produces a formatted report string
|
|
suitable for delivery to messaging providers.
|
|
|
|
The daily_summary, weekly_audit, and synthetic_run kinds query the real
|
|
investigation pipeline through the runner registered in
|
|
:mod:`platform.scheduler.investigation_runner`. When the pipeline returns no
|
|
findings, a clear quiet-period message is delivered. Pipeline failures raise
|
|
RuntimeError so the executor records FAILED in cron logs without masking
|
|
outages as success.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from platform.scheduler.investigation_runner import invoke_investigation_runner
|
|
from platform.scheduler.types import ScheduledTask, TaskKind
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Keys that should never be forwarded to the investigation pipeline
|
|
_CREDENTIAL_KEYS = frozenset({"bot_token", "access_token", "api_key", "webhook_url", "secret"})
|
|
|
|
|
|
def build_message(task: ScheduledTask) -> str:
|
|
"""Build the report message for a scheduled task based on its kind.
|
|
|
|
Returns the formatted message string. Raises RuntimeError on unrecoverable
|
|
pipeline failures for kinds that invoke the investigation graph.
|
|
"""
|
|
builders = {
|
|
TaskKind.DAILY_SUMMARY: _build_daily_summary,
|
|
TaskKind.WEEKLY_AUDIT: _build_weekly_audit,
|
|
TaskKind.INCIDENT_WINDOW_REPLAY: _build_incident_window_replay,
|
|
TaskKind.SYNTHETIC_RUN: _build_synthetic_run,
|
|
TaskKind.CUSTOM_INVESTIGATION: _build_custom_investigation,
|
|
}
|
|
builder = builders.get(task.kind)
|
|
if builder is None:
|
|
return f"⚠️ Unknown task kind: {task.kind}"
|
|
return builder(task)
|
|
|
|
|
|
def _build_daily_summary(task: ScheduledTask) -> str:
|
|
"""Build a daily reliability digest by running the investigation pipeline.
|
|
|
|
Queries the pipeline with a 'daily_summary' source over the configured
|
|
window. Returns the pipeline report if available. Distinguishes between
|
|
'pipeline ran but found nothing' vs 'pipeline failed' in the fallback.
|
|
"""
|
|
now = datetime.now(UTC)
|
|
window_start = now - timedelta(hours=task.window_hours)
|
|
|
|
try:
|
|
alert_payload = {
|
|
"source": "scheduled_daily_summary",
|
|
"task_id": task.id,
|
|
"window_hours": task.window_hours,
|
|
"kind": task.kind.value,
|
|
"window_start": window_start.isoformat(),
|
|
"window_end": now.isoformat(),
|
|
}
|
|
result = invoke_investigation_runner(alert_payload)
|
|
if result and result.get("report"):
|
|
return str(result["report"])
|
|
# Pipeline ran successfully but returned no report — genuinely quiet
|
|
except Exception as exc:
|
|
logger.error("Daily summary pipeline query failed for task %s: %s", task.id, exc)
|
|
raise RuntimeError(
|
|
f"Daily summary failed for task {task.id}. Check logs for details."
|
|
) from exc
|
|
|
|
return (
|
|
f"📊 <b>Daily Reliability Summary</b>\n\n"
|
|
f"Period: {window_start.strftime('%Y-%m-%d %H:%M')} → "
|
|
f"{now.strftime('%Y-%m-%d %H:%M')} UTC\n"
|
|
f"Window: {task.window_hours}h\n\n"
|
|
f"✅ No active incidents detected in the monitoring window.\n\n"
|
|
f"<i>Generated by OpenSRE scheduled delivery</i>"
|
|
)
|
|
|
|
|
|
def _build_weekly_audit(task: ScheduledTask) -> str:
|
|
"""Build a weekly noisy-alert audit by running the investigation pipeline.
|
|
|
|
Queries the pipeline with a 'weekly_audit' source over the configured
|
|
window. Returns the pipeline report if available. Distinguishes between
|
|
'pipeline ran but found nothing' vs 'pipeline failed' in the fallback.
|
|
"""
|
|
now = datetime.now(UTC)
|
|
window_start = now - timedelta(hours=task.window_hours)
|
|
|
|
try:
|
|
alert_payload = {
|
|
"source": "scheduled_weekly_audit",
|
|
"task_id": task.id,
|
|
"window_hours": task.window_hours,
|
|
"kind": task.kind.value,
|
|
"window_start": window_start.isoformat(),
|
|
"window_end": now.isoformat(),
|
|
}
|
|
result = invoke_investigation_runner(alert_payload)
|
|
if result and result.get("report"):
|
|
return str(result["report"])
|
|
except Exception as exc:
|
|
logger.error("Weekly audit pipeline query failed for task %s: %s", task.id, exc)
|
|
raise RuntimeError(
|
|
f"Weekly audit failed for task {task.id}. Check logs for details."
|
|
) from exc
|
|
|
|
return (
|
|
f"📋 <b>Weekly Alert Audit</b>\n\n"
|
|
f"Period: {window_start.strftime('%Y-%m-%d')} → "
|
|
f"{now.strftime('%Y-%m-%d')} UTC\n\n"
|
|
f"✅ No noisy or actionable alerts found for the past {task.window_hours}h.\n\n"
|
|
f"<i>Generated by OpenSRE scheduled delivery</i>"
|
|
)
|
|
|
|
|
|
def _build_incident_window_replay(task: ScheduledTask) -> str:
|
|
"""Build an incident window replay report.
|
|
|
|
Attempts to run the investigation pipeline over the configured window.
|
|
On failure, raises RuntimeError so the executor records the failure
|
|
without leaking exception details to the chat.
|
|
"""
|
|
try:
|
|
alert_payload = {
|
|
"source": "scheduled_replay",
|
|
"task_id": task.id,
|
|
"window_hours": task.window_hours,
|
|
"kind": task.kind.value,
|
|
}
|
|
result = invoke_investigation_runner(alert_payload)
|
|
if result and result.get("report"):
|
|
return str(result["report"])
|
|
return (
|
|
f"🔄 <b>Incident Window Replay</b>\n\n"
|
|
f"Window: {task.window_hours}h\n"
|
|
f"No incidents found in replay window.\n\n"
|
|
f"<i>Generated by OpenSRE scheduled delivery</i>"
|
|
)
|
|
except Exception as exc:
|
|
logger.error("Incident window replay failed for task %s: %s", task.id, exc)
|
|
raise RuntimeError(
|
|
f"Incident window replay failed for task {task.id}. Check logs for details."
|
|
) from exc
|
|
|
|
|
|
def _build_synthetic_run(task: ScheduledTask) -> str:
|
|
"""Build a synthetic test run summary by executing the synthetic suite.
|
|
|
|
Runs the synthetic test suite and reports results. On failure, raises
|
|
RuntimeError so the executor records the failure without leaking
|
|
exception details to the chat.
|
|
"""
|
|
now = datetime.now(UTC)
|
|
|
|
try:
|
|
alert_payload = {
|
|
"source": "scheduled_synthetic",
|
|
"task_id": task.id,
|
|
"kind": task.kind.value,
|
|
"window_hours": task.window_hours,
|
|
}
|
|
result = invoke_investigation_runner(alert_payload)
|
|
if result and result.get("report"):
|
|
return str(result["report"])
|
|
except Exception as exc:
|
|
logger.error("Synthetic run failed for task %s: %s", task.id, exc)
|
|
raise RuntimeError(
|
|
f"Synthetic run failed for task {task.id}. Check logs for details."
|
|
) from exc
|
|
|
|
return (
|
|
f"🧪 <b>Synthetic Test Summary</b>\n\n"
|
|
f"Run time: {now.strftime('%Y-%m-%d %H:%M')} UTC\n\n"
|
|
f"No synthetic test results available.\n"
|
|
f"Configure synthetic probes to see results here.\n\n"
|
|
f"<i>Generated by OpenSRE scheduled delivery</i>"
|
|
)
|
|
|
|
|
|
def _build_custom_investigation(task: ScheduledTask) -> str:
|
|
"""Run a custom investigation and return the report.
|
|
|
|
On failure, raises RuntimeError so the executor records the failure
|
|
without leaking exception details to the chat.
|
|
"""
|
|
try:
|
|
# Strip credential keys before passing params to the pipeline
|
|
safe_params = {k: v for k, v in task.params.items() if k not in _CREDENTIAL_KEYS}
|
|
alert_payload = {
|
|
"source": "scheduled_custom",
|
|
"task_id": task.id,
|
|
"window_hours": task.window_hours,
|
|
"kind": task.kind.value,
|
|
**safe_params,
|
|
}
|
|
result = invoke_investigation_runner(alert_payload)
|
|
if result and result.get("report"):
|
|
return str(result["report"])
|
|
return (
|
|
f"🔍 <b>Custom Investigation</b>\n\n"
|
|
f"Task: {task.id}\n"
|
|
f"No findings from custom investigation.\n\n"
|
|
f"<i>Generated by OpenSRE scheduled delivery</i>"
|
|
)
|
|
except Exception as exc:
|
|
logger.error("Custom investigation failed for task %s: %s", task.id, exc)
|
|
raise RuntimeError(
|
|
f"Custom investigation failed for task {task.id}. Check logs for details."
|
|
) from exc
|
|
|
|
|
|
__all__ = ["build_message"]
|