Files
tracer-cloud--opensre/platform/scheduler/tasks.py
T
wehub-resource-sync 4b6817381b
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 13:10:45 +08:00

220 lines
8.3 KiB
Python

"""Per-kind message builders for scheduled tasks.
Each task kind maps to a function that produces a formatted report string
suitable for delivery to messaging providers.
The daily_summary, weekly_audit, and synthetic_run kinds query the real
investigation pipeline through the runner registered in
:mod:`platform.scheduler.investigation_runner`. When the pipeline returns no
findings, a clear quiet-period message is delivered. Pipeline failures raise
RuntimeError so the executor records FAILED in cron logs without masking
outages as success.
"""
from __future__ import annotations
import logging
from datetime import UTC, datetime, timedelta
from platform.scheduler.investigation_runner import invoke_investigation_runner
from platform.scheduler.types import ScheduledTask, TaskKind
logger = logging.getLogger(__name__)
# Keys that should never be forwarded to the investigation pipeline
_CREDENTIAL_KEYS = frozenset({"bot_token", "access_token", "api_key", "webhook_url", "secret"})
def build_message(task: ScheduledTask) -> str:
"""Build the report message for a scheduled task based on its kind.
Returns the formatted message string. Raises RuntimeError on unrecoverable
pipeline failures for kinds that invoke the investigation graph.
"""
builders = {
TaskKind.DAILY_SUMMARY: _build_daily_summary,
TaskKind.WEEKLY_AUDIT: _build_weekly_audit,
TaskKind.INCIDENT_WINDOW_REPLAY: _build_incident_window_replay,
TaskKind.SYNTHETIC_RUN: _build_synthetic_run,
TaskKind.CUSTOM_INVESTIGATION: _build_custom_investigation,
}
builder = builders.get(task.kind)
if builder is None:
return f"⚠️ Unknown task kind: {task.kind}"
return builder(task)
def _build_daily_summary(task: ScheduledTask) -> str:
"""Build a daily reliability digest by running the investigation pipeline.
Queries the pipeline with a 'daily_summary' source over the configured
window. Returns the pipeline report if available. Distinguishes between
'pipeline ran but found nothing' vs 'pipeline failed' in the fallback.
"""
now = datetime.now(UTC)
window_start = now - timedelta(hours=task.window_hours)
try:
alert_payload = {
"source": "scheduled_daily_summary",
"task_id": task.id,
"window_hours": task.window_hours,
"kind": task.kind.value,
"window_start": window_start.isoformat(),
"window_end": now.isoformat(),
}
result = invoke_investigation_runner(alert_payload)
if result and result.get("report"):
return str(result["report"])
# Pipeline ran successfully but returned no report — genuinely quiet
except Exception as exc:
logger.error("Daily summary pipeline query failed for task %s: %s", task.id, exc)
raise RuntimeError(
f"Daily summary failed for task {task.id}. Check logs for details."
) from exc
return (
f"📊 <b>Daily Reliability Summary</b>\n\n"
f"Period: {window_start.strftime('%Y-%m-%d %H:%M')} → "
f"{now.strftime('%Y-%m-%d %H:%M')} UTC\n"
f"Window: {task.window_hours}h\n\n"
f"✅ No active incidents detected in the monitoring window.\n\n"
f"<i>Generated by OpenSRE scheduled delivery</i>"
)
def _build_weekly_audit(task: ScheduledTask) -> str:
"""Build a weekly noisy-alert audit by running the investigation pipeline.
Queries the pipeline with a 'weekly_audit' source over the configured
window. Returns the pipeline report if available. Distinguishes between
'pipeline ran but found nothing' vs 'pipeline failed' in the fallback.
"""
now = datetime.now(UTC)
window_start = now - timedelta(hours=task.window_hours)
try:
alert_payload = {
"source": "scheduled_weekly_audit",
"task_id": task.id,
"window_hours": task.window_hours,
"kind": task.kind.value,
"window_start": window_start.isoformat(),
"window_end": now.isoformat(),
}
result = invoke_investigation_runner(alert_payload)
if result and result.get("report"):
return str(result["report"])
except Exception as exc:
logger.error("Weekly audit pipeline query failed for task %s: %s", task.id, exc)
raise RuntimeError(
f"Weekly audit failed for task {task.id}. Check logs for details."
) from exc
return (
f"📋 <b>Weekly Alert Audit</b>\n\n"
f"Period: {window_start.strftime('%Y-%m-%d')} → "
f"{now.strftime('%Y-%m-%d')} UTC\n\n"
f"✅ No noisy or actionable alerts found for the past {task.window_hours}h.\n\n"
f"<i>Generated by OpenSRE scheduled delivery</i>"
)
def _build_incident_window_replay(task: ScheduledTask) -> str:
"""Build an incident window replay report.
Attempts to run the investigation pipeline over the configured window.
On failure, raises RuntimeError so the executor records the failure
without leaking exception details to the chat.
"""
try:
alert_payload = {
"source": "scheduled_replay",
"task_id": task.id,
"window_hours": task.window_hours,
"kind": task.kind.value,
}
result = invoke_investigation_runner(alert_payload)
if result and result.get("report"):
return str(result["report"])
return (
f"🔄 <b>Incident Window Replay</b>\n\n"
f"Window: {task.window_hours}h\n"
f"No incidents found in replay window.\n\n"
f"<i>Generated by OpenSRE scheduled delivery</i>"
)
except Exception as exc:
logger.error("Incident window replay failed for task %s: %s", task.id, exc)
raise RuntimeError(
f"Incident window replay failed for task {task.id}. Check logs for details."
) from exc
def _build_synthetic_run(task: ScheduledTask) -> str:
"""Build a synthetic test run summary by executing the synthetic suite.
Runs the synthetic test suite and reports results. On failure, raises
RuntimeError so the executor records the failure without leaking
exception details to the chat.
"""
now = datetime.now(UTC)
try:
alert_payload = {
"source": "scheduled_synthetic",
"task_id": task.id,
"kind": task.kind.value,
"window_hours": task.window_hours,
}
result = invoke_investigation_runner(alert_payload)
if result and result.get("report"):
return str(result["report"])
except Exception as exc:
logger.error("Synthetic run failed for task %s: %s", task.id, exc)
raise RuntimeError(
f"Synthetic run failed for task {task.id}. Check logs for details."
) from exc
return (
f"🧪 <b>Synthetic Test Summary</b>\n\n"
f"Run time: {now.strftime('%Y-%m-%d %H:%M')} UTC\n\n"
f"No synthetic test results available.\n"
f"Configure synthetic probes to see results here.\n\n"
f"<i>Generated by OpenSRE scheduled delivery</i>"
)
def _build_custom_investigation(task: ScheduledTask) -> str:
"""Run a custom investigation and return the report.
On failure, raises RuntimeError so the executor records the failure
without leaking exception details to the chat.
"""
try:
# Strip credential keys before passing params to the pipeline
safe_params = {k: v for k, v in task.params.items() if k not in _CREDENTIAL_KEYS}
alert_payload = {
"source": "scheduled_custom",
"task_id": task.id,
"window_hours": task.window_hours,
"kind": task.kind.value,
**safe_params,
}
result = invoke_investigation_runner(alert_payload)
if result and result.get("report"):
return str(result["report"])
return (
f"🔍 <b>Custom Investigation</b>\n\n"
f"Task: {task.id}\n"
f"No findings from custom investigation.\n\n"
f"<i>Generated by OpenSRE scheduled delivery</i>"
)
except Exception as exc:
logger.error("Custom investigation failed for task %s: %s", task.id, exc)
raise RuntimeError(
f"Custom investigation failed for task {task.id}. Check logs for details."
) from exc
__all__ = ["build_message"]