Files
tracer-cloud--opensre/tools/investigation/reporting/formatters/report.py
T
wehub-resource-sync 4b6817381b
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 13:10:45 +08:00

879 lines
32 KiB
Python

"""RCA report formatting for Slack (mrkdwn / Block Kit) and Telegram (HTML)."""
import html
import re
from tools.investigation.reporting.context import ReportContext
from tools.investigation.reporting.formatters.base import (
format_html_link,
format_slack_link,
)
from tools.investigation.reporting.formatters.evidence import (
format_cited_evidence_section,
format_cited_evidence_section_html,
)
from tools.investigation.reporting.formatters.infrastructure import (
build_investigation_trace,
format_pod_line,
get_failed_pods,
)
from tools.investigation.reporting.urls.aws import build_cloudwatch_url
def render_cloudwatch_link(ctx: ReportContext) -> str:
"""Render CloudWatch logs link if available in context."""
cw_url = ctx.get("cloudwatch_logs_url")
cw_group = ctx.get("cloudwatch_log_group")
cw_stream = ctx.get("cloudwatch_log_stream")
if cw_url:
return f"\n*{format_slack_link('CloudWatch Logs', cw_url)}*\n"
elif cw_group and cw_stream:
url = build_cloudwatch_url(ctx)
view_link = format_slack_link("CloudWatch Logs", url) if url else None
if view_link:
return f"\n*{view_link}*\n"
return f"\n*CloudWatch Logs:*\n* Log Group: {cw_group}\n* Log Stream: {cw_stream}\n"
return ""
def _format_provenance_lines(ctx: ReportContext) -> list[str]:
provenance = ctx.get("source_provenance") or {}
lines: list[str] = []
for source_name, entry in provenance.items():
label = entry.get("label") or source_name.title()
summary = entry.get("summary") or ""
if summary:
lines.append(f"• {label}: {summary}")
return lines
def _format_correlation_lines(ctx: ReportContext) -> tuple[list[str], list[str]]:
correlation = ctx.get("correlation") or {}
if not isinstance(correlation, dict):
return [], []
raw_signals = correlation.get("correlated_signals") or []
raw_drivers = correlation.get("most_likely_causal_drivers") or []
signal_lines: list[str] = []
for signal in raw_signals:
if not isinstance(signal, dict):
continue
name = signal.get("name") or "unknown"
source = signal.get("source") or "unknown"
score = signal.get("score")
score_text = f" score={float(score):.2f}" if isinstance(score, int | float) else ""
signal_lines.append(f"• {name} ({source}{score_text})")
driver_lines: list[str] = []
for driver in raw_drivers:
if not isinstance(driver, dict):
continue
name = driver.get("name") or "unknown"
confidence = driver.get("confidence")
rationale = driver.get("rationale") or ""
confidence_text = (
f" confidence={float(confidence):.2f}" if isinstance(confidence, int | float) else ""
)
suffix = f" — {_sanitize_for_slack(str(rationale))}" if rationale else ""
driver_lines.append(f"• {name}{confidence_text}{suffix}")
return signal_lines, driver_lines
# ---------------------------------------------------------------------------
# Shared section helpers — called by both text and block renderers
# ---------------------------------------------------------------------------
def _render_claim_lines(ctx: ReportContext) -> tuple[list[str], list[str]]:
"""Return (validated_lines, non_validated_lines) as plain mrkdwn bullet strings.
Each validated line includes evidence citations like [E1, E2]. Both renderers
(format_slack_message and build_slack_blocks) call this to avoid duplicating
the catalog-lookup and link-formatting logic.
"""
catalog = ctx.get("evidence_catalog") or {}
evidence = ctx.get("evidence") or {}
validated_lines: list[str] = []
for claim_data in ctx.get("validated_claims", []):
claim = claim_data.get("claim", "")
claim = _resolve_evidence_tags(claim, evidence)
claim = _sanitize_for_slack(claim)
evidence_ids = claim_data.get("evidence_ids", [])
evidence_labels = claim_data.get("evidence_labels", [])
evidence_list: list[str] = []
if evidence_ids:
for eid in evidence_ids:
entry = catalog.get(eid, {})
disp = entry.get("display_id", eid)
url = entry.get("url")
evidence_list.append(format_slack_link(disp, url) if url else disp)
elif evidence_labels:
evidence_list = list(evidence_labels)
ev_str = f" [{', '.join(evidence_list)}]" if evidence_list else ""
validated_lines.append(f"\u2022 {claim}{ev_str}")
non_validated_lines: list[str] = [
f"\u2022 {_sanitize_for_slack(cd.get('claim', ''))}"
for cd in ctx.get("non_validated_claims", [])
]
return validated_lines, non_validated_lines
def _sanitize_for_slack(text: str) -> str:
"""Convert markdown formatting to Slack mrkdwn.
Slack does not render # headers, ** bold, or other standard markdown.
This converts common patterns to Slack-native formatting.
"""
result = re.sub(r"^#{1,6}\s+(.+)$", r"*\1*", text, flags=re.MULTILINE)
result = re.sub(r"\*\*(.+?)\*\*", r"*\1*", result)
return result
def _to_telegram_html_body(text: str) -> str:
"""Convert mixed Slack-style text to Telegram HTML via the shared formatter."""
from integrations.telegram.formatting import markdown_to_telegram_html
return markdown_to_telegram_html(text)
def _norm_banner_key(text: str) -> str:
return re.sub(r"\s+", " ", text.strip().lower())
def _telegram_baseline_repeats_header(ctx: ReportContext, root_cause_sentence: str) -> bool:
"""True when the derived root-cause line only repeats alert metadata already in the header."""
alert = (ctx.get("alert_name") or "").strip()
pipeline = (ctx.get("pipeline_name") or "").strip()
if not alert or not pipeline:
return False
s = root_cause_sentence.strip()
if len(s) > 220:
return False
rc = _norm_banner_key(s)
if _norm_banner_key(alert) not in rc or _norm_banner_key(pipeline) not in rc:
return False
if "because" in rc or "due to" in rc or "caused" in rc:
return False
if "severity" in rc:
return True
return len(s) < 120
def _severity_telegram_header(ctx: ReportContext) -> str:
"""Severity emoji row aligned with Hermes Telegram sink conventions."""
raw = (ctx.get("severity") or "").strip()
lower = raw.lower()
emoji = {
"critical": "🔴",
"crit": "🔴",
"high": "🟠",
"error": "🟠",
"medium": "🟡",
"warning": "🟡",
"warn": "🟡",
"low": "🟢",
"info": "🟢",
"none": "⚪",
"healthy": "🟢",
"normal": "🟢",
}.get(lower, "⚠️")
display_sev = raw.upper() if raw else "UNKNOWN"
alert = html.escape(str(ctx.get("alert_name") or "Alert"))
pipeline = html.escape(str(ctx.get("pipeline_name") or "unknown"))
return f"{emoji} <b>{alert}</b> · {pipeline}\n<i>severity: {html.escape(display_sev)}</i>"
def _render_claim_lines_telegram(ctx: ReportContext) -> tuple[list[str], list[str]]:
catalog = ctx.get("evidence_catalog") or {}
evidence = ctx.get("evidence") or {}
validated_lines: list[str] = []
for claim_data in ctx.get("validated_claims", []):
claim = claim_data.get("claim", "")
claim = _resolve_evidence_tags(claim, evidence)
claim = _sanitize_for_slack(claim)
evidence_ids = claim_data.get("evidence_ids", [])
evidence_labels = claim_data.get("evidence_labels", [])
evidence_list: list[str] = []
if evidence_ids:
for eid in evidence_ids:
entry = catalog.get(eid, {})
disp = entry.get("display_id", eid)
url = entry.get("url")
evidence_list.append(format_html_link(str(disp), url or None))
elif evidence_labels:
evidence_list = [html.escape(str(x)) for x in evidence_labels]
ev_str = f" [{', '.join(evidence_list)}]" if evidence_list else ""
validated_lines.append(f"• {_to_telegram_html_body(claim)}{ev_str}")
non_validated_lines: list[str] = []
for cd in ctx.get("non_validated_claims", []):
raw = _sanitize_for_slack(cd.get("claim", ""))
non_validated_lines.append(f"• {_to_telegram_html_body(raw)}")
return validated_lines, non_validated_lines
def render_cloudwatch_link_html(ctx: ReportContext) -> str:
"""Telegram-HTML CloudWatch deep link, mirroring :func:`render_cloudwatch_link`."""
cw_url = ctx.get("cloudwatch_logs_url")
cw_group = ctx.get("cloudwatch_log_group")
cw_stream = ctx.get("cloudwatch_log_stream")
if cw_url:
safe = html.escape(str(cw_url), quote=True)
return f'\n<b>CloudWatch</b>: <a href="{safe}">View logs</a>\n'
if cw_group and cw_stream:
url = build_cloudwatch_url(ctx)
if url:
safe = html.escape(str(url), quote=True)
return f'\n<b>CloudWatch</b>: <a href="{safe}">View logs</a>\n'
return (
f"\n<b>CloudWatch Logs</b>\n"
f"Log Group: {html.escape(str(cw_group))}\n"
f"Log Stream: {html.escape(str(cw_stream))}\n"
)
return ""
def _mrkdwn_section(text: str) -> "dict | None":
"""Build a Slack Block Kit section block with sanitized mrkdwn text.
Slack section blocks have a 3000 char limit per text field.
Returns None when text is empty — caller must skip None results.
"""
sanitized = _sanitize_for_slack(text).strip()
if not sanitized:
return None
if len(sanitized) > 2990:
sanitized = sanitized[:2987] + "..."
return {
"type": "section",
"text": {"type": "mrkdwn", "text": sanitized},
}
# ---------------------------------------------------------------------------
# Evidence tag resolution helpers
# ---------------------------------------------------------------------------
# Maps LLM source name → ordered list of evidence dict keys to try for a log message
_EVIDENCE_LOG_KEYS: dict[str, list[str]] = {
"datadog_logs": ["datadog_error_logs", "datadog_logs"],
"datadog": ["datadog_error_logs", "datadog_logs"],
"grafana_logs": ["grafana_error_logs", "grafana_logs"],
"grafana": ["grafana_error_logs", "grafana_logs"],
"cloudwatch_logs": ["cloudwatch_logs"],
"cloudwatch": ["cloudwatch_logs"],
}
def _extract_log_message(entry: object) -> str:
"""Extract a plain string message from a log entry that may be a dict or a string."""
if isinstance(entry, dict):
return (entry.get("message") or "").strip()
return str(entry).strip()
def _resolve_evidence_tags(text: str, evidence: dict) -> str:
"""Replace [evidence: source] tags with the actual log message in a code span.
Tries error logs first, then all logs for the named source. If no message
is found the tag is removed silently to avoid leaking raw LLM annotations.
"""
def _replace(m: re.Match) -> str:
source = m.group(1).strip().lower()
for key in _EVIDENCE_LOG_KEYS.get(source, []):
logs = evidence.get(key) or []
if logs:
msg = _extract_log_message(logs[0])
if msg:
return f": `{msg}`"
return ""
return re.sub(r"\s*\[(?i:evidence):\s*([^\]]+)\]", _replace, text).strip()
def _get_top_error_log(evidence: dict) -> str | None:
"""Return the first error log message from available evidence sources."""
for key in (
"datadog_error_logs",
"datadog_logs",
"grafana_error_logs",
"grafana_logs",
"cloudwatch_logs",
):
logs = evidence.get(key) or []
if logs:
msg = _extract_log_message(logs[0])
if msg:
return msg
return None
# ---------------------------------------------------------------------------
# Root cause derivation helpers
# ---------------------------------------------------------------------------
def _first_sentence(text: str) -> str:
"""Return the first sentence from text, normalized to one line."""
cleaned = re.sub(r"(?:^|\s)#{1,6}\s+", " ", text, flags=re.MULTILINE)
cleaned = re.sub(
r"\b(?:Problem Statement|Summary|Context|Description|Overview)\b[:\s]*",
"",
cleaned,
flags=re.IGNORECASE,
)
normalized = " ".join(cleaned.split()).strip()
if not normalized:
return ""
parts = re.split(r"(?<=[.?!])\s+", normalized, maxsplit=1)
sentence = parts[0]
sentence = sentence.rstrip(".?!")
return sentence
def _is_speculative(text: str) -> bool:
speculative_terms = (" may ", " might ", " possibly", " possible ", " likely ")
lower = f" {text.lower()} "
return any(term in lower for term in speculative_terms)
def _remove_speculative_words(text: str) -> str:
speculative = ("may", "might", "likely", "probably", "possibly")
words = text.split()
filtered = [w for w in words if w.lower() not in speculative]
return " ".join(filtered)
def _derive_root_cause_sentence(ctx: ReportContext) -> str:
"""Derive a concise, single-sentence root cause with causal preference."""
root_cause_text = str(ctx.get("root_cause", "") or "").strip()
root_cause_text = re.sub(r"\s*\[(?i:evidence):[^\]]*\]", "", root_cause_text).strip()
validated_claims = ctx.get("validated_claims", [])
if root_cause_text:
sentence = _first_sentence(root_cause_text)
if sentence and not _is_speculative(sentence):
return sentence
causal_connectors = (
" because ",
" due to ",
" caused ",
" resulted in ",
" led to ",
" root cause ",
" failure triggered ",
)
for claim_data in validated_claims:
claim = claim_data.get("claim", "") or ""
claim = re.sub(r"\s*\[(?i:evidence):[^\]]*\]", "", claim).strip()
lower = f" {claim.lower()} "
if any(connector in lower for connector in causal_connectors):
sentence = _first_sentence(claim)
if sentence:
return _first_sentence(_remove_speculative_words(sentence))
if root_cause_text:
sentence = _first_sentence(root_cause_text)
if sentence:
return sentence
if validated_claims:
claim = validated_claims[0].get("claim", "") or ""
claim = re.sub(r"\s*\[(?i:evidence):[^\]]*\]", "", claim).strip()
sentence = _first_sentence(claim)
if sentence:
return sentence
return ""
def _normalize_triage_summary(summary: str) -> str:
"""Remove the marker when the model included it inside the parsed summary."""
return re.sub(r"^(?i:triage complete:)\s*", "", summary).strip()
def _format_incident_command_block(ctx: ReportContext) -> str:
"""Render incident-command summary fields when diagnose extracted them."""
triage = _normalize_triage_summary(str(ctx.get("triage_summary") or "").strip())
status = str(ctx.get("incident_status") or "").strip()
hypotheses = [
str(item).strip()
for item in (ctx.get("investigation_hypotheses") or [])
if str(item).strip()
]
verification = [
str(item).strip() for item in (ctx.get("verification_summary") or []) if str(item).strip()
]
follow_ups = [
str(item).strip() for item in (ctx.get("follow_up_questions") or []) if str(item).strip()
]
tradeoffs = str(ctx.get("remediation_tradeoffs") or "").strip()
if not any((triage, status, hypotheses, verification, follow_ups, tradeoffs)):
return ""
lines = ["\n## Incident Command\n"]
if triage:
lines.append(f"Triage complete: {_sanitize_for_slack(triage)}\n")
if status:
lines.append(f"{_sanitize_for_slack(status)}\n")
if hypotheses:
lines.append(
"*Hypotheses:*\n"
+ "\n".join(f"• {_sanitize_for_slack(item)}" for item in hypotheses)
+ "\n"
)
if verification:
lines.append(
"*Verification:*\n"
+ "\n".join(f"• {_sanitize_for_slack(item)}" for item in verification)
+ "\n"
)
if follow_ups:
lines.append(
"*Follow-up questions:*\n"
+ "\n".join(f"• {_sanitize_for_slack(item)}" for item in follow_ups)
+ "\n"
)
if tradeoffs:
lines.append(f"*Remediation trade-offs:*\n{_sanitize_for_slack(tradeoffs)}\n")
return "".join(lines)
# ---------------------------------------------------------------------------
# Text renderer (Slack mrkdwn fallback + terminal + ingest report_md)
# ---------------------------------------------------------------------------
def format_slack_message(ctx: ReportContext) -> str:
"""Format a plain-text Slack message for the RCA report.
Used as the `text` fallback (notifications, accessibility, terminal, ingest)
when Block Kit blocks are the primary rendered content.
"""
alert_id = ctx.get("alert_id")
duration_seconds = ctx.get("investigation_duration_seconds")
root_cause_sentence = _derive_root_cause_sentence(ctx)
if not root_cause_sentence:
root_cause_sentence = "Not determined (insufficient evidence)."
# Start the report directly with the root cause sentence, without a "Root Cause"
# heading line, so that section headings below can carry the visual emphasis.
conclusion_block = f"{root_cause_sentence}\n"
top_log = _get_top_error_log(ctx.get("evidence") or {})
if top_log:
conclusion_block += f"`{top_log}`\n"
conclusion_block += _format_incident_command_block(ctx)
validated_lines, non_validated_lines = _render_claim_lines(ctx)
if validated_lines:
# Use a larger markdown heading so that "Findings" stands out as a section.
conclusion_block += "\n## Findings\n" + "\n".join(validated_lines) + "\n"
if non_validated_lines:
conclusion_block += (
"\n*Non-Validated Claims (Inferred):*\n" + "\n".join(non_validated_lines) + "\n"
)
correlation_signal_lines, correlation_driver_lines = _format_correlation_lines(ctx)
if correlation_signal_lines or correlation_driver_lines:
conclusion_block += "\n## Upstream Correlation\n"
if correlation_signal_lines:
conclusion_block += (
"*Correlated signals:*\n" + "\n".join(correlation_signal_lines) + "\n"
)
if correlation_driver_lines:
conclusion_block += (
"*Most likely causal drivers:*\n" + "\n".join(correlation_driver_lines) + "\n"
)
provenance_lines = _format_provenance_lines(ctx)
provenance_block = ""
if provenance_lines:
provenance_block = (
"\n*Provenance:*\n" + _sanitize_for_slack("\n".join(provenance_lines)) + "\n"
)
remediation_steps = ctx.get("remediation_steps", [])
remediation_block = ""
if remediation_steps:
remediation_block = (
"\n## Recommended Actions\n"
+ "\n".join(f"• {_sanitize_for_slack(s)}" for s in remediation_steps)
+ "\n"
)
trace_steps = build_investigation_trace(ctx)
trace_block = (
"\n## Investigation Trace\n" + "\n".join(trace_steps) + "\n" if trace_steps else ""
)
cited_section = _sanitize_for_slack(format_cited_evidence_section(ctx))
cloudwatch_link = render_cloudwatch_link(ctx)
meta_lines = []
if duration_seconds is not None:
meta_lines.append(f"Timing: {duration_seconds}s")
if alert_id:
meta_lines.append(f"*Alert ID:* {alert_id}")
meta_block = "\n" + "\n".join(meta_lines) if meta_lines else ""
# Do not prefix with a separate [RCA] title line; the consumer can render
# section headings (Root Cause text, Findings, Investigation Trace) with
# larger fonts as needed.
return f"""{conclusion_block}{provenance_block}{remediation_block}{trace_block}
{cited_section}
{cloudwatch_link}{meta_block}
"""
def format_telegram_message(ctx: ReportContext) -> str:
"""Format an HTML RCA message for Telegram (:meth:`parse_mode` ``HTML``).
Uses Telegram-supported tags and a Hermes-style severity emoji header, instead
of Slack mrkdwn (``<url|label>``, ``##`` headings) which render as plain text
without ``parse_mode``.
"""
duration_seconds = ctx.get("investigation_duration_seconds")
alert_id = ctx.get("alert_id")
derived_rc = _derive_root_cause_sentence(ctx)
root_cause_sentence = derived_rc or "Not determined (insufficient evidence)."
parts: list[str] = [_severity_telegram_header(ctx)]
top_log = _get_top_error_log(ctx.get("evidence") or {})
baseline_noise = (
derived_rc
and _telegram_baseline_repeats_header(ctx, derived_rc)
and root_cause_sentence != "Not determined (insufficient evidence)."
)
if baseline_noise and not top_log:
pass
elif baseline_noise and top_log:
parts.append("<code>" + html.escape(top_log) + "</code>")
else:
rc = _to_telegram_html_body(_sanitize_for_slack(root_cause_sentence))
if top_log:
rc += "\n<code>" + html.escape(top_log) + "</code>"
parts.append(rc)
triage = _normalize_triage_summary(str(ctx.get("triage_summary") or "").strip())
status = str(ctx.get("incident_status") or "").strip()
hypotheses = [
str(item).strip()
for item in (ctx.get("investigation_hypotheses") or [])
if str(item).strip()
]
verification = [
str(item).strip() for item in (ctx.get("verification_summary") or []) if str(item).strip()
]
follow_ups = [
str(item).strip() for item in (ctx.get("follow_up_questions") or []) if str(item).strip()
]
tradeoffs = str(ctx.get("remediation_tradeoffs") or "").strip()
if any((triage, status, hypotheses, verification, follow_ups, tradeoffs)):
ic_parts: list[str] = ["<b>Incident Command</b>"]
if triage:
ic_parts.append(
"Triage complete: " + _to_telegram_html_body(_sanitize_for_slack(triage))
)
if status:
ic_parts.append(_to_telegram_html_body(_sanitize_for_slack(status)))
if hypotheses:
ic_parts.append(
"<b>Hypotheses</b>\n"
+ "\n".join(
"• " + _to_telegram_html_body(_sanitize_for_slack(item)) for item in hypotheses
)
)
if verification:
ic_parts.append(
"<b>Verification</b>\n"
+ "\n".join(
"• " + _to_telegram_html_body(_sanitize_for_slack(item))
for item in verification
)
)
if follow_ups:
ic_parts.append(
"<b>Follow-up questions</b>\n"
+ "\n".join(
"• " + _to_telegram_html_body(_sanitize_for_slack(item)) for item in follow_ups
)
)
if tradeoffs:
ic_parts.append(
"<b>Remediation trade-offs</b>\n"
+ _to_telegram_html_body(_sanitize_for_slack(tradeoffs))
)
parts.append("\n".join(ic_parts))
validated_lines, non_validated_lines = _render_claim_lines_telegram(ctx)
if validated_lines:
parts.append("<b>Findings</b>\n" + "\n".join(validated_lines))
if non_validated_lines:
parts.append("<b>Non-Validated Claims (Inferred)</b>\n" + "\n".join(non_validated_lines))
provenance_lines = _format_provenance_lines(ctx)
if provenance_lines:
prov = "\n".join(
"• " + _to_telegram_html_body(_sanitize_for_slack(pl.lstrip("• ").strip()))
for pl in provenance_lines
)
parts.append("<b>Provenance</b>\n" + prov)
remediation_steps = ctx.get("remediation_steps", [])
if remediation_steps:
ra = "\n".join(
"• " + _to_telegram_html_body(_sanitize_for_slack(str(step)))
for step in remediation_steps
)
parts.append("<b>Recommended Actions</b>\n" + ra)
trace_steps = build_investigation_trace(ctx)
if trace_steps:
tr = "\n".join(_to_telegram_html_body(step) for step in trace_steps)
parts.append("<b>Investigation Trace</b>\n" + tr)
cited_block = format_cited_evidence_section_html(ctx).strip()
if cited_block:
parts.append(cited_block)
cw_block = render_cloudwatch_link_html(ctx).strip()
if cw_block:
parts.append(cw_block)
meta_bits: list[str] = []
if duration_seconds is not None:
meta_bits.append(f"Timing: {duration_seconds}s")
if alert_id:
meta_bits.append(f"Alert ID: {alert_id}")
if meta_bits:
parts.append("<i>" + html.escape(" | ".join(meta_bits)) + "</i>")
return "\n\n".join(p for p in parts if p)
def format_whatsapp_message(ctx: ReportContext) -> str:
"""Format a plain-text RCA message for WhatsApp (mobile-friendly).
WhatsApp supports basic formatting (*bold*, _italic_, `code`) but we
keep the message plain and structured for maximum compatibility and
readability on small screens.
"""
duration_seconds = ctx.get("investigation_duration_seconds")
alert_id = ctx.get("alert_id")
derived_rc = _derive_root_cause_sentence(ctx)
root_cause_sentence = derived_rc or "Not determined (insufficient evidence)."
parts: list[str] = []
# Severity header
severity = ctx.get("severity", "")
if severity:
parts.append(f"[{severity.upper()}] OpenSRE Investigation")
else:
parts.append("OpenSRE Investigation")
# Root cause + top log
top_log = _get_top_error_log(ctx.get("evidence") or {})
if top_log:
parts.append(f"{root_cause_sentence}\nTop log: {top_log}")
else:
parts.append(root_cause_sentence)
# Findings
validated_lines, non_validated_lines = _render_claim_lines(ctx)
if validated_lines:
parts.append("*Findings*\n" + "\n".join(validated_lines))
if non_validated_lines:
parts.append("*Inferred Claims*\n" + "\n".join(non_validated_lines))
# Provenance
provenance_lines = _format_provenance_lines(ctx)
if provenance_lines:
parts.append("*Provenance*\n" + "\n".join(provenance_lines))
# Recommended actions
remediation_steps = ctx.get("remediation_steps", [])
if remediation_steps:
parts.append("*Recommended Actions*\n" + "\n".join(f"• {s}" for s in remediation_steps))
# Investigation trace
trace_steps = build_investigation_trace(ctx)
if trace_steps:
parts.append("*Investigation Trace*\n" + "\n".join(trace_steps))
# Meta
meta_bits: list[str] = []
if duration_seconds is not None:
meta_bits.append(f"Timing: {duration_seconds}s")
if alert_id:
meta_bits.append(f"Alert ID: {alert_id}")
if meta_bits:
parts.append(" | ".join(meta_bits))
return "\n\n".join(p for p in parts if p)
# ---------------------------------------------------------------------------
# Block Kit renderer (Slack interactive cards)
# ---------------------------------------------------------------------------
def build_slack_blocks(ctx: ReportContext) -> list[dict]:
"""Build Slack Block Kit blocks for the RCA report.
Produces a clean, well-structured message using Slack's native
formatting: header, sections with mrkdwn, dividers, and context blocks.
"""
from typing import Any
duration_seconds = ctx.get("investigation_duration_seconds")
alert_id = ctx.get("alert_id")
root_cause_sentence = _derive_root_cause_sentence(ctx)
blocks: list[dict[str, Any]] = []
def _add(block: "dict[str, Any] | None") -> None:
if block is not None:
blocks.append(block)
# ── Root Cause
if not root_cause_sentence:
root_cause_sentence = "Not determined (insufficient evidence)"
rc_text = root_cause_sentence
top_log = _get_top_error_log(ctx.get("evidence") or {})
if top_log:
rc_text += f"\n`{top_log}`"
_add(_mrkdwn_section(rc_text))
# ── Failed Pods ──
datadog_site = ctx.get("datadog_site", "datadoghq.com")
all_pods = get_failed_pods(ctx)
pod_lines = [
line for p in all_pods[:5] if (line := format_pod_line(p, datadog_site, bullet="\u2022 "))
]
if len(all_pods) > 5:
pod_lines.append(f"• ... and {len(all_pods) - 5} more pods")
if pod_lines:
blocks.append({"type": "divider"})
blocks.append(
{
"type": "header",
"text": {"type": "plain_text", "text": "Failed Pods"},
}
)
_add(_mrkdwn_section("\n".join(pod_lines)))
# ── Validated Claims (Findings) and Non-Validated Claims ──
validated_lines, non_validated_lines = _render_claim_lines(ctx)
if validated_lines:
blocks.append({"type": "divider"})
blocks.append(
{
"type": "header",
"text": {"type": "plain_text", "text": "Findings"},
}
)
_add(_mrkdwn_section("\n".join(validated_lines)))
if non_validated_lines:
_add(_mrkdwn_section("*Inferred (not yet validated)*\n" + "\n".join(non_validated_lines)))
correlation_signal_lines, correlation_driver_lines = _format_correlation_lines(ctx)
if correlation_signal_lines or correlation_driver_lines:
blocks.append({"type": "divider"})
blocks.append(
{
"type": "header",
"text": {"type": "plain_text", "text": "Upstream Correlation"},
}
)
if correlation_signal_lines:
_add(_mrkdwn_section("*Correlated signals:*\n" + "\n".join(correlation_signal_lines)))
if correlation_driver_lines:
_add(
_mrkdwn_section(
"*Most likely causal drivers:*\n" + "\n".join(correlation_driver_lines)
)
)
provenance_lines = _format_provenance_lines(ctx)
if provenance_lines:
blocks.append({"type": "divider"})
blocks.append(
{
"type": "header",
"text": {"type": "plain_text", "text": "Provenance"},
}
)
_add(_mrkdwn_section("\n".join(provenance_lines)))
# ── Recommended Actions ──
remediation_steps = ctx.get("remediation_steps", [])
if remediation_steps:
blocks.append({"type": "divider"})
blocks.append(
{
"type": "header",
"text": {"type": "plain_text", "text": "Recommended Actions"},
}
)
_add(_mrkdwn_section("\n".join(f"• {_sanitize_for_slack(s)}" for s in remediation_steps)))
# ── Investigation Trace ──
trace_steps = build_investigation_trace(ctx)
if trace_steps:
blocks.append({"type": "divider"})
blocks.append(
{
"type": "header",
"text": {"type": "plain_text", "text": "Investigation Trace"},
}
)
_add(_mrkdwn_section("\n".join(trace_steps)))
# ── Cited Evidence ──
cited_section = format_cited_evidence_section(ctx).strip()
if cited_section:
blocks.append({"type": "divider"})
_add(_mrkdwn_section(cited_section))
# ── CloudWatch link ──
cw_link = render_cloudwatch_link(ctx).strip()
if cw_link:
_add(_mrkdwn_section(cw_link))
# ── Meta context (duration / alert) at the bottom ──
meta_parts = []
if duration_seconds is not None:
meta_parts.append(f"Analyzed in {duration_seconds}s")
if alert_id:
meta_parts.append(f"Alert: {alert_id}")
if meta_parts:
blocks.append({"type": "divider"})
blocks.append(
{
"type": "context",
"elements": [{"type": "mrkdwn", "text": " | ".join(meta_parts)}],
}
)
# Slack hard-limits messages to 50 blocks — truncate from the middle to keep
# the header (first block) and meta/actions (last 2 blocks) intact.
if len(blocks) > 50:
blocks = blocks[:48] + blocks[-2:]
return blocks