chore: import upstream snapshot with attribution

core/tool_framework/utils/__init__.py

@@ -0,0 +1,11 @@
+"""Tool utilities — code-host helpers, data validation, and database warnings."""
+
+from core.tool_framework.utils.code_host_unavailable import code_host_unavailable_payload
+from core.tool_framework.utils.data_validation import validate_host_metrics
+from core.tool_framework.utils.db_warnings import default_db_warning
+
+__all__ = [
+    "code_host_unavailable_payload",
+    "default_db_warning",
+    "validate_host_metrics",
+]

core/tool_framework/utils/code_host_unavailable.py

@@ -0,0 +1,21 @@
+"""Shared unavailable payload helper for code-host investigation tools."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def code_host_unavailable_payload(
+    *,
+    source: str,
+    integration_name: str,
+    empty_key: str,
+    empty_value: Any,
+) -> dict[str, Any]:
+    """Return a standardized unavailable payload for code-host tools."""
+    return {
+        "source": source,
+        "available": False,
+        "error": f"{integration_name} integration is not configured.",
+        empty_key: empty_value,
+    }

core/tool_framework/utils/data_validation.py

@@ -0,0 +1,411 @@
+"""Data validators - sanitize API responses before LLM sees them."""
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class ValidationIssue:
+    """Represents a data quality issue found during validation."""
+
+    field: str
+    raw_value: Any
+    issue_type: str  # "impossible_value", "unit_mismatch", "missing_data", etc.
+    severity: str  # "error", "warning", "info"
+    explanation: str
+    suggested_fix: str | None = None
+
+
+class MetricsValidator:
+    """Validates and normalizes metrics data from APIs."""
+
+    issues: list[ValidationIssue]
+
+    def __init__(self) -> None:
+        self.issues = []
+
+    def validate_metrics(self, metrics: dict) -> dict:
+        """
+        Validate and normalize metrics, flagging impossible values.
+
+        Handles different API response structures:
+        - Flat structure: {"cpu": 95, "ram": 8471740416, "disk": 50}
+        - Nested structure: {"memory": {"percent": 8471740416}, "cpu": {"percent": 95}}
+        - List structure: {"data": [{"cpu": 95, "ram": 8471740416}]}
+
+        Returns:
+            Normalized metrics dict with added 'data_quality_issues' key
+        """
+        normalized = metrics.copy() if isinstance(metrics, dict) else {}
+        self.issues = []
+
+        # Handle list structure (common in API responses)
+        if "data" in normalized and isinstance(normalized["data"], list):
+            validated_data, _ = _validate_data_list(normalized["data"], self._validate_flat_metrics)
+            normalized["data"] = validated_data
+            # Also check aggregated values if present
+            if "max_cpu" in normalized or "max_ram" in normalized:
+                normalized = self._validate_flat_metrics(normalized)
+
+        # Check nested memory metrics
+        if "memory" in normalized:
+            normalized["memory"] = self._validate_memory_metric(normalized["memory"])
+
+        # Check CPU metrics
+        if "cpu" in normalized:
+            normalized["cpu"] = self._validate_cpu_metric(normalized["cpu"])
+
+        # Check disk metrics
+        if "disk" in normalized:
+            normalized["disk"] = self._validate_disk_metric(normalized["disk"])
+
+        # Check for flat structure metrics (cpu, ram, disk at top level)
+        normalized = self._validate_flat_metrics(normalized)
+
+        # Check for percentage fields at top level
+        for key in ["percent", "percentage", "usage_percent"]:
+            if key in normalized:
+                value = normalized[key]
+                if isinstance(value, int | float) and value > 100:
+                    self._flag_impossible_percentage(key, value, normalized)
+
+        # Attach validation issues to the response
+        if self.issues:
+            normalized["data_quality_issues"] = [
+                {
+                    "field": issue.field,
+                    "raw_value": issue.raw_value,
+                    "issue": issue.issue_type,
+                    "severity": issue.severity,
+                    "explanation": issue.explanation,
+                    "suggested_fix": issue.suggested_fix,
+                }
+                for issue in self.issues
+            ]
+
+        return normalized
+
+    def _validate_memory_metric(self, memory_data: dict | Any) -> dict:
+        """Validate and fix memory metrics with intelligent unit inference."""
+        if not isinstance(memory_data, dict):
+            return {"raw": memory_data, "validated": False}
+
+        normalized = memory_data.copy()
+
+        # Check for impossible percentage values
+        if "percent" in memory_data:
+            raw_percent = memory_data["percent"]
+
+            if isinstance(raw_percent, int | float) and raw_percent > 100:
+                # Infer the most likely unit and interpretation
+                interpretation = self._infer_memory_unit(raw_percent)
+
+                self.issues.append(
+                    ValidationIssue(
+                        field="memory.percent",
+                        raw_value=raw_percent,
+                        issue_type="impossible_percentage",
+                        severity="error",
+                        explanation=interpretation["explanation"],
+                        suggested_fix=interpretation["suggested_fix"],
+                    )
+                )
+
+                # Add interpretation hints to the normalized data
+                normalized["percent_interpretation"] = interpretation
+                normalized["percent_invalid"] = True
+                normalized["percent_raw"] = raw_percent
+                normalized["percent"] = None  # Mark as invalid, LLM should use interpretation
+
+        # Also check for "ram" field (common in API responses)
+        if "ram" in memory_data:
+            raw_ram = memory_data["ram"]
+            if isinstance(raw_ram, int | float) and raw_ram > 100:
+                interpretation = self._infer_memory_unit(raw_ram)
+                self.issues.append(
+                    ValidationIssue(
+                        field="memory.ram",
+                        raw_value=raw_ram,
+                        issue_type="impossible_percentage",
+                        severity="error",
+                        explanation=interpretation["explanation"],
+                        suggested_fix=interpretation["suggested_fix"],
+                    )
+                )
+                normalized["ram_interpretation"] = interpretation
+                normalized["ram_invalid"] = True
+                normalized["ram_raw"] = raw_ram
+                normalized["ram"] = None
+
+        return normalized
+
+    def _infer_memory_unit(self, value: float) -> dict:
+        """
+        Infer the most likely unit for a memory value that's labeled as percentage.
+
+        Returns interpretation with likely unit, explanation, and suggested fix.
+        """
+        # Common memory sizes in bytes
+        if value >= 1024**3:  # >= 1GB
+            gb_value = value / (1024**3)
+            interpretation = {
+                "likely_unit": "bytes",
+                "likely_value_gb": round(gb_value, 2),
+                "likely_value_mb": round(value / (1024**2), 2),
+                "explanation": (
+                    f"Value {value:,.0f} labeled as 'percent' is impossible (>100%). "
+                    f"This is most likely a unit error where bytes are reported as percentage. "
+                    f"Interpretation: {gb_value:.2f} GB ({value / (1024**2):,.0f} MB) of memory used. "
+                    f"To determine actual percentage, divide by total memory: "
+                    f"percent = (used_bytes / total_memory_bytes) * 100"
+                ),
+                "suggested_fix": (
+                    f"Treat this as {gb_value:.2f} GB of memory used, not a percentage. "
+                    f"Compare against instance type memory limits to determine if memory was exhausted."
+                ),
+            }
+        elif value >= 1024**2:  # >= 1MB
+            mb_value = value / (1024**2)
+            interpretation = {
+                "likely_unit": "bytes",
+                "likely_value_mb": round(mb_value, 2),
+                "likely_value_gb": round(value / (1024**3), 2),
+                "explanation": (
+                    f"Value {value:,.0f} labeled as 'percent' is impossible (>100%). "
+                    f"This is likely a unit error where bytes are reported as percentage. "
+                    f"Interpretation: {mb_value:.2f} MB ({value / (1024**3):.2f} GB) of memory used."
+                ),
+                "suggested_fix": (
+                    f"Treat this as {mb_value:.2f} MB of memory used, not a percentage."
+                ),
+            }
+        else:
+            # Value is > 100 but < 1MB - might be a different error
+            interpretation = {
+                "likely_unit": "unknown",
+                "explanation": (
+                    f"Value {value:,.0f} labeled as 'percent' exceeds 100% but is unusually small. "
+                    f"This suggests a data collection or unit conversion error."
+                ),
+                "suggested_fix": "Verify the metric source and unit conversion logic",
+            }
+
+        return interpretation
+
+    def _validate_cpu_metric(self, cpu_data: dict | Any) -> dict:
+        """Validate CPU metrics."""
+        if not isinstance(cpu_data, dict):
+            return {"raw": cpu_data, "validated": False}
+
+        normalized = cpu_data.copy()
+
+        if "percent" in cpu_data:
+            raw_percent = cpu_data["percent"]
+
+            # CPU can legitimately exceed 100% (multi-core)
+            # But beyond 1000% is suspicious for most workloads
+            if isinstance(raw_percent, int | float) and raw_percent > 1000:
+                self.issues.append(
+                    ValidationIssue(
+                        field="cpu.percent",
+                        raw_value=raw_percent,
+                        issue_type="suspicious_value",
+                        severity="warning",
+                        explanation=(
+                            f"CPU usage reported as {raw_percent}% which is unusually high. "
+                            f"While multi-core systems can exceed 100%, values over 1000% "
+                            f"suggest a data collection error or misconfigured metric."
+                        ),
+                        suggested_fix="Verify CPU metric calculation and core count normalization",
+                    )
+                )
+                # Cap at reasonable value or mark as suspicious
+                normalized["percent_suspicious"] = True
+                normalized["percent_raw"] = raw_percent
+
+        return normalized
+
+    def _validate_disk_metric(self, disk_data: dict | Any) -> dict:
+        """Validate disk metrics."""
+        if not isinstance(disk_data, dict):
+            return {"raw": disk_data, "validated": False}
+
+        normalized = disk_data.copy()
+
+        if "percent" in disk_data:
+            raw_percent = disk_data["percent"]
+
+            if isinstance(raw_percent, int | float) and raw_percent > 100:
+                self.issues.append(
+                    ValidationIssue(
+                        field="disk.percent",
+                        raw_value=raw_percent,
+                        issue_type="impossible_percentage",
+                        severity="error",
+                        explanation=(
+                            f"Disk usage reported as {raw_percent}% which is impossible. "
+                            f"This indicates a data collection or unit conversion error."
+                        ),
+                        suggested_fix="Verify disk metric calculation and units",
+                    )
+                )
+                normalized["percent"] = None
+                normalized["percent_invalid"] = True
+                normalized["percent_raw"] = raw_percent
+
+        return normalized
+
+    def _validate_flat_metrics(self, metrics: dict) -> dict:
+        """
+        Validate metrics in flat structure (cpu, ram, disk at top level).
+
+        Common API format: {"cpu": 95.28, "ram": 8471740416, "disk": 50}
+        """
+        normalized = metrics.copy()
+
+        # Check "ram" field (often used instead of "memory")
+        if "ram" in normalized:
+            raw_ram = normalized["ram"]
+            if isinstance(raw_ram, int | float) and raw_ram > 100:
+                interpretation = self._infer_memory_unit(raw_ram)
+                self.issues.append(
+                    ValidationIssue(
+                        field="ram",
+                        raw_value=raw_ram,
+                        issue_type="impossible_percentage",
+                        severity="error",
+                        explanation=interpretation["explanation"],
+                        suggested_fix=interpretation["suggested_fix"],
+                    )
+                )
+                normalized["ram_interpretation"] = interpretation
+                normalized["ram_invalid"] = True
+                normalized["ram_raw"] = raw_ram
+                # Don't set to None - keep raw value but mark as invalid
+                # LLM can use interpretation to understand it
+
+        # Check "max_ram" if present
+        if "max_ram" in normalized:
+            raw_max_ram = normalized["max_ram"]
+            if isinstance(raw_max_ram, int | float) and raw_max_ram > 100:
+                interpretation = self._infer_memory_unit(raw_max_ram)
+                self.issues.append(
+                    ValidationIssue(
+                        field="max_ram",
+                        raw_value=raw_max_ram,
+                        issue_type="impossible_percentage",
+                        severity="error",
+                        explanation=interpretation["explanation"],
+                        suggested_fix=interpretation["suggested_fix"],
+                    )
+                )
+                normalized["max_ram_interpretation"] = interpretation
+                normalized["max_ram_invalid"] = True
+                normalized["max_ram_raw"] = raw_max_ram
+
+        return normalized
+
+    def _flag_impossible_percentage(self, field: str, value: Any, data: dict) -> None:
+        """Flag an impossible percentage value."""
+        if isinstance(value, int | float) and value > 100:
+            # For memory-related fields, use intelligent inference
+            if "memory" in field.lower() or "ram" in field.lower():
+                interpretation = self._infer_memory_unit(value)
+                self.issues.append(
+                    ValidationIssue(
+                        field=field,
+                        raw_value=value,
+                        issue_type="impossible_percentage",
+                        severity="error",
+                        explanation=interpretation["explanation"],
+                        suggested_fix=interpretation["suggested_fix"],
+                    )
+                )
+                data[f"{field}_interpretation"] = interpretation
+            else:
+                self.issues.append(
+                    ValidationIssue(
+                        field=field,
+                        raw_value=value,
+                        issue_type="impossible_percentage",
+                        severity="error",
+                        explanation=(
+                            f"Field '{field}' has value {value}% which exceeds 100% and is impossible. "
+                            f"This is likely a data collection error or unit mismatch."
+                        ),
+                        suggested_fix="Verify the metric source and unit conversion",
+                    )
+                )
+            data[f"{field}_invalid"] = True
+            data[f"{field}_raw"] = value
+
+
+def _validate_data_list(
+    data_points: list[Any],
+    validate_fn: Callable[[dict], dict],
+) -> tuple[list[Any], list[dict]]:
+    """Iterate a list of data points, validate each dict entry, and collect issues.
+
+    Returns ``(validated_points, collected_issues)`` where issues are stripped
+    from each point's ``data_quality_issues`` key and merged into the returned list.
+    Points that are not dicts are passed through unchanged.
+    """
+    validated: list[Any] = []
+    issues: list[dict] = []
+    for point in data_points:
+        if isinstance(point, dict):
+            result = validate_fn(point.copy())
+            point_issues = result.pop("data_quality_issues", None)
+            if isinstance(point_issues, list):
+                issues.extend(point_issues)
+            validated.append(result)
+        else:
+            validated.append(point)
+    return validated, issues
+
+
+def validate_host_metrics(metrics: dict | Any) -> dict:
+    """
+    Validate host metrics data.
+
+    Handles different API response structures:
+    - List structure: {"success": True, "data": [{"cpu": 95, "ram": 8471740416, "disk": 50}]}
+    - Flat structure: {"cpu": 95, "ram": 8471740416}
+    - Nested structure: {"memory": {"percent": 8471740416}}
+
+    Args:
+        metrics: Raw metrics data from API
+
+    Returns:
+        Validated and normalized metrics dict with interpretation hints
+    """
+    validator = MetricsValidator()
+    if not isinstance(metrics, dict):
+        return {
+            "raw": metrics,
+            "validated": False,
+            "data_quality_issues": [
+                {
+                    "field": "root",
+                    "raw_value": metrics,
+                    "issue": "invalid_format",
+                    "severity": "error",
+                    "explanation": "Metrics data is not in expected dictionary format",
+                }
+            ],
+        }
+
+    # Handle list-based structure (common in API responses)
+    if "data" in metrics and isinstance(metrics["data"], list):
+        validated_data, all_issues = _validate_data_list(
+            metrics["data"], validator.validate_metrics
+        )
+        result = metrics.copy()
+        result["data"] = validated_data
+        if all_issues:
+            result["data_quality_issues"] = all_issues
+        return result
+
+    # Handle flat or nested structure
+    return validator.validate_metrics(metrics)

core/tool_framework/utils/db_warnings.py

@@ -0,0 +1,10 @@
+"""Shared warning helper for SQL tools that default to a system database."""
+
+from __future__ import annotations
+
+
+def default_db_warning(db_name: str) -> str:
+    return (
+        f"WARNING: No database was specified; defaulted to '{db_name}'. "
+        "Results may not reflect application data."
+    )

core/tool_framework/utils/integration_sources.py

@@ -0,0 +1,27 @@
+"""Adapt resolved integration configs to the legacy tool availability contract."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+def availability_view(resolved_integrations: dict[str, Any]) -> dict[str, Any]:
+    """Convert classified integration configs into dicts tools can consume."""
+    view: dict[str, Any] = {}
+    for key, value in resolved_integrations.items():
+        if key.startswith("_"):
+            view[key] = value
+            continue
+        if isinstance(value, BaseModel):
+            item = value.model_dump(exclude_none=True)
+            item.setdefault("connection_verified", True)
+            view[key] = item
+        elif isinstance(value, dict) and value:
+            item = dict(value)
+            item.setdefault("connection_verified", True)
+            view[key] = item
+        else:
+            view[key] = value
+    return view

core/tool_framework/utils/mcp_params.py

@@ -0,0 +1,34 @@
+"""Normalize MCP integration source dicts for tool param extraction.
+
+MCP-backed tools read connection settings from the verified integration
+source (e.g. ``posthog_mcp``, ``sentry_mcp``, ``openclaw``). Catalog and
+runtime configs may use prefixed keys (``posthog_url``) or short aliases
+(``url``). These helpers pick the first non-empty value across alias keys
+and coerce list fields into stripped string lists.
+"""
+
+from __future__ import annotations
+
+__all__ = ["first_list", "first_string", "string_list"]
+
+
+def string_list(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item).strip() for item in value if str(item).strip()]
+
+
+def first_string(source: dict[str, object], *keys: str) -> str | None:
+    for key in keys:
+        value = str(source.get(key, "")).strip()
+        if value:
+            return value
+    return None
+
+
+def first_list(source: dict[str, object], *keys: str) -> list[str]:
+    for key in keys:
+        values = string_list(source.get(key, []))
+        if values:
+            return values
+    return []

core/tool_framework/utils/mcp_tool_listing.py

@@ -0,0 +1,128 @@
+"""Bounded, model-safe rendering of MCP server tool catalogs.
+
+MCP servers (PostHog, Sentry, OpenClaw, ...) can expose dozens to hundreds of
+tools, each carrying a full JSON input schema. A discovery tool that returns
+every tool *with* its schema produces a payload many times larger than any
+model's context window; the agent's context-budget enforcer then trims the
+listing away before the model sees a single tool name, and the model loops
+re-calling the discovery tool and guessing tool names that don't exist.
+
+This module renders a compact, bounded view by default (names + truncated
+descriptions, no schemas) and lets callers narrow with a name filter or pull
+the full schema for a small, specific set of tools. It is shared by every
+``list_*_tools`` MCP discovery tool so they behave identically.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Defaults sized to keep even a full catalog dump well under model context
+# windows (the live PostHog server alone is ~580k estimated tokens unbounded).
+MAX_DESCRIPTION_CHARS = 160
+MAX_TOOLS_RETURNED = 80
+MAX_SCHEMAS_RETURNED = 15
+
+
+def _truncate_description(description: str) -> str:
+    cleaned = " ".join(description.split())
+    if len(cleaned) <= MAX_DESCRIPTION_CHARS:
+        return cleaned
+    return cleaned[: MAX_DESCRIPTION_CHARS - 1].rstrip() + "\u2026"
+
+
+def _filter_tools(
+    tools: list[dict[str, object]],
+    name_filter: str,
+) -> list[dict[str, object]]:
+    """Keep tools whose name or description matches any whitespace/comma term."""
+    terms = [term for term in re.split(r"[,\s]+", name_filter.lower()) if term]
+    if not terms:
+        return tools
+    matched: list[dict[str, object]] = []
+    for descriptor in tools:
+        haystack = f"{descriptor.get('name', '')} {descriptor.get('description', '')}".lower()
+        if any(term in haystack for term in terms):
+            matched.append(descriptor)
+    return matched
+
+
+def _summarize_tool(
+    descriptor: dict[str, object],
+    *,
+    include_schema: bool,
+) -> dict[str, object]:
+    summary: dict[str, object] = {
+        "name": str(descriptor.get("name", "")).strip(),
+        "description": _truncate_description(str(descriptor.get("description", "") or "")),
+    }
+    schema = descriptor.get("input_schema")
+    if include_schema and schema is not None:
+        summary["input_schema"] = schema
+    return summary
+
+
+def build_mcp_tool_listing(
+    tools: list[dict[str, object]],
+    *,
+    name_filter: str | None,
+    include_schema: bool,
+    filter_example: str = "events query sql",
+) -> dict[str, object]:
+    """Render a bounded, model-safe view of discovered MCP tools.
+
+    ``filter_example`` only affects the human-readable ``notes`` hint so each
+    integration can suggest filter terms relevant to its own tool catalog.
+    """
+    total = len(tools)
+    filtered = _filter_tools(tools, name_filter) if name_filter else list(tools)
+    returned = filtered[:MAX_TOOLS_RETURNED]
+    # Only attach full schemas when the result set is small enough that doing so
+    # keeps the payload bounded. Otherwise schemas would reintroduce the very
+    # context blow-up this listing exists to prevent.
+    attach_schema = include_schema and len(returned) <= MAX_SCHEMAS_RETURNED
+
+    summaries = [
+        _summarize_tool(descriptor, include_schema=attach_schema) for descriptor in returned
+    ]
+    notes: list[str] = []
+    if len(filtered) > len(returned):
+        notes.append(
+            f"Showing {len(returned)} of {len(filtered)} matching tools; "
+            "pass name_filter to narrow the list."
+        )
+    elif total > len(returned):
+        notes.append(
+            f"Showing {len(returned)} of {total} tools; pass name_filter "
+            f"(e.g. '{filter_example}') to narrow the list."
+        )
+    if include_schema and not attach_schema:
+        notes.append(
+            "input_schema omitted because too many tools matched; narrow to "
+            f"{MAX_SCHEMAS_RETURNED} or fewer tools with name_filter to include schemas."
+        )
+    if not include_schema:
+        notes.append(
+            "Schemas omitted to save context; call again with include_schema=true and a "
+            "name_filter once you know which tool you need."
+        )
+
+    listing: dict[str, object] = {
+        "total_tools": total,
+        "matched_tools": len(filtered),
+        "returned_tools": len(summaries),
+        "tools": summaries,
+    }
+    if name_filter:
+        listing["name_filter"] = name_filter
+    if notes:
+        listing["notes"] = " ".join(notes)
+    return listing
+
+
+__all__ = [
+    "MAX_DESCRIPTION_CHARS",
+    "MAX_SCHEMAS_RETURNED",
+    "MAX_TOOLS_RETURNED",
+    "build_mcp_tool_listing",
+]

core/tool_framework/utils/sql_wrapper.py

@@ -0,0 +1,52 @@
+"""Shared wrapper helper for repeated SQL-tool flow pattern.
+
+Centralizes the common pattern of:
+  1. Resolving database config (with optional default fallback)
+  2. Calling a vendor-specific query/process function
+  3. Injecting optional warning when database was defaulted
+  4. Returning result dict
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from core.tool_framework.utils.db_warnings import default_db_warning
+
+
+def call_db_tool_with_default_db_warning[T](
+    database: str | None,
+    default_db_name: str,
+    config_resolver: Callable[..., T],
+    resolver_kwargs: dict[str, Any],
+    db_caller: Callable[[T], dict[str, Any]],
+) -> dict[str, Any]:
+    """Wrapper for repeated SQL-tool flow: resolve, call, warn (optional), return.
+
+    Args:
+        database: The database parameter from the tool invocation, may be None.
+        default_db_name: The default database name if `database` is None (e.g., 'master', 'postgres', 'mysql').
+        config_resolver: Function that builds a config object from kwargs (e.g., resolve_azure_sql_config).
+        resolver_kwargs: Keyword arguments to pass to config_resolver.
+        db_caller: Function that takes the config and returns a dict result.
+
+    Returns:
+        The result dict from db_caller, with optional 'default_db_warning' key injected if database was None.
+    """
+    _db_defaulted = database is None
+    if database is None:
+        database = default_db_name
+
+    # Resolve config using the vendor-specific resolver
+    kwargs = {**resolver_kwargs, "database": database}
+    config = config_resolver(**kwargs)
+
+    # Call the vendor-specific query/process function
+    result = db_caller(config)
+
+    # Inject warning if database was defaulted
+    if _db_defaulted:
+        result["default_db_warning"] = default_db_warning(default_db_name)
+
+    return result