chore: import upstream snapshot with attribution
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:10:45 +08:00
commit 4b6817381b
3933 changed files with 525247 additions and 0 deletions
View File
+145
View File
@@ -0,0 +1,145 @@
"""Abstract base class for all investigation tool actions."""
from __future__ import annotations
from abc import ABC
from collections.abc import Sequence
from typing import Any, ClassVar
from pydantic import BaseModel
from config.constants.investigation import DEFAULT_APPROVAL_EXPIRY_SECONDS
from core.domain.types.evidence import EvidenceSource
from core.domain.types.retrieval import RetrievalControls
from core.domain.types.tools import ToolSurface
from core.tool_framework.metadata import EvidenceType, SideEffectLevel, ToolMetadata
from core.tool_framework.registry_metadata import BaseToolRegistryMetadata
class BaseTool(ABC):
"""Abstract base class for every investigation tool.
Subclass contract
-----------------
* Declare all metadata as **ClassVars** (``name``, ``description``,
``input_schema``, ``source``, etc.). ``__init_subclass__`` validates
them through ``ToolMetadata`` on class creation, so missing or
ill-typed declarations fail at import time rather than at runtime.
* Implement ``run(**kwargs)`` — *not* declared here to avoid forcing a
fixed signature on every subclass. The planner invokes the tool
through ``__call__``, which delegates to ``run`` via
``telemetry.invoke_tool`` so exceptions are always captured and
converted to a structured ``{"error": ..., "exception_type": ...}``
dict rather than propagating to the agent loop.
* Override ``is_available`` and ``extract_params`` when the tool
requires specific data-source checks or needs to pull kwargs from the
investigation sources dict.
* Do **not** declare ``run`` with positional arguments — the call site
always uses keyword arguments: ``tool_instance.run(**kwargs)``.
"""
name: ClassVar[str]
description: ClassVar[str]
display_name: ClassVar[str | None] = None
input_schema: ClassVar[dict[str, Any]] # JSON Schema — consumed by LLM planner
input_model: ClassVar[type[BaseModel] | None] = None
source: ClassVar[EvidenceSource]
source_id: ClassVar[str | None] = None
evidence_type: ClassVar[EvidenceType | None] = None
side_effect_level: ClassVar[SideEffectLevel | None] = None
use_cases: ClassVar[Sequence[str]] = ()
examples: ClassVar[Sequence[str]] = ()
anti_examples: ClassVar[Sequence[str]] = ()
requires: ClassVar[Sequence[str]] = ()
outputs: ClassVar[dict[str, str]] = {} # Output field -> description (optional, for prompting)
output_schema: ClassVar[dict[str, Any] | None] = None
output_model: ClassVar[type[BaseModel] | None] = None
injected_params: ClassVar[Sequence[str]] = ()
retrieval_controls: ClassVar[RetrievalControls] = (
RetrievalControls()
) # Declares supported controls
surfaces: ClassVar[tuple[ToolSurface, ...]] = ("investigation",)
tags: ClassVar[Sequence[str]] = ()
parallel_safe: ClassVar[bool] = True
requires_approval: ClassVar[bool] = False # Whether this tool needs approval from messaging
approval_reason: ClassVar[str] = "" # Human-readable reason for requiring approval
approval_expiry_seconds: ClassVar[int] = DEFAULT_APPROVAL_EXPIRY_SECONDS
accepts_runtime_context: ClassVar[bool] = False
def __init_subclass__(cls, **kwargs: Any) -> None:
super().__init_subclass__(**kwargs)
metadata = cls.metadata()
cls.name = metadata.name
cls.description = metadata.description
cls.display_name = metadata.display_name
cls.input_schema = metadata.input_schema
cls.source = metadata.source
cls.source_id = metadata.source_id
cls.evidence_type = metadata.evidence_type
cls.side_effect_level = metadata.side_effect_level
cls.use_cases = tuple(metadata.use_cases)
cls.examples = tuple(metadata.examples)
cls.anti_examples = tuple(metadata.anti_examples)
cls.requires = tuple(metadata.requires)
cls.outputs = metadata.outputs
cls.output_schema = metadata.output_schema
cls.injected_params = tuple(metadata.injected_params)
cls.retrieval_controls = metadata.retrieval_controls
registry = cls.registry_metadata()
cls.surfaces = registry.surfaces
cls.tags = registry.tags
cls.parallel_safe = registry.parallel_safe
@classmethod
def metadata(cls) -> ToolMetadata:
"""Return validated tool metadata for this subclass."""
return ToolMetadata.model_validate(
{
"name": getattr(cls, "name", ""),
"description": getattr(cls, "description", ""),
"display_name": getattr(cls, "display_name", None),
"input_schema": getattr(cls, "input_schema", {}),
"source_id": getattr(cls, "source_id", None),
"source": getattr(cls, "source", ""),
"evidence_type": getattr(cls, "evidence_type", None),
"side_effect_level": getattr(cls, "side_effect_level", None),
"use_cases": list(getattr(cls, "use_cases", [])),
"examples": list(getattr(cls, "examples", [])),
"anti_examples": list(getattr(cls, "anti_examples", [])),
"requires": list(getattr(cls, "requires", [])),
"outputs": dict(getattr(cls, "outputs", {})),
"output_schema": getattr(cls, "output_schema", None),
"injected_params": list(getattr(cls, "injected_params", [])),
"retrieval_controls": getattr(cls, "retrieval_controls", RetrievalControls()),
}
)
@classmethod
def registry_metadata(cls) -> BaseToolRegistryMetadata:
"""Return validated registry/runtime metadata for this subclass."""
return BaseToolRegistryMetadata.model_validate(
{
"surfaces": getattr(cls, "surfaces", ("investigation",)),
"tags": tuple(getattr(cls, "tags", ())),
"parallel_safe": getattr(cls, "parallel_safe", True),
}
)
def __call__(self, **kwargs: Any) -> dict[str, Any]:
from core.tool_framework.telemetry import invoke_tool
return invoke_tool(self.run, name=self.name, source=str(self.source), kwargs=kwargs) # type: ignore[attr-defined, no-any-return]
def is_available(self, _sources: dict[str, dict]) -> bool:
"""Return True when required data sources are present.
Override per tool. Default allows the tool to always run.
"""
return True
def extract_params(self, _sources: dict[str, dict]) -> dict[str, Any]:
"""Extract the kwargs to pass to ``run()`` from the available sources.
Override per tool. Default returns an empty dict.
"""
return {}
+68
View File
@@ -0,0 +1,68 @@
"""Validated metadata schema for registered tools.
``ToolMetadata`` is the Pydantic model that enforces the tool contract at
class-definition time (via ``BaseTool.__init_subclass__``) and at
registration time (via ``RegisteredTool.__post_init__``). Keeping it here,
separate from the abstract base class, means the validation schema can
evolve independently of the dispatch protocol in ``BaseTool``.
"""
from __future__ import annotations
from typing import Any, Literal
from pydantic import Field, field_validator
from config.strict_config import StrictConfigModel
from core.domain.types.evidence import EvidenceSource
from core.domain.types.retrieval import RetrievalControls
EvidenceType = Literal[
"logs",
"metrics",
"traces",
"events",
"topology",
"deployment_metadata",
"query_stats",
"artifact",
"other",
]
SideEffectLevel = Literal["none", "read_only", "mutating", "external"]
class ToolMetadata(StrictConfigModel):
"""Strict schema for tool metadata declared on BaseTool subclasses."""
name: str
description: str
display_name: str | None = None
input_schema: dict[str, Any]
source: EvidenceSource
source_id: str | None = None
evidence_type: EvidenceType | None = None
side_effect_level: SideEffectLevel | None = None
use_cases: list[str] = Field(default_factory=list)
examples: list[str] = Field(default_factory=list)
anti_examples: list[str] = Field(default_factory=list)
requires: list[str] = Field(default_factory=list)
outputs: dict[str, str] = Field(default_factory=dict)
output_schema: dict[str, Any] | None = None
injected_params: list[str] = Field(default_factory=list)
retrieval_controls: RetrievalControls = Field(
default_factory=RetrievalControls,
description="Declares which structured retrieval controls this tool supports",
)
@field_validator("name", "description", "display_name")
@classmethod
def _require_non_empty_strings(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = value.strip()
if not normalized:
raise ValueError("must be a non-empty string")
return normalized
__all__ = ["EvidenceType", "SideEffectLevel", "ToolMetadata"]
+344
View File
@@ -0,0 +1,344 @@
"""Shared runtime tool definition for class-based and function-based tools."""
from __future__ import annotations
import inspect
from collections.abc import Callable, Iterable
from copy import deepcopy
from dataclasses import dataclass, field
from typing import Any, cast
from pydantic import BaseModel
from config.constants.investigation import DEFAULT_APPROVAL_EXPIRY_SECONDS
from core.domain.types.evidence import EvidenceSource
from core.domain.types.retrieval import RetrievalControls
from core.domain.types.tools import ToolSurface
from core.tool_framework.base import BaseTool
from core.tool_framework.metadata import EvidenceType, SideEffectLevel, ToolMetadata
from core.tool_framework.registry_metadata import normalize_surfaces
from core.tool_framework.schema import (
_value_matches_schema,
infer_input_schema,
model_to_json_schema,
)
REGISTERED_TOOL_ATTR = "__opensre_registered_tool__"
_DEFAULT_SURFACES: tuple[ToolSurface, ...] = ("investigation",)
def _always_available(_sources: dict[str, dict]) -> bool:
return True
def _extract_no_params(_sources: dict[str, dict]) -> dict[str, Any]:
return {}
def _normalize_surfaces(surfaces: Iterable[str] | None) -> tuple[ToolSurface, ...]:
"""Backward-compatible alias for registry surface normalization."""
return normalize_surfaces(surfaces)
@dataclass
class RegisteredTool:
"""Uniform runtime representation shared by all registered tools."""
name: str
description: str
input_schema: dict[str, Any]
source: EvidenceSource
run: Callable[..., Any] = field(repr=False)
display_name: str | None = None
source_id: str | None = None
evidence_type: EvidenceType | None = None
side_effect_level: SideEffectLevel | None = None
surfaces: tuple[ToolSurface, ...] = _DEFAULT_SURFACES
use_cases: list[str] = field(default_factory=list)
examples: list[str] = field(default_factory=list)
anti_examples: list[str] = field(default_factory=list)
requires: list[str] = field(default_factory=list)
outputs: dict[str, str] = field(default_factory=dict)
output_schema: dict[str, Any] | None = None
injected_params: tuple[str, ...] = ()
retrieval_controls: RetrievalControls = field(
default_factory=RetrievalControls,
)
is_available: Callable[[dict[str, dict]], bool] = field(
default=_always_available,
repr=False,
)
extract_params: Callable[[dict[str, dict]], dict[str, Any]] = field(
default=_extract_no_params,
repr=False,
)
tags: tuple[str, ...] = ()
requires_approval: bool = False
approval_reason: str = ""
approval_expiry_seconds: int = DEFAULT_APPROVAL_EXPIRY_SECONDS
parallel_safe: bool = True
accepts_runtime_context: bool = False
origin_module: str = ""
origin_name: str = ""
skill_guidance: str = ""
def __post_init__(self) -> None:
metadata = ToolMetadata.model_validate(
{
"name": self.name,
"description": self.description,
"display_name": self.display_name,
"input_schema": self.input_schema,
"source": self.source,
"source_id": self.source_id,
"evidence_type": self.evidence_type,
"side_effect_level": self.side_effect_level,
"use_cases": self.use_cases,
"examples": self.examples,
"anti_examples": self.anti_examples,
"requires": self.requires,
"outputs": self.outputs,
"output_schema": self.output_schema,
"injected_params": list(self.injected_params),
"retrieval_controls": self.retrieval_controls,
}
)
self.name = metadata.name
self.description = metadata.description
self.display_name = metadata.display_name
self.input_schema = metadata.input_schema
self.source = metadata.source
self.source_id = metadata.source_id
self.evidence_type = metadata.evidence_type
self.side_effect_level = metadata.side_effect_level
self.use_cases = metadata.use_cases
self.examples = metadata.examples
self.anti_examples = metadata.anti_examples
self.requires = metadata.requires
self.outputs = metadata.outputs
self.output_schema = metadata.output_schema
self.injected_params = tuple(metadata.injected_params)
self.retrieval_controls = metadata.retrieval_controls
self.surfaces = _normalize_surfaces(self.surfaces)
if not callable(self.run):
raise TypeError("run must be callable")
if not callable(self.is_available):
raise TypeError("is_available must be callable")
if not callable(self.extract_params):
raise TypeError("extract_params must be callable")
@property
def inputs(self) -> dict[str, str]:
props = self.input_schema.get("properties", {})
return {
param: str(info.get("description", info.get("type", "")))
for param, info in props.items()
}
@property
def public_input_schema(self) -> dict[str, Any]:
"""Return a schema exposed to the model (without injected params)."""
schema = deepcopy(self.input_schema)
properties = schema.get("properties")
if not isinstance(properties, dict):
return schema
for injected in self.injected_params:
properties.pop(injected, None)
required = schema.get("required")
if isinstance(required, list):
schema["required"] = [name for name in required if name not in self.injected_params]
return schema
def validate_public_input(self, payload: dict[str, Any]) -> str | None:
"""Validate model-provided input against this tool's public schema."""
schema = self.public_input_schema
if schema.get("type") != "object":
return f"{self.name} exposes a non-object input schema."
if not isinstance(payload, dict):
return f"{self.name} expected object input."
properties = schema.get("properties")
if not isinstance(properties, dict):
properties = {}
required = schema.get("required")
if not isinstance(required, list):
required = []
missing = [name for name in required if name not in payload]
if missing:
return f"{self.name} missing required args: {', '.join(sorted(missing))}."
if schema.get("additionalProperties") is False:
extra = sorted(name for name in payload if name not in properties)
if extra:
return f"{self.name} got unexpected args: {', '.join(extra)}."
for key, value in payload.items():
prop_schema = properties.get(key)
if not isinstance(prop_schema, dict):
continue
if not _value_matches_schema(value, prop_schema):
return f"{self.name}.{key} has invalid type/value."
return None
def __call__(self, **kwargs: Any) -> Any:
from core.tool_framework.telemetry import invoke_tool
return invoke_tool(self.run, name=self.name, source=str(self.source), kwargs=kwargs)
@classmethod
def from_base_tool(
cls,
tool: BaseTool,
*,
surfaces: Iterable[str] | None = None,
retrieval_controls: RetrievalControls | None = None,
tags: tuple[str, ...] | None = None,
requires_approval: bool | None = None,
approval_reason: str | None = None,
approval_expiry_seconds: int | None = None,
parallel_safe: bool | None = None,
accepts_runtime_context: bool | None = None,
) -> RegisteredTool:
metadata = tool.metadata()
input_model = cast(type[BaseModel] | None, getattr(tool, "input_model", None))
output_model = cast(type[BaseModel] | None, getattr(tool, "output_model", None))
resolved_input_schema = (
model_to_json_schema(input_model) if input_model else metadata.input_schema
)
resolved_output_schema = (
model_to_json_schema(output_model) if output_model else metadata.output_schema
)
registry = tool.registry_metadata()
resolved_surfaces = (
normalize_surfaces(surfaces) if surfaces is not None else registry.surfaces
)
resolved_tags = tags if tags is not None else registry.tags
return cls(
name=metadata.name,
description=metadata.description,
display_name=metadata.display_name,
input_schema=resolved_input_schema,
source=metadata.source,
source_id=metadata.source_id,
evidence_type=metadata.evidence_type,
side_effect_level=metadata.side_effect_level,
use_cases=metadata.use_cases,
examples=metadata.examples,
anti_examples=metadata.anti_examples,
requires=metadata.requires,
outputs=metadata.outputs,
output_schema=resolved_output_schema,
injected_params=tuple(metadata.injected_params),
retrieval_controls=retrieval_controls or metadata.retrieval_controls,
surfaces=resolved_surfaces,
run=tool.run, # type: ignore[attr-defined]
is_available=tool.is_available,
extract_params=tool.extract_params,
tags=resolved_tags,
requires_approval=bool(
requires_approval
if requires_approval is not None
else tool.__class__.requires_approval
),
approval_reason=str(
approval_reason if approval_reason is not None else tool.__class__.approval_reason
),
approval_expiry_seconds=int(
approval_expiry_seconds
if approval_expiry_seconds is not None
else tool.__class__.approval_expiry_seconds
),
parallel_safe=bool(
parallel_safe if parallel_safe is not None else tool.__class__.parallel_safe
),
accepts_runtime_context=bool(
accepts_runtime_context
if accepts_runtime_context is not None
else tool.__class__.accepts_runtime_context
),
origin_module=tool.__class__.__module__,
origin_name=tool.__class__.__name__,
)
@classmethod
def from_function(
cls,
func: Callable[..., Any],
*,
name: str | None = None,
description: str | None = None,
display_name: str | None = None,
input_schema: dict[str, Any] | None = None,
input_model: type[BaseModel] | None = None,
source: EvidenceSource | None,
source_id: str | None = None,
evidence_type: EvidenceType | None = None,
side_effect_level: SideEffectLevel | None = None,
surfaces: Iterable[str] | None = None,
use_cases: list[str] | None = None,
examples: list[str] | None = None,
anti_examples: list[str] | None = None,
requires: list[str] | None = None,
outputs: dict[str, str] | None = None,
output_schema: dict[str, Any] | None = None,
output_model: type[BaseModel] | None = None,
injected_params: tuple[str, ...] | None = None,
retrieval_controls: RetrievalControls | None = None,
is_available: Callable[[dict[str, dict]], bool] | None = None,
extract_params: Callable[[dict[str, dict]], dict[str, Any]] | None = None,
tags: tuple[str, ...] | None = None,
requires_approval: bool | None = None,
approval_reason: str | None = None,
approval_expiry_seconds: int | None = None,
parallel_safe: bool | None = None,
accepts_runtime_context: bool | None = None,
) -> RegisteredTool:
if source is None:
raise ValueError("Function tools must declare a source.")
resolved_input_schema = (
input_schema
or (model_to_json_schema(input_model) if input_model is not None else None)
or infer_input_schema(func)
)
resolved_output_schema = output_schema or (
model_to_json_schema(output_model) if output_model is not None else None
)
inferred_description = inspect.getdoc(func) or func.__name__.replace("_", " ")
return cls(
name=name or func.__name__,
description=description or inferred_description,
display_name=display_name,
input_schema=resolved_input_schema,
source=source,
source_id=source_id,
evidence_type=evidence_type,
side_effect_level=side_effect_level,
surfaces=_normalize_surfaces(surfaces),
use_cases=list(use_cases or []),
examples=list(examples or []),
anti_examples=list(anti_examples or []),
requires=list(requires or []),
outputs=dict(outputs or {}),
output_schema=resolved_output_schema,
injected_params=tuple(injected_params or ()),
retrieval_controls=retrieval_controls or RetrievalControls(),
run=func,
is_available=is_available or _always_available,
extract_params=extract_params or _extract_no_params,
tags=tags or (),
requires_approval=bool(requires_approval),
approval_reason=approval_reason or "",
approval_expiry_seconds=(
approval_expiry_seconds
if approval_expiry_seconds is not None
else DEFAULT_APPROVAL_EXPIRY_SECONDS
),
parallel_safe=True if parallel_safe is None else bool(parallel_safe),
accepts_runtime_context=bool(accepts_runtime_context),
origin_module=func.__module__,
origin_name=func.__name__,
)
+83
View File
@@ -0,0 +1,83 @@
"""Validated registry/runtime metadata for ``BaseTool`` subclasses.
``BaseToolRegistryMetadata`` covers fields the tool registry uses to decide
where a tool is exposed and how it executes, separate from the planner/evidence
contract in ``ToolMetadata``.
"""
from __future__ import annotations
from collections.abc import Iterable
from typing import cast, get_args
from pydantic import Field, field_validator
from config.strict_config import StrictConfigModel
from core.domain.types.tools import ToolSurface
_DEFAULT_SURFACES: tuple[ToolSurface, ...] = ("investigation",)
_VALID_SURFACES = set(get_args(ToolSurface))
def normalize_surfaces(surfaces: Iterable[str] | None) -> tuple[ToolSurface, ...]:
"""Normalize and validate tool surface names."""
if surfaces is None:
return _DEFAULT_SURFACES
normalized: list[ToolSurface] = []
for raw_surface in surfaces:
surface = str(raw_surface).strip().lower()
if surface not in _VALID_SURFACES:
valid = ", ".join(sorted(_VALID_SURFACES))
raise ValueError(f"Unsupported tool surface '{surface}'. Expected one of: {valid}.")
typed_surface = cast(ToolSurface, surface)
if typed_surface not in normalized:
normalized.append(typed_surface)
return tuple(normalized) or _DEFAULT_SURFACES
def normalize_tags(tags: Iterable[str] | None) -> tuple[str, ...]:
"""Normalize optional planner tags into a deduplicated tuple."""
if tags is None:
return ()
normalized: list[str] = []
for raw_tag in tags:
tag = str(raw_tag).strip()
if tag and tag not in normalized:
normalized.append(tag)
return tuple(normalized)
class BaseToolRegistryMetadata(StrictConfigModel):
"""Registry/runtime metadata declared on ``BaseTool`` subclasses."""
surfaces: tuple[ToolSurface, ...] = Field(default=_DEFAULT_SURFACES)
tags: tuple[str, ...] = ()
parallel_safe: bool = True
@field_validator("surfaces", mode="before")
@classmethod
def _coerce_surfaces(cls, value: object) -> tuple[ToolSurface, ...]:
if value is None:
return normalize_surfaces(None)
if isinstance(value, str):
return normalize_surfaces((value,))
return normalize_surfaces(cast(Iterable[str], value))
@field_validator("tags", mode="before")
@classmethod
def _coerce_tags(cls, value: object) -> tuple[str, ...]:
if value is None:
return ()
if isinstance(value, str):
return normalize_tags((value,))
return normalize_tags(cast(Iterable[str], value))
__all__ = [
"BaseToolRegistryMetadata",
"normalize_surfaces",
"normalize_tags",
]
+159
View File
@@ -0,0 +1,159 @@
"""JSON schema inference and value-validation helpers for registered tools.
Inference: build a JSON Schema object from a Python function signature
(``infer_input_schema``) or a Pydantic model (``model_to_json_schema``).
Validation: check that a runtime value satisfies a JSON Schema fragment
(``_value_matches_schema``). Used by ``RegisteredTool.validate_public_input``
to reject bad planner payloads before they reach ``run()``.
"""
from __future__ import annotations
import inspect
from collections.abc import Callable
from types import NoneType, UnionType
from typing import Any, Union, get_args, get_origin, get_type_hints
from pydantic import BaseModel
__all__ = [
"infer_input_schema",
"model_to_json_schema",
]
def _strip_optional(annotation: Any) -> tuple[Any, bool]:
origin = get_origin(annotation)
if origin is None:
return annotation, False
args = tuple(arg for arg in get_args(annotation) if arg is not NoneType)
if len(args) != len(get_args(annotation)):
if len(args) == 1:
return args[0], True
return args, True
return annotation, False
def _annotation_to_json_schema(annotation: Any) -> dict[str, Any]:
base_annotation, is_optional = _strip_optional(annotation)
origin = get_origin(base_annotation)
if base_annotation in (inspect.Signature.empty, Any):
schema: dict[str, Any] = {}
elif base_annotation is str:
schema = {"type": "string"}
elif base_annotation is int:
schema = {"type": "integer"}
elif base_annotation is float:
schema = {"type": "number"}
elif base_annotation is bool:
schema = {"type": "boolean"}
elif base_annotation is dict or origin is dict:
schema = {"type": "object"}
elif base_annotation is list or origin in (list, set, tuple):
schema = {"type": "array"}
elif origin is Union or isinstance(base_annotation, UnionType): # noqa: UP007
# Non-optional union (e.g. int | str): emit oneOf over each member.
schema = {"oneOf": [_annotation_to_json_schema(a) for a in get_args(base_annotation)]}
elif isinstance(base_annotation, tuple):
# Residual tuple produced by _strip_optional for X | Y | None with >1 non-None arm.
schema = {"oneOf": [_annotation_to_json_schema(a) for a in base_annotation]}
else:
schema = {"type": "string"}
if is_optional:
schema["nullable"] = True
return schema
def infer_input_schema(func: Callable[..., Any]) -> dict[str, Any]:
"""Infer a minimal JSON schema from a function signature."""
properties: dict[str, Any] = {}
required: list[str] = []
type_hints = get_type_hints(func)
for param in inspect.signature(func).parameters.values():
if param.kind in (
inspect.Parameter.POSITIONAL_ONLY,
inspect.Parameter.VAR_POSITIONAL,
inspect.Parameter.VAR_KEYWORD,
):
continue
if param.name.startswith("_"):
continue
resolved_annotation = type_hints.get(param.name, param.annotation)
schema = _annotation_to_json_schema(resolved_annotation)
properties[param.name] = schema
_, is_optional = _strip_optional(resolved_annotation)
if param.default is inspect.Signature.empty and not is_optional:
required.append(param.name)
return {
"type": "object",
"properties": properties,
"required": required,
}
def model_to_json_schema(model: type[BaseModel]) -> dict[str, Any]:
"""Convert a Pydantic model to a JSON object schema for tools."""
schema = model.model_json_schema()
if not isinstance(schema, dict):
return {"type": "object", "properties": {}, "required": [], "additionalProperties": False}
schema.setdefault("type", "object")
if schema.get("type") == "object":
schema.setdefault("properties", {})
schema.setdefault("required", [])
schema.setdefault("additionalProperties", False)
return schema
def _json_type_matches(value: Any, schema_type: str) -> bool:
if schema_type == "string":
return isinstance(value, str)
if schema_type == "integer":
return isinstance(value, int) and not isinstance(value, bool)
if schema_type == "number":
return isinstance(value, (int, float)) and not isinstance(value, bool)
if schema_type == "boolean":
return isinstance(value, bool)
if schema_type == "array":
return isinstance(value, list)
if schema_type == "object":
return isinstance(value, dict)
return True
def _value_matches_schema(value: Any, schema: dict[str, Any]) -> bool:
if value is None and bool(schema.get("nullable")):
return True
if "enum" in schema and value not in schema.get("enum", []):
return False
one_of = schema.get("oneOf")
if isinstance(one_of, list) and one_of:
return any(
isinstance(option, dict) and _value_matches_schema(value, option) for option in one_of
)
any_of = schema.get("anyOf")
if isinstance(any_of, list) and any_of:
return any(
isinstance(option, dict) and _value_matches_schema(value, option) for option in any_of
)
schema_type = schema.get("type")
if isinstance(schema_type, str):
return _json_type_matches(value, schema_type)
if isinstance(schema_type, list):
return any(
isinstance(item, str) and _json_type_matches(value, item) for item in schema_type
)
return True
+264
View File
@@ -0,0 +1,264 @@
"""Declarative model guidance that can be attached to registered tools."""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Literal
import yaml
MAX_SKILL_NAME_LENGTH = 64
MAX_SKILL_DESCRIPTION_LENGTH = 1024
_SKILL_NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
SkillDiagnosticCode = Literal[
"read_failed",
"parse_failed",
"invalid_metadata",
"unknown_tool",
]
@dataclass(frozen=True)
class SkillDiagnostic:
"""Warning produced while loading declarative tool guidance."""
type: Literal["warning"]
code: SkillDiagnosticCode
message: str
path: str
@dataclass(frozen=True)
class SkillGuidance:
"""Markdown guidance that applies to a known set of registered tools."""
name: str
description: str
content: str
file_path: str
tool_names: tuple[str, ...]
disable_model_invocation: bool = False
@dataclass(frozen=True)
class SkillGuidanceLoadResult:
"""Result of loading one explicit SKILL.md file."""
skill: SkillGuidance | None
diagnostics: list[SkillDiagnostic] = field(default_factory=list)
def load_tool_skill_guidance(
file_path: str | Path,
*,
known_tool_names: set[str] | frozenset[str] | None = None,
) -> SkillGuidanceLoadResult:
"""Load one explicit ``SKILL.md`` file.
Missing files are skipped without diagnostics so optional skill guidance can be
introduced per tool family without becoming a registry hard dependency.
"""
path = Path(file_path)
if not path.exists():
return SkillGuidanceLoadResult(skill=None)
if not path.is_file():
return SkillGuidanceLoadResult(
skill=None,
diagnostics=[
_diagnostic("invalid_metadata", "skill path is not a file", path),
],
)
try:
raw = path.read_text(encoding="utf-8")
except OSError as exc:
return SkillGuidanceLoadResult(
skill=None,
diagnostics=[_diagnostic("read_failed", str(exc), path)],
)
parsed = _parse_frontmatter(raw, path)
if parsed.diagnostics:
return SkillGuidanceLoadResult(skill=None, diagnostics=parsed.diagnostics)
frontmatter = parsed.frontmatter
diagnostics = _validate_frontmatter(path, frontmatter, known_tool_names)
name = _string_value(frontmatter.get("name"))
description = _string_value(frontmatter.get("description"))
tool_names = _tool_names(frontmatter.get("tools"))
if not name or not description or not tool_names:
return SkillGuidanceLoadResult(skill=None, diagnostics=diagnostics)
return SkillGuidanceLoadResult(
skill=SkillGuidance(
name=name,
description=description,
content=parsed.body,
file_path=str(path),
tool_names=tool_names,
disable_model_invocation=frontmatter.get("disable-model-invocation") is True,
),
diagnostics=diagnostics,
)
def _xml_attr(value: str) -> str:
"""Escape a string for safe use inside an XML double-quoted attribute value."""
return (
value.replace("&", "&amp;").replace('"', "&quot;").replace("<", "&lt;").replace(">", "&gt;")
)
def format_tool_skill_guidance(skill: SkillGuidance) -> str:
"""Format skill guidance for inclusion in model-facing tool descriptions."""
skill_dir = str(Path(skill.file_path).parent)
return (
f'<skill name="{_xml_attr(skill.name)}" description="{_xml_attr(skill.description)}" '
f'location="{_xml_attr(skill.file_path)}">\n'
f"Use this skill when the request matches the description above.\n"
f"References are relative to {skill_dir}.\n\n"
f"{skill.content.strip()}\n"
"</skill>"
)
@dataclass(frozen=True)
class _ParsedFrontmatter:
frontmatter: dict[str, Any]
body: str
diagnostics: list[SkillDiagnostic] = field(default_factory=list)
def _parse_frontmatter(content: str, path: Path) -> _ParsedFrontmatter:
normalized = content.replace("\r\n", "\n").replace("\r", "\n")
if not normalized.startswith("---"):
return _ParsedFrontmatter(frontmatter={}, body=normalized.strip())
end_index = normalized.find("\n---", 3)
if end_index == -1:
return _ParsedFrontmatter(frontmatter={}, body=normalized.strip())
yaml_content = normalized[4:end_index]
body = normalized[end_index + 4 :].strip()
try:
loaded = yaml.safe_load(yaml_content) or {}
except yaml.YAMLError as exc:
return _ParsedFrontmatter(
frontmatter={},
body="",
diagnostics=[_diagnostic("parse_failed", str(exc), path)],
)
if not isinstance(loaded, dict):
return _ParsedFrontmatter(
frontmatter={},
body="",
diagnostics=[
_diagnostic("invalid_metadata", "frontmatter must be a mapping", path),
],
)
return _ParsedFrontmatter(frontmatter=loaded, body=body)
def _validate_frontmatter(
path: Path,
frontmatter: dict[str, Any],
known_tool_names: set[str] | frozenset[str] | None,
) -> list[SkillDiagnostic]:
diagnostics: list[SkillDiagnostic] = []
name = _string_value(frontmatter.get("name"))
description = _string_value(frontmatter.get("description"))
tool_names = _tool_names(frontmatter.get("tools"))
if not name:
diagnostics.append(_diagnostic("invalid_metadata", "name is required", path))
else:
if len(name) > MAX_SKILL_NAME_LENGTH:
diagnostics.append(
_diagnostic(
"invalid_metadata",
f"name exceeds {MAX_SKILL_NAME_LENGTH} characters ({len(name)})",
path,
)
)
if not _SKILL_NAME_RE.match(name):
diagnostics.append(
_diagnostic(
"invalid_metadata",
"name must be lowercase kebab-case using a-z, 0-9, and hyphens",
path,
)
)
if not description:
diagnostics.append(_diagnostic("invalid_metadata", "description is required", path))
elif len(description) > MAX_SKILL_DESCRIPTION_LENGTH:
diagnostics.append(
_diagnostic(
"invalid_metadata",
(
f"description exceeds {MAX_SKILL_DESCRIPTION_LENGTH} "
f"characters ({len(description)})"
),
path,
)
)
if not tool_names:
diagnostics.append(
_diagnostic("invalid_metadata", "tools must be a non-empty list of names", path)
)
elif known_tool_names is not None:
for tool_name in tool_names:
if tool_name not in known_tool_names:
diagnostics.append(
_diagnostic(
"unknown_tool",
f"tool {tool_name!r} is not registered",
path,
)
)
return diagnostics
def _tool_names(value: Any) -> tuple[str, ...]:
if not isinstance(value, list):
return ()
names: list[str] = []
for item in value:
if not isinstance(item, str):
return ()
name = item.strip()
if not name:
return ()
if name not in names:
names.append(name)
return tuple(names)
def _string_value(value: Any) -> str:
return value.strip() if isinstance(value, str) else ""
def _diagnostic(code: SkillDiagnosticCode, message: str, path: Path) -> SkillDiagnostic:
return SkillDiagnostic(type="warning", code=code, message=message, path=str(path))
__all__ = [
"MAX_SKILL_DESCRIPTION_LENGTH",
"MAX_SKILL_NAME_LENGTH",
"SkillDiagnostic",
"SkillDiagnosticCode",
"SkillGuidance",
"SkillGuidanceLoadResult",
"format_tool_skill_guidance",
"load_tool_skill_guidance",
]
+88
View File
@@ -0,0 +1,88 @@
"""Shared error-reporting helpers for tool call sites.
``report_run_error`` is for tools that deliberately swallow exceptions and
return a degraded ``{"available": False, ...}`` dict. It turns a silent
swallow into a structured log entry plus Sentry event.
``invoke_tool`` is the unified dispatch wrapper used by ``BaseTool.__call__``
and ``RegisteredTool.__call__``. It owns the single try/except + error-capture
contract so both call paths behave identically.
"""
from __future__ import annotations
import logging
from collections.abc import Callable
from typing import Any, Literal
from platform.observability.errors.boundary import report_exception
ToolErrorSeverity = Literal["error", "warning"]
_DEFAULT_LOGGER = logging.getLogger("tools")
def report_run_error(
exc: BaseException,
*,
tool_name: str,
source: str,
component: str,
method: str | None = None,
severity: ToolErrorSeverity = "error",
logger: logging.Logger | None = None,
extras: dict[str, Any] | None = None,
) -> None:
"""Log + Sentry-capture an error swallowed by a tool wrapper.
``tool_name`` and ``source`` come from the tool's metadata (the
``name=``/``source=`` arguments of ``@tool`` or the corresponding
``BaseTool`` ClassVars). ``component`` should identify the call site —
typically ``"<module>.<function_or_class>"`` — so Sentry groups events
per tool implementation, not per top-level surface tag.
"""
tags: dict[str, str] = {
"surface": "tool",
"tool_name": tool_name,
"source": source,
"component": component,
}
if method:
tags["method"] = method
report_exception(
exc,
logger=logger or _DEFAULT_LOGGER,
message=f"Tool {tool_name} failed: {type(exc).__name__}: {exc}",
severity=severity,
tags=tags,
extras=extras,
)
def invoke_tool(
run_fn: Callable[..., Any],
*,
name: str,
source: str,
kwargs: dict[str, Any],
) -> Any:
"""Call ``run_fn(**kwargs)`` and capture any exception via ``report_exception``.
Returns the run result on success, or
``{"error": ..., "exception_type": ...}`` on failure — the shape both
``BaseTool.__call__`` and ``RegisteredTool.__call__`` have always returned.
"""
try:
return run_fn(**kwargs)
except Exception as exc:
report_exception(
exc,
logger=_DEFAULT_LOGGER,
message=f"Tool {name} failed: {type(exc).__name__}: {exc}",
severity="error",
tags={"surface": "tool", "tool_name": name, "source": source},
)
return {"error": str(exc), "exception_type": type(exc).__name__}
__all__ = ["ToolErrorSeverity", "invoke_tool", "report_run_error"]
+265
View File
@@ -0,0 +1,265 @@
"""Tool decorator and compatibility helper for lightweight tool registration."""
from __future__ import annotations
from collections.abc import Callable
from typing import Any, cast, overload
from pydantic import BaseModel
from core.domain.types.evidence import EvidenceSource
from core.domain.types.retrieval import RetrievalControls
from core.tool_framework.base import BaseTool
from core.tool_framework.metadata import EvidenceType, SideEffectLevel
from core.tool_framework.registered_tool import REGISTERED_TOOL_ATTR, RegisteredTool
@overload
def tool(
func: BaseTool,
*,
name: str | None = None,
description: str | None = None,
display_name: str | None = None,
input_schema: dict[str, Any] | None = None,
input_model: type[BaseModel] | None = None,
source: EvidenceSource | None = None,
source_id: str | None = None,
evidence_type: EvidenceType | None = None,
side_effect_level: SideEffectLevel | None = None,
surfaces: tuple[str, ...] | None = None,
use_cases: list[str] | None = None,
examples: list[str] | None = None,
anti_examples: list[str] | None = None,
requires: list[str] | None = None,
outputs: dict[str, str] | None = None,
output_schema: dict[str, Any] | None = None,
output_model: type[BaseModel] | None = None,
injected_params: tuple[str, ...] | None = None,
retrieval_controls: RetrievalControls | None = None,
is_available: Callable[[dict[str, dict]], bool] | None = None,
extract_params: Callable[[dict[str, dict]], dict[str, Any]] | None = None,
tags: tuple[str, ...] | None = None,
requires_approval: bool | None = None,
approval_reason: str | None = None,
approval_expiry_seconds: int | None = None,
parallel_safe: bool | None = None,
accepts_runtime_context: bool | None = None,
) -> BaseTool:
pass
@overload
def tool[F: Callable[..., Any]](
func: F,
*,
name: str | None = None,
description: str | None = None,
display_name: str | None = None,
input_schema: dict[str, Any] | None = None,
input_model: type[BaseModel] | None = None,
source: EvidenceSource | None = None,
source_id: str | None = None,
evidence_type: EvidenceType | None = None,
side_effect_level: SideEffectLevel | None = None,
surfaces: tuple[str, ...] | None = None,
use_cases: list[str] | None = None,
examples: list[str] | None = None,
anti_examples: list[str] | None = None,
requires: list[str] | None = None,
outputs: dict[str, str] | None = None,
output_schema: dict[str, Any] | None = None,
output_model: type[BaseModel] | None = None,
injected_params: tuple[str, ...] | None = None,
retrieval_controls: RetrievalControls | None = None,
is_available: Callable[[dict[str, dict]], bool] | None = None,
extract_params: Callable[[dict[str, dict]], dict[str, Any]] | None = None,
tags: tuple[str, ...] | None = None,
requires_approval: bool | None = None,
approval_reason: str | None = None,
approval_expiry_seconds: int | None = None,
parallel_safe: bool | None = None,
accepts_runtime_context: bool | None = None,
) -> F:
pass
@overload
def tool[F: Callable[..., Any]](
func: None = None,
*,
name: str | None = None,
description: str | None = None,
display_name: str | None = None,
input_schema: dict[str, Any] | None = None,
input_model: type[BaseModel] | None = None,
source: EvidenceSource | None = None,
source_id: str | None = None,
evidence_type: EvidenceType | None = None,
side_effect_level: SideEffectLevel | None = None,
surfaces: tuple[str, ...] | None = None,
use_cases: list[str] | None = None,
examples: list[str] | None = None,
anti_examples: list[str] | None = None,
requires: list[str] | None = None,
outputs: dict[str, str] | None = None,
output_schema: dict[str, Any] | None = None,
output_model: type[BaseModel] | None = None,
injected_params: tuple[str, ...] | None = None,
retrieval_controls: RetrievalControls | None = None,
is_available: Callable[[dict[str, dict]], bool] | None = None,
extract_params: Callable[[dict[str, dict]], dict[str, Any]] | None = None,
tags: tuple[str, ...] | None = None,
requires_approval: bool | None = None,
approval_reason: str | None = None,
approval_expiry_seconds: int | None = None,
parallel_safe: bool | None = None,
accepts_runtime_context: bool | None = None,
) -> Callable[[F], F]:
pass
def tool[F: Callable[..., Any]](
func: F | BaseTool | None = None,
*,
name: str | None = None,
description: str | None = None,
display_name: str | None = None,
input_schema: dict[str, Any] | None = None,
input_model: type[BaseModel] | None = None,
source: EvidenceSource | None = None,
source_id: str | None = None,
evidence_type: EvidenceType | None = None,
side_effect_level: SideEffectLevel | None = None,
surfaces: tuple[str, ...] | None = None,
use_cases: list[str] | None = None,
examples: list[str] | None = None,
anti_examples: list[str] | None = None,
requires: list[str] | None = None,
outputs: dict[str, str] | None = None,
output_schema: dict[str, Any] | None = None,
output_model: type[BaseModel] | None = None,
injected_params: tuple[str, ...] | None = None,
retrieval_controls: RetrievalControls | None = None,
is_available: Callable[[dict[str, dict]], bool] | None = None,
extract_params: Callable[[dict[str, dict]], dict[str, Any]] | None = None,
tags: tuple[str, ...] | None = None,
requires_approval: bool | None = None,
approval_reason: str | None = None,
approval_expiry_seconds: int | None = None,
parallel_safe: bool | None = None,
accepts_runtime_context: bool | None = None,
) -> Any:
"""Register a lightweight function tool or annotate an existing BaseTool.
Backward compatibility:
- ``tool(existing_base_tool)`` keeps working as a no-op.
- ``tool(plain_function)`` with no metadata remains a no-op.
"""
def should_register_function() -> bool:
return any(
[
name is not None,
description is not None,
display_name is not None,
input_schema is not None,
input_model is not None,
source is not None,
source_id is not None,
evidence_type is not None,
side_effect_level is not None,
surfaces is not None,
bool(use_cases),
bool(examples),
bool(anti_examples),
bool(requires),
bool(outputs),
output_schema is not None,
output_model is not None,
bool(injected_params),
retrieval_controls is not None,
is_available is not None,
extract_params is not None,
bool(tags),
requires_approval is not None,
approval_reason is not None,
approval_expiry_seconds is not None,
parallel_safe is not None,
accepts_runtime_context is not None,
]
)
def attach(target: F | BaseTool) -> F | BaseTool:
if isinstance(target, BaseTool):
if (
surfaces is not None
or retrieval_controls is not None
or tags is not None
or requires_approval is not None
or approval_reason is not None
or approval_expiry_seconds is not None
or parallel_safe is not None
or accepts_runtime_context is not None
):
setattr(
target,
REGISTERED_TOOL_ATTR,
RegisteredTool.from_base_tool(
target,
surfaces=surfaces,
retrieval_controls=retrieval_controls,
tags=tags,
requires_approval=requires_approval,
approval_reason=approval_reason,
approval_expiry_seconds=approval_expiry_seconds,
parallel_safe=parallel_safe,
accepts_runtime_context=accepts_runtime_context,
),
)
return target
if should_register_function():
setattr(
target,
REGISTERED_TOOL_ATTR,
RegisteredTool.from_function(
target,
name=name,
description=description,
display_name=display_name,
input_schema=input_schema,
input_model=input_model,
source=source,
source_id=source_id,
evidence_type=evidence_type,
side_effect_level=side_effect_level,
surfaces=surfaces,
use_cases=use_cases,
examples=examples,
anti_examples=anti_examples,
requires=requires,
outputs=outputs,
output_schema=output_schema,
output_model=output_model,
injected_params=injected_params,
retrieval_controls=retrieval_controls,
is_available=is_available,
extract_params=extract_params,
tags=tags,
requires_approval=requires_approval,
approval_reason=approval_reason,
approval_expiry_seconds=approval_expiry_seconds,
parallel_safe=parallel_safe,
accepts_runtime_context=accepts_runtime_context,
),
)
return target
if func is None:
def wrapper(inner: F) -> F:
return cast(F, attach(inner))
return wrapper
return attach(func)
+11
View File
@@ -0,0 +1,11 @@
"""Tool utilities — code-host helpers, data validation, and database warnings."""
from core.tool_framework.utils.code_host_unavailable import code_host_unavailable_payload
from core.tool_framework.utils.data_validation import validate_host_metrics
from core.tool_framework.utils.db_warnings import default_db_warning
__all__ = [
"code_host_unavailable_payload",
"default_db_warning",
"validate_host_metrics",
]
@@ -0,0 +1,21 @@
"""Shared unavailable payload helper for code-host investigation tools."""
from __future__ import annotations
from typing import Any
def code_host_unavailable_payload(
*,
source: str,
integration_name: str,
empty_key: str,
empty_value: Any,
) -> dict[str, Any]:
"""Return a standardized unavailable payload for code-host tools."""
return {
"source": source,
"available": False,
"error": f"{integration_name} integration is not configured.",
empty_key: empty_value,
}
@@ -0,0 +1,411 @@
"""Data validators - sanitize API responses before LLM sees them."""
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
@dataclass
class ValidationIssue:
"""Represents a data quality issue found during validation."""
field: str
raw_value: Any
issue_type: str # "impossible_value", "unit_mismatch", "missing_data", etc.
severity: str # "error", "warning", "info"
explanation: str
suggested_fix: str | None = None
class MetricsValidator:
"""Validates and normalizes metrics data from APIs."""
issues: list[ValidationIssue]
def __init__(self) -> None:
self.issues = []
def validate_metrics(self, metrics: dict) -> dict:
"""
Validate and normalize metrics, flagging impossible values.
Handles different API response structures:
- Flat structure: {"cpu": 95, "ram": 8471740416, "disk": 50}
- Nested structure: {"memory": {"percent": 8471740416}, "cpu": {"percent": 95}}
- List structure: {"data": [{"cpu": 95, "ram": 8471740416}]}
Returns:
Normalized metrics dict with added 'data_quality_issues' key
"""
normalized = metrics.copy() if isinstance(metrics, dict) else {}
self.issues = []
# Handle list structure (common in API responses)
if "data" in normalized and isinstance(normalized["data"], list):
validated_data, _ = _validate_data_list(normalized["data"], self._validate_flat_metrics)
normalized["data"] = validated_data
# Also check aggregated values if present
if "max_cpu" in normalized or "max_ram" in normalized:
normalized = self._validate_flat_metrics(normalized)
# Check nested memory metrics
if "memory" in normalized:
normalized["memory"] = self._validate_memory_metric(normalized["memory"])
# Check CPU metrics
if "cpu" in normalized:
normalized["cpu"] = self._validate_cpu_metric(normalized["cpu"])
# Check disk metrics
if "disk" in normalized:
normalized["disk"] = self._validate_disk_metric(normalized["disk"])
# Check for flat structure metrics (cpu, ram, disk at top level)
normalized = self._validate_flat_metrics(normalized)
# Check for percentage fields at top level
for key in ["percent", "percentage", "usage_percent"]:
if key in normalized:
value = normalized[key]
if isinstance(value, int | float) and value > 100:
self._flag_impossible_percentage(key, value, normalized)
# Attach validation issues to the response
if self.issues:
normalized["data_quality_issues"] = [
{
"field": issue.field,
"raw_value": issue.raw_value,
"issue": issue.issue_type,
"severity": issue.severity,
"explanation": issue.explanation,
"suggested_fix": issue.suggested_fix,
}
for issue in self.issues
]
return normalized
def _validate_memory_metric(self, memory_data: dict | Any) -> dict:
"""Validate and fix memory metrics with intelligent unit inference."""
if not isinstance(memory_data, dict):
return {"raw": memory_data, "validated": False}
normalized = memory_data.copy()
# Check for impossible percentage values
if "percent" in memory_data:
raw_percent = memory_data["percent"]
if isinstance(raw_percent, int | float) and raw_percent > 100:
# Infer the most likely unit and interpretation
interpretation = self._infer_memory_unit(raw_percent)
self.issues.append(
ValidationIssue(
field="memory.percent",
raw_value=raw_percent,
issue_type="impossible_percentage",
severity="error",
explanation=interpretation["explanation"],
suggested_fix=interpretation["suggested_fix"],
)
)
# Add interpretation hints to the normalized data
normalized["percent_interpretation"] = interpretation
normalized["percent_invalid"] = True
normalized["percent_raw"] = raw_percent
normalized["percent"] = None # Mark as invalid, LLM should use interpretation
# Also check for "ram" field (common in API responses)
if "ram" in memory_data:
raw_ram = memory_data["ram"]
if isinstance(raw_ram, int | float) and raw_ram > 100:
interpretation = self._infer_memory_unit(raw_ram)
self.issues.append(
ValidationIssue(
field="memory.ram",
raw_value=raw_ram,
issue_type="impossible_percentage",
severity="error",
explanation=interpretation["explanation"],
suggested_fix=interpretation["suggested_fix"],
)
)
normalized["ram_interpretation"] = interpretation
normalized["ram_invalid"] = True
normalized["ram_raw"] = raw_ram
normalized["ram"] = None
return normalized
def _infer_memory_unit(self, value: float) -> dict:
"""
Infer the most likely unit for a memory value that's labeled as percentage.
Returns interpretation with likely unit, explanation, and suggested fix.
"""
# Common memory sizes in bytes
if value >= 1024**3: # >= 1GB
gb_value = value / (1024**3)
interpretation = {
"likely_unit": "bytes",
"likely_value_gb": round(gb_value, 2),
"likely_value_mb": round(value / (1024**2), 2),
"explanation": (
f"Value {value:,.0f} labeled as 'percent' is impossible (>100%). "
f"This is most likely a unit error where bytes are reported as percentage. "
f"Interpretation: {gb_value:.2f} GB ({value / (1024**2):,.0f} MB) of memory used. "
f"To determine actual percentage, divide by total memory: "
f"percent = (used_bytes / total_memory_bytes) * 100"
),
"suggested_fix": (
f"Treat this as {gb_value:.2f} GB of memory used, not a percentage. "
f"Compare against instance type memory limits to determine if memory was exhausted."
),
}
elif value >= 1024**2: # >= 1MB
mb_value = value / (1024**2)
interpretation = {
"likely_unit": "bytes",
"likely_value_mb": round(mb_value, 2),
"likely_value_gb": round(value / (1024**3), 2),
"explanation": (
f"Value {value:,.0f} labeled as 'percent' is impossible (>100%). "
f"This is likely a unit error where bytes are reported as percentage. "
f"Interpretation: {mb_value:.2f} MB ({value / (1024**3):.2f} GB) of memory used."
),
"suggested_fix": (
f"Treat this as {mb_value:.2f} MB of memory used, not a percentage."
),
}
else:
# Value is > 100 but < 1MB - might be a different error
interpretation = {
"likely_unit": "unknown",
"explanation": (
f"Value {value:,.0f} labeled as 'percent' exceeds 100% but is unusually small. "
f"This suggests a data collection or unit conversion error."
),
"suggested_fix": "Verify the metric source and unit conversion logic",
}
return interpretation
def _validate_cpu_metric(self, cpu_data: dict | Any) -> dict:
"""Validate CPU metrics."""
if not isinstance(cpu_data, dict):
return {"raw": cpu_data, "validated": False}
normalized = cpu_data.copy()
if "percent" in cpu_data:
raw_percent = cpu_data["percent"]
# CPU can legitimately exceed 100% (multi-core)
# But beyond 1000% is suspicious for most workloads
if isinstance(raw_percent, int | float) and raw_percent > 1000:
self.issues.append(
ValidationIssue(
field="cpu.percent",
raw_value=raw_percent,
issue_type="suspicious_value",
severity="warning",
explanation=(
f"CPU usage reported as {raw_percent}% which is unusually high. "
f"While multi-core systems can exceed 100%, values over 1000% "
f"suggest a data collection error or misconfigured metric."
),
suggested_fix="Verify CPU metric calculation and core count normalization",
)
)
# Cap at reasonable value or mark as suspicious
normalized["percent_suspicious"] = True
normalized["percent_raw"] = raw_percent
return normalized
def _validate_disk_metric(self, disk_data: dict | Any) -> dict:
"""Validate disk metrics."""
if not isinstance(disk_data, dict):
return {"raw": disk_data, "validated": False}
normalized = disk_data.copy()
if "percent" in disk_data:
raw_percent = disk_data["percent"]
if isinstance(raw_percent, int | float) and raw_percent > 100:
self.issues.append(
ValidationIssue(
field="disk.percent",
raw_value=raw_percent,
issue_type="impossible_percentage",
severity="error",
explanation=(
f"Disk usage reported as {raw_percent}% which is impossible. "
f"This indicates a data collection or unit conversion error."
),
suggested_fix="Verify disk metric calculation and units",
)
)
normalized["percent"] = None
normalized["percent_invalid"] = True
normalized["percent_raw"] = raw_percent
return normalized
def _validate_flat_metrics(self, metrics: dict) -> dict:
"""
Validate metrics in flat structure (cpu, ram, disk at top level).
Common API format: {"cpu": 95.28, "ram": 8471740416, "disk": 50}
"""
normalized = metrics.copy()
# Check "ram" field (often used instead of "memory")
if "ram" in normalized:
raw_ram = normalized["ram"]
if isinstance(raw_ram, int | float) and raw_ram > 100:
interpretation = self._infer_memory_unit(raw_ram)
self.issues.append(
ValidationIssue(
field="ram",
raw_value=raw_ram,
issue_type="impossible_percentage",
severity="error",
explanation=interpretation["explanation"],
suggested_fix=interpretation["suggested_fix"],
)
)
normalized["ram_interpretation"] = interpretation
normalized["ram_invalid"] = True
normalized["ram_raw"] = raw_ram
# Don't set to None - keep raw value but mark as invalid
# LLM can use interpretation to understand it
# Check "max_ram" if present
if "max_ram" in normalized:
raw_max_ram = normalized["max_ram"]
if isinstance(raw_max_ram, int | float) and raw_max_ram > 100:
interpretation = self._infer_memory_unit(raw_max_ram)
self.issues.append(
ValidationIssue(
field="max_ram",
raw_value=raw_max_ram,
issue_type="impossible_percentage",
severity="error",
explanation=interpretation["explanation"],
suggested_fix=interpretation["suggested_fix"],
)
)
normalized["max_ram_interpretation"] = interpretation
normalized["max_ram_invalid"] = True
normalized["max_ram_raw"] = raw_max_ram
return normalized
def _flag_impossible_percentage(self, field: str, value: Any, data: dict) -> None:
"""Flag an impossible percentage value."""
if isinstance(value, int | float) and value > 100:
# For memory-related fields, use intelligent inference
if "memory" in field.lower() or "ram" in field.lower():
interpretation = self._infer_memory_unit(value)
self.issues.append(
ValidationIssue(
field=field,
raw_value=value,
issue_type="impossible_percentage",
severity="error",
explanation=interpretation["explanation"],
suggested_fix=interpretation["suggested_fix"],
)
)
data[f"{field}_interpretation"] = interpretation
else:
self.issues.append(
ValidationIssue(
field=field,
raw_value=value,
issue_type="impossible_percentage",
severity="error",
explanation=(
f"Field '{field}' has value {value}% which exceeds 100% and is impossible. "
f"This is likely a data collection error or unit mismatch."
),
suggested_fix="Verify the metric source and unit conversion",
)
)
data[f"{field}_invalid"] = True
data[f"{field}_raw"] = value
def _validate_data_list(
data_points: list[Any],
validate_fn: Callable[[dict], dict],
) -> tuple[list[Any], list[dict]]:
"""Iterate a list of data points, validate each dict entry, and collect issues.
Returns ``(validated_points, collected_issues)`` where issues are stripped
from each point's ``data_quality_issues`` key and merged into the returned list.
Points that are not dicts are passed through unchanged.
"""
validated: list[Any] = []
issues: list[dict] = []
for point in data_points:
if isinstance(point, dict):
result = validate_fn(point.copy())
point_issues = result.pop("data_quality_issues", None)
if isinstance(point_issues, list):
issues.extend(point_issues)
validated.append(result)
else:
validated.append(point)
return validated, issues
def validate_host_metrics(metrics: dict | Any) -> dict:
"""
Validate host metrics data.
Handles different API response structures:
- List structure: {"success": True, "data": [{"cpu": 95, "ram": 8471740416, "disk": 50}]}
- Flat structure: {"cpu": 95, "ram": 8471740416}
- Nested structure: {"memory": {"percent": 8471740416}}
Args:
metrics: Raw metrics data from API
Returns:
Validated and normalized metrics dict with interpretation hints
"""
validator = MetricsValidator()
if not isinstance(metrics, dict):
return {
"raw": metrics,
"validated": False,
"data_quality_issues": [
{
"field": "root",
"raw_value": metrics,
"issue": "invalid_format",
"severity": "error",
"explanation": "Metrics data is not in expected dictionary format",
}
],
}
# Handle list-based structure (common in API responses)
if "data" in metrics and isinstance(metrics["data"], list):
validated_data, all_issues = _validate_data_list(
metrics["data"], validator.validate_metrics
)
result = metrics.copy()
result["data"] = validated_data
if all_issues:
result["data_quality_issues"] = all_issues
return result
# Handle flat or nested structure
return validator.validate_metrics(metrics)
+10
View File
@@ -0,0 +1,10 @@
"""Shared warning helper for SQL tools that default to a system database."""
from __future__ import annotations
def default_db_warning(db_name: str) -> str:
return (
f"WARNING: No database was specified; defaulted to '{db_name}'. "
"Results may not reflect application data."
)
@@ -0,0 +1,27 @@
"""Adapt resolved integration configs to the legacy tool availability contract."""
from __future__ import annotations
from typing import Any
from pydantic import BaseModel
def availability_view(resolved_integrations: dict[str, Any]) -> dict[str, Any]:
"""Convert classified integration configs into dicts tools can consume."""
view: dict[str, Any] = {}
for key, value in resolved_integrations.items():
if key.startswith("_"):
view[key] = value
continue
if isinstance(value, BaseModel):
item = value.model_dump(exclude_none=True)
item.setdefault("connection_verified", True)
view[key] = item
elif isinstance(value, dict) and value:
item = dict(value)
item.setdefault("connection_verified", True)
view[key] = item
else:
view[key] = value
return view
+34
View File
@@ -0,0 +1,34 @@
"""Normalize MCP integration source dicts for tool param extraction.
MCP-backed tools read connection settings from the verified integration
source (e.g. ``posthog_mcp``, ``sentry_mcp``, ``openclaw``). Catalog and
runtime configs may use prefixed keys (``posthog_url``) or short aliases
(``url``). These helpers pick the first non-empty value across alias keys
and coerce list fields into stripped string lists.
"""
from __future__ import annotations
__all__ = ["first_list", "first_string", "string_list"]
def string_list(value: object) -> list[str]:
if not isinstance(value, list):
return []
return [str(item).strip() for item in value if str(item).strip()]
def first_string(source: dict[str, object], *keys: str) -> str | None:
for key in keys:
value = str(source.get(key, "")).strip()
if value:
return value
return None
def first_list(source: dict[str, object], *keys: str) -> list[str]:
for key in keys:
values = string_list(source.get(key, []))
if values:
return values
return []
@@ -0,0 +1,128 @@
"""Bounded, model-safe rendering of MCP server tool catalogs.
MCP servers (PostHog, Sentry, OpenClaw, ...) can expose dozens to hundreds of
tools, each carrying a full JSON input schema. A discovery tool that returns
every tool *with* its schema produces a payload many times larger than any
model's context window; the agent's context-budget enforcer then trims the
listing away before the model sees a single tool name, and the model loops
re-calling the discovery tool and guessing tool names that don't exist.
This module renders a compact, bounded view by default (names + truncated
descriptions, no schemas) and lets callers narrow with a name filter or pull
the full schema for a small, specific set of tools. It is shared by every
``list_*_tools`` MCP discovery tool so they behave identically.
"""
from __future__ import annotations
import re
# Defaults sized to keep even a full catalog dump well under model context
# windows (the live PostHog server alone is ~580k estimated tokens unbounded).
MAX_DESCRIPTION_CHARS = 160
MAX_TOOLS_RETURNED = 80
MAX_SCHEMAS_RETURNED = 15
def _truncate_description(description: str) -> str:
cleaned = " ".join(description.split())
if len(cleaned) <= MAX_DESCRIPTION_CHARS:
return cleaned
return cleaned[: MAX_DESCRIPTION_CHARS - 1].rstrip() + "\u2026"
def _filter_tools(
tools: list[dict[str, object]],
name_filter: str,
) -> list[dict[str, object]]:
"""Keep tools whose name or description matches any whitespace/comma term."""
terms = [term for term in re.split(r"[,\s]+", name_filter.lower()) if term]
if not terms:
return tools
matched: list[dict[str, object]] = []
for descriptor in tools:
haystack = f"{descriptor.get('name', '')} {descriptor.get('description', '')}".lower()
if any(term in haystack for term in terms):
matched.append(descriptor)
return matched
def _summarize_tool(
descriptor: dict[str, object],
*,
include_schema: bool,
) -> dict[str, object]:
summary: dict[str, object] = {
"name": str(descriptor.get("name", "")).strip(),
"description": _truncate_description(str(descriptor.get("description", "") or "")),
}
schema = descriptor.get("input_schema")
if include_schema and schema is not None:
summary["input_schema"] = schema
return summary
def build_mcp_tool_listing(
tools: list[dict[str, object]],
*,
name_filter: str | None,
include_schema: bool,
filter_example: str = "events query sql",
) -> dict[str, object]:
"""Render a bounded, model-safe view of discovered MCP tools.
``filter_example`` only affects the human-readable ``notes`` hint so each
integration can suggest filter terms relevant to its own tool catalog.
"""
total = len(tools)
filtered = _filter_tools(tools, name_filter) if name_filter else list(tools)
returned = filtered[:MAX_TOOLS_RETURNED]
# Only attach full schemas when the result set is small enough that doing so
# keeps the payload bounded. Otherwise schemas would reintroduce the very
# context blow-up this listing exists to prevent.
attach_schema = include_schema and len(returned) <= MAX_SCHEMAS_RETURNED
summaries = [
_summarize_tool(descriptor, include_schema=attach_schema) for descriptor in returned
]
notes: list[str] = []
if len(filtered) > len(returned):
notes.append(
f"Showing {len(returned)} of {len(filtered)} matching tools; "
"pass name_filter to narrow the list."
)
elif total > len(returned):
notes.append(
f"Showing {len(returned)} of {total} tools; pass name_filter "
f"(e.g. '{filter_example}') to narrow the list."
)
if include_schema and not attach_schema:
notes.append(
"input_schema omitted because too many tools matched; narrow to "
f"{MAX_SCHEMAS_RETURNED} or fewer tools with name_filter to include schemas."
)
if not include_schema:
notes.append(
"Schemas omitted to save context; call again with include_schema=true and a "
"name_filter once you know which tool you need."
)
listing: dict[str, object] = {
"total_tools": total,
"matched_tools": len(filtered),
"returned_tools": len(summaries),
"tools": summaries,
}
if name_filter:
listing["name_filter"] = name_filter
if notes:
listing["notes"] = " ".join(notes)
return listing
__all__ = [
"MAX_DESCRIPTION_CHARS",
"MAX_SCHEMAS_RETURNED",
"MAX_TOOLS_RETURNED",
"build_mcp_tool_listing",
]
+52
View File
@@ -0,0 +1,52 @@
"""Shared wrapper helper for repeated SQL-tool flow pattern.
Centralizes the common pattern of:
1. Resolving database config (with optional default fallback)
2. Calling a vendor-specific query/process function
3. Injecting optional warning when database was defaulted
4. Returning result dict
"""
from __future__ import annotations
from collections.abc import Callable
from typing import Any
from core.tool_framework.utils.db_warnings import default_db_warning
def call_db_tool_with_default_db_warning[T](
database: str | None,
default_db_name: str,
config_resolver: Callable[..., T],
resolver_kwargs: dict[str, Any],
db_caller: Callable[[T], dict[str, Any]],
) -> dict[str, Any]:
"""Wrapper for repeated SQL-tool flow: resolve, call, warn (optional), return.
Args:
database: The database parameter from the tool invocation, may be None.
default_db_name: The default database name if `database` is None (e.g., 'master', 'postgres', 'mysql').
config_resolver: Function that builds a config object from kwargs (e.g., resolve_azure_sql_config).
resolver_kwargs: Keyword arguments to pass to config_resolver.
db_caller: Function that takes the config and returns a dict result.
Returns:
The result dict from db_caller, with optional 'default_db_warning' key injected if database was None.
"""
_db_defaulted = database is None
if database is None:
database = default_db_name
# Resolve config using the vendor-specific resolver
kwargs = {**resolver_kwargs, "database": database}
config = config_resolver(**kwargs)
# Call the vendor-specific query/process function
result = db_caller(config)
# Inject warning if database was defaulted
if _db_defaulted:
result["default_db_warning"] = default_db_warning(default_db_name)
return result