wehub-skill-sync
529 lines
18 KiB
Python
529 lines
18 KiB
Python
"""
|
|
Tool Description Engineering -- Generation and Evaluation Utilities.
|
|
|
|
Use when: building, auditing, or iterating on tool descriptions for agent
|
|
systems. Provides templates for structured descriptions, a scoring evaluator
|
|
that flags vague or incomplete descriptions, error-message generators that
|
|
produce agent-recoverable responses, and a builder that assembles complete
|
|
tool schemas.
|
|
|
|
Typical workflow:
|
|
1. Define a tool spec with ``ToolSchemaBuilder``.
|
|
2. Generate a rendered description with ``generate_tool_description``.
|
|
3. Score the description with ``ToolDescriptionEvaluator.evaluate``.
|
|
4. Generate error templates with ``ErrorMessageGenerator.generate``.
|
|
|
|
Example::
|
|
|
|
builder = ToolSchemaBuilder("get_customer")
|
|
builder.set_description("Retrieve customer record", "Full details...")
|
|
builder.add_parameter("customer_id", "string", "CUST-######", required=True)
|
|
schema = builder.build()
|
|
|
|
desc = generate_tool_description(schema)
|
|
scores = ToolDescriptionEvaluator().evaluate(desc, schema)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Dict, List, Optional, Protocol, Sequence
|
|
import json
|
|
import re
|
|
|
|
|
|
__all__ = [
|
|
"generate_tool_description",
|
|
"generate_usage_context",
|
|
"ToolDescriptionEvaluator",
|
|
"ErrorMessageGenerator",
|
|
"ToolSchemaBuilder",
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Protocols -- lightweight structural typing for tool specs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ToolSpec(Protocol):
|
|
"""Structural interface expected by generation helpers.
|
|
|
|
Use when: passing tool metadata objects that were not built with
|
|
``ToolSchemaBuilder`` (e.g., third-party specs).
|
|
"""
|
|
|
|
name: str
|
|
description: str
|
|
triggers: Sequence[str]
|
|
examples: Sequence[Any]
|
|
parameters: Sequence[Dict[str, Any]]
|
|
returns: Dict[str, Any]
|
|
errors: Sequence[Dict[str, Any]]
|
|
|
|
|
|
@dataclass
|
|
class _BuiltToolSpec:
|
|
"""Concrete implementation of ToolSpec returned by ToolSchemaBuilder.build()."""
|
|
|
|
name: str
|
|
description: str
|
|
triggers: List[str]
|
|
examples: List[Dict[str, str]]
|
|
parameters: List[Dict[str, Any]]
|
|
returns: Dict[str, Any]
|
|
errors: List[Dict[str, Any]]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Description Templates
|
|
# ---------------------------------------------------------------------------
|
|
|
|
TOOL_DESCRIPTION_TEMPLATE: str = """
|
|
## {tool_name}
|
|
|
|
{detailed_description}
|
|
|
|
### When to Use
|
|
{usage_context}
|
|
|
|
### Parameters
|
|
{parameters_description}
|
|
|
|
### Returns
|
|
{returns_description}
|
|
|
|
### Errors
|
|
{errors_description}
|
|
"""
|
|
|
|
PARAM_TEMPLATE: str = """
|
|
- **{param_name}** ({param_type}{required_label})
|
|
|
|
{param_description}
|
|
{default_label}
|
|
"""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Generation helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def generate_tool_description(tool_spec: ToolSpec) -> str:
|
|
"""Render a complete markdown tool description from *tool_spec*.
|
|
|
|
Use when: producing human-readable or agent-injectable documentation
|
|
from a structured spec object.
|
|
"""
|
|
description: str = TOOL_DESCRIPTION_TEMPLATE.format(
|
|
tool_name=tool_spec.name,
|
|
detailed_description=tool_spec.description,
|
|
usage_context=generate_usage_context(tool_spec),
|
|
parameters_description=_generate_parameters(tool_spec.parameters),
|
|
returns_description=_generate_returns(tool_spec.returns),
|
|
errors_description=_generate_errors(tool_spec.errors),
|
|
)
|
|
return description
|
|
|
|
|
|
def generate_usage_context(tool_spec: ToolSpec) -> str:
|
|
"""Build the 'When to Use' section from triggers and examples.
|
|
|
|
Use when: the caller needs only the usage-context fragment rather
|
|
than the full rendered description.
|
|
"""
|
|
contexts: list[str] = []
|
|
|
|
for trigger in tool_spec.triggers:
|
|
contexts.append(f"- When {trigger}")
|
|
|
|
if tool_spec.examples:
|
|
contexts.append("\n**Examples**:\n")
|
|
for example in tool_spec.examples:
|
|
if isinstance(example, dict):
|
|
contexts.append(f"- Input: {example.get('input', '')}")
|
|
contexts.append(f" Output: {example.get('tool_call', '')}")
|
|
else:
|
|
contexts.append(f"- {example}")
|
|
|
|
return "\n".join(contexts)
|
|
|
|
|
|
def _generate_parameters(parameters: Sequence[Dict[str, Any]]) -> str:
|
|
"""Render parameter list to markdown."""
|
|
parts: list[str] = []
|
|
for p in parameters:
|
|
required_label = " | required" if p.get("required") else " | optional"
|
|
default = p.get("default")
|
|
default_label = f"Default: {default}" if default is not None else ""
|
|
parts.append(
|
|
f"- **{p['name']}** ({p['type']}{required_label})\n"
|
|
f" {p['description']}\n"
|
|
f" {default_label}".rstrip()
|
|
)
|
|
return "\n".join(parts)
|
|
|
|
|
|
def _generate_returns(returns: Optional[Dict[str, Any]]) -> str:
|
|
"""Render the returns section to markdown."""
|
|
if not returns:
|
|
return "No return value documented."
|
|
desc = returns.get("description", "")
|
|
rtype = returns.get("type", "object")
|
|
return f"{rtype} -- {desc}"
|
|
|
|
|
|
def _generate_errors(errors: Sequence[Dict[str, Any]]) -> str:
|
|
"""Render error definitions to markdown."""
|
|
if not errors:
|
|
return "No error conditions documented."
|
|
parts: list[str] = []
|
|
for err in errors:
|
|
parts.append(f"- **{err['code']}**: {err['description']} -- {err.get('resolution', '')}")
|
|
return "\n".join(parts)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Evaluator
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ToolDescriptionEvaluator:
|
|
"""Score a rendered description against quality criteria.
|
|
|
|
Use when: auditing existing tool descriptions for clarity,
|
|
completeness, accuracy, actionability, and consistency.
|
|
"""
|
|
|
|
CRITERIA: List[str] = [
|
|
"clarity",
|
|
"completeness",
|
|
"accuracy",
|
|
"actionability",
|
|
"consistency",
|
|
]
|
|
|
|
def evaluate(self, description: str, tool_spec: ToolSpec) -> Dict[str, float]:
|
|
"""Return per-criterion scores (0.0 -- 1.0) for *description*.
|
|
|
|
Use when: running automated quality checks on tool descriptions
|
|
before deploying them into an agent system.
|
|
"""
|
|
results: Dict[str, float] = {
|
|
"clarity": self._check_clarity(description),
|
|
"completeness": self._check_completeness(description, tool_spec),
|
|
"accuracy": self._check_accuracy(description, tool_spec),
|
|
"actionability": self._check_actionability(description),
|
|
"consistency": self._check_consistency(description, tool_spec),
|
|
}
|
|
return results
|
|
|
|
# -- private scoring helpers ------------------------------------------
|
|
|
|
def _check_clarity(self, description: str) -> float:
|
|
"""Score description clarity (0-1).
|
|
|
|
Use when: detecting vague or ambiguous language that would
|
|
confuse an agent during tool selection.
|
|
"""
|
|
vague_terms: list[str] = ["help", "assist", "thing", "stuff", "handle"]
|
|
vague_count: int = sum(1 for term in vague_terms if term in description.lower())
|
|
|
|
ambiguous: list[str] = ["it", "this", "that"]
|
|
ambiguous_count: int = sum(1 for term in ambiguous if f" {term} " in description)
|
|
|
|
clarity: float = 1.0 - (vague_count * 0.1) - (ambiguous_count * 0.05)
|
|
return max(0.0, clarity)
|
|
|
|
def _check_completeness(self, description: str, tool_spec: ToolSpec) -> float:
|
|
"""Score presence of required sections (0-1).
|
|
|
|
Use when: verifying a description has all mandatory sections
|
|
before publishing.
|
|
"""
|
|
required_patterns: list[tuple[str, str]] = [
|
|
("description", r"## " + re.escape(str(getattr(tool_spec, "name", "")))),
|
|
("parameters", r"### Parameters"),
|
|
("returns", r"### Returns"),
|
|
("errors", r"### Errors"),
|
|
]
|
|
present: int = sum(
|
|
1 for _, pattern in required_patterns if re.search(pattern, description)
|
|
)
|
|
return present / len(required_patterns)
|
|
|
|
def _check_accuracy(self, description: str, tool_spec: ToolSpec) -> float:
|
|
"""Score alignment between description text and spec metadata.
|
|
|
|
Use when: detecting description rot where the text no longer
|
|
matches the current tool spec.
|
|
"""
|
|
score = 1.0
|
|
# Check that tool name appears in description
|
|
if hasattr(tool_spec, "name") and tool_spec.name not in description:
|
|
score -= 0.3
|
|
# Check parameter names appear
|
|
if hasattr(tool_spec, "parameters"):
|
|
for param in tool_spec.parameters:
|
|
pname = param.get("name", "") if isinstance(param, dict) else ""
|
|
if pname and pname not in description:
|
|
score -= 0.15
|
|
return max(0.0, score)
|
|
|
|
def _check_actionability(self, description: str) -> float:
|
|
"""Score whether the description contains actionable cues.
|
|
|
|
Use when: confirming agents can determine correct usage from
|
|
the description alone.
|
|
"""
|
|
signals: list[str] = ["Use when", "Returns", "Errors", "Args", "Parameters"]
|
|
found: int = sum(1 for s in signals if s in description)
|
|
return min(1.0, found / max(1, len(signals)))
|
|
|
|
def _check_consistency(self, description: str, tool_spec: ToolSpec) -> float:
|
|
"""Score naming and formatting consistency.
|
|
|
|
Use when: checking that parameter and section naming follows
|
|
conventions across the tool collection.
|
|
"""
|
|
# Penalise mixed naming styles (camelCase vs snake_case)
|
|
camel = len(re.findall(r"[a-z][A-Z]", description))
|
|
snake = len(re.findall(r"[a-z]_[a-z]", description))
|
|
if camel > 0 and snake > 0:
|
|
return 0.5
|
|
return 1.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Error Message Generator
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ErrorMessageGenerator:
|
|
"""Produce structured, agent-recoverable error messages.
|
|
|
|
Use when: building error responses that tell agents what went wrong,
|
|
why, and how to correct the call.
|
|
"""
|
|
|
|
TEMPLATES: Dict[str, str] = {
|
|
"NOT_FOUND": json.dumps({
|
|
"error": "{error_code}",
|
|
"message": "{specific_message}",
|
|
"resolution": "{how_to_resolve}",
|
|
"example": "{correct_format}",
|
|
}, indent=2),
|
|
|
|
"INVALID_INPUT": json.dumps({
|
|
"error": "{error_code}",
|
|
"message": "Invalid {field}: {received_value}",
|
|
"expected_format": "{expected_format}",
|
|
"resolution": "Provide value matching {expected_format}",
|
|
}, indent=2),
|
|
|
|
"RATE_LIMITED": json.dumps({
|
|
"error": "{error_code}",
|
|
"message": "Rate limit exceeded",
|
|
"retry_after": "{seconds}",
|
|
"resolution": "Wait {seconds} seconds before retrying",
|
|
}, indent=2),
|
|
}
|
|
|
|
def generate(self, error_type: str, context: Dict[str, str]) -> str:
|
|
"""Render an error message for *error_type* using *context* values.
|
|
|
|
Use when: a tool needs to return a structured error that an agent
|
|
can parse and act on.
|
|
"""
|
|
template: str = self.TEMPLATES.get(error_type, self.TEMPLATES["INVALID_INPUT"])
|
|
return template.format(**context)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Schema Builder
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ToolSchemaBuilder:
|
|
"""Fluent builder for complete tool schemas.
|
|
|
|
Use when: defining a new tool's schema programmatically and want
|
|
compile-time structure rather than hand-written dictionaries.
|
|
"""
|
|
|
|
def __init__(self, name: str) -> None:
|
|
self.name: str = name
|
|
self.description: str = ""
|
|
self.detailed_description: str = ""
|
|
self.parameters: List[Dict[str, Any]] = []
|
|
self.returns: Optional[Dict[str, Any]] = None
|
|
self.errors: List[Dict[str, str]] = []
|
|
self._triggers: List[str] = []
|
|
self._examples: List[Dict[str, str]] = []
|
|
|
|
def set_description(self, short: str, detailed: str) -> "ToolSchemaBuilder":
|
|
"""Set short and detailed description sections.
|
|
|
|
Use when: providing both a one-line summary and a full
|
|
multi-paragraph description for the tool.
|
|
"""
|
|
self.description = short
|
|
self.detailed_description = detailed
|
|
return self
|
|
|
|
def add_parameter(
|
|
self,
|
|
name: str,
|
|
param_type: str,
|
|
description: str,
|
|
required: bool = False,
|
|
default: Optional[Any] = None,
|
|
enum: Optional[List[str]] = None,
|
|
) -> "ToolSchemaBuilder":
|
|
"""Append a parameter definition.
|
|
|
|
Use when: declaring each accepted input for the tool.
|
|
"""
|
|
self.parameters.append({
|
|
"name": name,
|
|
"type": param_type,
|
|
"description": description,
|
|
"required": required,
|
|
"default": default,
|
|
"enum": enum,
|
|
})
|
|
return self
|
|
|
|
def set_returns(
|
|
self,
|
|
return_type: str,
|
|
description: str,
|
|
properties: Dict[str, Any],
|
|
) -> "ToolSchemaBuilder":
|
|
"""Define the return value schema.
|
|
|
|
Use when: documenting what the tool sends back on success.
|
|
"""
|
|
self.returns = {
|
|
"type": return_type,
|
|
"description": description,
|
|
"properties": properties,
|
|
}
|
|
return self
|
|
|
|
def add_error(
|
|
self,
|
|
code: str,
|
|
description: str,
|
|
resolution: str,
|
|
) -> "ToolSchemaBuilder":
|
|
"""Register an error condition with recovery guidance.
|
|
|
|
Use when: enumerating known failure modes so agents can
|
|
handle them gracefully.
|
|
"""
|
|
self.errors.append({
|
|
"code": code,
|
|
"description": description,
|
|
"resolution": resolution,
|
|
})
|
|
return self
|
|
|
|
def build(self) -> "_BuiltToolSpec":
|
|
"""Assemble and return the complete tool spec.
|
|
|
|
Use when: the builder is fully configured and the schema is
|
|
ready for registration, serialization, or passing to
|
|
``generate_tool_description``.
|
|
|
|
Returns a ``_BuiltToolSpec`` object that satisfies the ``ToolSpec``
|
|
protocol, so it can be used directly with ``generate_tool_description``
|
|
and ``ToolDescriptionEvaluator``.
|
|
"""
|
|
return _BuiltToolSpec(
|
|
name=self.name,
|
|
description=self.detailed_description or self.description,
|
|
triggers=self._triggers,
|
|
examples=self._examples,
|
|
parameters=list(self.parameters),
|
|
returns=self.returns or {},
|
|
errors=list(self.errors),
|
|
)
|
|
|
|
def add_trigger(self, trigger: str) -> "ToolSchemaBuilder":
|
|
"""Add an activation trigger for the tool.
|
|
|
|
Use when: documenting when agents should select this tool.
|
|
"""
|
|
self._triggers.append(trigger)
|
|
return self
|
|
|
|
def add_example(
|
|
self, input_text: str, tool_call: str
|
|
) -> "ToolSchemaBuilder":
|
|
"""Add a usage example.
|
|
|
|
Use when: providing concrete input/output pairs that help agents
|
|
understand expected usage.
|
|
"""
|
|
self._examples.append({"input": input_text, "tool_call": tool_call})
|
|
return self
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI entry point
|
|
# ---------------------------------------------------------------------------
|
|
|
|
if __name__ == "__main__":
|
|
# Quick demo: build a schema, render it, and evaluate it.
|
|
builder = ToolSchemaBuilder("get_customer")
|
|
builder.set_description(
|
|
"Retrieve customer record by ID",
|
|
"Fetches a customer object from the primary datastore. "
|
|
"Supports concise and detailed response formats.",
|
|
)
|
|
builder.add_parameter(
|
|
"customer_id", "string",
|
|
'Customer identifier in CUST-###### format (e.g., "CUST-000001")',
|
|
required=True,
|
|
)
|
|
builder.add_parameter(
|
|
"format", "string",
|
|
'"concise" for key fields, "detailed" for complete record',
|
|
required=False,
|
|
default="concise",
|
|
enum=["concise", "detailed"],
|
|
)
|
|
builder.set_returns(
|
|
"object",
|
|
"Customer object with requested fields",
|
|
{"id": {"type": "string"}, "name": {"type": "string"}},
|
|
)
|
|
builder.add_error("NOT_FOUND", "Customer ID not in datastore", "Verify ID format and retry")
|
|
builder.add_error("INVALID_FORMAT", "ID does not match CUST-######", "Use CUST-###### pattern")
|
|
|
|
spec = builder.build()
|
|
|
|
print("=== Built Spec ===")
|
|
print(f"Name: {spec.name}")
|
|
print(f"Parameters: {[p['name'] for p in spec.parameters]}")
|
|
print(f"Errors: {[e['code'] for e in spec.errors]}")
|
|
|
|
# Generate and evaluate description
|
|
description = generate_tool_description(spec)
|
|
print("\n=== Generated Description ===")
|
|
print(description)
|
|
|
|
evaluator = ToolDescriptionEvaluator()
|
|
scores = evaluator.evaluate(description, spec)
|
|
print("\n=== Evaluation Scores ===")
|
|
for criterion, score in scores.items():
|
|
print(f" {criterion}: {score:.2f}")
|
|
|
|
# Generate an error message example
|
|
err_gen = ErrorMessageGenerator()
|
|
err_msg = err_gen.generate("NOT_FOUND", {
|
|
"error_code": "NOT_FOUND",
|
|
"specific_message": "No customer with ID CUST-999999",
|
|
"how_to_resolve": "Check ID and retry",
|
|
"correct_format": "CUST-######",
|
|
})
|
|
print("\n=== Sample Error Message ===")
|
|
print(err_msg)
|