chore: import zh skill tool-design
This commit is contained in:
wehub-skill-sync
@@ -0,0 +1,528 @@
|
||||
"""
|
||||
Tool Description Engineering -- Generation and Evaluation Utilities.
|
||||
|
||||
Use when: building, auditing, or iterating on tool descriptions for agent
|
||||
systems. Provides templates for structured descriptions, a scoring evaluator
|
||||
that flags vague or incomplete descriptions, error-message generators that
|
||||
produce agent-recoverable responses, and a builder that assembles complete
|
||||
tool schemas.
|
||||
|
||||
Typical workflow:
|
||||
1. Define a tool spec with ``ToolSchemaBuilder``.
|
||||
2. Generate a rendered description with ``generate_tool_description``.
|
||||
3. Score the description with ``ToolDescriptionEvaluator.evaluate``.
|
||||
4. Generate error templates with ``ErrorMessageGenerator.generate``.
|
||||
|
||||
Example::
|
||||
|
||||
builder = ToolSchemaBuilder("get_customer")
|
||||
builder.set_description("Retrieve customer record", "Full details...")
|
||||
builder.add_parameter("customer_id", "string", "CUST-######", required=True)
|
||||
schema = builder.build()
|
||||
|
||||
desc = generate_tool_description(schema)
|
||||
scores = ToolDescriptionEvaluator().evaluate(desc, schema)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Protocol, Sequence
|
||||
import json
|
||||
import re
|
||||
|
||||
|
||||
__all__ = [
|
||||
"generate_tool_description",
|
||||
"generate_usage_context",
|
||||
"ToolDescriptionEvaluator",
|
||||
"ErrorMessageGenerator",
|
||||
"ToolSchemaBuilder",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocols -- lightweight structural typing for tool specs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ToolSpec(Protocol):
|
||||
"""Structural interface expected by generation helpers.
|
||||
|
||||
Use when: passing tool metadata objects that were not built with
|
||||
``ToolSchemaBuilder`` (e.g., third-party specs).
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
triggers: Sequence[str]
|
||||
examples: Sequence[Any]
|
||||
parameters: Sequence[Dict[str, Any]]
|
||||
returns: Dict[str, Any]
|
||||
errors: Sequence[Dict[str, Any]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _BuiltToolSpec:
|
||||
"""Concrete implementation of ToolSpec returned by ToolSchemaBuilder.build()."""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
triggers: List[str]
|
||||
examples: List[Dict[str, str]]
|
||||
parameters: List[Dict[str, Any]]
|
||||
returns: Dict[str, Any]
|
||||
errors: List[Dict[str, Any]]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Description Templates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TOOL_DESCRIPTION_TEMPLATE: str = """
|
||||
## {tool_name}
|
||||
|
||||
{detailed_description}
|
||||
|
||||
### When to Use
|
||||
{usage_context}
|
||||
|
||||
### Parameters
|
||||
{parameters_description}
|
||||
|
||||
### Returns
|
||||
{returns_description}
|
||||
|
||||
### Errors
|
||||
{errors_description}
|
||||
"""
|
||||
|
||||
PARAM_TEMPLATE: str = """
|
||||
- **{param_name}** ({param_type}{required_label})
|
||||
|
||||
{param_description}
|
||||
{default_label}
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Generation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate_tool_description(tool_spec: ToolSpec) -> str:
|
||||
"""Render a complete markdown tool description from *tool_spec*.
|
||||
|
||||
Use when: producing human-readable or agent-injectable documentation
|
||||
from a structured spec object.
|
||||
"""
|
||||
description: str = TOOL_DESCRIPTION_TEMPLATE.format(
|
||||
tool_name=tool_spec.name,
|
||||
detailed_description=tool_spec.description,
|
||||
usage_context=generate_usage_context(tool_spec),
|
||||
parameters_description=_generate_parameters(tool_spec.parameters),
|
||||
returns_description=_generate_returns(tool_spec.returns),
|
||||
errors_description=_generate_errors(tool_spec.errors),
|
||||
)
|
||||
return description
|
||||
|
||||
|
||||
def generate_usage_context(tool_spec: ToolSpec) -> str:
|
||||
"""Build the 'When to Use' section from triggers and examples.
|
||||
|
||||
Use when: the caller needs only the usage-context fragment rather
|
||||
than the full rendered description.
|
||||
"""
|
||||
contexts: list[str] = []
|
||||
|
||||
for trigger in tool_spec.triggers:
|
||||
contexts.append(f"- When {trigger}")
|
||||
|
||||
if tool_spec.examples:
|
||||
contexts.append("\n**Examples**:\n")
|
||||
for example in tool_spec.examples:
|
||||
if isinstance(example, dict):
|
||||
contexts.append(f"- Input: {example.get('input', '')}")
|
||||
contexts.append(f" Output: {example.get('tool_call', '')}")
|
||||
else:
|
||||
contexts.append(f"- {example}")
|
||||
|
||||
return "\n".join(contexts)
|
||||
|
||||
|
||||
def _generate_parameters(parameters: Sequence[Dict[str, Any]]) -> str:
|
||||
"""Render parameter list to markdown."""
|
||||
parts: list[str] = []
|
||||
for p in parameters:
|
||||
required_label = " | required" if p.get("required") else " | optional"
|
||||
default = p.get("default")
|
||||
default_label = f"Default: {default}" if default is not None else ""
|
||||
parts.append(
|
||||
f"- **{p['name']}** ({p['type']}{required_label})\n"
|
||||
f" {p['description']}\n"
|
||||
f" {default_label}".rstrip()
|
||||
)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _generate_returns(returns: Optional[Dict[str, Any]]) -> str:
|
||||
"""Render the returns section to markdown."""
|
||||
if not returns:
|
||||
return "No return value documented."
|
||||
desc = returns.get("description", "")
|
||||
rtype = returns.get("type", "object")
|
||||
return f"{rtype} -- {desc}"
|
||||
|
||||
|
||||
def _generate_errors(errors: Sequence[Dict[str, Any]]) -> str:
|
||||
"""Render error definitions to markdown."""
|
||||
if not errors:
|
||||
return "No error conditions documented."
|
||||
parts: list[str] = []
|
||||
for err in errors:
|
||||
parts.append(f"- **{err['code']}**: {err['description']} -- {err.get('resolution', '')}")
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Evaluator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ToolDescriptionEvaluator:
|
||||
"""Score a rendered description against quality criteria.
|
||||
|
||||
Use when: auditing existing tool descriptions for clarity,
|
||||
completeness, accuracy, actionability, and consistency.
|
||||
"""
|
||||
|
||||
CRITERIA: List[str] = [
|
||||
"clarity",
|
||||
"completeness",
|
||||
"accuracy",
|
||||
"actionability",
|
||||
"consistency",
|
||||
]
|
||||
|
||||
def evaluate(self, description: str, tool_spec: ToolSpec) -> Dict[str, float]:
|
||||
"""Return per-criterion scores (0.0 -- 1.0) for *description*.
|
||||
|
||||
Use when: running automated quality checks on tool descriptions
|
||||
before deploying them into an agent system.
|
||||
"""
|
||||
results: Dict[str, float] = {
|
||||
"clarity": self._check_clarity(description),
|
||||
"completeness": self._check_completeness(description, tool_spec),
|
||||
"accuracy": self._check_accuracy(description, tool_spec),
|
||||
"actionability": self._check_actionability(description),
|
||||
"consistency": self._check_consistency(description, tool_spec),
|
||||
}
|
||||
return results
|
||||
|
||||
# -- private scoring helpers ------------------------------------------
|
||||
|
||||
def _check_clarity(self, description: str) -> float:
|
||||
"""Score description clarity (0-1).
|
||||
|
||||
Use when: detecting vague or ambiguous language that would
|
||||
confuse an agent during tool selection.
|
||||
"""
|
||||
vague_terms: list[str] = ["help", "assist", "thing", "stuff", "handle"]
|
||||
vague_count: int = sum(1 for term in vague_terms if term in description.lower())
|
||||
|
||||
ambiguous: list[str] = ["it", "this", "that"]
|
||||
ambiguous_count: int = sum(1 for term in ambiguous if f" {term} " in description)
|
||||
|
||||
clarity: float = 1.0 - (vague_count * 0.1) - (ambiguous_count * 0.05)
|
||||
return max(0.0, clarity)
|
||||
|
||||
def _check_completeness(self, description: str, tool_spec: ToolSpec) -> float:
|
||||
"""Score presence of required sections (0-1).
|
||||
|
||||
Use when: verifying a description has all mandatory sections
|
||||
before publishing.
|
||||
"""
|
||||
required_patterns: list[tuple[str, str]] = [
|
||||
("description", r"## " + re.escape(str(getattr(tool_spec, "name", "")))),
|
||||
("parameters", r"### Parameters"),
|
||||
("returns", r"### Returns"),
|
||||
("errors", r"### Errors"),
|
||||
]
|
||||
present: int = sum(
|
||||
1 for _, pattern in required_patterns if re.search(pattern, description)
|
||||
)
|
||||
return present / len(required_patterns)
|
||||
|
||||
def _check_accuracy(self, description: str, tool_spec: ToolSpec) -> float:
|
||||
"""Score alignment between description text and spec metadata.
|
||||
|
||||
Use when: detecting description rot where the text no longer
|
||||
matches the current tool spec.
|
||||
"""
|
||||
score = 1.0
|
||||
# Check that tool name appears in description
|
||||
if hasattr(tool_spec, "name") and tool_spec.name not in description:
|
||||
score -= 0.3
|
||||
# Check parameter names appear
|
||||
if hasattr(tool_spec, "parameters"):
|
||||
for param in tool_spec.parameters:
|
||||
pname = param.get("name", "") if isinstance(param, dict) else ""
|
||||
if pname and pname not in description:
|
||||
score -= 0.15
|
||||
return max(0.0, score)
|
||||
|
||||
def _check_actionability(self, description: str) -> float:
|
||||
"""Score whether the description contains actionable cues.
|
||||
|
||||
Use when: confirming agents can determine correct usage from
|
||||
the description alone.
|
||||
"""
|
||||
signals: list[str] = ["Use when", "Returns", "Errors", "Args", "Parameters"]
|
||||
found: int = sum(1 for s in signals if s in description)
|
||||
return min(1.0, found / max(1, len(signals)))
|
||||
|
||||
def _check_consistency(self, description: str, tool_spec: ToolSpec) -> float:
|
||||
"""Score naming and formatting consistency.
|
||||
|
||||
Use when: checking that parameter and section naming follows
|
||||
conventions across the tool collection.
|
||||
"""
|
||||
# Penalise mixed naming styles (camelCase vs snake_case)
|
||||
camel = len(re.findall(r"[a-z][A-Z]", description))
|
||||
snake = len(re.findall(r"[a-z]_[a-z]", description))
|
||||
if camel > 0 and snake > 0:
|
||||
return 0.5
|
||||
return 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error Message Generator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ErrorMessageGenerator:
|
||||
"""Produce structured, agent-recoverable error messages.
|
||||
|
||||
Use when: building error responses that tell agents what went wrong,
|
||||
why, and how to correct the call.
|
||||
"""
|
||||
|
||||
TEMPLATES: Dict[str, str] = {
|
||||
"NOT_FOUND": json.dumps({
|
||||
"error": "{error_code}",
|
||||
"message": "{specific_message}",
|
||||
"resolution": "{how_to_resolve}",
|
||||
"example": "{correct_format}",
|
||||
}, indent=2),
|
||||
|
||||
"INVALID_INPUT": json.dumps({
|
||||
"error": "{error_code}",
|
||||
"message": "Invalid {field}: {received_value}",
|
||||
"expected_format": "{expected_format}",
|
||||
"resolution": "Provide value matching {expected_format}",
|
||||
}, indent=2),
|
||||
|
||||
"RATE_LIMITED": json.dumps({
|
||||
"error": "{error_code}",
|
||||
"message": "Rate limit exceeded",
|
||||
"retry_after": "{seconds}",
|
||||
"resolution": "Wait {seconds} seconds before retrying",
|
||||
}, indent=2),
|
||||
}
|
||||
|
||||
def generate(self, error_type: str, context: Dict[str, str]) -> str:
|
||||
"""Render an error message for *error_type* using *context* values.
|
||||
|
||||
Use when: a tool needs to return a structured error that an agent
|
||||
can parse and act on.
|
||||
"""
|
||||
template: str = self.TEMPLATES.get(error_type, self.TEMPLATES["INVALID_INPUT"])
|
||||
return template.format(**context)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schema Builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ToolSchemaBuilder:
|
||||
"""Fluent builder for complete tool schemas.
|
||||
|
||||
Use when: defining a new tool's schema programmatically and want
|
||||
compile-time structure rather than hand-written dictionaries.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str) -> None:
|
||||
self.name: str = name
|
||||
self.description: str = ""
|
||||
self.detailed_description: str = ""
|
||||
self.parameters: List[Dict[str, Any]] = []
|
||||
self.returns: Optional[Dict[str, Any]] = None
|
||||
self.errors: List[Dict[str, str]] = []
|
||||
self._triggers: List[str] = []
|
||||
self._examples: List[Dict[str, str]] = []
|
||||
|
||||
def set_description(self, short: str, detailed: str) -> "ToolSchemaBuilder":
|
||||
"""Set short and detailed description sections.
|
||||
|
||||
Use when: providing both a one-line summary and a full
|
||||
multi-paragraph description for the tool.
|
||||
"""
|
||||
self.description = short
|
||||
self.detailed_description = detailed
|
||||
return self
|
||||
|
||||
def add_parameter(
|
||||
self,
|
||||
name: str,
|
||||
param_type: str,
|
||||
description: str,
|
||||
required: bool = False,
|
||||
default: Optional[Any] = None,
|
||||
enum: Optional[List[str]] = None,
|
||||
) -> "ToolSchemaBuilder":
|
||||
"""Append a parameter definition.
|
||||
|
||||
Use when: declaring each accepted input for the tool.
|
||||
"""
|
||||
self.parameters.append({
|
||||
"name": name,
|
||||
"type": param_type,
|
||||
"description": description,
|
||||
"required": required,
|
||||
"default": default,
|
||||
"enum": enum,
|
||||
})
|
||||
return self
|
||||
|
||||
def set_returns(
|
||||
self,
|
||||
return_type: str,
|
||||
description: str,
|
||||
properties: Dict[str, Any],
|
||||
) -> "ToolSchemaBuilder":
|
||||
"""Define the return value schema.
|
||||
|
||||
Use when: documenting what the tool sends back on success.
|
||||
"""
|
||||
self.returns = {
|
||||
"type": return_type,
|
||||
"description": description,
|
||||
"properties": properties,
|
||||
}
|
||||
return self
|
||||
|
||||
def add_error(
|
||||
self,
|
||||
code: str,
|
||||
description: str,
|
||||
resolution: str,
|
||||
) -> "ToolSchemaBuilder":
|
||||
"""Register an error condition with recovery guidance.
|
||||
|
||||
Use when: enumerating known failure modes so agents can
|
||||
handle them gracefully.
|
||||
"""
|
||||
self.errors.append({
|
||||
"code": code,
|
||||
"description": description,
|
||||
"resolution": resolution,
|
||||
})
|
||||
return self
|
||||
|
||||
def build(self) -> "_BuiltToolSpec":
|
||||
"""Assemble and return the complete tool spec.
|
||||
|
||||
Use when: the builder is fully configured and the schema is
|
||||
ready for registration, serialization, or passing to
|
||||
``generate_tool_description``.
|
||||
|
||||
Returns a ``_BuiltToolSpec`` object that satisfies the ``ToolSpec``
|
||||
protocol, so it can be used directly with ``generate_tool_description``
|
||||
and ``ToolDescriptionEvaluator``.
|
||||
"""
|
||||
return _BuiltToolSpec(
|
||||
name=self.name,
|
||||
description=self.detailed_description or self.description,
|
||||
triggers=self._triggers,
|
||||
examples=self._examples,
|
||||
parameters=list(self.parameters),
|
||||
returns=self.returns or {},
|
||||
errors=list(self.errors),
|
||||
)
|
||||
|
||||
def add_trigger(self, trigger: str) -> "ToolSchemaBuilder":
|
||||
"""Add an activation trigger for the tool.
|
||||
|
||||
Use when: documenting when agents should select this tool.
|
||||
"""
|
||||
self._triggers.append(trigger)
|
||||
return self
|
||||
|
||||
def add_example(
|
||||
self, input_text: str, tool_call: str
|
||||
) -> "ToolSchemaBuilder":
|
||||
"""Add a usage example.
|
||||
|
||||
Use when: providing concrete input/output pairs that help agents
|
||||
understand expected usage.
|
||||
"""
|
||||
self._examples.append({"input": input_text, "tool_call": tool_call})
|
||||
return self
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Quick demo: build a schema, render it, and evaluate it.
|
||||
builder = ToolSchemaBuilder("get_customer")
|
||||
builder.set_description(
|
||||
"Retrieve customer record by ID",
|
||||
"Fetches a customer object from the primary datastore. "
|
||||
"Supports concise and detailed response formats.",
|
||||
)
|
||||
builder.add_parameter(
|
||||
"customer_id", "string",
|
||||
'Customer identifier in CUST-###### format (e.g., "CUST-000001")',
|
||||
required=True,
|
||||
)
|
||||
builder.add_parameter(
|
||||
"format", "string",
|
||||
'"concise" for key fields, "detailed" for complete record',
|
||||
required=False,
|
||||
default="concise",
|
||||
enum=["concise", "detailed"],
|
||||
)
|
||||
builder.set_returns(
|
||||
"object",
|
||||
"Customer object with requested fields",
|
||||
{"id": {"type": "string"}, "name": {"type": "string"}},
|
||||
)
|
||||
builder.add_error("NOT_FOUND", "Customer ID not in datastore", "Verify ID format and retry")
|
||||
builder.add_error("INVALID_FORMAT", "ID does not match CUST-######", "Use CUST-###### pattern")
|
||||
|
||||
spec = builder.build()
|
||||
|
||||
print("=== Built Spec ===")
|
||||
print(f"Name: {spec.name}")
|
||||
print(f"Parameters: {[p['name'] for p in spec.parameters]}")
|
||||
print(f"Errors: {[e['code'] for e in spec.errors]}")
|
||||
|
||||
# Generate and evaluate description
|
||||
description = generate_tool_description(spec)
|
||||
print("\n=== Generated Description ===")
|
||||
print(description)
|
||||
|
||||
evaluator = ToolDescriptionEvaluator()
|
||||
scores = evaluator.evaluate(description, spec)
|
||||
print("\n=== Evaluation Scores ===")
|
||||
for criterion, score in scores.items():
|
||||
print(f" {criterion}: {score:.2f}")
|
||||
|
||||
# Generate an error message example
|
||||
err_gen = ErrorMessageGenerator()
|
||||
err_msg = err_gen.generate("NOT_FOUND", {
|
||||
"error_code": "NOT_FOUND",
|
||||
"specific_message": "No customer with ID CUST-999999",
|
||||
"how_to_resolve": "Check ID and retry",
|
||||
"correct_format": "CUST-######",
|
||||
})
|
||||
print("\n=== Sample Error Message ===")
|
||||
print(err_msg)
|
||||
Reference in New Issue
Block a user