wehub-skill-sync
393 lines
13 KiB
Python
393 lines
13 KiB
Python
"""Advanced Evaluation Example
|
|
|
|
Use when: building LLM-as-judge evaluation pipelines, comparing model outputs
|
|
with position-bias mitigation, or generating domain-specific scoring rubrics.
|
|
|
|
This module demonstrates the three core evaluation patterns from the
|
|
advanced-evaluation skill: direct scoring, pairwise comparison with position
|
|
swapping, and rubric generation. All functions use pseudocode-style examples
|
|
that work across Python environments without specific dependencies.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
__all__ = [
|
|
"direct_scoring_example",
|
|
"pairwise_comparison_example",
|
|
"rubric_generation_example",
|
|
]
|
|
|
|
|
|
# =============================================================================
|
|
# DIRECT SCORING EXAMPLE
|
|
# =============================================================================
|
|
|
|
|
|
def direct_scoring_example() -> dict[str, Any]:
|
|
"""Rate a single response against defined criteria using direct scoring.
|
|
|
|
Use when: evaluating objective criteria like factual accuracy, instruction
|
|
following, or toxicity where a clear ground truth or rubric exists.
|
|
|
|
Returns:
|
|
Dictionary containing per-criterion scores, evidence, justifications,
|
|
and a weighted summary.
|
|
"""
|
|
|
|
# Input
|
|
prompt: str = "Explain quantum entanglement to a high school student"
|
|
response: str = (
|
|
"Quantum entanglement is like having two magical coins that are connected. "
|
|
"When you flip one and it lands on heads, the other instantly shows tails, "
|
|
'no matter how far apart they are. Scientists call this "spooky action at a distance."'
|
|
)
|
|
|
|
criteria: list[dict[str, Any]] = [
|
|
{"name": "Accuracy", "description": "Scientific correctness", "weight": 0.4},
|
|
{"name": "Clarity", "description": "Understandable for audience", "weight": 0.3},
|
|
{"name": "Engagement", "description": "Interesting and memorable", "weight": 0.3},
|
|
]
|
|
|
|
# System prompt for the evaluator
|
|
system_prompt: str = (
|
|
"You are an expert evaluator. Assess the response against each criterion.\n\n"
|
|
"For each criterion:\n"
|
|
"1. Find specific evidence in the response\n"
|
|
"2. Score according to the rubric (1-5 scale)\n"
|
|
"3. Justify your score with evidence\n"
|
|
"4. Suggest one specific improvement\n\n"
|
|
"Be objective and consistent. Base scores on explicit evidence."
|
|
)
|
|
|
|
# User prompt structure
|
|
user_prompt: str = f"""## Original Prompt
|
|
{prompt}
|
|
|
|
## Response to Evaluate
|
|
{response}
|
|
|
|
## Criteria
|
|
1. **Accuracy** (weight: 0.4): Scientific correctness
|
|
2. **Clarity** (weight: 0.3): Understandable for audience
|
|
3. **Engagement** (weight: 0.3): Interesting and memorable
|
|
|
|
## Output Format
|
|
Respond with valid JSON:
|
|
{{
|
|
"scores": [
|
|
{{
|
|
"criterion": "Accuracy",
|
|
"score": 4,
|
|
"evidence": ["quote or observation"],
|
|
"justification": "why this score",
|
|
"improvement": "specific suggestion"
|
|
}}
|
|
],
|
|
"summary": {{
|
|
"assessment": "overall quality summary",
|
|
"strengths": ["strength 1"],
|
|
"weaknesses": ["weakness 1"]
|
|
}}
|
|
}}"""
|
|
|
|
# Expected output structure
|
|
expected_output: dict[str, Any] = {
|
|
"scores": [
|
|
{
|
|
"criterion": "Accuracy",
|
|
"score": 4,
|
|
"evidence": ["Correctly uses analogy", "Mentions spooky action at a distance"],
|
|
"justification": "Core concept is correct, analogy is appropriate",
|
|
"improvement": "Could mention it's a quantum mechanical phenomenon",
|
|
},
|
|
{
|
|
"criterion": "Clarity",
|
|
"score": 5,
|
|
"evidence": ["Simple coin analogy", "No jargon"],
|
|
"justification": "Appropriate for high school level",
|
|
"improvement": "None needed",
|
|
},
|
|
{
|
|
"criterion": "Engagement",
|
|
"score": 4,
|
|
"evidence": ["Magical coins", "Spooky action quote"],
|
|
"justification": "Memorable imagery and Einstein quote",
|
|
"improvement": "Could add a real-world application",
|
|
},
|
|
],
|
|
"summary": {
|
|
"assessment": "Good explanation suitable for the target audience",
|
|
"strengths": ["Clear analogy", "Age-appropriate language"],
|
|
"weaknesses": ["Could be more comprehensive"],
|
|
},
|
|
}
|
|
|
|
# Calculate weighted score
|
|
total_weight: float = sum(c["weight"] for c in criteria)
|
|
weighted_score: float = sum(
|
|
s["score"] * next(c["weight"] for c in criteria if c["name"] == s["criterion"])
|
|
for s in expected_output["scores"]
|
|
) / total_weight
|
|
|
|
print(f"Weighted Score: {weighted_score:.2f}/5")
|
|
return expected_output
|
|
|
|
|
|
# =============================================================================
|
|
# PAIRWISE COMPARISON WITH POSITION BIAS MITIGATION
|
|
# =============================================================================
|
|
|
|
|
|
def pairwise_comparison_example() -> dict[str, Any]:
|
|
"""Compare two responses with position-swapped bias mitigation.
|
|
|
|
Use when: evaluating subjective preferences like tone, style, or
|
|
persuasiveness where pairwise comparison achieves higher human-judge
|
|
agreement than direct scoring.
|
|
|
|
Returns:
|
|
Dictionary containing the winner, confidence score, and whether
|
|
position consistency was achieved across both passes.
|
|
"""
|
|
|
|
prompt: str = "Explain machine learning to a beginner"
|
|
|
|
response_a: str = (
|
|
"Machine learning is a subset of artificial intelligence that enables "
|
|
"systems to learn and improve from experience without being explicitly "
|
|
"programmed. It uses statistical techniques to give computers the ability "
|
|
"to identify patterns in data."
|
|
)
|
|
|
|
response_b: str = (
|
|
"Imagine teaching a dog a new trick. You show the dog what to do, give "
|
|
"treats when it's right, and eventually it learns. Machine learning works "
|
|
"similarly - we show computers lots of examples, tell them when they're "
|
|
"right, and they learn to recognize patterns on their own."
|
|
)
|
|
|
|
criteria: list[str] = ["clarity", "accessibility", "accuracy"]
|
|
|
|
# System prompt emphasizing bias awareness
|
|
system_prompt: str = (
|
|
"You are an expert evaluator comparing two AI responses.\n\n"
|
|
"CRITICAL INSTRUCTIONS:\n"
|
|
"- Do NOT prefer responses because they are longer\n"
|
|
"- Do NOT prefer responses based on position (first vs second)\n"
|
|
"- Focus ONLY on quality according to the specified criteria\n"
|
|
"- Ties are acceptable when responses are genuinely equivalent"
|
|
)
|
|
|
|
# Build evaluation prompt for a given ordering
|
|
def evaluate_pass(
|
|
first_response: str,
|
|
second_response: str,
|
|
first_label: str,
|
|
second_label: str,
|
|
) -> str:
|
|
"""Build evaluation prompt for one pass of position-swapped comparison.
|
|
|
|
Use when: constructing the prompt for a single evaluation pass before
|
|
swapping response positions for bias mitigation.
|
|
"""
|
|
return f"""## Original Prompt
|
|
{prompt}
|
|
|
|
## Response {first_label}
|
|
{first_response}
|
|
|
|
## Response {second_label}
|
|
{second_response}
|
|
|
|
## Comparison Criteria
|
|
{', '.join(criteria)}
|
|
|
|
## Output Format
|
|
{{
|
|
"comparison": [
|
|
{{"criterion": "clarity", "winner": "A|B|TIE", "reasoning": "..."}}
|
|
],
|
|
"result": {{
|
|
"winner": "A|B|TIE",
|
|
"confidence": 0.0-1.0,
|
|
"reasoning": "overall reasoning"
|
|
}}
|
|
}}"""
|
|
|
|
# Position bias mitigation protocol
|
|
print("Pass 1: A in first position")
|
|
pass1_result: dict[str, Any] = {"winner": "B", "confidence": 0.8}
|
|
|
|
print("Pass 2: B in first position (swapped)")
|
|
pass2_result: dict[str, Any] = {"winner": "A", "confidence": 0.75} # A because B was first
|
|
|
|
# Map pass2 result back (swap labels)
|
|
def map_winner(winner: str) -> str:
|
|
"""Map winner label after position swap."""
|
|
return {"A": "B", "B": "A", "TIE": "TIE"}[winner]
|
|
|
|
pass2_mapped: str = map_winner(pass2_result["winner"])
|
|
print(f"Pass 2 mapped winner: {pass2_mapped}")
|
|
|
|
# Check consistency
|
|
consistent: bool = pass1_result["winner"] == pass2_mapped
|
|
|
|
final_result: dict[str, Any]
|
|
if consistent:
|
|
final_result = {
|
|
"winner": pass1_result["winner"],
|
|
"confidence": (pass1_result["confidence"] + pass2_result["confidence"]) / 2,
|
|
"position_consistent": True,
|
|
}
|
|
else:
|
|
final_result = {
|
|
"winner": "TIE",
|
|
"confidence": 0.5,
|
|
"position_consistent": False,
|
|
"bias_detected": True,
|
|
}
|
|
|
|
print(f"\nFinal Result: {final_result}")
|
|
return final_result
|
|
|
|
|
|
# =============================================================================
|
|
# RUBRIC GENERATION
|
|
# =============================================================================
|
|
|
|
|
|
def rubric_generation_example() -> dict[str, Any]:
|
|
"""Generate a domain-specific scoring rubric for consistent evaluation.
|
|
|
|
Use when: establishing evaluation standards for a new criterion, reducing
|
|
scoring variance (rubrics cut variance by 40-60%), or onboarding new
|
|
evaluators to an existing evaluation pipeline.
|
|
|
|
Returns:
|
|
Dictionary containing score levels, characteristics, examples,
|
|
scoring guidelines, and edge case handling.
|
|
"""
|
|
|
|
criterion_name: str = "Code Readability"
|
|
criterion_description: str = "How easy the code is to understand and maintain"
|
|
domain: str = "software engineering"
|
|
scale: str = "1-5"
|
|
strictness: str = "balanced"
|
|
|
|
system_prompt: str = (
|
|
f"You are an expert in creating evaluation rubrics.\n"
|
|
f"Create clear, actionable rubrics with distinct boundaries between levels.\n\n"
|
|
f"Strictness: {strictness}\n"
|
|
f"- lenient: Lower bar for passing scores\n"
|
|
f"- balanced: Fair, typical expectations\n"
|
|
f"- strict: High standards, critical evaluation"
|
|
)
|
|
|
|
user_prompt: str = f"""Create a scoring rubric for:
|
|
|
|
**Criterion**: {criterion_name}
|
|
**Description**: {criterion_description}
|
|
**Scale**: {scale}
|
|
**Domain**: {domain}
|
|
|
|
Generate:
|
|
1. Clear descriptions for each score level
|
|
2. Specific characteristics that define each level
|
|
3. Brief example text for each level
|
|
4. General scoring guidelines
|
|
5. Edge cases with guidance"""
|
|
|
|
# Expected rubric structure
|
|
rubric: dict[str, Any] = {
|
|
"criterion": criterion_name,
|
|
"scale": {"min": 1, "max": 5},
|
|
"levels": [
|
|
{
|
|
"score": 1,
|
|
"label": "Poor",
|
|
"description": "Code is difficult to understand without significant effort",
|
|
"characteristics": [
|
|
"No meaningful variable or function names",
|
|
"No comments or documentation",
|
|
"Deeply nested or convoluted logic",
|
|
],
|
|
"example": "def f(x): return x[0]*x[1]+x[2]",
|
|
},
|
|
{
|
|
"score": 3,
|
|
"label": "Adequate",
|
|
"description": "Code is understandable with some effort",
|
|
"characteristics": [
|
|
"Most variables have meaningful names",
|
|
"Basic comments for complex sections",
|
|
"Logic is followable but could be cleaner",
|
|
],
|
|
"example": (
|
|
"def calc_total(items): # calculate sum\n"
|
|
" total = 0\n"
|
|
" for i in items: total += i\n"
|
|
" return total"
|
|
),
|
|
},
|
|
{
|
|
"score": 5,
|
|
"label": "Excellent",
|
|
"description": "Code is immediately clear and maintainable",
|
|
"characteristics": [
|
|
"All names are descriptive and consistent",
|
|
"Comprehensive documentation",
|
|
"Clean, modular structure",
|
|
],
|
|
"example": (
|
|
"def calculate_total_price(items: List[Item]) -> Decimal:\n"
|
|
" '''Calculate the total price of all items.'''\n"
|
|
" return sum(item.price for item in items)"
|
|
),
|
|
},
|
|
],
|
|
"scoring_guidelines": [
|
|
"Focus on readability, not cleverness",
|
|
"Consider the intended audience (team skill level)",
|
|
"Consistency matters more than style preference",
|
|
],
|
|
"edge_cases": [
|
|
{
|
|
"situation": "Code uses domain-specific abbreviations",
|
|
"guidance": "Score based on readability for domain experts, not general audience",
|
|
},
|
|
{
|
|
"situation": "Code is auto-generated",
|
|
"guidance": "Apply same standards but note in evaluation",
|
|
},
|
|
],
|
|
}
|
|
|
|
print("Generated Rubric:")
|
|
for level in rubric["levels"]:
|
|
print(f" {level['score']}: {level['label']} - {level['description']}")
|
|
|
|
return rubric
|
|
|
|
|
|
# =============================================================================
|
|
# MAIN
|
|
# =============================================================================
|
|
|
|
if __name__ == "__main__":
|
|
print("=" * 60)
|
|
print("DIRECT SCORING EXAMPLE")
|
|
print("=" * 60)
|
|
direct_scoring_example()
|
|
|
|
print("\n" + "=" * 60)
|
|
print("PAIRWISE COMPARISON EXAMPLE")
|
|
print("=" * 60)
|
|
pairwise_comparison_example()
|
|
|
|
print("\n" + "=" * 60)
|
|
print("RUBRIC GENERATION EXAMPLE")
|
|
print("=" * 60)
|
|
rubric_generation_example()
|