wehub-resource-sync
296 lines
13 KiB
Python
296 lines
13 KiB
Python
import os
|
||
import time
|
||
import base64
|
||
import requests
|
||
from typing import List, Dict, Any, Optional
|
||
|
||
from dotenv import load_dotenv
|
||
from groundx import GroundX, Document
|
||
from openai import OpenAI
|
||
|
||
import opik
|
||
from opik.evaluation import evaluate
|
||
from opik.evaluation.metrics import BaseMetric, GEval
|
||
|
||
load_dotenv()
|
||
|
||
def evaluate_invoice_parsing(model_output: str, expected_output: str, question: str):
|
||
"""
|
||
Evaluate invoice parsing results using Comet Opik's GEval metrics.
|
||
|
||
Args:
|
||
model_output (str): The model's extracted answer
|
||
expected_output (str): The expected correct answer
|
||
question (str): The question that was asked
|
||
|
||
Returns:
|
||
dict: A dictionary containing evaluation results with the following structure:
|
||
{
|
||
"overall_score": float, # Score on 0-10 scale
|
||
"reason": str, # Detailed reason for the score
|
||
"passed": bool, # Whether score >= 7.0 (70% threshold)
|
||
"error": str, optional # Error message if evaluation fails
|
||
}
|
||
"""
|
||
try:
|
||
# Validate input
|
||
if not model_output or not expected_output:
|
||
raise ValueError("Model output and expected output cannot be empty")
|
||
|
||
# Build the context string that includes both expected and model answers
|
||
context = f"QUESTION: {question}\nEXPECTED_ANSWER: {expected_output}\nMODEL_ANSWER: {model_output}"
|
||
|
||
# Define rubric scoring criteria for invoice parsing
|
||
accuracy_rubric_text = (
|
||
"Score 0-2: Completely incorrect or missing information\n"
|
||
"Score 3-5: Partially correct but missing key details\n"
|
||
"Score 6-8: Mostly correct with minor inaccuracies\n"
|
||
"Score 9-10: Completely accurate and matches expected answer"
|
||
)
|
||
|
||
# Invoice Parsing Accuracy Metric
|
||
accuracy_metric = GEval(
|
||
task_introduction=(
|
||
"You are an expert judge evaluating invoice parsing accuracy. "
|
||
"Compare the model's extracted answer against the expected answer. "
|
||
"Focus on whether the model correctly extracted the requested information. "
|
||
"Use the following rubric to assign scores:"
|
||
),
|
||
evaluation_criteria=(
|
||
"EVALUATION STEPS:\n"
|
||
"1. Read the QUESTION carefully to understand what information was requested.\n"
|
||
"2. Compare the EXPECTED_ANSWER with the MODEL_ANSWER.\n"
|
||
"3. Check if the model extracted the correct information.\n"
|
||
"4. Consider formatting differences (e.g., '$8.45' vs '8.45' vs 'The bill total is $8.45').\n"
|
||
"5. For 'data not available' cases, check if the model correctly indicated missing information.\n\n"
|
||
"SCORING RUBRIC:\n"
|
||
f"{accuracy_rubric_text}\n\n"
|
||
"SPECIAL CASES:\n"
|
||
"- If expected answer is 'data not available', the model should indicate information is not present\n"
|
||
"- If expected answer has a specific value, the model should match it exactly or semantically\n"
|
||
"- Ignore extra words like 'The bill total is' or 'The account number is' - focus on the actual value\n\n"
|
||
"Return only a score between 0 and 10, and a concise reason that references the rubric."
|
||
),
|
||
name="Invoice Parsing Accuracy",
|
||
)
|
||
|
||
# Run evaluation using direct GEval scoring
|
||
accuracy_result = accuracy_metric.score(output=context)
|
||
|
||
# Convert score from Opik's 0-1 scale to 0-10 scale
|
||
accuracy_score = accuracy_result.value * 10
|
||
|
||
# Return results
|
||
return {
|
||
"overall_score": accuracy_score,
|
||
"reason": accuracy_result.reason,
|
||
"passed": accuracy_score >= 7.0, # 70% threshold
|
||
}
|
||
|
||
except Exception as e:
|
||
# Error handling
|
||
return {
|
||
"error": f"Error evaluating invoice parsing: {str(e)}",
|
||
"overall_score": 0.0,
|
||
"reason": "Evaluation failed",
|
||
"passed": False,
|
||
}
|
||
|
||
|
||
class EvaluatorGEval:
|
||
"""Evaluator that scores answers using GEval (LLM judge with rubric)."""
|
||
|
||
def __init__(self, groundx_api_key: str, openai_api_key: str, comet_api_key: Optional[str] = None):
|
||
self.gx = GroundX(api_key=groundx_api_key)
|
||
self.oa = OpenAI(api_key=openai_api_key)
|
||
self.opik = opik.Opik(api_key=comet_api_key) if comet_api_key else opik.Opik()
|
||
self.bucket_id = self._ensure_bucket()
|
||
|
||
# ---------------- bucket helpers -----------------
|
||
def _ensure_bucket(self, name: str = "gx_eval") -> str:
|
||
for b in self.gx.buckets.list().buckets:
|
||
if b.name == name:
|
||
return b.bucket_id
|
||
return self.gx.buckets.create(name=name).bucket.bucket_id
|
||
|
||
# ---------------- ingestion helpers --------------
|
||
def process_invoice(self, file_path: str, file_name: str, mime_type: str) -> Dict[str, Any]:
|
||
ingest = self.gx.ingest(documents=[
|
||
Document(
|
||
bucket_id=int(self.bucket_id),
|
||
file_name=file_name,
|
||
file_path=file_path,
|
||
file_type=mime_type.split("/")[-1],
|
||
)
|
||
])
|
||
self._poll_until_complete(ingest.ingest.process_id)
|
||
return self._fetch_xray(file_name)
|
||
|
||
def _poll_until_complete(self, pid: str, timeout: int = 600):
|
||
start = time.time()
|
||
while True:
|
||
status = self.gx.documents.get_processing_status_by_id(process_id=pid).ingest.status
|
||
if status in {"complete", "error", "cancelled"}:
|
||
break
|
||
if time.time() - start > timeout:
|
||
raise TimeoutError("Ground X processing timed out.")
|
||
time.sleep(3)
|
||
if status != "complete":
|
||
raise RuntimeError(f"Ingest finished with status {status}")
|
||
|
||
def _fetch_xray(self, expected_name: str):
|
||
docs = self.gx.documents.lookup(id=self.bucket_id).documents
|
||
doc = next((d for d in docs if getattr(d, "file_name", None) == expected_name), docs[0])
|
||
if getattr(doc, "xray_url", None):
|
||
r = requests.get(doc.xray_url)
|
||
r.raise_for_status()
|
||
return r.json()
|
||
raise RuntimeError("X-Ray URL missing")
|
||
|
||
# ---------------- context & prompting ------------
|
||
@staticmethod
|
||
def _context(xray: Dict[str, Any]) -> str:
|
||
parts = []
|
||
if s := xray.get("fileSummary"):
|
||
parts.append(f"Summary: {s}")
|
||
for page in xray.get("documentPages", [])[:2]:
|
||
texts = [ch.get("text", "")[:500] for ch in page.get("chunks", [])[:3] if ch.get("text")]
|
||
if texts:
|
||
parts.append("Document Content: " + " ".join(texts))
|
||
return "\n\n".join(parts)
|
||
|
||
def _gpt_ctx(self, q: str, ctx: str) -> str:
|
||
prompt = (
|
||
"You are an AI assistant analysing an invoice.\n\n" + ctx +
|
||
f"\n\nUser Question: {q}\n\nAnswer concisely. If unknown reply 'data not available'."
|
||
)
|
||
resp = self.oa.chat.completions.create(
|
||
model="gpt-4o",
|
||
messages=[{"role": "user", "content": prompt}],
|
||
max_tokens=300,
|
||
temperature=0.3,
|
||
)
|
||
return resp.choices[0].message.content
|
||
|
||
def _gpt_direct(self, q: str, raw: bytes, mime: str) -> str:
|
||
img_uri = None
|
||
if mime.startswith("image/"):
|
||
img_uri = f"data:{mime};base64,{base64.b64encode(raw).decode()}"
|
||
print(f"DEBUG: Using image file directly")
|
||
elif mime == "application/pdf":
|
||
try:
|
||
import fitz
|
||
page = fitz.open(stream=raw, filetype="pdf").load_page(0)
|
||
png = page.get_pixmap(dpi=180).tobytes("png")
|
||
img_uri = f"data:image/png;base64,{base64.b64encode(png).decode()}"
|
||
print(f"DEBUG: Successfully converted PDF to image")
|
||
except Exception as e:
|
||
print(f"DEBUG: PDF conversion failed: {e}")
|
||
# Fallback: try to extract text from PDF
|
||
try:
|
||
import fitz
|
||
doc = fitz.open(stream=raw, filetype="pdf")
|
||
text = ""
|
||
for page in doc:
|
||
text += page.get_text()
|
||
doc.close()
|
||
# Send text instead of image
|
||
resp = self.oa.chat.completions.create(
|
||
model="gpt-4o",
|
||
messages=[
|
||
{"role": "system", "content": "You are an expert document analyser. Analyze the following invoice text and answer the question."},
|
||
{"role": "user", "content": f"Invoice text:\n{text}\n\nQuestion: {q}"}
|
||
],
|
||
max_tokens=300,
|
||
temperature=0.3,
|
||
)
|
||
return resp.choices[0].message.content
|
||
except Exception as e2:
|
||
print(f"DEBUG: Text extraction also failed: {e2}")
|
||
return "Error: Unable to process PDF document"
|
||
|
||
user_content: List[Dict[str, Any]] = [{"type": "text", "text": f"Parse the invoice and answer: {q}"}]
|
||
if img_uri:
|
||
user_content.append({"type": "image_url", "image_url": {"url": img_uri}})
|
||
print(f"DEBUG: Sending image to GPT-4o")
|
||
else:
|
||
print(f"DEBUG: No image available, sending text only")
|
||
|
||
resp = self.oa.chat.completions.create(
|
||
model="gpt-4o",
|
||
messages=[{"role": "system", "content": "You are an expert document analyser."},
|
||
{"role": "user", "content": user_content}],
|
||
max_tokens=300,
|
||
temperature=0.3,
|
||
)
|
||
return resp.choices[0].message.content
|
||
|
||
# ---------------- evaluation ---------------------
|
||
def run(self, invoices: List[Dict[str, Any]], questions: List[str]):
|
||
dataset = self.opik.get_or_create_dataset("GroundX vs GPT4o")
|
||
samples = []
|
||
for inv in invoices:
|
||
ctx = self._context(inv["xray_data"])
|
||
for q in questions:
|
||
expected_output = inv.get("expected_outputs", {}).get(q, "")
|
||
samples.append({
|
||
"input": f"Invoice {inv['name']} – {q}",
|
||
"context": ctx,
|
||
"invoice_name": inv["name"],
|
||
"expected_output": expected_output,
|
||
})
|
||
# Validation output for GEval parameters
|
||
print(f"DEBUG: Question='{q}', Expected='{expected_output}' for {inv['name']}")
|
||
|
||
dataset.clear(); dataset.insert(samples)
|
||
|
||
def gx_task(sample):
|
||
output = self._gpt_ctx(sample["input"], sample["context"])
|
||
print(f"DEBUG: GroundX output for '{sample['input']}': '{output}'")
|
||
return {"output": output}
|
||
|
||
def gpt_task(sample):
|
||
inv = next(i for i in invoices if i["name"] == sample["invoice_name"])
|
||
output = self._gpt_direct(sample["input"], inv["raw_bytes"], inv["mime_type"])
|
||
print(f"DEBUG: GPT-4o output for '{sample['input']}': '{output}'")
|
||
return {"output": output}
|
||
|
||
gx_results = []
|
||
gpt_results = []
|
||
|
||
for i, sample in enumerate(samples):
|
||
model_output = gx_task(sample)["output"]
|
||
expected_output = sample["expected_output"]
|
||
question = sample["input"]
|
||
|
||
gx_eval_result = evaluate_invoice_parsing(model_output, expected_output, question)
|
||
gx_results.append(gx_eval_result)
|
||
|
||
gpt_model_output = gpt_task(sample)["output"]
|
||
gpt_eval_result = evaluate_invoice_parsing(gpt_model_output, expected_output, question)
|
||
gpt_results.append(gpt_eval_result)
|
||
|
||
# Log evaluation results to Opik dataset
|
||
sample["groundx_score"] = gx_eval_result["overall_score"]
|
||
sample["groundx_reason"] = gx_eval_result["reason"]
|
||
sample["gpt4o_score"] = gpt_eval_result["overall_score"]
|
||
sample["gpt4o_reason"] = gpt_eval_result["reason"]
|
||
sample["groundx_output"] = model_output
|
||
sample["gpt4o_output"] = gpt_model_output
|
||
|
||
# Update dataset with evaluation results
|
||
dataset.clear()
|
||
dataset.insert(samples)
|
||
|
||
return {"groundx_parsing": gx_results, "gpt4o_direct": gpt_results, "dataset": dataset}
|
||
|
||
|
||
def create_evaluator_geval() -> EvaluatorGEval:
|
||
load_dotenv()
|
||
gx_key = os.environ.get("GROUNDX_API_KEY")
|
||
oa_key = os.environ.get("OPENAI_API_KEY")
|
||
comet_key = os.environ.get("COMET_API_KEY")
|
||
if not gx_key or not oa_key:
|
||
raise ValueError("API keys missing")
|
||
return EvaluatorGEval(gx_key, oa_key, comet_key)
|