import os import time import base64 import requests from typing import List, Dict, Any, Optional from dotenv import load_dotenv from groundx import GroundX, Document from openai import OpenAI import opik from opik.evaluation import evaluate from opik.evaluation.metrics import BaseMetric, GEval load_dotenv() def evaluate_invoice_parsing(model_output: str, expected_output: str, question: str): """ Evaluate invoice parsing results using Comet Opik's GEval metrics. Args: model_output (str): The model's extracted answer expected_output (str): The expected correct answer question (str): The question that was asked Returns: dict: A dictionary containing evaluation results with the following structure: { "overall_score": float, # Score on 0-10 scale "reason": str, # Detailed reason for the score "passed": bool, # Whether score >= 7.0 (70% threshold) "error": str, optional # Error message if evaluation fails } """ try: # Validate input if not model_output or not expected_output: raise ValueError("Model output and expected output cannot be empty") # Build the context string that includes both expected and model answers context = f"QUESTION: {question}\nEXPECTED_ANSWER: {expected_output}\nMODEL_ANSWER: {model_output}" # Define rubric scoring criteria for invoice parsing accuracy_rubric_text = ( "Score 0-2: Completely incorrect or missing information\n" "Score 3-5: Partially correct but missing key details\n" "Score 6-8: Mostly correct with minor inaccuracies\n" "Score 9-10: Completely accurate and matches expected answer" ) # Invoice Parsing Accuracy Metric accuracy_metric = GEval( task_introduction=( "You are an expert judge evaluating invoice parsing accuracy. " "Compare the model's extracted answer against the expected answer. " "Focus on whether the model correctly extracted the requested information. " "Use the following rubric to assign scores:" ), evaluation_criteria=( "EVALUATION STEPS:\n" "1. Read the QUESTION carefully to understand what information was requested.\n" "2. Compare the EXPECTED_ANSWER with the MODEL_ANSWER.\n" "3. Check if the model extracted the correct information.\n" "4. Consider formatting differences (e.g., '$8.45' vs '8.45' vs 'The bill total is $8.45').\n" "5. For 'data not available' cases, check if the model correctly indicated missing information.\n\n" "SCORING RUBRIC:\n" f"{accuracy_rubric_text}\n\n" "SPECIAL CASES:\n" "- If expected answer is 'data not available', the model should indicate information is not present\n" "- If expected answer has a specific value, the model should match it exactly or semantically\n" "- Ignore extra words like 'The bill total is' or 'The account number is' - focus on the actual value\n\n" "Return only a score between 0 and 10, and a concise reason that references the rubric." ), name="Invoice Parsing Accuracy", ) # Run evaluation using direct GEval scoring accuracy_result = accuracy_metric.score(output=context) # Convert score from Opik's 0-1 scale to 0-10 scale accuracy_score = accuracy_result.value * 10 # Return results return { "overall_score": accuracy_score, "reason": accuracy_result.reason, "passed": accuracy_score >= 7.0, # 70% threshold } except Exception as e: # Error handling return { "error": f"Error evaluating invoice parsing: {str(e)}", "overall_score": 0.0, "reason": "Evaluation failed", "passed": False, } class EvaluatorGEval: """Evaluator that scores answers using GEval (LLM judge with rubric).""" def __init__(self, groundx_api_key: str, openai_api_key: str, comet_api_key: Optional[str] = None): self.gx = GroundX(api_key=groundx_api_key) self.oa = OpenAI(api_key=openai_api_key) self.opik = opik.Opik(api_key=comet_api_key) if comet_api_key else opik.Opik() self.bucket_id = self._ensure_bucket() # ---------------- bucket helpers ----------------- def _ensure_bucket(self, name: str = "gx_eval") -> str: for b in self.gx.buckets.list().buckets: if b.name == name: return b.bucket_id return self.gx.buckets.create(name=name).bucket.bucket_id # ---------------- ingestion helpers -------------- def process_invoice(self, file_path: str, file_name: str, mime_type: str) -> Dict[str, Any]: ingest = self.gx.ingest(documents=[ Document( bucket_id=int(self.bucket_id), file_name=file_name, file_path=file_path, file_type=mime_type.split("/")[-1], ) ]) self._poll_until_complete(ingest.ingest.process_id) return self._fetch_xray(file_name) def _poll_until_complete(self, pid: str, timeout: int = 600): start = time.time() while True: status = self.gx.documents.get_processing_status_by_id(process_id=pid).ingest.status if status in {"complete", "error", "cancelled"}: break if time.time() - start > timeout: raise TimeoutError("Ground X processing timed out.") time.sleep(3) if status != "complete": raise RuntimeError(f"Ingest finished with status {status}") def _fetch_xray(self, expected_name: str): docs = self.gx.documents.lookup(id=self.bucket_id).documents doc = next((d for d in docs if getattr(d, "file_name", None) == expected_name), docs[0]) if getattr(doc, "xray_url", None): r = requests.get(doc.xray_url) r.raise_for_status() return r.json() raise RuntimeError("X-Ray URL missing") # ---------------- context & prompting ------------ @staticmethod def _context(xray: Dict[str, Any]) -> str: parts = [] if s := xray.get("fileSummary"): parts.append(f"Summary: {s}") for page in xray.get("documentPages", [])[:2]: texts = [ch.get("text", "")[:500] for ch in page.get("chunks", [])[:3] if ch.get("text")] if texts: parts.append("Document Content: " + " ".join(texts)) return "\n\n".join(parts) def _gpt_ctx(self, q: str, ctx: str) -> str: prompt = ( "You are an AI assistant analysing an invoice.\n\n" + ctx + f"\n\nUser Question: {q}\n\nAnswer concisely. If unknown reply 'data not available'." ) resp = self.oa.chat.completions.create( model="gpt-4o", messages=[{"role": "user", "content": prompt}], max_tokens=300, temperature=0.3, ) return resp.choices[0].message.content def _gpt_direct(self, q: str, raw: bytes, mime: str) -> str: img_uri = None if mime.startswith("image/"): img_uri = f"data:{mime};base64,{base64.b64encode(raw).decode()}" print(f"DEBUG: Using image file directly") elif mime == "application/pdf": try: import fitz page = fitz.open(stream=raw, filetype="pdf").load_page(0) png = page.get_pixmap(dpi=180).tobytes("png") img_uri = f"data:image/png;base64,{base64.b64encode(png).decode()}" print(f"DEBUG: Successfully converted PDF to image") except Exception as e: print(f"DEBUG: PDF conversion failed: {e}") # Fallback: try to extract text from PDF try: import fitz doc = fitz.open(stream=raw, filetype="pdf") text = "" for page in doc: text += page.get_text() doc.close() # Send text instead of image resp = self.oa.chat.completions.create( model="gpt-4o", messages=[ {"role": "system", "content": "You are an expert document analyser. Analyze the following invoice text and answer the question."}, {"role": "user", "content": f"Invoice text:\n{text}\n\nQuestion: {q}"} ], max_tokens=300, temperature=0.3, ) return resp.choices[0].message.content except Exception as e2: print(f"DEBUG: Text extraction also failed: {e2}") return "Error: Unable to process PDF document" user_content: List[Dict[str, Any]] = [{"type": "text", "text": f"Parse the invoice and answer: {q}"}] if img_uri: user_content.append({"type": "image_url", "image_url": {"url": img_uri}}) print(f"DEBUG: Sending image to GPT-4o") else: print(f"DEBUG: No image available, sending text only") resp = self.oa.chat.completions.create( model="gpt-4o", messages=[{"role": "system", "content": "You are an expert document analyser."}, {"role": "user", "content": user_content}], max_tokens=300, temperature=0.3, ) return resp.choices[0].message.content # ---------------- evaluation --------------------- def run(self, invoices: List[Dict[str, Any]], questions: List[str]): dataset = self.opik.get_or_create_dataset("GroundX vs GPT4o") samples = [] for inv in invoices: ctx = self._context(inv["xray_data"]) for q in questions: expected_output = inv.get("expected_outputs", {}).get(q, "") samples.append({ "input": f"Invoice {inv['name']} – {q}", "context": ctx, "invoice_name": inv["name"], "expected_output": expected_output, }) # Validation output for GEval parameters print(f"DEBUG: Question='{q}', Expected='{expected_output}' for {inv['name']}") dataset.clear(); dataset.insert(samples) def gx_task(sample): output = self._gpt_ctx(sample["input"], sample["context"]) print(f"DEBUG: GroundX output for '{sample['input']}': '{output}'") return {"output": output} def gpt_task(sample): inv = next(i for i in invoices if i["name"] == sample["invoice_name"]) output = self._gpt_direct(sample["input"], inv["raw_bytes"], inv["mime_type"]) print(f"DEBUG: GPT-4o output for '{sample['input']}': '{output}'") return {"output": output} gx_results = [] gpt_results = [] for i, sample in enumerate(samples): model_output = gx_task(sample)["output"] expected_output = sample["expected_output"] question = sample["input"] gx_eval_result = evaluate_invoice_parsing(model_output, expected_output, question) gx_results.append(gx_eval_result) gpt_model_output = gpt_task(sample)["output"] gpt_eval_result = evaluate_invoice_parsing(gpt_model_output, expected_output, question) gpt_results.append(gpt_eval_result) # Log evaluation results to Opik dataset sample["groundx_score"] = gx_eval_result["overall_score"] sample["groundx_reason"] = gx_eval_result["reason"] sample["gpt4o_score"] = gpt_eval_result["overall_score"] sample["gpt4o_reason"] = gpt_eval_result["reason"] sample["groundx_output"] = model_output sample["gpt4o_output"] = gpt_model_output # Update dataset with evaluation results dataset.clear() dataset.insert(samples) return {"groundx_parsing": gx_results, "gpt4o_direct": gpt_results, "dataset": dataset} def create_evaluator_geval() -> EvaluatorGEval: load_dotenv() gx_key = os.environ.get("GROUNDX_API_KEY") oa_key = os.environ.get("OPENAI_API_KEY") comet_key = os.environ.get("COMET_API_KEY") if not gx_key or not oa_key: raise ValueError("API keys missing") return EvaluatorGEval(gx_key, oa_key, comet_key)