chore: import upstream snapshot with attribution

2026-07-13 12:37:47 +08:00
commit 7653f56fed
1422 changed files with 359026 additions and 0 deletions
@@ -0,0 +1,76 @@
+fallback_response = (
+    "I'm sorry, I couldn't find an answer for that — can I help with something else?"
+)
+
+
+def codex_validated_query(query_engine, project, user_query):
+    """
+    Validates a user query against a RAG pipeline response using Cleanlab Codex.
+
+    Args:
+        query_engine: The RAG pipeline query engine.
+        project: The Cleanlab Codex project instance.
+        user_query: The user's query string.
+
+    Returns:
+        A tuple containing an emoji representing trustworthiness, the trust score, and the final response.
+    """
+    # Step 1: Get response from your RAG pipeline
+    response_obj = query_engine.query(user_query)
+    initial_response = str(response_obj)
+
+    # Step 2: Convert to message format
+    context = response_obj.source_nodes
+    context_str = "\n".join([n.node.text for n in context])
+
+    prompt_template = (
+        "Context information is below.\n"
+        "---------------------\n"
+        "{context}\n"
+        "---------------------\n"
+        "Given the context information above, I want you to think step by step to answer the query in a crisp manner. "
+        "First, carefully check if the answer can be found in the provided context. "
+        "If the answer is available in the context, use that information to respond. "
+        "If the answer is not available in the context or the context is insufficient, "
+        "you may use your own knowledge to provide a helpful response. "
+        "Only say 'I don't know!' if you cannot answer the question using either the context or your general knowledge.\n"
+        "Query: {query}\n"
+        "Answer: "
+    )
+
+    user_prompt = prompt_template.format(context=context_str, query=user_query)
+    messages = [
+        {
+            "role": "user",
+            "content": user_prompt,
+        }
+    ]
+
+    # Step 3: Validate with Codex
+    result = project.validate(
+        messages=messages,
+        query=user_query,
+        context=context_str,
+        response=initial_response,
+    )
+
+    # Step 4: Return Codex-evaluated final response
+    final_response = (
+        result.expert_answer
+        if result.expert_answer and result.escalated_to_sme
+        else fallback_response if result.should_guardrail else initial_response
+    )
+
+    # Step 5: Return both final response and full validation info
+    trust_score = result.model_dump()["eval_scores"]["trustworthiness"]["score"]
+
+    # Determine emoji based on score
+    if trust_score >= 0.8:
+        emoji = "🟢"
+    elif trust_score >= 0.5:
+        emoji = "🟡"
+    else:
+        emoji = "🔴"
+
+    # Return emoji, trust score, and final response
+    return emoji, trust_score, final_response