wehub-resource-sync
79 lines
3.0 KiB
Python
79 lines
3.0 KiB
Python
|
|
""" This example demonstrates evidence and fact checking capabilities in the Prompt class. """
|
|
|
|
import os
|
|
|
|
from llmware.prompts import Prompt
|
|
from llmware.setup import Setup
|
|
from llmware.configs import LLMWareConfig
|
|
from llmware.dataset_tools import Datasets
|
|
|
|
|
|
def contract_analysis_with_fact_checking (model_name):
|
|
|
|
# Load the llmware sample files
|
|
print (f"\n > Loading the llmware sample files...")
|
|
sample_files_path = Setup().load_sample_files()
|
|
contracts_path = os.path.join(sample_files_path,"Agreements")
|
|
|
|
print (f"\n > Loading model {model_name}...")
|
|
|
|
prompter = Prompt().load_model(model_name, temperature=0.0, sample=False)
|
|
|
|
research = {"topic": "base salary", "prompt": "What is the executive's base salary?"}
|
|
|
|
for i, contract in enumerate(os.listdir(contracts_path)):
|
|
|
|
print("\nAnalyzing Contract - ", str(i+1), contract)
|
|
print("Question: ", research["prompt"])
|
|
|
|
# contract is parsed, text-chunked, and then filtered by "base salary'
|
|
source = prompter.add_source_document(contracts_path, contract, query=research["topic"])
|
|
|
|
# calling the LLM with 'source' information from the contract automatically packaged into the prompt
|
|
responses = prompter.prompt_with_source(research["prompt"], prompt_name="default_with_context")
|
|
|
|
# run several fact checks
|
|
ev_numbers = prompter.evidence_check_numbers(responses)
|
|
ev_sources = prompter.evidence_check_sources(responses)
|
|
ev_stats = prompter.evidence_comparison_stats(responses)
|
|
z = prompter.classify_not_found_response(responses, parse_response=True, evidence_match=True,ask_the_model=False)
|
|
|
|
for r, response in enumerate(responses):
|
|
print("LLM Response: ", response["llm_response"])
|
|
print("Numbers: ", ev_numbers[r]["fact_check"])
|
|
print("Sources: ", ev_sources[r]["source_review"])
|
|
print("Stats: ", ev_stats[r]["comparison_stats"])
|
|
print("Not Found Check: ", z[r])
|
|
|
|
# We're done with this contract, clear the source from the prompt
|
|
prompter.clear_source_materials()
|
|
|
|
# Save jsonl report to jsonl to /prompt_history folder
|
|
print("\nPrompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id))
|
|
prompter.save_state()
|
|
|
|
# Optional - builds a dataset from prompt history that is 'model-training-ready'
|
|
ds = Datasets().build_gen_ds_from_prompt_history(prompt_wrapper="human_bot")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
hf_model_list = ["llmware/bling-1b-0.1",
|
|
"llmware/bling-1.4b-0.1",
|
|
"llmware/bling-falcon-1b-0.1",
|
|
"llmware/bling-sheared-llama-1.3b-0.1",
|
|
"bling-phi-3-gguf"]
|
|
|
|
model_name = hf_model_list[0]
|
|
|
|
# to use a 3rd party model, select model_name, e.g., "gpt-4"
|
|
# --if model requires an api_key, then it must be set as an os.environ variable
|
|
# --see example on 'set_model_api_keys.py'
|
|
|
|
contract_analysis_with_fact_checking(model_name)
|
|
|
|
|