wehub-resource-sync
50 lines
1.7 KiB
Python
50 lines
1.7 KiB
Python
import argparse
|
|
import json
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Get agent's gaia score")
|
|
parser.add_argument('--file', default='/Users/tangjiabin/Documents/reasoning/metachain/evaluation_results/gaia/system_triage_agent/claude-3-5-sonnet-20241022_maxiter/output.jsonl', type=str, help="Path to the agent's output.jsonl")
|
|
args = parser.parse_args()
|
|
this_log = args.file
|
|
outs = []
|
|
with open(this_log, 'r') as f:
|
|
lines = f.readlines()
|
|
for line in lines:
|
|
outs.append(json.loads(line))
|
|
print(f'Reading {this_log}')
|
|
print(f'Metadata:\n {outs[0]["metadata"]}')
|
|
|
|
total = 0
|
|
success = 0
|
|
l1_total = 0
|
|
l1_success = 0
|
|
l2_total = 0
|
|
l2_success = 0
|
|
l3_total = 0
|
|
l3_success = 0
|
|
for out in outs:
|
|
total += 1
|
|
if out['test_result']['score']:
|
|
success += 1
|
|
if out['instance']['Level'] == "1":
|
|
l1_total += 1
|
|
if out['test_result']['score']:
|
|
l1_success += 1
|
|
elif out['instance']['Level'] == "2":
|
|
l2_total += 1
|
|
if out['test_result']['score']:
|
|
l2_success += 1
|
|
elif out['instance']['Level'] == "3":
|
|
l3_total += 1
|
|
if out['test_result']['score']:
|
|
l3_success += 1
|
|
print(f'Success rate: {success}/{total} = {success/total * 100:.4f}%')
|
|
print(f'L1 success rate: {l1_success}/{l1_total} = {l1_success/l1_total * 100:.4f}%')
|
|
print(f'L2 success rate: {l2_success}/{l2_total} = {l2_success/l2_total * 100:.4f}%')
|
|
print(f'L3 success rate: {l3_success}/{l3_total} = {l3_success/l3_total * 100:.4f}%')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|