wehub-resource-sync
405 lines
16 KiB
Python
405 lines
16 KiB
Python
"""
|
|
Evaluation of -.tar.gz file.
|
|
Yu Fang - March 2019
|
|
"""
|
|
|
|
import os
|
|
import xml.dom.minidom
|
|
|
|
# from eval import eval
|
|
|
|
reg_gt_path = os.path.abspath("data/test")
|
|
reg_gt_path_archival = os.path.abspath("data/test")
|
|
reg_gt_path_modern = os.path.abspath("data/test")
|
|
str_gt_path_1 = os.path.abspath("data/test")
|
|
str_gt_path_2 = os.path.abspath("data/test")
|
|
str_gt_path_archival = os.path.abspath("data/test")
|
|
str_gt_path_modern = os.path.abspath("data/test")
|
|
|
|
import xml.dom.minidom
|
|
# from functools import cmp_to_key
|
|
from os.path import join as osj
|
|
from .data_structure import *
|
|
|
|
|
|
class eval:
|
|
STR = "-str"
|
|
REG = "-reg"
|
|
DEFAULT_ENCODING = "UTF-8"
|
|
# reg_gt_path = "./annotations/trackA/"
|
|
# str_gt_path = "./annotations/trackB/"
|
|
# reg_gt_path = os.path.abspath("data/test")
|
|
# reg_gt_path_archival = os.path.abspath("data/test")
|
|
# reg_gt_path_modern = os.path.abspath("data/test")
|
|
# str_gt_path_1 = os.path.abspath("data/test")
|
|
# str_gt_path_2 = os.path.abspath("data/test")
|
|
# str_gt_path_archival = os.path.abspath("data/test")
|
|
# str_gt_path_modern = os.path.abspath("data/test")
|
|
|
|
# dummyDom = xml.dom.minidom.parse("./dummyXML.xml")
|
|
|
|
def __init__(self, track, res_path):
|
|
self.return_result = None
|
|
self.reg = True
|
|
self.str = False
|
|
|
|
self.resultFile = res_path
|
|
self.inPrefix = os.path.split(res_path)[-1].split(".")[0][:-7]
|
|
|
|
if track == "-trackA":
|
|
self.reg = True
|
|
self.GTFile = osj(reg_gt_path, self.inPrefix + ".xml")
|
|
# self.GTFile = osj(self.reg_gt_path, self.inPrefix)
|
|
elif track == "-trackA1": # archival documents
|
|
self.reg = True
|
|
self.GTFile = osj(reg_gt_path_archival, self.inPrefix + ".xml")
|
|
elif track == "-trackA2": # modern documents
|
|
self.reg = True
|
|
self.GTFile = osj(reg_gt_path_modern, self.inPrefix + ".xml")
|
|
elif track == "-trackB1":
|
|
self.str = True
|
|
self.GTFile = osj(str_gt_path_1, self.inPrefix + ".xml")
|
|
# self.GTFile = osj(self.str_gt_path_1, self.inPrefix)
|
|
elif track == "-trackB2":
|
|
self.str = True
|
|
self.GTFile = osj(str_gt_path_2, self.inPrefix + ".xml")
|
|
# print(self.GTFile)
|
|
# self.GTFile = osj(self.str_gt_path_2, self.inPrefix)
|
|
elif track == "-trackB2_a":
|
|
self.str = True
|
|
self.GTFile = osj(str_gt_path_archival, self.inPrefix + ".xml")
|
|
elif track == "-trackB2_m":
|
|
self.str = True
|
|
self.GTFile = osj(str_gt_path_modern, self.inPrefix + ".xml")
|
|
else:
|
|
print(track)
|
|
print("Not a valid track, please check your spelling.")
|
|
|
|
# self.resultFile = res_path
|
|
# self.inPrefix = os.path.split(res_path)[-1].split("-")[0]
|
|
|
|
# if self.str:
|
|
# # self.GTFile = osj(self.str_gt_path, self.inPrefix + "-str.xml")
|
|
# self.GTFile = osj(self.str_gt_path, self.inPrefix + ".xml")
|
|
# elif self.reg:
|
|
# # self.GTFile = osj(self.reg_gt_path, self.inPrefix + "-reg.xml")
|
|
# self.GTFile = osj(self.reg_gt_path, self.inPrefix + ".xml")
|
|
# else:
|
|
# print("Not a valid track, please check your spelling.")
|
|
|
|
self.gene_ret_lst()
|
|
|
|
@property
|
|
def result(self):
|
|
return self.return_result
|
|
|
|
def gene_ret_lst(self):
|
|
ret_lst = []
|
|
for iou in [0.6, 0.7, 0.8, 0.9]:
|
|
temp = self.compute_retVal(iou)
|
|
ret_lst.append(temp)
|
|
# ret_lst.append(self.compute_retVal(iou))
|
|
|
|
ret_lst.append(self.inPrefix + ".xml")
|
|
# ret_lst.append(self.inPrefix)
|
|
# print("Done processing {}\n".format(self.resultFile))
|
|
self.return_result = ret_lst
|
|
|
|
def compute_retVal(self, iou):
|
|
gt_dom = xml.dom.minidom.parse(self.GTFile)
|
|
# incorrect submission format handling
|
|
try:
|
|
result_dom = xml.dom.minidom.parse(self.resultFile)
|
|
except Exception as e:
|
|
# result_dom = xml.dom.minidom.parse(dummyDom)
|
|
gt_tables = eval.get_table_list(gt_dom)
|
|
retVal = ResultStructure(truePos=0, gtTotal=len(gt_tables), resTotal=0)
|
|
return retVal
|
|
|
|
# result_dom = xml.dom.minidom.parse(self.resultFile)
|
|
if self.reg:
|
|
ret = self.evaluate_result_reg(gt_dom, result_dom, iou)
|
|
return ret
|
|
if self.str:
|
|
ret = self.evaluate_result_str(gt_dom, result_dom, iou)
|
|
return ret
|
|
|
|
@staticmethod
|
|
def get_table_list(dom):
|
|
"""
|
|
return a list of Table objects corresponding to the table element of the DOM.
|
|
"""
|
|
return [Table(_nd) for _nd in dom.documentElement.getElementsByTagName("table")]
|
|
|
|
@staticmethod
|
|
def evaluate_result_reg(gt_dom, result_dom, iou_value):
|
|
# parse the tables in input elements
|
|
gt_tables = eval.get_table_list(gt_dom)
|
|
result_tables = eval.get_table_list(result_dom)
|
|
# duplicate result table list
|
|
remaining_tables = result_tables.copy()
|
|
|
|
# map the tables in gt and result file
|
|
table_matches = [] # @param: table_matches - list of mapping of tables in gt and res file, in order (gt, res)
|
|
for gtt in gt_tables:
|
|
for rest in remaining_tables:
|
|
if gtt.compute_table_iou(rest) >= iou_value:
|
|
remaining_tables.remove(rest)
|
|
table_matches.append((gtt, rest))
|
|
break
|
|
|
|
assert len(table_matches) <= len(gt_tables)
|
|
assert len(table_matches) <= len(result_tables)
|
|
|
|
retVal = ResultStructure(truePos=len(table_matches), gtTotal=len(gt_tables), resTotal=len(result_tables))
|
|
return retVal
|
|
|
|
@staticmethod
|
|
def evaluate_result_str(gt_dom, result_dom, iou_value, table_iou_value=0.8):
|
|
# parse the tables in input elements
|
|
gt_tables = eval.get_table_list(gt_dom)
|
|
result_tables = eval.get_table_list(result_dom)
|
|
|
|
# duplicate result table list
|
|
remaining_tables = result_tables.copy()
|
|
gt_remaining = gt_tables.copy()
|
|
|
|
# map the tables in gt and result file
|
|
table_matches = [] # @param: table_matches - list of mapping of tables in gt and res file, in order (gt, res)
|
|
for gtt in gt_remaining:
|
|
for rest in remaining_tables:
|
|
# note: for structural analysis, use 0.8 for table mapping
|
|
if gtt.compute_table_iou(rest) >= table_iou_value:
|
|
table_matches.append((gtt, rest))
|
|
remaining_tables.remove(rest) # unsafe... should be ok with the break below
|
|
gt_remaining.remove(gtt)
|
|
break
|
|
|
|
total_gt_relation, total_res_relation, total_correct_relation = 0, 0, 0
|
|
for gt_table, ress_table in table_matches:
|
|
|
|
# set up the cell mapping for matching tables
|
|
cell_mapping = gt_table.find_cell_mapping(ress_table, iou_value)
|
|
# set up the adj relations, convert the one for result table to a dictionary for faster searching
|
|
gt_AR = gt_table.find_adj_relations()
|
|
total_gt_relation += len(gt_AR)
|
|
|
|
res_AR = ress_table.find_adj_relations()
|
|
total_res_relation += len(res_AR)
|
|
|
|
if False: # for DEBUG
|
|
Table.printCellMapping(cell_mapping)
|
|
Table.printAdjacencyRelationList(gt_AR, "GT")
|
|
Table.printAdjacencyRelationList(res_AR, "run")
|
|
|
|
# Now map GT adjacency relations to result
|
|
lMappedAR = []
|
|
for ar in gt_AR:
|
|
try:
|
|
resFromCell = cell_mapping[ar.fromText]
|
|
resToCell = cell_mapping[ar.toText]
|
|
# make a mapped adjacency relation
|
|
lMappedAR.append(AdjRelation(resFromCell, resToCell, ar.direction))
|
|
except:
|
|
# no mapping is possible
|
|
pass
|
|
|
|
# compare two list of adjacency relation
|
|
correct_dect = 0
|
|
for ar1 in res_AR:
|
|
for ar2 in lMappedAR:
|
|
if ar1.isEqual(ar2):
|
|
correct_dect += 1
|
|
break
|
|
|
|
total_correct_relation += correct_dect
|
|
|
|
# handle gt_relations in unmatched gt table
|
|
for gtt_remain in gt_remaining:
|
|
total_gt_relation += len(gtt_remain.find_adj_relations())
|
|
|
|
# handle gt_relation in unmatched res table
|
|
for res_remain in remaining_tables:
|
|
total_res_relation += len(res_remain.find_adj_relations())
|
|
|
|
retVal = ResultStructure(truePos=total_correct_relation, gtTotal=total_gt_relation, resTotal=total_res_relation)
|
|
return retVal
|
|
|
|
# calculate the gt adj_relations of the missing file
|
|
# @param: file_lst - list of missing ground truth file
|
|
# @param: cur_gt_num - current total of ground truth objects (tables / cells)
|
|
def process_missing_files(track, gt_file_lst, cur_gt_num):
|
|
if track in ["-trackA", "-trackA1", "-trackA2"]:
|
|
gt_file_lst_full = [osj(reg_gt_path, filename) for filename in gt_file_lst]
|
|
for file in gt_file_lst_full:
|
|
if os.path.split(file)[-1].split(".")[-1] == "xml":
|
|
gt_dom = xml.dom.minidom.parse(file)
|
|
gt_root = gt_dom.documentElement
|
|
# tables = []
|
|
table_elements = gt_root.getElementsByTagName("table")
|
|
for res_table in table_elements:
|
|
# t = Table(res_table)
|
|
# tables.append(t)
|
|
cur_gt_num += 1
|
|
return cur_gt_num
|
|
elif track == "-trackB1":
|
|
gt_file_lst_full = [osj(str_gt_path_1, filename) for filename in gt_file_lst]
|
|
for file in gt_file_lst_full:
|
|
if os.path.split(file)[-1].split(".")[-1] == "xml":
|
|
gt_dom = xml.dom.minidom.parse(file)
|
|
gt_root = gt_dom.documentElement
|
|
tables = []
|
|
table_elements = gt_root.getElementsByTagName("table")
|
|
for res_table in table_elements:
|
|
t = Table(res_table)
|
|
tables.append(t)
|
|
for table in tables:
|
|
cur_gt_num += len(table.find_adj_relations())
|
|
return cur_gt_num
|
|
elif track == "-trackB2":
|
|
gt_file_lst_full = [osj(str_gt_path_2, filename) for filename in gt_file_lst]
|
|
for file in gt_file_lst_full:
|
|
if os.path.split(file)[-1].split(".")[-1] == "xml":
|
|
gt_dom = xml.dom.minidom.parse(file)
|
|
gt_root = gt_dom.documentElement
|
|
tables = []
|
|
table_elements = gt_root.getElementsByTagName("table")
|
|
for res_table in table_elements:
|
|
t = Table(res_table)
|
|
tables.append(t)
|
|
for table in tables:
|
|
cur_gt_num += len(table.find_adj_relations())
|
|
return cur_gt_num
|
|
|
|
def calc(F1):
|
|
sum_a = 0.6 * F1[0] + 0.7 * F1[1] + 0.8 * F1[2] + 0.9 * F1[3]
|
|
sum_b = 0.6 + 0.7 + 0.8 + 0.9
|
|
|
|
return sum_a / sum_b
|
|
|
|
def calc_table_score(result_path):
|
|
# measure = eval(*sys.argv[1:])
|
|
|
|
gt_file_lst = os.listdir(reg_gt_path_archival)
|
|
track = "-trackA1"
|
|
untar_path = result_path
|
|
|
|
res_lst = []
|
|
for root, files, dirs in os.walk(untar_path):
|
|
for name in dirs:
|
|
if name.split(".")[-1] == "xml":
|
|
cur_filepath = osj(os.path.abspath(root), name)
|
|
res_lst.append(eval(track, cur_filepath))
|
|
# printing for debug
|
|
# print("Processing... {}".format(name))
|
|
# print("DONE WITH FILE PROCESSING\n")
|
|
# note: results are stored as list of each when iou at [0.6, 0.7, 0.8, 0.9, gt_filename]
|
|
# gt number should be the same for all files
|
|
gt_num = 0
|
|
correct_six, res_six = 0, 0
|
|
correct_seven, res_seven = 0, 0
|
|
correct_eight, res_eight = 0, 0
|
|
correct_nine, res_nine = 0, 0
|
|
|
|
|
|
for each_file in res_lst:
|
|
# print(each_file)
|
|
try:
|
|
gt_file_lst.remove(each_file.result[-1])
|
|
if each_file.result[-1].replace('.xml', '.jpg') in gt_file_lst:
|
|
gt_file_lst.remove(each_file.result[-1].replace('.xml', '.jpg'))
|
|
correct_six += each_file.result[0].truePos
|
|
gt_num += each_file.result[0].gtTotal
|
|
res_six += each_file.result[0].resTotal
|
|
# print("{} {} {}".format(each_file.result[0].truePos, each_file.result[0].gtTotal, each_file.result[0].resTotal))
|
|
|
|
correct_seven += each_file.result[1].truePos
|
|
res_seven += each_file.result[1].resTotal
|
|
|
|
correct_eight += each_file.result[2].truePos
|
|
res_eight += each_file.result[2].resTotal
|
|
|
|
correct_nine += each_file.result[3].truePos
|
|
res_nine += each_file.result[3].resTotal
|
|
except:
|
|
print("Error occur in processing result list.")
|
|
print(each_file.result[-1])
|
|
break
|
|
# print(each_file.result[-1])
|
|
# print(each_file)
|
|
|
|
# for file in gt_file_lst:
|
|
# if file.split(".") != "xml":
|
|
# gt_file_lst.remove(file)
|
|
# # print(gt_file_lst)
|
|
|
|
for i in range(len(gt_file_lst) - 1, -1, -1):
|
|
if gt_file_lst[i].split(".")[-1] != "xml":
|
|
del gt_file_lst[i]
|
|
|
|
if len(gt_file_lst) > 0:
|
|
print("\nWarning: missing result annotations for file: {}\n".format(gt_file_lst))
|
|
gt_total = process_missing_files(track, gt_file_lst, gt_num)
|
|
else:
|
|
gt_total = gt_num
|
|
|
|
|
|
try:
|
|
# print("Evaluation of {}".format(track.replace("-", "")))
|
|
# iou @ 0.6
|
|
p_six = correct_six / res_six
|
|
r_six = correct_six / gt_total
|
|
f1_six = 2 * p_six * r_six / (p_six + r_six)
|
|
print("IOU @ 0.6 -\nprecision: {}\nrecall: {}\nf1: {}".format(p_six, r_six, f1_six))
|
|
print("correct: {}, gt: {}, res: {}\n".format(correct_six, gt_total, res_six))
|
|
|
|
# iou @ 0.7
|
|
p_seven = correct_seven / res_seven
|
|
r_seven = correct_seven / gt_total
|
|
f1_seven = 2 * p_seven * r_seven / (p_seven + r_seven)
|
|
print("IOU @ 0.7 -\nprecision: {}\nrecall: {}\nf1: {}".format(p_seven, r_seven, f1_seven))
|
|
print("correct: {}, gt: {}, res: {}\n".format(correct_seven, gt_total, res_seven))
|
|
|
|
# iou @ 0.8
|
|
p_eight = correct_eight / res_eight
|
|
r_eight = correct_eight / gt_total
|
|
f1_eight = 2 * p_eight * r_eight / (p_eight + r_eight)
|
|
print("IOU @ 0.8 -\nprecision: {}\nrecall: {}\nf1: {}".format(p_eight, r_eight, f1_eight))
|
|
print("correct: {}, gt: {}, res: {}\n".format(correct_eight, gt_total, res_eight))
|
|
|
|
# iou @ 0.9
|
|
p_nine = correct_nine / res_nine
|
|
r_nine = correct_nine / gt_total
|
|
f1_nine = 2 * p_nine * r_nine / (p_nine + r_nine)
|
|
print("IOU @ 0.9 -\nprecision: {}\nrecall: {}\nf1: {}".format(p_nine, r_nine, f1_nine))
|
|
print("correct: {}, gt: {}, res: {}".format(correct_nine, gt_total, res_nine))
|
|
|
|
F1 = [f1_six, f1_seven, f1_eight, f1_nine]
|
|
wF1 = calc(F1)
|
|
|
|
print("Average weight F1: {}".format(wF1))
|
|
|
|
return {
|
|
'p_six':p_six * 100,
|
|
"r_six":r_six * 100,
|
|
"f1_six":f1_six * 100,
|
|
"p_seven":p_seven * 100,
|
|
"r_seven":r_seven * 100,
|
|
"f1_seven":f1_seven * 100,
|
|
"p_eight":p_eight * 100,
|
|
"r_eight":r_eight * 100,
|
|
"f1_eight":f1_eight * 100,
|
|
"p_nine":p_nine * 100,
|
|
"r_nine":r_nine * 100,
|
|
"f1_nine":f1_nine * 100,
|
|
"wF1":wF1 * 100
|
|
}
|
|
except ZeroDivisionError:
|
|
print(
|
|
"Error: zero devision error found, (possible that no adjacency relations are found), please check the file input.")
|
|
return {"wF1": 0}
|
|
|
|
|
|
if __name__=="__main__":
|
|
pass
|