wehub-resource-sync
324 lines
14 KiB
Python
324 lines
14 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
import math
|
|
from collections import namedtuple
|
|
import numpy as np
|
|
from shapely.geometry import Polygon
|
|
|
|
|
|
class DetectionDetEvalEvaluator(object):
|
|
def __init__(
|
|
self,
|
|
area_recall_constraint=0.8, area_precision_constraint=0.4,
|
|
ev_param_ind_center_diff_thr=1,
|
|
mtype_oo_o=1.0, mtype_om_o=0.8, mtype_om_m=1.0
|
|
):
|
|
|
|
|
|
self.area_recall_constraint = area_recall_constraint
|
|
self.area_precision_constraint = area_precision_constraint
|
|
self.ev_param_ind_center_diff_thr = ev_param_ind_center_diff_thr
|
|
self.mtype_oo_o = mtype_oo_o
|
|
self.mtype_om_o = mtype_om_o
|
|
self.mtype_om_m = mtype_om_m
|
|
|
|
def evaluate_image(self, gt, pred):
|
|
|
|
def get_union(pD,pG):
|
|
return Polygon(pD).union(Polygon(pG)).area
|
|
|
|
def get_intersection_over_union(pD,pG):
|
|
return get_intersection(pD, pG) / get_union(pD, pG)
|
|
|
|
def get_intersection(pD,pG):
|
|
return Polygon(pD).intersection(Polygon(pG)).area
|
|
|
|
def one_to_one_match(row, col):
|
|
cont = 0
|
|
for j in range(len(recallMat[0])):
|
|
if recallMat[row,j] >= self.area_recall_constraint and precisionMat[row,j] >= self.area_precision_constraint:
|
|
cont = cont +1
|
|
if (cont != 1):
|
|
return False
|
|
cont = 0
|
|
for i in range(len(recallMat)):
|
|
if recallMat[i,col] >= self.area_recall_constraint and precisionMat[i,col] >= self.area_precision_constraint:
|
|
cont = cont +1
|
|
if (cont != 1):
|
|
return False
|
|
|
|
if recallMat[row,col] >= self.area_recall_constraint and precisionMat[row,col] >= self.area_precision_constraint:
|
|
return True
|
|
return False
|
|
|
|
def num_overlaps_gt(gtNum):
|
|
cont = 0
|
|
for detNum in range(len(detRects)):
|
|
if detNum not in detDontCareRectsNum:
|
|
if recallMat[gtNum,detNum] > 0 :
|
|
cont = cont +1
|
|
return cont
|
|
|
|
def num_overlaps_det(detNum):
|
|
cont = 0
|
|
for gtNum in range(len(recallMat)):
|
|
if gtNum not in gtDontCareRectsNum:
|
|
if recallMat[gtNum,detNum] > 0 :
|
|
cont = cont +1
|
|
return cont
|
|
|
|
def is_single_overlap(row, col):
|
|
if num_overlaps_gt(row)==1 and num_overlaps_det(col)==1:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def one_to_many_match(gtNum):
|
|
many_sum = 0
|
|
detRects = []
|
|
for detNum in range(len(recallMat[0])):
|
|
if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and detNum not in detDontCareRectsNum:
|
|
if precisionMat[gtNum,detNum] >= self.area_precision_constraint:
|
|
many_sum += recallMat[gtNum,detNum]
|
|
detRects.append(detNum)
|
|
if round(many_sum,4) >= self.area_recall_constraint:
|
|
return True,detRects
|
|
else:
|
|
return False,[]
|
|
|
|
def many_to_one_match(detNum):
|
|
many_sum = 0
|
|
gtRects = []
|
|
for gtNum in range(len(recallMat)):
|
|
if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and gtNum not in gtDontCareRectsNum:
|
|
if recallMat[gtNum,detNum] >= self.area_recall_constraint:
|
|
many_sum += precisionMat[gtNum,detNum]
|
|
gtRects.append(gtNum)
|
|
if round(many_sum,4) >= self.area_precision_constraint:
|
|
return True,gtRects
|
|
else:
|
|
return False,[]
|
|
|
|
def center_distance(r1, r2):
|
|
return ((np.mean(r1, axis=0) - np.mean(r2, axis=0)) ** 2).sum() ** 0.5
|
|
|
|
def diag(r):
|
|
r = np.array(r)
|
|
return ((r[:, 0].max() - r[:, 0].min()) ** 2 + (r[:, 1].max() - r[:, 1].min()) ** 2) ** 0.5
|
|
|
|
perSampleMetrics = {}
|
|
|
|
recall = 0
|
|
precision = 0
|
|
hmean = 0
|
|
recallAccum = 0.
|
|
precisionAccum = 0.
|
|
gtRects = []
|
|
detRects = []
|
|
gtPolPoints = []
|
|
detPolPoints = []
|
|
gtDontCareRectsNum = []#Array of Ground Truth Rectangles' keys marked as don't Care
|
|
detDontCareRectsNum = []#Array of Detected Rectangles' matched with a don't Care GT
|
|
pairs = []
|
|
evaluationLog = ""
|
|
|
|
recallMat = np.empty([1,1])
|
|
precisionMat = np.empty([1,1])
|
|
|
|
for n in range(len(gt)):
|
|
points = gt[n]['points']
|
|
# transcription = gt[n]['text']
|
|
dontCare = gt[n]['ignore']
|
|
|
|
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
|
continue
|
|
|
|
gtRects.append(points)
|
|
gtPolPoints.append(points)
|
|
if dontCare:
|
|
gtDontCareRectsNum.append( len(gtRects)-1 )
|
|
|
|
evaluationLog += "GT rectangles: " + str(len(gtRects)) + (" (" + str(len(gtDontCareRectsNum)) + " don't care)\n" if len(gtDontCareRectsNum)>0 else "\n")
|
|
|
|
for n in range(len(pred)):
|
|
points = pred[n]['points']
|
|
|
|
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
|
continue
|
|
|
|
detRect = points
|
|
detRects.append(detRect)
|
|
detPolPoints.append(points)
|
|
if len(gtDontCareRectsNum)>0 :
|
|
for dontCareRectNum in gtDontCareRectsNum:
|
|
dontCareRect = gtRects[dontCareRectNum]
|
|
intersected_area = get_intersection(dontCareRect,detRect)
|
|
rdDimensions = Polygon(detRect).area
|
|
if (rdDimensions==0) :
|
|
precision = 0
|
|
else:
|
|
precision= intersected_area / rdDimensions
|
|
if (precision > self.area_precision_constraint):
|
|
detDontCareRectsNum.append( len(detRects)-1 )
|
|
break
|
|
|
|
evaluationLog += "DET rectangles: " + str(len(detRects)) + (" (" + str(len(detDontCareRectsNum)) + " don't care)\n" if len(detDontCareRectsNum)>0 else "\n")
|
|
|
|
if len(gtRects)==0:
|
|
recall = 1
|
|
precision = 0 if len(detRects)>0 else 1
|
|
|
|
if len(detRects)>0:
|
|
#Calculate recall and precision matrixs
|
|
outputShape=[len(gtRects),len(detRects)]
|
|
recallMat = np.empty(outputShape)
|
|
precisionMat = np.empty(outputShape)
|
|
gtRectMat = np.zeros(len(gtRects),np.int8)
|
|
detRectMat = np.zeros(len(detRects),np.int8)
|
|
for gtNum in range(len(gtRects)):
|
|
for detNum in range(len(detRects)):
|
|
rG = gtRects[gtNum]
|
|
rD = detRects[detNum]
|
|
intersected_area = get_intersection(rG,rD)
|
|
rgDimensions = Polygon(rG).area
|
|
rdDimensions = Polygon(rD).area
|
|
recallMat[gtNum,detNum] = 0 if rgDimensions==0 else intersected_area / rgDimensions
|
|
precisionMat[gtNum,detNum] = 0 if rdDimensions==0 else intersected_area / rdDimensions
|
|
|
|
# Find one-to-one matches
|
|
evaluationLog += "Find one-to-one matches\n"
|
|
for gtNum in range(len(gtRects)):
|
|
for detNum in range(len(detRects)):
|
|
if gtRectMat[gtNum] == 0 and detRectMat[detNum] == 0 and gtNum not in gtDontCareRectsNum and detNum not in detDontCareRectsNum :
|
|
match = one_to_one_match(gtNum, detNum)
|
|
if match is True :
|
|
#in deteval we have to make other validation before mark as one-to-one
|
|
if is_single_overlap(gtNum, detNum) is True :
|
|
rG = gtRects[gtNum]
|
|
rD = detRects[detNum]
|
|
normDist = center_distance(rG, rD);
|
|
normDist /= diag(rG) + diag(rD);
|
|
normDist *= 2.0;
|
|
if normDist < self.ev_param_ind_center_diff_thr:
|
|
gtRectMat[gtNum] = 1
|
|
detRectMat[detNum] = 1
|
|
recallAccum += self.mtype_oo_o
|
|
precisionAccum += self.mtype_oo_o
|
|
pairs.append({'gt':gtNum,'det':detNum,'type':'OO'})
|
|
evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(detNum) + "\n"
|
|
else:
|
|
evaluationLog += "Match Discarded GT #" + str(gtNum) + " with Det #" + str(detNum) + " normDist: " + str(normDist) + " \n"
|
|
else:
|
|
evaluationLog += "Match Discarded GT #" + str(gtNum) + " with Det #" + str(detNum) + " not single overlap\n"
|
|
# Find one-to-many matches
|
|
evaluationLog += "Find one-to-many matches\n"
|
|
for gtNum in range(len(gtRects)):
|
|
if gtNum not in gtDontCareRectsNum:
|
|
match,matchesDet = one_to_many_match(gtNum)
|
|
if match is True :
|
|
evaluationLog += "num_overlaps_gt=" + str(num_overlaps_gt(gtNum))
|
|
#in deteval we have to make other validation before mark as one-to-one
|
|
if num_overlaps_gt(gtNum)>=2 :
|
|
gtRectMat[gtNum] = 1
|
|
recallAccum += (self.mtype_oo_o if len(matchesDet)==1 else self.mtype_om_o)
|
|
precisionAccum += (self.mtype_oo_o if len(matchesDet)==1 else self.mtype_om_o*len(matchesDet))
|
|
pairs.append({'gt':gtNum,'det':matchesDet,'type': 'OO' if len(matchesDet)==1 else 'OM'})
|
|
for detNum in matchesDet :
|
|
detRectMat[detNum] = 1
|
|
evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(matchesDet) + "\n"
|
|
else:
|
|
evaluationLog += "Match Discarded GT #" + str(gtNum) + " with Det #" + str(matchesDet) + " not single overlap\n"
|
|
|
|
# Find many-to-one matches
|
|
evaluationLog += "Find many-to-one matches\n"
|
|
for detNum in range(len(detRects)):
|
|
if detNum not in detDontCareRectsNum:
|
|
match,matchesGt = many_to_one_match(detNum)
|
|
if match is True :
|
|
#in deteval we have to make other validation before mark as one-to-one
|
|
if num_overlaps_det(detNum)>=2 :
|
|
detRectMat[detNum] = 1
|
|
recallAccum += (self.mtype_oo_o if len(matchesGt)==1 else self.mtype_om_m*len(matchesGt))
|
|
precisionAccum += (self.mtype_oo_o if len(matchesGt)==1 else self.mtype_om_m)
|
|
pairs.append({'gt':matchesGt,'det':detNum,'type': 'OO' if len(matchesGt)==1 else 'MO'})
|
|
for gtNum in matchesGt :
|
|
gtRectMat[gtNum] = 1
|
|
evaluationLog += "Match GT #" + str(matchesGt) + " with Det #" + str(detNum) + "\n"
|
|
else:
|
|
evaluationLog += "Match Discarded GT #" + str(matchesGt) + " with Det #" + str(detNum) + " not single overlap\n"
|
|
|
|
numGtCare = (len(gtRects) - len(gtDontCareRectsNum))
|
|
if numGtCare == 0:
|
|
recall = float(1)
|
|
precision = float(0) if len(detRects)>0 else float(1)
|
|
else:
|
|
recall = float(recallAccum) / numGtCare
|
|
precision = float(0) if (len(detRects) - len(detDontCareRectsNum))==0 else float(precisionAccum) / (len(detRects) - len(detDontCareRectsNum))
|
|
hmean = 0 if (precision + recall)==0 else 2.0 * precision * recall / (precision + recall)
|
|
|
|
numGtCare = len(gtRects) - len(gtDontCareRectsNum)
|
|
numDetCare = len(detRects) - len(detDontCareRectsNum)
|
|
|
|
perSampleMetrics = {
|
|
'precision':precision,
|
|
'recall':recall,
|
|
'hmean':hmean,
|
|
'pairs':pairs,
|
|
'recallMat':[] if len(detRects)>100 else recallMat.tolist(),
|
|
'precisionMat':[] if len(detRects)>100 else precisionMat.tolist(),
|
|
'gtPolPoints':gtPolPoints,
|
|
'detPolPoints':detPolPoints,
|
|
'gtCare': numGtCare,
|
|
'detCare': numDetCare,
|
|
'gtDontCare':gtDontCareRectsNum,
|
|
'detDontCare':detDontCareRectsNum,
|
|
'recallAccum':recallAccum,
|
|
'precisionAccum':precisionAccum,
|
|
'evaluationLog': evaluationLog
|
|
}
|
|
|
|
return perSampleMetrics
|
|
|
|
def combine_results(self, results):
|
|
numGt = 0
|
|
numDet = 0
|
|
methodRecallSum = 0
|
|
methodPrecisionSum = 0
|
|
|
|
for result in results:
|
|
numGt += result['gtCare']
|
|
numDet += result['detCare']
|
|
methodRecallSum += result['recallAccum']
|
|
methodPrecisionSum += result['precisionAccum']
|
|
|
|
methodRecall = 0 if numGt==0 else methodRecallSum/numGt
|
|
methodPrecision = 0 if numDet==0 else methodPrecisionSum/numDet
|
|
methodHmean = 0 if methodRecall + methodPrecision==0 else 2* methodRecall * methodPrecision / (methodRecall + methodPrecision)
|
|
|
|
methodMetrics = {'precision':methodPrecision, 'recall':methodRecall,'hmean': methodHmean }
|
|
|
|
return methodMetrics
|
|
|
|
|
|
if __name__=='__main__':
|
|
evaluator = DetectionDetEvalEvaluator()
|
|
gts = [[{
|
|
'points': [(0, 0), (1, 0), (1, 1), (0, 1)],
|
|
'text': 1234,
|
|
'ignore': False,
|
|
}, {
|
|
'points': [(2, 2), (3, 2), (3, 3), (2, 3)],
|
|
'text': 5678,
|
|
'ignore': True,
|
|
}]]
|
|
preds = [[{
|
|
'points': [(0.1, 0.1), (1, 0), (1, 1), (0, 1)],
|
|
'text': 123,
|
|
'ignore': False,
|
|
}]]
|
|
results = []
|
|
for gt, pred in zip(gts, preds):
|
|
results.append(evaluator.evaluate_image(gt, pred))
|
|
metrics = evaluator.combine_results(results)
|
|
print(metrics)
|