wehub-resource-sync
235 lines
9.3 KiB
Python
235 lines
9.3 KiB
Python
import matplotlib.patheffects as PathEffects
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
|
|
from . import colors
|
|
|
|
xlabel_names = {
|
|
"remove absolute": "Fraction removed",
|
|
"remove positive": "Fraction removed",
|
|
"remove negative": "Fraction removed",
|
|
"keep absolute": "Fraction kept",
|
|
"keep positive": "Fraction kept",
|
|
"keep negative": "Fraction kept",
|
|
"explanation error": "Explanation error as std dev.",
|
|
"compute time": "Seconds per. sample",
|
|
}
|
|
|
|
|
|
def benchmark(benchmark, show=True):
|
|
"""Plot a BenchmarkResult or list of such results."""
|
|
if hasattr(benchmark, "__iter__"):
|
|
benchmark = list(benchmark)
|
|
|
|
# see if we have multiple metrics or just a single metric
|
|
single_metric = True
|
|
metric_name = None
|
|
has_curves = True
|
|
for b in benchmark:
|
|
if metric_name is None:
|
|
metric_name = b.metric
|
|
elif metric_name != b.metric:
|
|
single_metric = False
|
|
|
|
if b.curve_x is None or b.curve_y is None:
|
|
has_curves = False
|
|
|
|
methods = list({b.method for b in benchmark})
|
|
methods.sort()
|
|
method_color = {}
|
|
|
|
for i, m in enumerate(methods):
|
|
method_color[m] = colors.red_blue_circle(i / len(methods))
|
|
|
|
# plot a single metric benchmark result
|
|
if single_metric and has_curves:
|
|
benchmark.sort(key=lambda b: -b.value_sign * b.value)
|
|
for i, b in enumerate(benchmark):
|
|
plt.fill_between(
|
|
b.curve_x,
|
|
b.curve_y - b.curve_y_std,
|
|
b.curve_y + b.curve_y_std,
|
|
color=method_color[b.method],
|
|
alpha=0.1,
|
|
linewidth=0,
|
|
)
|
|
for i, b in enumerate(benchmark):
|
|
plt.plot(
|
|
b.curve_x,
|
|
b.curve_y,
|
|
color=method_color[b.method],
|
|
linewidth=2,
|
|
label=b.method + f" ({b.value:0.3})",
|
|
)
|
|
# plt.fill_between(b.curve_x, b.curve_y - b.curve_y_std, b.curve_y + b.curve_y_std, color=method_color[b.method], alpha=0.2)
|
|
ax = plt.gca()
|
|
ax.set_xlabel(xlabel_names[metric_name], fontsize=13)
|
|
ax.set_ylabel("Model output", fontsize=13)
|
|
ax.xaxis.set_ticks_position("bottom")
|
|
ax.yaxis.set_ticks_position("left")
|
|
ax.spines["right"].set_visible(False)
|
|
ax.spines["top"].set_visible(False)
|
|
plt.title(metric_name.capitalize())
|
|
plt.legend(fontsize=11)
|
|
if show:
|
|
plt.show()
|
|
|
|
elif single_metric:
|
|
benchmark.sort(key=lambda b: -b.value_sign * b.value)
|
|
|
|
values = np.array([b.value for b in benchmark])
|
|
total_width = 0.7
|
|
bar_width = total_width
|
|
# for i, b in enumerate(benchmark):
|
|
# ypos_offset = 0#- ((i - len(values) / 2) * bar_width + bar_width / 2)
|
|
plt.barh(
|
|
np.arange(len(values)),
|
|
values,
|
|
bar_width,
|
|
align="center",
|
|
color=[method_color[b.method] for b in benchmark],
|
|
edgecolor=(1, 1, 1, 0.8),
|
|
)
|
|
# plt.plot(
|
|
# b.curve_x, b.curve_y,
|
|
# color=method_color[b.method],
|
|
# linewidth=2,
|
|
# label=b.method + f" ({b.value:0.3})"
|
|
# )
|
|
ax = plt.gca()
|
|
ax.set_yticks(np.arange(len(methods)))
|
|
ax.set_yticklabels([b.method for b in benchmark], rotation=0, fontsize=11)
|
|
ax.set_xlabel(xlabel_names[metric_name], fontsize=13)
|
|
# ax.set_ylabel("Model output", fontsize=13)
|
|
ax.xaxis.set_ticks_position("bottom")
|
|
ax.yaxis.set_ticks_position("left")
|
|
ax.spines["right"].set_visible(False)
|
|
ax.spines["top"].set_visible(False)
|
|
plt.title(metric_name.capitalize())
|
|
# plt.legend(fontsize=11)
|
|
plt.gca().invert_yaxis()
|
|
if show:
|
|
plt.show()
|
|
|
|
# plot a multi-metric benchmark result
|
|
else:
|
|
# get a list of all the metrics in the order they first appear
|
|
metrics = []
|
|
for b in benchmark:
|
|
if b.metric not in metrics:
|
|
metrics.append(b.metric)
|
|
|
|
# compute normalized values
|
|
max_value = {n: -np.inf for n in metrics}
|
|
min_value = {n: np.inf for n in metrics}
|
|
for b in benchmark:
|
|
if max_value[b.metric] < b.value_sign * b.value:
|
|
max_value[b.metric] = b.value_sign * b.value
|
|
if min_value[b.metric] > b.value_sign * b.value:
|
|
min_value[b.metric] = b.value_sign * b.value
|
|
norm_values = {}
|
|
for b in benchmark:
|
|
norm_values[b.full_name] = (b.value_sign * b.value - min_value[b.metric]) / (
|
|
max_value[b.metric] - min_value[b.metric]
|
|
)
|
|
|
|
# compute the average value for each method and sort by it
|
|
# global_values = {}
|
|
# global_counts = {}
|
|
# for b in benchmark:
|
|
# global_values[b.method] = global_values.get(b.method, 0) + norm_values[b.full_name]
|
|
# global_counts[b.method] = global_counts.get(b.method, 0) + 1
|
|
# for k in global_values:
|
|
# global_values[k] /= global_counts[k]
|
|
|
|
# sort by the first and then second metric
|
|
metric_0 = {}
|
|
metric_1 = {}
|
|
for b in benchmark:
|
|
if b.metric == metrics[0]:
|
|
metric_0[b.method] = b.value
|
|
elif b.metric == metrics[1]:
|
|
metric_1[b.method] = b.value
|
|
|
|
methods.sort(key=lambda method: (np.round(metric_0[method], 3), metric_1[method]))
|
|
xs = [-0.03 * (len(methods) - 1)] + list(range(len(metrics) + 1))
|
|
for i, method in enumerate(methods):
|
|
scores = [1 - i / (len(methods) - 1), 1 - i / (len(methods) - 1)]
|
|
values = [None, None]
|
|
for metric in metrics:
|
|
for b in benchmark:
|
|
if b.method == method and b.metric == metric:
|
|
scores.append(norm_values[b.full_name])
|
|
values.append(b.value)
|
|
plt.plot(xs, scores, color=method_color[method], label=method)
|
|
|
|
for x, y, value in zip(xs, scores, values):
|
|
if value is None:
|
|
continue
|
|
label = f"{value:.2f}"
|
|
txt = plt.annotate(
|
|
label, # this is the text
|
|
(x, y), # these are the coordinates to position the label
|
|
textcoords="offset points", # how to position the text
|
|
xytext=(0, -3), # distance from text to points (x,y)
|
|
ha="center", # horizontal alignment can be left, right or center
|
|
color=method_color[method],
|
|
fontsize=9,
|
|
)
|
|
txt.set_path_effects([PathEffects.withStroke(linewidth=5, foreground="w")])
|
|
|
|
ax = plt.gca()
|
|
ax.set_yticks([1 - i / (len(methods) - 1) for i in range(len(methods))])
|
|
ax.set_yticklabels(methods, rotation=0, fontsize=11)
|
|
|
|
ax.set_xticks(np.arange(len(metrics) + 1))
|
|
# from matplotlib import rcParams
|
|
# rcParams['text.latex.preamble'] = [r'\boldmath']
|
|
ax.set_xticklabels([""] + [m.capitalize() for m in metrics], rotation=45, ha="left", fontsize=11)
|
|
|
|
ax.xaxis.tick_top()
|
|
plt.grid(which="major", axis="x", linestyle="--")
|
|
ax.spines["top"].set_visible(False)
|
|
ax.spines["bottom"].set_visible(False)
|
|
ax.spines["right"].set_visible(False)
|
|
ax.spines["left"].set_visible(False)
|
|
ax.yaxis.set_ticks_position("none")
|
|
ax.xaxis.set_ticks_position("none")
|
|
plt.xlim(xs[0], len(metrics))
|
|
# for l in ax.get_xticklabels():
|
|
# l.set_fontweight('bold')
|
|
ax.get_xticklabels()[1].set_fontweight("bold")
|
|
# plt.gca().invert_yaxis()
|
|
# plt.ylabel("\nAll scores are relative")
|
|
# ax.yaxis.set_label_position("right")
|
|
if show:
|
|
plt.show()
|
|
|
|
# plot a single benchmark result
|
|
else:
|
|
plt.fill_between(
|
|
benchmark.curve_x,
|
|
benchmark.curve_y - benchmark.curve_y_std,
|
|
benchmark.curve_y + benchmark.curve_y_std,
|
|
color=colors.blue_rgb,
|
|
alpha=0.1,
|
|
linewidth=0,
|
|
)
|
|
plt.plot(
|
|
benchmark.curve_x,
|
|
benchmark.curve_y,
|
|
color=colors.blue_rgb,
|
|
linewidth=2,
|
|
label=benchmark.method + f" ({benchmark.value:0.3})",
|
|
)
|
|
ax = plt.gca()
|
|
ax.set_xlabel(xlabel_names[benchmark.metric], fontsize=13)
|
|
ax.set_ylabel("Model output", fontsize=13)
|
|
ax.xaxis.set_ticks_position("bottom")
|
|
ax.yaxis.set_ticks_position("left")
|
|
ax.spines["right"].set_visible(False)
|
|
ax.spines["top"].set_visible(False)
|
|
plt.legend(fontsize=11)
|
|
if show:
|
|
plt.show()
|