wehub-resource-sync
141 lines
5.5 KiB
Python
141 lines
5.5 KiB
Python
import json
|
|
import argparse
|
|
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
|
|
colors = ['#FFEBAD', '#DCD7C1', '#BFB1D0', '#A7C0DE', '#6C91C2', '#F46F43']
|
|
|
|
|
|
def draw_double_histogram(correct_data, incorrect_data, file_path):
|
|
# Create histogram
|
|
plt.figure(figsize=(5, 4))
|
|
plt.hist(correct_data, bins=10, alpha=0.7, label='Correct', color='blue')
|
|
plt.hist(incorrect_data, bins=10, alpha=0.7, label='Incorrect', color='red')
|
|
|
|
# Add labels and title
|
|
plt.xlabel('Top-1 Frequency Averaged by Number of Test Cases')
|
|
plt.ylabel('No. of Data Points')
|
|
# plt.title('Frequency Distribution of Correct and Incorrect Data')
|
|
plt.legend(loc='upper right')
|
|
|
|
# Show the plot
|
|
# plt.show()
|
|
plt.savefig(file_path)
|
|
|
|
|
|
def draw_histogram(data, labels, file_path):
|
|
# Create bar chart for the single group of data
|
|
# Create a bar chart for the three metrics
|
|
plt.figure(figsize=(4, 4))
|
|
# bars = plt.bar(labels, data, color='#5ea69c', width=0.4)
|
|
# bars = plt.bar(labels, data, color=colors[5], width=0.4)
|
|
bars = plt.bar(labels, data, color="#1E90FF", width=0.4)
|
|
# Add text labels above each bar to indicate the values
|
|
for bar in bars:
|
|
yval = bar.get_height()
|
|
plt.text(bar.get_x() + bar.get_width() / 2, yval + 0.005, round(yval, 2), ha='center', va='bottom', fontsize=16)
|
|
|
|
plt.tick_params(axis='x', labelsize=15) # Change the font size of the x-axis scale
|
|
plt.tick_params(axis='y', labelsize=15) # Change the font size of the y-axis scale
|
|
|
|
# Add labels and title
|
|
# plt.xlabel('Category')
|
|
plt.ylabel('Pass@1', fontsize=16)
|
|
# plt.title('Accuracy Comparison Across Different Metrics')
|
|
|
|
plt.ylim(bottom=15)
|
|
|
|
plt.savefig(file_path, bbox_inches='tight', dpi=800, format='png')
|
|
|
|
|
|
def draw_line_plot(correct_data, incorrect_data, file_path):
|
|
# Compute histogram data
|
|
correct_hist, correct_bins = np.histogram(correct_data, bins=10)
|
|
incorrect_hist, incorrect_bins = np.histogram(incorrect_data, bins=10)
|
|
|
|
# Get the center of each bin for plotting
|
|
correct_bin_centers = 0.5 * (correct_bins[1:] + correct_bins[:-1])
|
|
incorrect_bin_centers = 0.5 * (incorrect_bins[1:] + incorrect_bins[:-1])
|
|
|
|
# Create the line plot
|
|
plt.figure(figsize=(4, 4))
|
|
# plt.plot(correct_bin_centers, correct_hist, label='Correct', color='#658873', marker='.', linestyle='-')
|
|
# plt.plot(incorrect_bin_centers, incorrect_hist, label='Incorrect', color='#d2bfa5', marker='.', linestyle='-')
|
|
plt.plot(correct_bin_centers, correct_hist, label='Correct', color=colors[2], marker='.', linestyle='-')
|
|
plt.plot(incorrect_bin_centers, incorrect_hist, label='Incorrect', color=colors[4], marker='.', linestyle='-')
|
|
|
|
# Change the size of the font in the legent
|
|
plt.legend(prop={"size": 15})
|
|
plt.tick_params(axis='x', labelsize=15) # Change the font size of the x-axis scale
|
|
plt.tick_params(axis='y', labelsize=15) # Change the font size of the y-axis scale
|
|
|
|
# Add labels and title
|
|
# plt.xlabel('Top-1 Averaged Frequency', fontsize=16)
|
|
plt.ylabel('No. of Data Points', fontsize=16)
|
|
plt.legend(loc='upper left')
|
|
|
|
plt.savefig(file_path, bbox_inches='tight', dpi=800, format='png')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--freq_file", type=str)
|
|
parser.add_argument("--output_file", type=str)
|
|
args = parser.parse_args()
|
|
|
|
data = json.load(open(args.freq_file))
|
|
|
|
pos_freqs = []
|
|
neg_freqs = []
|
|
difficulties = set([item["difficulty"] for item in data])
|
|
diff_freqs = {
|
|
diff: {"pos": [], "neg": []} for diff in difficulties
|
|
}
|
|
for item in data:
|
|
if item["prog_sc_res"]:
|
|
pos_freqs.append(item["tot_freq"])
|
|
diff_freqs[item["difficulty"]]["pos"].append(item["tot_freq"])
|
|
else:
|
|
neg_freqs.append(item["tot_freq"])
|
|
diff_freqs[item["difficulty"]]["neg"].append(item["tot_freq"])
|
|
|
|
# paint(pos_freqs, neg_freqs, args.output_file.replace(".png", "_all.png"))
|
|
draw_line_plot(pos_freqs, neg_freqs, args.output_file.replace(".png", "_all.png"))
|
|
for diff, freqs in diff_freqs.items():
|
|
# paint(freqs["pos"], freqs["neg"], args.output_file.replace(".png", f"_{diff}.png"))
|
|
draw_line_plot(freqs["pos"], freqs["neg"], args.output_file.replace(".png", f"_{diff}.png"))
|
|
|
|
sc = 0
|
|
prog_sc = 0
|
|
first_res = 0
|
|
diff_res = {
|
|
diff: {"sc": 0, "prog_sc": 0, "first_res": 0} for diff in difficulties
|
|
}
|
|
for item in data:
|
|
if item["sc_res"]:
|
|
sc += 1
|
|
diff_res[item["difficulty"]]["sc"] += 1
|
|
if item["prog_sc_res"]:
|
|
prog_sc += 1
|
|
diff_res[item["difficulty"]]["prog_sc"] += 1
|
|
if item["res"]:
|
|
first_res += 1
|
|
diff_res[item["difficulty"]]["first_res"] += 1
|
|
|
|
print(f"Self-consistency: {sc}/{len(data)} = {sc / len(data)}")
|
|
print(f"Program self-consistency: {prog_sc}/{len(data)} = {prog_sc / len(data)}")
|
|
print(f"First res: {first_res}/{len(data)} = {first_res / len(data)}")
|
|
for diff, res in diff_res.items():
|
|
print(f"Difficulty {diff}:")
|
|
print(f"Self-consistency: {res['sc']}/{len(data)} = {res['sc'] / len(data)}")
|
|
print(f"Program self-consistency: {res['prog_sc']}/{len(data)} = {res['prog_sc'] / len(data)}")
|
|
print(f"First res: {res['first_res']}/{len(data)} = {res['first_res'] / len(data)}")
|
|
|
|
draw_histogram([19.24, prog_sc / len(data) * 100, sc / len(data) * 100],
|
|
["G.D.", "S.C. - P", "S.C. - T"], args.output_file.replace(".png", "_res.png"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|