wehub-resource-sync
54 lines
1.6 KiB
Python
54 lines
1.6 KiB
Python
"""Base class for benchmark core metric extraction."""
|
|
|
|
from abc import ABC, abstractmethod
|
|
|
|
|
|
class BenchmarkProcessor(ABC):
|
|
"""Base class for benchmark core metric extraction."""
|
|
|
|
# Metrics where higher values are better (default assumption)
|
|
# Override in subclass if needed
|
|
HIGHER_IS_BETTER: set[str] = {
|
|
"accuracy",
|
|
"exact_match",
|
|
"f1",
|
|
"f1_score",
|
|
"macro_f1",
|
|
"correct_rate",
|
|
"success_rate",
|
|
"gold_hit_rate",
|
|
"score",
|
|
"scaffold_hard",
|
|
"kendall_tau",
|
|
"ROUGE-L",
|
|
}
|
|
|
|
@classmethod
|
|
@abstractmethod
|
|
def match(cls, benchmark_name: str) -> bool:
|
|
"""Check if this processor handles the given benchmark."""
|
|
pass
|
|
|
|
@classmethod
|
|
@abstractmethod
|
|
def get_core_metric(cls, accuracy_summary: dict) -> tuple[str, float, bool] | None:
|
|
"""Extract core metric name, value, and direction from accuracy_summary.
|
|
|
|
Args:
|
|
accuracy_summary: {dataset_name: {metric: value, ...}, ...}
|
|
|
|
Returns:
|
|
(metric_name, value, higher_is_better) or None
|
|
- metric_name: includes "(average)" suffix if multiple datasets
|
|
- value: the score
|
|
- higher_is_better: True if higher values are better, False otherwise
|
|
"""
|
|
pass
|
|
|
|
@classmethod
|
|
def is_higher_better(cls, metric_name: str) -> bool:
|
|
"""Check if higher values are better for this metric."""
|
|
# Remove (average) suffix for checking
|
|
base_metric = metric_name.replace(" (average)", "").strip()
|
|
return base_metric.lower() in {m.lower() for m in cls.HIGHER_IS_BETTER}
|