wehub-resource-sync
104 lines
4.5 KiB
Python
104 lines
4.5 KiB
Python
from typing import Dict, List, Optional
|
|
|
|
from torch.utils.data import Dataset
|
|
|
|
from transformers import Seq2SeqTrainer, is_torch_tpu_available
|
|
from transformers.trainer_utils import PredictionOutput
|
|
|
|
|
|
if is_torch_tpu_available():
|
|
import torch_xla.core.xla_model as xm
|
|
import torch_xla.debug.metrics as met
|
|
|
|
|
|
class QuestionAnsweringSeq2SeqTrainer(Seq2SeqTrainer):
|
|
def __init__(self, *args, eval_examples=None, post_process_function=None, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self.eval_examples = eval_examples
|
|
self.post_process_function = post_process_function
|
|
|
|
# def evaluate(self, eval_dataset=None, eval_examples=None, ignore_keys=None, metric_key_prefix: str = "eval"):
|
|
def evaluate(
|
|
self,
|
|
eval_dataset: Optional[Dataset] = None,
|
|
eval_examples=None,
|
|
ignore_keys: Optional[List[str]] = None,
|
|
metric_key_prefix: str = "eval",
|
|
max_length: Optional[int] = None,
|
|
num_beams: Optional[int] = None,
|
|
) -> Dict[str, float]:
|
|
self._max_length = max_length if max_length is not None else self.args.generation_max_length
|
|
self._num_beams = num_beams if num_beams is not None else self.args.generation_num_beams
|
|
|
|
eval_dataset = self.eval_dataset if eval_dataset is None else eval_dataset
|
|
eval_dataloader = self.get_eval_dataloader(eval_dataset)
|
|
eval_examples = self.eval_examples if eval_examples is None else eval_examples
|
|
|
|
# Temporarily disable metric computation, we will do it in the loop here.
|
|
compute_metrics = self.compute_metrics
|
|
self.compute_metrics = None
|
|
eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
|
|
try:
|
|
output = eval_loop(
|
|
eval_dataloader,
|
|
description="Evaluation",
|
|
# No point gathering the predictions if there are no metrics, otherwise we defer to
|
|
# self.args.prediction_loss_only
|
|
prediction_loss_only=True if compute_metrics is None else None,
|
|
ignore_keys=ignore_keys,
|
|
)
|
|
finally:
|
|
self.compute_metrics = compute_metrics
|
|
|
|
if self.post_process_function is not None and self.compute_metrics is not None:
|
|
eval_preds = self.post_process_function(eval_examples, eval_dataset, output)
|
|
metrics = self.compute_metrics(eval_preds)
|
|
|
|
# Prefix all keys with metric_key_prefix + '_'
|
|
for key in list(metrics.keys()):
|
|
if not key.startswith(f"{metric_key_prefix}_"):
|
|
metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)
|
|
|
|
self.log(metrics)
|
|
else:
|
|
metrics = {}
|
|
|
|
if self.args.tpu_metrics_debug or self.args.debug:
|
|
# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
|
|
xm.master_print(met.metrics_report())
|
|
|
|
self.control = self.callback_handler.on_evaluate(self.args, self.state, self.control, metrics)
|
|
return metrics
|
|
|
|
def predict(self, predict_dataset, predict_examples, ignore_keys=None, metric_key_prefix: str = "test"):
|
|
predict_dataloader = self.get_test_dataloader(predict_dataset)
|
|
|
|
# Temporarily disable metric computation, we will do it in the loop here.
|
|
compute_metrics = self.compute_metrics
|
|
self.compute_metrics = None
|
|
eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
|
|
try:
|
|
output = eval_loop(
|
|
predict_dataloader,
|
|
description="Prediction",
|
|
# No point gathering the predictions if there are no metrics, otherwise we defer to
|
|
# self.args.prediction_loss_only
|
|
prediction_loss_only=True if compute_metrics is None else None,
|
|
ignore_keys=ignore_keys,
|
|
)
|
|
finally:
|
|
self.compute_metrics = compute_metrics
|
|
|
|
if self.post_process_function is None or self.compute_metrics is None:
|
|
return output
|
|
|
|
predictions = self.post_process_function(predict_examples, predict_dataset, output.predictions, "predict")
|
|
metrics = self.compute_metrics(predictions)
|
|
|
|
# Prefix all keys with metric_key_prefix + '_'
|
|
for key in list(metrics.keys()):
|
|
if not key.startswith(f"{metric_key_prefix}_"):
|
|
metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)
|
|
|
|
return PredictionOutput(predictions=predictions.predictions, label_ids=predictions.label_ids, metrics=metrics)
|