wehub-resource-sync
228 lines
9.4 KiB
Python
228 lines
9.4 KiB
Python
# -*- coding:utf-8 -*-
|
|
# Author: hankcs
|
|
# Date: 2020-06-12 17:56
|
|
from typing import Optional, Iterable
|
|
|
|
import torch
|
|
|
|
from hanlp.metrics.metric import Metric
|
|
|
|
|
|
class CategoricalAccuracy(Metric):
|
|
"""
|
|
Categorical Top-K accuracy. Assumes integer labels, with
|
|
each item to be classified having a single correct class.
|
|
Tie break enables equal distribution of scores among the
|
|
classes with same maximum predicted scores.
|
|
Copied from AllenNLP and added several methods.
|
|
"""
|
|
|
|
def __init__(self, top_k: int = 1, tie_break: bool = False) -> None:
|
|
if top_k > 1 and tie_break:
|
|
raise ValueError(
|
|
"Tie break in Categorical Accuracy can be done only for maximum (top_k = 1)"
|
|
)
|
|
if top_k <= 0:
|
|
raise ValueError("top_k passed to Categorical Accuracy must be > 0")
|
|
self._top_k = top_k
|
|
self._tie_break = tie_break
|
|
self.correct_count = 0.0
|
|
self.total_count = 0.0
|
|
|
|
def __call__(
|
|
self,
|
|
predictions: torch.Tensor,
|
|
gold_labels: torch.Tensor,
|
|
mask: Optional[torch.BoolTensor] = None,
|
|
):
|
|
"""
|
|
# Parameters
|
|
|
|
predictions : `torch.Tensor`, required.
|
|
A tensor of predictions of shape (batch_size, ..., num_classes).
|
|
gold_labels : `torch.Tensor`, required.
|
|
A tensor of integer class label of shape (batch_size, ...). It must be the same
|
|
shape as the `predictions` tensor without the `num_classes` dimension.
|
|
mask : `torch.BoolTensor`, optional (default = `None`).
|
|
A masking tensor the same size as `gold_labels`.
|
|
"""
|
|
predictions, gold_labels, mask = self.detach_tensors(predictions, gold_labels, mask)
|
|
|
|
# Some sanity checks.
|
|
num_classes = predictions.size(-1)
|
|
if gold_labels.dim() != predictions.dim() - 1:
|
|
raise ValueError(
|
|
"gold_labels must have dimension == predictions.size() - 1 but "
|
|
"found tensor of shape: {}".format(predictions.size())
|
|
)
|
|
if (gold_labels >= num_classes).any():
|
|
raise ValueError(
|
|
"A gold label passed to Categorical Accuracy contains an id >= {}, "
|
|
"the number of classes.".format(num_classes)
|
|
)
|
|
|
|
predictions = predictions.view((-1, num_classes))
|
|
gold_labels = gold_labels.view(-1).long()
|
|
if not self._tie_break:
|
|
# Top K indexes of the predictions (or fewer, if there aren't K of them).
|
|
# Special case topk == 1, because it's common and .max() is much faster than .topk().
|
|
if self._top_k == 1:
|
|
top_k = predictions.max(-1)[1].unsqueeze(-1)
|
|
else:
|
|
top_k = predictions.topk(min(self._top_k, predictions.shape[-1]), -1)[1]
|
|
|
|
# This is of shape (batch_size, ..., top_k).
|
|
correct = top_k.eq(gold_labels.unsqueeze(-1)).float()
|
|
else:
|
|
# prediction is correct if gold label falls on any of the max scores. distribute score by tie_counts
|
|
max_predictions = predictions.max(-1)[0]
|
|
max_predictions_mask = predictions.eq(max_predictions.unsqueeze(-1))
|
|
# max_predictions_mask is (rows X num_classes) and gold_labels is (batch_size)
|
|
# ith entry in gold_labels points to index (0-num_classes) for ith row in max_predictions
|
|
# For each row check if index pointed by gold_label is was 1 or not (among max scored classes)
|
|
correct = max_predictions_mask[
|
|
torch.arange(gold_labels.numel(), device=gold_labels.device).long(), gold_labels
|
|
].float()
|
|
tie_counts = max_predictions_mask.sum(-1)
|
|
correct /= tie_counts.float()
|
|
correct.unsqueeze_(-1)
|
|
|
|
if mask is not None:
|
|
correct *= mask.view(-1, 1)
|
|
self.total_count += mask.sum()
|
|
else:
|
|
self.total_count += gold_labels.numel()
|
|
self.correct_count += correct.sum()
|
|
|
|
@property
|
|
def score(self):
|
|
if self.total_count > 1e-12:
|
|
accuracy = float(self.correct_count) / float(self.total_count)
|
|
else:
|
|
accuracy = 0.0
|
|
return accuracy
|
|
|
|
def __repr__(self) -> str:
|
|
return f'Accuracy:{self.score:.2%}'
|
|
|
|
@staticmethod
|
|
def detach_tensors(*tensors: torch.Tensor) -> Iterable[torch.Tensor]:
|
|
"""
|
|
If you actually passed gradient-tracking Tensors to a Metric, there will be
|
|
a huge memory leak, because it will prevent garbage collection for the computation
|
|
graph. This method ensures the tensors are detached.
|
|
"""
|
|
# Check if it's actually a tensor in case something else was passed.
|
|
return (x.detach() if isinstance(x, torch.Tensor) else x for x in tensors)
|
|
|
|
def reset(self):
|
|
self.correct_count = 0.0
|
|
self.total_count = 0.0
|
|
|
|
|
|
class BooleanAccuracy(Metric):
|
|
"""
|
|
Just checks batch-equality of two tensors and computes an accuracy metric based on that.
|
|
That is, if your prediction has shape (batch_size, dim_1, ..., dim_n), this metric considers that
|
|
as a set of `batch_size` predictions and checks that each is *entirely* correct across the remaining dims.
|
|
This means the denominator in the accuracy computation is `batch_size`, with the caveat that predictions
|
|
that are totally masked are ignored (in which case the denominator is the number of predictions that have
|
|
at least one unmasked element).
|
|
|
|
This is similar to [`CategoricalAccuracy`](./categorical_accuracy.md), if you've already done a `.max()`
|
|
on your predictions. If you have categorical output, though, you should typically just use
|
|
`CategoricalAccuracy`. The reason you might want to use this instead is if you've done
|
|
some kind of constrained inference and don't have a prediction tensor that matches the API of
|
|
`CategoricalAccuracy`, which assumes a final dimension of size `num_classes`.
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
self._correct_count = 0.0
|
|
self._total_count = 0.0
|
|
|
|
def __call__(
|
|
self,
|
|
predictions: torch.Tensor,
|
|
gold_labels: torch.Tensor,
|
|
mask: Optional[torch.BoolTensor] = None,
|
|
):
|
|
"""
|
|
# Parameters
|
|
|
|
predictions : `torch.Tensor`, required.
|
|
A tensor of predictions of shape (batch_size, ...).
|
|
gold_labels : `torch.Tensor`, required.
|
|
A tensor of the same shape as `predictions`.
|
|
mask : `torch.BoolTensor`, optional (default = `None`).
|
|
A tensor of the same shape as `predictions`.
|
|
"""
|
|
predictions, gold_labels, mask = self.detach_tensors(predictions, gold_labels, mask)
|
|
|
|
# Some sanity checks.
|
|
if gold_labels.size() != predictions.size():
|
|
raise ValueError(
|
|
f"gold_labels must have shape == predictions.size() but "
|
|
f"found tensor of shape: {gold_labels.size()}"
|
|
)
|
|
if mask is not None and mask.size() != predictions.size():
|
|
raise ValueError(
|
|
f"mask must have shape == predictions.size() but "
|
|
f"found tensor of shape: {mask.size()}"
|
|
)
|
|
|
|
batch_size = predictions.size(0)
|
|
|
|
if mask is not None:
|
|
# We can multiply by the mask up front, because we're just checking equality below, and
|
|
# this way everything that's masked will be equal.
|
|
predictions = predictions * mask
|
|
gold_labels = gold_labels * mask
|
|
|
|
# We want to skip predictions that are completely masked;
|
|
# so we'll keep predictions that aren't.
|
|
keep = mask.view(batch_size, -1).max(dim=1)[0]
|
|
else:
|
|
keep = torch.ones(batch_size, device=predictions.device).bool()
|
|
|
|
predictions = predictions.view(batch_size, -1)
|
|
gold_labels = gold_labels.view(batch_size, -1)
|
|
|
|
# At this point, predictions is (batch_size, rest_of_dims_combined),
|
|
# so .eq -> .prod will be 1 if every element of the instance prediction is correct
|
|
# and 0 if at least one element of the instance prediction is wrong.
|
|
# Because of how we're handling masking, masked positions are automatically "correct".
|
|
correct = predictions.eq(gold_labels).prod(dim=1).float()
|
|
|
|
# Since masked positions are correct, we need to explicitly exclude instance predictions
|
|
# where the entire prediction is masked (because they look "correct").
|
|
self._correct_count += (correct * keep).sum()
|
|
self._total_count += keep.sum()
|
|
|
|
def get_metric(self, reset: bool = False):
|
|
"""
|
|
# Returns
|
|
|
|
The accumulated accuracy.
|
|
"""
|
|
if self._total_count > 0:
|
|
accuracy = float(self._correct_count) / float(self._total_count)
|
|
else:
|
|
accuracy = 0.0
|
|
if reset:
|
|
self.reset()
|
|
return accuracy
|
|
|
|
def reset(self):
|
|
self._correct_count = 0.0
|
|
self._total_count = 0.0
|
|
|
|
@staticmethod
|
|
def detach_tensors(*tensors: torch.Tensor) -> Iterable[torch.Tensor]:
|
|
"""
|
|
If you actually passed gradient-tracking Tensors to a Metric, there will be
|
|
a huge memory leak, because it will prevent garbage collection for the computation
|
|
graph. This method ensures the tensors are detached.
|
|
"""
|
|
# Check if it's actually a tensor in case something else was passed.
|
|
return (x.detach() if isinstance(x, torch.Tensor) else x for x in tensors)
|