chore: import upstream snapshot with attribution

labml_nn/transformers/utils.py

@@ -0,0 +1,27 @@
+"""
+---
+title: Utilities for Transformer
+summary: A bunch of utility functions and classes for transformers.
+---
+
+# Utilities for Transformer
+"""
+
+import torch
+
+
+def subsequent_mask(seq_len):
+    """
+    ## Subsequent mask to mask out data from future (subsequent) time steps
+    """
+    mask = torch.tril(torch.ones(seq_len, seq_len)).to(torch.bool).unsqueeze(-1)
+    return mask
+
+
+def _subsequent_mask():
+    from labml.logger import inspect
+    inspect(subsequent_mask(10)[:, :, 0])
+
+
+if __name__ == '__main__':
+    _subsequent_mask()