chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"<h1>Neural Networks Activations</h1>\n<ul><li><a href=\"fta/index.html\">Fuzzy Tiling Activations</a> </li>\n<li>\ud83d\udea7 <a href=\"swish/index.html\">Swish</a></li></ul>\n": "<h1>\u30cb\u30e5\u30fc\u30e9\u30eb\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3</h1>\n<ul><li><a href=\"fta/index.html\">\u30d5\u30a1\u30b8\u30fc\u30bf\u30a4\u30ea\u30f3\u30b0\u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3</a></li>\n<li>\ud83d\udea7 <a href=\"swish/index.html\">\u30b9\u30a6\u30a3\u30c3\u30b7\u30e5</a></li></ul>\n",
|
||||
"A set of PyTorch implementations/tutorials related to neural network activations": "\u30cb\u30e5\u30fc\u30e9\u30eb\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u306e\u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3\u306b\u95a2\u9023\u3059\u308bPyTorch\u306e\u5b9f\u88c5/\u30c1\u30e5\u30fc\u30c8\u30ea\u30a2\u30eb\u306e\u30bb\u30c3\u30c8",
|
||||
"Neural Network Activation Functions": "\u30cb\u30e5\u30fc\u30e9\u30eb\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u8d77\u52d5\u6a5f\u80fd"
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"<h1>Neural Networks Activations</h1>\n<ul><li><a href=\"fta/index.html\">Fuzzy Tiling Activations</a> </li>\n<li>\ud83d\udea7 <a href=\"swish/index.html\">Swish</a></li></ul>\n": "<h1>\u0dc3\u0dca\u0db1\u0dcf\u0dba\u0dd4\u0d9a\u0da2\u0dcf\u0dbd \u0dc3\u0d9a\u0dca\u0dbb\u0dd3\u0dba</h1>\n<ul><li><a href=\"fta/index.html\">\u0db1\u0ddc\u0db4\u0dd0\u0dc4\u0dd0\u0daf\u0dd2\u0dbd\u0dd2 \u0da7\u0dba\u0dd2\u0dbd\u0dca \u0d9a\u0dca\u0dbb\u0dd2\u0dba\u0dcf\u0d9a\u0dcf\u0dbb\u0d9a\u0db8\u0dca</a> </li>\n<li>\ud83d\udea7 <a href=\"swish/index.html\">\u0dc3\u0dca\u0dc0\u0dd2\u0dc2\u0dca</a></li></ul>\n",
|
||||
"A set of PyTorch implementations/tutorials related to neural network activations": "\u0dc3\u0dca\u0db1\u0dcf\u0dba\u0dd4\u0d9a \u0da2\u0dcf\u0dbd \u0dc3\u0d9a\u0dca\u0dbb\u0dd3\u0dba \u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dca \u0dc0\u0dbd\u0da7 \u0d85\u0daf\u0dcf\u0dc5 PyTorch \u0d9a\u0dca\u0dbb\u0dd2\u0dba\u0dcf\u0dad\u0dca\u0db8\u0d9a \u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dca/\u0db1\u0dd2\u0db6\u0db1\u0dca\u0db0\u0db1 \u0dc3\u0db8\u0dd6\u0dc4\u0dba\u0d9a\u0dca",
|
||||
"Neural Network Activation Functions": "\u0dc3\u0dca\u0db1\u0dcf\u0dba\u0dd4\u0d9a \u0da2\u0dcf\u0dbd \u0dc3\u0d9a\u0dca\u0dbb\u0dd3\u0dba \u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dda \u0d9a\u0dcf\u0dbb\u0dca\u0dba\u0dba\u0db1\u0dca"
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"<h1>Neural Networks Activations</h1>\n<ul><li><a href=\"fta/index.html\">Fuzzy Tiling Activations</a> </li>\n<li>\ud83d\udea7 <a href=\"swish/index.html\">Swish</a></li></ul>\n": "<h1>\u795e\u7ecf\u7f51\u7edc\u6fc0\u6d3b</h1>\n<ul><li><a href=\"fta/index.html\">\u6a21\u7cca\u5e73\u94fa\u6fc0\u6d3b</a></li>\n<li>\ud83d\udea7 <a href=\"swish/index.html\">Swish</a></li></ul>\n",
|
||||
"A set of PyTorch implementations/tutorials related to neural network activations": "\u4e00\u7ec4\u4e0e\u795e\u7ecf\u7f51\u7edc\u6fc0\u6d3b\u76f8\u5173\u7684 PyTorch \u5b9e\u73b0/\u6559\u7a0b",
|
||||
"Neural Network Activation Functions": "\u795e\u7ecf\u7f51\u7edc\u6fc0\u6d3b\u51fd\u6570"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"<h1><a href=\"index.html\">Fuzzy Tiling Activation</a> Experiment</h1>\n<p><a href=\"https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/activations/fta/experiment.ipynb\"><span translate=no>_^_0_^_</span></a></p>\n<p>Here we train a transformer that uses <a href=\"index.html\">Fuzzy Tiling Activation</a> in the <a href=\"../../transformers/feed_forward.html\">Feed-Forward Network</a>. We use it for a language model and train it on Tiny Shakespeare dataset for demonstration.</p>\n<p>However, this is probably not the ideal task for FTA, and we believe FTA is more suitable for modeling data with continuous variables.</p>\n": "<h1><a href=\"index.html\">\u30d5\u30a1\u30b8\u30fc\u30bf\u30a4\u30ea\u30f3\u30b0\u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3\u5b9f\u9a13</a></h1>\n<p><a href=\"https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/activations/fta/experiment.ipynb\"><span translate=no>_^_0_^_</span></a></p>\n<p><a href=\"../../transformers/feed_forward.html\">\u3053\u3053\u3067\u306f\u3001<a href=\"index.html\">\u30d5\u30a3\u30fc\u30c9\u30d5\u30a9\u30ef\u30fc\u30c9\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u3067\u30d5\u30a1\u30b8\u30fc\u30bf\u30a4\u30ea\u30f3\u30b0\u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3\u3092\u4f7f\u7528\u3059\u308b\u30c8\u30e9\u30f3\u30b9\u30d5\u30a9\u30fc\u30de\u30fc\u3092\u30c8\u30ec\u30fc\u30cb\u30f3\u30b0\u3057\u307e\u3059</a>\u3002</a>\u3053\u308c\u3092\u8a00\u8a9e\u30e2\u30c7\u30eb\u3068\u3057\u3066\u4f7f\u7528\u3057\u3001Tiny Shakespeare\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u3067\u30c8\u30ec\u30fc\u30cb\u30f3\u30b0\u3057\u3066\u30c7\u30e2\u30f3\u30b9\u30c8\u30ec\u30fc\u30b7\u30e7\u30f3\u3092\u884c\u3044\u307e\u3059</p>\u3002\n<p>\u305f\u3060\u3057\u3001\u3053\u308c\u306f\u304a\u305d\u3089\u304fFTA\u306b\u3068\u3063\u3066\u7406\u60f3\u7684\u306a\u30bf\u30b9\u30af\u3067\u306f\u306a\u304f\u3001\u9023\u7d9a\u5909\u6570\u3092\u542b\u3080\u30c7\u30fc\u30bf\u306e\u30e2\u30c7\u30eb\u5316\u306b\u306fFTA\u306e\u65b9\u304c\u9069\u3057\u3066\u3044\u308b\u3068\u8003\u3048\u3066\u3044\u307e\u3059\u3002</p>\n",
|
||||
"<h2>Auto-Regressive model</h2>\n<p>This is an autoregressive transformer model that uses Feed-Forward Networks with (Fuzzy Tiling Activations)(index.html).</p>\n": "<h2>\u81ea\u5df1\u56de\u5e30\u30e2\u30c7\u30eb</h2>\n<p>\u3053\u308c\u306f\u3001\u30d5\u30a3\u30fc\u30c9\u30d5\u30a9\u30ef\u30fc\u30c9\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u3068 (\u30d5\u30a1\u30b8\u30fc\u30bf\u30a4\u30ea\u30f3\u30b0\u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3) (index.html) \u3092\u4f7f\u7528\u3059\u308b\u81ea\u5df1\u56de\u5e30\u30c8\u30e9\u30f3\u30b9\u30d5\u30a9\u30fc\u30de\u30fc\u30e2\u30c7\u30eb\u3067\u3059\u3002</p>\n",
|
||||
"<h2>Configurations</h2>\n<p>This inherits from <a href=\"../../experiments/nlp_autoregression.html#NLPAutoRegressionConfigs\"><span translate=no>_^_0_^_</span></a></p>\n": "<h2>\u30b3\u30f3\u30d5\u30a3\u30ae\u30e5\u30ec\u30fc\u30b7\u30e7\u30f3</h2>\n<p>\u3053\u308c\u306f\u4ee5\u4e0b\u304b\u3089\u7d99\u627f\u3055\u308c\u307e\u3059 <a href=\"../../experiments/nlp_autoregression.html#NLPAutoRegressionConfigs\"><span translate=no>_^_0_^_</span></a></p>\n",
|
||||
"<h2>FFN module with <a href=\"index.html\">FTA</a> activation</h2>\n": "<h2><a href=\"index.html\">FTA</a> \u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3\u6a5f\u80fd\u4ed8\u304d FFN \u30e2\u30b8\u30e5\u30fc\u30eb</h2>\n",
|
||||
"<h4>Create and run the experiment</h4>\n": "<h4>\u5b9f\u9a13\u3092\u4f5c\u6210\u3057\u3066\u5b9f\u884c\u3059\u308b</h4>\n",
|
||||
"<h4>Initialize the model</h4>\n": "<h4>\u30e2\u30c7\u30eb\u3092\u521d\u671f\u5316</h4>\n",
|
||||
"<p> </p>\n": "<p></p>\n",
|
||||
"<p><span translate=no>_^_0_^_</span> </p>\n": "<p><span translate=no>_^_0_^_</span></p>\n",
|
||||
"<p><span translate=no>_^_0_^_</span> and <span translate=no>_^_1_^_</span> for DeepNorm </p>\n": "<p><span translate=no>_^_0_^_</span><span translate=no>_^_1_^_</span>\u305d\u3057\u3066\u30c7\u30a3\u30fc\u30d7\u30ce\u30fc\u30e0\u7528</p>\n",
|
||||
"<p>Activation function <span translate=no>_^_0_^_</span> </p>\n": "<p>\u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3\u6a5f\u80fd <span translate=no>_^_0_^_</span></p>\n",
|
||||
"<p>Adam optimizer with no warmup </p>\n": "<p>\u30a6\u30a9\u30fc\u30e0\u30a2\u30c3\u30d7\u306a\u3057\u306e Adam \u30aa\u30d7\u30c6\u30a3\u30de\u30a4\u30b6\u30fc</p>\n",
|
||||
"<p>Apply dropout </p>\n": "<p>\u30c9\u30ed\u30c3\u30d7\u30a2\u30a6\u30c8\u3092\u9069\u7528</p>\n",
|
||||
"<p>Batch size <span translate=no>_^_0_^_</span> </p>\n": "<p>\u30d0\u30c3\u30c1\u30b5\u30a4\u30ba <span translate=no>_^_0_^_</span></p>\n",
|
||||
"<p>Create FTA activation module </p>\n": "<p>FTA \u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u4f5c\u6210</p>\n",
|
||||
"<p>Create auto-regressive mask </p>\n": "<p>\u81ea\u52d5\u56de\u5e30\u30de\u30b9\u30af\u306e\u4f5c\u6210</p>\n",
|
||||
"<p>Create configs </p>\n": "<p>\u30b3\u30f3\u30d5\u30a3\u30b0\u306e\u4f5c\u6210</p>\n",
|
||||
"<p>Create experiment </p>\n": "<p>\u5b9f\u9a13\u3092\u4f5c\u6210</p>\n",
|
||||
"<p>Create the transformer. We re-use <a href=\"../../transformers/models.html#TransformerLayer\"><span translate=no>_^_0_^_</span></a> and <a href=\"../../transformers/mha.html\"><span translate=no>_^_1_^_</span></a> implementations. </p>\n": "<p>\u30c8\u30e9\u30f3\u30b9\u30d5\u30a9\u30fc\u30de\u30fc\u3092\u4f5c\u6210\u3057\u307e\u3059\u3002<a href=\"../../transformers/models.html#TransformerLayer\"><span translate=no>_^_0_^_</span><a href=\"../../transformers/mha.html\"><span translate=no>_^_1_^_</span></a></a>\u518d\u5229\u7528\u3057\u3066\u5b9f\u88c5\u3057\u307e\u3059</p>\u3002\n",
|
||||
"<p>Embedding size </p>\n": "<p>\u57cb\u3081\u8fbc\u307f\u30b5\u30a4\u30ba</p>\n",
|
||||
"<p>FTA </p>\n": "<p>\u81ea\u7531\u8cbf\u6613\u5354\u5b9a</p>\n",
|
||||
"<p>Feed forward layer size </p>\n": "<p>\u30d5\u30a3\u30fc\u30c9\u30d5\u30a9\u30ef\u30fc\u30c9\u30ec\u30a4\u30e4\u30fc\u30b5\u30a4\u30ba</p>\n",
|
||||
"<p>Get logits </p>\n": "<p>\u30ed\u30b8\u30c3\u30c8\u3092\u53d6\u5f97</p>\n",
|
||||
"<p>Get the token embeddings </p>\n": "<p>\u30c8\u30fc\u30af\u30f3\u306e\u57cb\u3081\u8fbc\u307f\u3092\u5165\u624b</p>\n",
|
||||
"<p>Hidden layer dropout </p>\n": "<p>\u96a0\u3057\u30ec\u30a4\u30e4\u30fc\u306e\u30c9\u30ed\u30c3\u30d7\u30a2\u30a6\u30c8</p>\n",
|
||||
"<p>Layer one parameterized by weight <span translate=no>_^_0_^_</span> and bias <span translate=no>_^_1_^_</span> </p>\n": "<p>\u91cd\u307f\u3068\u30d0\u30a4\u30a2\u30b9\u3067\u30d1\u30e9\u30e1\u30fc\u30bf\u5316\u3055\u308c\u305f\u30ec\u30a4\u30e4\u30fc 1 <span translate=no>_^_0_^_</span> <span translate=no>_^_1_^_</span></p>\n",
|
||||
"<p>Layer two parameterized by weight <span translate=no>_^_0_^_</span> and bias <span translate=no>_^_1_^_</span> </p>\n": "<p>\u91cd\u307f\u3068\u30d0\u30a4\u30a2\u30b9\u3067\u30d1\u30e9\u30e1\u30fc\u30bf\u5316\u3055\u308c\u305f\u30ec\u30a4\u30e4\u30fc 2 <span translate=no>_^_0_^_</span> <span translate=no>_^_1_^_</span></p>\n",
|
||||
"<p>Model </p>\n": "<p>\u30e2\u30c7\u30eb</p>\n",
|
||||
"<p>Move to the device </p>\n": "<p>\u30c7\u30d0\u30a4\u30b9\u306b\u79fb\u52d5</p>\n",
|
||||
"<p>Number of heads in the attention </p>\n": "<p>\u6ce8\u76ee\u3055\u308c\u3066\u3044\u308b\u30d8\u30c3\u30c9\u306e\u6570</p>\n",
|
||||
"<p>Number of layers </p>\n": "<p>\u30ec\u30a4\u30e4\u30fc\u6570</p>\n",
|
||||
"<p>Override configurations </p>\n": "<p>\u30aa\u30fc\u30d0\u30fc\u30e9\u30a4\u30c9\u8a2d\u5b9a</p>\n",
|
||||
"<p>Prompt separator is blank </p>\n": "<p>\u30d7\u30ed\u30f3\u30d7\u30c8\u30bb\u30d1\u30ec\u30fc\u30bf\u304c\u7a7a\u767d</p>\n",
|
||||
"<p>Readout layer </p>\n": "<p>\u8aad\u307f\u51fa\u3057\u5c64</p>\n",
|
||||
"<p>Return results </p>\n": "<p>\u7d50\u679c\u3092\u8fd4\u3059</p>\n",
|
||||
"<p>Run training </p>\n": "<p>\u30c8\u30ec\u30fc\u30cb\u30f3\u30b0\u3092\u5b9f\u884c</p>\n",
|
||||
"<p>Set model(s) for saving and loading </p>\n": "<p>\u4fdd\u5b58\u304a\u3088\u3073\u8aad\u307f\u8fbc\u307f\u7528\u306e\u30e2\u30c7\u30eb\u3092\u8a2d\u5b9a\u3057\u307e\u3059</p>\n",
|
||||
"<p>Size of each attention head </p>\n": "<p>\u5404\u30a2\u30c6\u30f3\u30b7\u30e7\u30f3\u30d8\u30c3\u30c9\u306e\u30b5\u30a4\u30ba</p>\n",
|
||||
"<p>Start the experiment </p>\n": "<p>\u5b9f\u9a13\u3092\u59cb\u3081\u308b</p>\n",
|
||||
"<p>Starting prompt for sampling </p>\n": "<p>\u30b5\u30f3\u30d7\u30ea\u30f3\u30b0\u306e\u958b\u59cb\u30d7\u30ed\u30f3\u30d7\u30c8</p>\n",
|
||||
"<p>Subsequent mask, will mask out tokens from seeing future tokens </p>\n": "<p>\u6b21\u306b\u30de\u30b9\u30af\u3059\u308b\u3068\u3001\u30c8\u30fc\u30af\u30f3\u304c\u30de\u30b9\u30af\u3055\u308c\u3001\u5c06\u6765\u306e\u30c8\u30fc\u30af\u30f3\u304c\u898b\u3048\u306a\u304f\u306a\u308a\u307e\u3059</p>\n",
|
||||
"<p>Switch between training and validation for <span translate=no>_^_0_^_</span> times per epoch </p>\n": "<p>\u30a8\u30dd\u30c3\u30af\u3054\u3068\u306b\u30c8\u30ec\u30fc\u30cb\u30f3\u30b0\u3068\u691c\u8a3c\u3092\u5207\u308a\u66ff\u3048\u308b <span translate=no>_^_0_^_</span></p>\n",
|
||||
"<p>The mask will be initialized on the first call </p>\n": "<p>\u30de\u30b9\u30af\u306f\u6700\u521d\u306e\u547c\u3073\u51fa\u3057\u3067\u521d\u671f\u5316\u3055\u308c\u307e\u3059</p>\n",
|
||||
"<p>Token embedding layer </p>\n": "<p>\u30c8\u30fc\u30af\u30f3\u57cb\u3081\u8fbc\u307f\u30ec\u30a4\u30e4\u30fc</p>\n",
|
||||
"<p>Train for 32 epochs </p>\n": "<p>32 \u30a8\u30dd\u30c3\u30af\u306e\u30c8\u30ec\u30fc\u30cb\u30f3\u30b0</p>\n",
|
||||
"<p>Transformer encoder </p>\n": "<p>\u30c8\u30e9\u30f3\u30b9\u30a8\u30f3\u30b3\u30fc\u30c0\u30fc</p>\n",
|
||||
"<p>Transformer with <span translate=no>_^_0_^_</span> layers </p>\n": "<p><span translate=no>_^_0_^_</span>\u5c64\u4ed8\u304d\u5909\u5727\u5668</p>\n",
|
||||
"<p>Use Tiny Shakespeare dataset </p>\n": "<p>\u30bf\u30a4\u30cb\u30fc\u30fb\u30b7\u30a7\u30a4\u30af\u30b9\u30d4\u30a2\u30fb\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u3092\u4f7f\u3046</p>\n",
|
||||
"<p>Use a context size of <span translate=no>_^_0_^_</span> </p>\n": "<p>\u30b3\u30f3\u30c6\u30ad\u30b9\u30c8\u30b5\u30a4\u30ba\u3092\u6b21\u306e\u5024\u306b\u3057\u3066\u304f\u3060\u3055\u3044 <span translate=no>_^_0_^_</span></p>\n",
|
||||
"<p>Use character level tokenizer </p>\n": "<p>\u30ad\u30e3\u30e9\u30af\u30bf\u30fc\u30ec\u30d9\u30eb\u306e\u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc\u3092\u4f7f\u3046</p>\n",
|
||||
"<ul><li><span translate=no>_^_0_^_</span> are the input tokens of shape <span translate=no>_^_1_^_</span></li></ul>\n": "<ul><li><span translate=no>_^_0_^_</span>\u5f62\u72b6\u306e\u5165\u529b\u30c8\u30fc\u30af\u30f3\u3067\u3059 <span translate=no>_^_1_^_</span></li></ul>\n",
|
||||
"<ul><li><span translate=no>_^_0_^_</span> is the number of tokens in the vocabulary </li>\n<li><span translate=no>_^_1_^_</span> is the embedding size </li>\n<li><span translate=no>_^_2_^_</span> is the number of transformer layers </li>\n<li><span translate=no>_^_3_^_</span> is the layer. We use <span translate=no>_^_4_^_</span> copies of this for the transformer.</li></ul>\n": "<ul><li><span translate=no>_^_0_^_</span>\u30dc\u30ad\u30e3\u30d6\u30e9\u30ea\u5185\u306e\u30c8\u30fc\u30af\u30f3\u306e\u6570\u3067\u3059</li>\n<li><span translate=no>_^_1_^_</span>\u306f\u57cb\u3081\u8fbc\u307f\u30b5\u30a4\u30ba</li>\n<li><span translate=no>_^_2_^_</span>\u5909\u5727\u5668\u5c64\u306e\u6570\u3067\u3059</li>\n<li><span translate=no>_^_3_^_</span>\u30ec\u30a4\u30e4\u30fc\u3067\u3059\u3002<span translate=no>_^_4_^_</span>\u5909\u5727\u5668\u306b\u306f\u3053\u308c\u306e\u30b3\u30d4\u30fc\u3092\u4f7f\u3044\u307e\u3059</li></ul>\u3002\n",
|
||||
"<ul><li><span translate=no>_^_0_^_</span> is the number of features in a token embedding </li>\n<li><span translate=no>_^_1_^_</span> is the number of features in the hidden layer of the FFN </li>\n<li><span translate=no>_^_2_^_</span> is FTA activation module </li>\n<li><span translate=no>_^_3_^_</span> is dropout probability for the hidden layer</li></ul>\n": "<ul><li><span translate=no>_^_0_^_</span>\u30c8\u30fc\u30af\u30f3\u57cb\u3081\u8fbc\u307f\u306b\u542b\u307e\u308c\u308b\u6a5f\u80fd\u306e\u6570</li>\n<li><span translate=no>_^_1_^_</span>\u306f FFN \u306e\u96a0\u308c\u30ec\u30a4\u30e4\u30fc\u306b\u3042\u308b\u30d5\u30a3\u30fc\u30c1\u30e3\u306e\u6570\u3067\u3059</li>\n<li><span translate=no>_^_2_^_</span>FTA \u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3\u30e2\u30b8\u30e5\u30fc\u30eb\u3067\u3059\u304b</li>\n<li><span translate=no>_^_3_^_</span>\u306f\u96a0\u308c\u5c64\u306e\u30c9\u30ed\u30c3\u30d7\u30a2\u30a6\u30c8\u78ba\u7387\u3067\u3059</li></ul>\n",
|
||||
"Fuzzy Tiling Activation Experiment": "\u30d5\u30a1\u30b8\u30fc\u30bf\u30a4\u30ea\u30f3\u30b0\u30a2\u30af\u30c6\u30a3\u30d9\u30fc\u30b7\u30e7\u30f3\u5b9f\u9a13",
|
||||
"Training a transformer with FTA in FFN on Tiny Shakespeare.": "\u30bf\u30a4\u30cb\u30fc\u30fb\u30b7\u30a7\u30a4\u30af\u30b9\u30d4\u30a2\u306eFFN\u3067\u30c8\u30e9\u30f3\u30b9\u30d5\u30a9\u30fc\u30de\u30fc\u3092FTA\u3067\u30c8\u30ec\u30fc\u30cb\u30f3\u30b0\u4e2d\u3002"
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"<h1><a href=\"index.html\">Fuzzy Tiling Activation</a> Experiment</h1>\n<p><a href=\"https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/activations/fta/experiment.ipynb\"><span translate=no>_^_0_^_</span></a></p>\n<p>Here we train a transformer that uses <a href=\"index.html\">Fuzzy Tiling Activation</a> in the <a href=\"../../transformers/feed_forward.html\">Feed-Forward Network</a>. We use it for a language model and train it on Tiny Shakespeare dataset for demonstration.</p>\n<p>However, this is probably not the ideal task for FTA, and we believe FTA is more suitable for modeling data with continuous variables.</p>\n": "<h1><a href=\"index.html\">\u0db1\u0ddc\u0db4\u0dd0\u0dc4\u0dd0\u0daf\u0dd2\u0dbd\u0dd2 \u0da7\u0dba\u0dd2\u0dbd\u0dca \u0dc3\u0d9a\u0dca\u0dbb\u0dd3\u0dba \u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dda</a> \u0d85\u0dad\u0dca\u0dc4\u0daf\u0dcf \u0db6\u0dd0\u0dbd\u0dd3\u0db8</h1>\n<p><a href=\"https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/activations/fta/experiment.ipynb\"><span translate=no>_^_0_^_</span></a></p>\n<p>\u0db8\u0dd9\u0db1\u0dca\u0db1 \u0d85\u0db4\u0dd2 <a href=\"../../transformers/feed_forward.html\">Feed-Forward \u0da2\u0dcf\u0dbd\u0dba\u0dda</a> <a href=\"index.html\">Fuzzy Tiling Activation</a> \u0db7\u0dcf\u0dc0\u0dd2\u0dad\u0dcf \u0d9a\u0dbb\u0db1 \u0da7\u0dca\u0dbb\u0dcf\u0db1\u0dca\u0dc3\u0dca\u0dc6\u0ddd\u0db8\u0dbb\u0dba\u0d9a\u0dca \u0db4\u0dd4\u0dc4\u0dd4\u0dab\u0dd4 \u0d9a\u0dbb\u0db8\u0dd4. \u0d85\u0db4\u0dd2 \u0d91\u0dba \u0db7\u0dcf\u0dc2\u0dcf \u0d86\u0d9a\u0dd8\u0dad\u0dd2\u0dba\u0d9a\u0dca \u0dc3\u0db3\u0dc4\u0dcf \u0db7\u0dcf\u0dc0\u0dd2\u0dad\u0dcf \u0d9a\u0dbb\u0db1 \u0d85\u0dad\u0dbb \u0db1\u0dd2\u0dbb\u0dd6\u0db4\u0dab\u0dba \u0dc3\u0db3\u0dc4\u0dcf \u0d9a\u0dd4\u0da9\u0dcf \u0dc2\u0dda\u0d9a\u0dca\u0dc3\u0dca\u0db4\u0dd2\u0dba\u0dbb\u0dca \u0daf\u0dad\u0dca\u0dad \u0d9a\u0da7\u0dca\u0da7\u0dbd\u0dba \u0db8\u0dad \u0d91\u0dba \u0db4\u0dd4\u0dc4\u0dd4\u0dab\u0dd4 \u0d9a\u0dbb\u0db8\u0dd4.</p>\n<p>\u0d9a\u0dd9\u0dc3\u0dda \u0dc0\u0dd9\u0dad\u0dad\u0dca, \u0db8\u0dd9\u0dba \u0db6\u0ddc\u0dc4\u0ddd \u0dc0\u0dd2\u0da7 FTA \u0dc3\u0db3\u0dc4\u0dcf \u0dc3\u0dd4\u0daf\u0dd4\u0dc3\u0dd4\u0db8 \u0d9a\u0dcf\u0dbb\u0dca\u0dba\u0dba \u0db1\u0ddc\u0dc0\u0db1 \u0d85\u0dad\u0dbb \u0d85\u0d9b\u0dab\u0dca\u0da9 \u0dc0\u0dd2\u0da0\u0dbd\u0dca\u0dba\u0dba\u0db1\u0dca \u0dc3\u0dc4\u0dd2\u0dad \u0daf\u0dad\u0dca\u0dad \u0d86\u0d9a\u0dd8\u0dad\u0dd2\u0d9a\u0dbb\u0dab\u0dba \u0dc3\u0db3\u0dc4\u0dcf FTA \u0dc0\u0da9\u0dcf\u0dad\u0dca \u0dc3\u0dd4\u0daf\u0dd4\u0dc3\u0dd4 \u0dba\u0dd0\u0dba\u0dd2 \u0d85\u0db4\u0dd2 \u0dc0\u0dd2\u0dc1\u0dca\u0dc0\u0dcf\u0dc3 \u0d9a\u0dbb\u0db8\u0dd4.</p>\n",
|
||||
"<h2>Auto-Regressive model</h2>\n<p>This is an autoregressive transformer model that uses Feed-Forward Networks with (Fuzzy Tiling Activations)(index.html).</p>\n": "<h2>\u0dc3\u0dca\u0dc0\u0dba\u0d82\u0d9a\u0dca\u0dbb\u0dd3\u0dba\u0db4\u0dca\u0dbb\u0dad\u0dd2\u0d9c\u0dcf\u0db8\u0dd3 \u0d86\u0d9a\u0dd8\u0dad\u0dd2\u0dba</h2>\n<p>\u0db8\u0dd9\u0dba(\u0db1\u0ddc\u0db4\u0dd0\u0dc4\u0dd0\u0daf\u0dd2\u0dbd\u0dd2 \u0da7\u0dba\u0dd2\u0dbd\u0dd2\u0d82 \u0d87\u0d9a\u0dca\u0da7\u0dd2\u0dc0\u0dda\u0dc2\u0db1\u0dca) (index.html) \u0dc3\u0db8\u0d9f \u0dc6\u0dd3\u0da9\u0dca-\u0dc6\u0ddd\u0dc0\u0dbb\u0dca\u0da9\u0dca \u0db1\u0dd9\u0da7\u0dca\u0dc0\u0dbb\u0dca\u0d9a\u0dca \u0db7\u0dcf\u0dc0\u0dd2\u0dad\u0dcf \u0d9a\u0dbb\u0db1 \u0dc3\u0dca\u0dc0\u0dba\u0d82\u0d9a\u0dca\u0dbb\u0dd3\u0dba \u0db4\u0dca\u0dbb\u0dad\u0dd2\u0d9c\u0dcf\u0db8\u0dd3 \u0da7\u0dca\u0dbb\u0dcf\u0db1\u0dca\u0dc3\u0dca\u0dc6\u0ddd\u0db8\u0dbb\u0dca \u0d86\u0d9a\u0dd8\u0dad\u0dd2\u0dba\u0d9a\u0dd2. </p>\n",
|
||||
"<h2>Configurations</h2>\n<p>This inherits from <a href=\"../../experiments/nlp_autoregression.html#NLPAutoRegressionConfigs\"><span translate=no>_^_0_^_</span></a></p>\n": "<h2>\u0dc0\u0dd2\u0db1\u0dca\u0dba\u0dcf\u0dc3\u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dca</h2>\n<p>\u0db8\u0dd9\u0dba\u0d8b\u0dbb\u0dd4\u0db8 \u0dc0\u0db1\u0dca\u0db1\u0dda <a href=\"../../experiments/nlp_autoregression.html#NLPAutoRegressionConfigs\"><span translate=no>_^_0_^_</span></a></p>\n",
|
||||
"<h2>FFN module with <a href=\"index.html\">FTA</a> activation</h2>\n": "<h2><a href=\"index.html\">FTA \u0dc3\u0d9a\u0dca\u0dbb\u0dd2\u0dba \u0d9a\u0dd2\u0dbb\u0dd3\u0db8 \u0dc3\u0db8\u0d9f FFN</a> \u0db8\u0ddc\u0da9\u0dd2\u0dba\u0dd4\u0dbd\u0dba</h2>\n",
|
||||
"<h4>Create and run the experiment</h4>\n": "<h4>\u0d85\u0dad\u0dca\u0dc4\u0daf\u0dcf\u0db6\u0dd0\u0dbd\u0dd3\u0db8 \u0db1\u0dd2\u0dbb\u0dca\u0db8\u0dcf\u0dab\u0dba \u0d9a\u0dbb \u0d9a\u0dca\u0dbb\u0dd2\u0dba\u0dcf\u0dad\u0dca\u0db8\u0d9a \u0d9a\u0dbb\u0db1\u0dca\u0db1</h4>\n",
|
||||
"<h4>Initialize the model</h4>\n": "<h4>\u0d86\u0d9a\u0dd8\u0dad\u0dd2\u0dba\u0d86\u0dbb\u0db8\u0dca\u0db7 \u0d9a\u0dbb\u0db1\u0dca\u0db1</h4>\n",
|
||||
"<p> </p>\n": "<p> </p>\n",
|
||||
"<p><span translate=no>_^_0_^_</span> </p>\n": "<p><span translate=no>_^_0_^_</span> </p>\n",
|
||||
"<p><span translate=no>_^_0_^_</span> and <span translate=no>_^_1_^_</span> for DeepNorm </p>\n": "<p><span translate=no>_^_0_^_</span> \u0dc3\u0dc4 \u0d9c\u0dd0\u0db9\u0dd4\u0dbb\u0dd4 \u0dc3\u0db8\u0dca\u0db8\u0dad\u0dba <span translate=no>_^_1_^_</span> \u0dc3\u0db3\u0dc4\u0dcf </p>\n",
|
||||
"<p>Activation function <span translate=no>_^_0_^_</span> </p>\n": "<p>\u0dc3\u0d9a\u0dca\u0dbb\u0dd2\u0dba\u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dda \u0d9a\u0dcf\u0dbb\u0dca\u0dba\u0dba <span translate=no>_^_0_^_</span> </p>\n",
|
||||
"<p>Adam optimizer with no warmup </p>\n": "<p>\u0d8b\u0db1\u0dd4\u0dc3\u0dd4\u0db8\u0dca\u0dc0\u0dd3\u0db8\u0d9a\u0dca \u0db1\u0ddc\u0db8\u0dd0\u0dad\u0dd2 \u0d86\u0daf\u0db8\u0dca \u0db4\u0dca\u0dbb\u0dc1\u0dc3\u0dca\u0dad\u0d9a\u0dbb\u0dab\u0dba </p>\n",
|
||||
"<p>Apply dropout </p>\n": "<p>\u0d85\u0dad\u0dc4\u0dd0\u0dbb\u0daf\u0dd0\u0db8\u0dd3\u0db8 \u0dba\u0ddc\u0daf\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Batch size <span translate=no>_^_0_^_</span> </p>\n": "<p>\u0d9a\u0dab\u0dca\u0da9\u0dcf\u0dba\u0db8\u0dca\u0db4\u0dca\u0dbb\u0db8\u0dcf\u0dab\u0dba <span translate=no>_^_0_^_</span> </p>\n",
|
||||
"<p>Create FTA activation module </p>\n": "<p>FTA\u0dc3\u0d9a\u0dca\u0dbb\u0dd2\u0dba \u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dda \u0db8\u0ddc\u0da9\u0dd2\u0dba\u0dd4\u0dbd\u0dba \u0dc3\u0dcf\u0daf\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Create auto-regressive mask </p>\n": "<p>\u0dc3\u0dca\u0dc0\u0dba\u0d82\u0d9a\u0dca\u0dbb\u0dd3\u0dba\u0db4\u0dca\u0dbb\u0dad\u0dd2\u0d9c\u0dcf\u0db8\u0dd3 \u0dc0\u0dd9\u0dc3\u0dca \u0db8\u0dd4\u0dc4\u0dd4\u0dab\u0d9a\u0dca \u0dc3\u0dcf\u0daf\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Create configs </p>\n": "<p>\u0dc0\u0dd2\u0db1\u0dca\u0dba\u0dcf\u0dc3\u0dc3\u0dcf\u0daf\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Create experiment </p>\n": "<p>\u0d85\u0dad\u0dca\u0dc4\u0daf\u0dcf\u0db6\u0dd0\u0dbd\u0dd3\u0db8 \u0dc3\u0dcf\u0daf\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Create the transformer. We re-use <a href=\"../../transformers/models.html#TransformerLayer\"><span translate=no>_^_0_^_</span></a> and <a href=\"../../transformers/mha.html\"><span translate=no>_^_1_^_</span></a> implementations. </p>\n": "<p>\u0da7\u0dca\u0dbb\u0dcf\u0db1\u0dca\u0dc3\u0dca\u0dc6\u0ddd\u0db8\u0dbb\u0dba\u0dc3\u0dcf\u0daf\u0db1\u0dca\u0db1. \u0d85\u0db4\u0dd2 \u0db1\u0dd0\u0dc0\u0dad \u0db7\u0dcf\u0dc0\u0dd2\u0dad\u0dcf \u0d9a\u0dd2\u0dbb\u0dd3\u0db8 <a href=\"../../transformers/models.html#TransformerLayer\"><span translate=no>_^_0_^_</span><a href=\"../../transformers/mha.html\"><span translate=no>_^_1_^_</span></a> </a> \u0dc3\u0dc4 \u0d9a\u0dca\u0dbb\u0dd2\u0dba\u0dcf\u0dad\u0dca\u0db8\u0d9a \u0d9a\u0dd2\u0dbb\u0dd3\u0db8. </p>\n",
|
||||
"<p>Embedding size </p>\n": "<p>\u0d9a\u0dcf\u0dc0\u0dd0\u0daf\u0dca\u0daf\u0dd3\u0db8\u0db4\u0dca\u0dbb\u0db8\u0dcf\u0dab\u0dba </p>\n",
|
||||
"<p>FTA </p>\n": "<p>FTA </p>\n",
|
||||
"<p>Feed forward layer size </p>\n": "<p>\u0d89\u0daf\u0dd2\u0dbb\u0dd2\u0dc3\u0dca\u0dae\u0dbb\u0dba\u0dda \u0db4\u0dca\u0dbb\u0db8\u0dcf\u0dab\u0dba \u0db4\u0ddd\u0dc2\u0dab\u0dba \u0d9a\u0dbb\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Get logits </p>\n": "<p>\u0db4\u0dd2\u0dc0\u0dd2\u0dc3\u0dd4\u0db8\u0dca\u0dbd\u0db6\u0dcf \u0d9c\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Get the token embeddings </p>\n": "<p>\u0da7\u0ddd\u0d9a\u0db1\u0dca\u0d9a\u0dcf\u0dc0\u0dd0\u0daf\u0dca\u0daf\u0dd3\u0db8\u0dca \u0dbd\u0db6\u0dcf \u0d9c\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Hidden layer dropout </p>\n": "<p>\u0dc3\u0dd0\u0d9f\u0dc0\u0dd4\u0dab\u0dd4\u0dc3\u0dca\u0dae\u0dbb \u0dc4\u0dd0\u0dbd\u0dd3\u0db8 </p>\n",
|
||||
"<p>Layer one parameterized by weight <span translate=no>_^_0_^_</span> and bias <span translate=no>_^_1_^_</span> </p>\n": "<p>\u0db6\u0dbb <span translate=no>_^_0_^_</span> \u0dc4\u0dcf \u0db1\u0dd0\u0db9\u0dd4\u0dbb\u0dd4\u0dc0 \u0d85\u0db1\u0dd4\u0dc0 \u0db4\u0dbb\u0dcf\u0db8\u0dd2\u0dad\u0dd2\u0d9a\u0dbb\u0dab\u0dba \u0d9a\u0dbb\u0db1 \u0dbd\u0daf \u0d91\u0d9a\u0dca \u0dc3\u0dca\u0dae\u0dbb\u0dba <span translate=no>_^_1_^_</span> </p>\n",
|
||||
"<p>Layer two parameterized by weight <span translate=no>_^_0_^_</span> and bias <span translate=no>_^_1_^_</span> </p>\n": "<p>\u0db6\u0dbb <span translate=no>_^_0_^_</span> \u0dc4\u0dcf \u0db1\u0dd0\u0db9\u0dd4\u0dbb\u0dd4\u0dc0 \u0d85\u0db1\u0dd4\u0dc0 \u0db4\u0dbb\u0dcf\u0db8\u0dd2\u0dad\u0dd2\u0d9a\u0dbb\u0dab\u0dba \u0d9a\u0dbb\u0db1 \u0dbd\u0daf \u0dc3\u0dca\u0dae\u0dbb \u0daf\u0dd9\u0d9a\u0d9a\u0dca <span translate=no>_^_1_^_</span> </p>\n",
|
||||
"<p>Model </p>\n": "<p>\u0d86\u0d9a\u0dd8\u0dad\u0dd2\u0dba </p>\n",
|
||||
"<p>Move to the device </p>\n": "<p>\u0d8b\u0db4\u0dcf\u0d82\u0d9c\u0dba\u0dc0\u0dd9\u0dad \u0d9c\u0dd9\u0db1 \u0dba\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Number of heads in the attention </p>\n": "<p>\u0d85\u0dc0\u0db0\u0dcf\u0db1\u0dba\u0dba\u0ddc\u0db8\u0dd4 \u0d9a\u0dbb\u0db1 \u0dc4\u0dd2\u0dc3\u0dca \u0d9c\u0dab\u0db1 </p>\n",
|
||||
"<p>Number of layers </p>\n": "<p>\u0dc3\u0dca\u0dae\u0dbb\u0d9c\u0dab\u0db1 </p>\n",
|
||||
"<p>Override configurations </p>\n": "<p>\u0dc0\u0dd2\u0db1\u0dca\u0dba\u0dcf\u0dc3\u0dba\u0db1\u0dca\u0d85\u0db7\u0dd2\u0db6\u0dc0\u0dcf \u0dba\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Prompt separator is blank </p>\n": "<p>\u0d9a\u0da9\u0dd2\u0db1\u0db8\u0dca\u0db6\u0dd9\u0daf\u0dd4\u0db8\u0dca\u0d9a\u0dbb\u0dd4 \u0dc4\u0dd2\u0dc3\u0dca \u0dba </p>\n",
|
||||
"<p>Readout layer </p>\n": "<p>\u0d9a\u0dd2\u0dba\u0dc0\u0dd3\u0db8\u0dda\u0dc3\u0dca\u0dae\u0dbb\u0dba </p>\n",
|
||||
"<p>Return results </p>\n": "<p>\u0d86\u0db4\u0dc3\u0dd4\u0db4\u0dca\u0dbb\u0dad\u0dd2\u0db5\u0dbd </p>\n",
|
||||
"<p>Run training </p>\n": "<p>\u0db4\u0dd4\u0dc4\u0dd4\u0dab\u0dd4\u0db0\u0dcf\u0dc0\u0db1\u0dba </p>\n",
|
||||
"<p>Set model(s) for saving and loading </p>\n": "<p>\u0d89\u0dad\u0dd2\u0dbb\u0dd2\u0d9a\u0dd2\u0dbb\u0dd3\u0db8 \u0dc3\u0dc4 \u0db4\u0dd0\u0da7\u0dc0\u0dd3\u0db8 \u0dc3\u0db3\u0dc4\u0dcf \u0d86\u0d9a\u0dd8\u0dad\u0dd2\u0dba (\u0dba) \u0dc3\u0d9a\u0dc3\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Size of each attention head </p>\n": "<p>\u0d91\u0d9a\u0dca\u0d91\u0d9a\u0dca \u0d85\u0dc0\u0db0\u0dcf\u0db1\u0dba \u0dc4\u0dd2\u0dc3 \u0db4\u0dca\u0dbb\u0db8\u0dcf\u0dab\u0dba </p>\n",
|
||||
"<p>Start the experiment </p>\n": "<p>\u0d85\u0dad\u0dca\u0dc4\u0daf\u0dcf\u0db6\u0dd0\u0dbd\u0dd3\u0db8 \u0d86\u0dbb\u0db8\u0dca\u0db7 \u0d9a\u0dbb\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Starting prompt for sampling </p>\n": "<p>\u0db1\u0dd2\u0dba\u0dd0\u0daf\u0dd3\u0db8\u0dc3\u0db3\u0dc4\u0dcf \u0dc0\u0dd2\u0db8\u0dc3\u0dd4\u0db8\u0d9a\u0dca \u0d86\u0dbb\u0db8\u0dca\u0db7 \u0d9a\u0dd2\u0dbb\u0dd3\u0db8 </p>\n",
|
||||
"<p>Subsequent mask, will mask out tokens from seeing future tokens </p>\n": "<p>\u0db4\u0dc3\u0dd4\u0d9a\u0dcf\u0dbd\u0dd3\u0db1\u0dc0\u0dd9\u0dc3\u0dca\u0db8\u0dd4\u0dc4\u0dd4\u0dab, \u0d85\u0db1\u0dcf\u0d9c\u0dad \u0da7\u0ddd\u0d9a\u0db1 \u0daf\u0dd0\u0d9a\u0dd3\u0db8\u0dd9\u0db1\u0dca \u0da7\u0ddd\u0d9a\u0db1 \u0dc0\u0dc3\u0d82 \u0d9a\u0dbb\u0db1\u0dd4 \u0d87\u0dad </p>\n",
|
||||
"<p>Switch between training and validation for <span translate=no>_^_0_^_</span> times per epoch </p>\n": "<p>\u0d91\u0d9a\u0dca <span translate=no>_^_0_^_</span> \u0dba\u0dd4\u0d9c\u0dba\u0d9a\u0da7 \u0dc0\u0dbb\u0d9a\u0dca \u0db4\u0dd4\u0dc4\u0dd4\u0dab\u0dd4\u0dc0 \u0dc3\u0dc4 \u0dc0\u0dbd\u0d82\u0d9c\u0dd4 \u0d9a\u0dd2\u0dbb\u0dd3\u0db8 \u0d85\u0dad\u0dbb \u0db8\u0dcf\u0dbb\u0dd4 \u0dc0\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>The mask will be initialized on the first call </p>\n": "<p>\u0db4\u0dc5\u0db8\u0dd4\u0d87\u0db8\u0dad\u0dd4\u0db8\u0dd9\u0db1\u0dca \u0dc0\u0dd9\u0dc3\u0dca\u0db8\u0dd4\u0dc4\u0dd4\u0dab \u0d86\u0dbb\u0db8\u0dca\u0db7 \u0d9a\u0dbb\u0db1\u0dd4 \u0d87\u0dad </p>\n",
|
||||
"<p>Token embedding layer </p>\n": "<p>\u0da7\u0ddd\u0d9a\u0db1\u0dca\u0d9a\u0dcf\u0dc0\u0dd0\u0daf\u0dca\u0daf\u0dd3\u0db8 \u0dc3\u0dca\u0dae\u0dbb\u0dba </p>\n",
|
||||
"<p>Train for 32 epochs </p>\n": "<p>32\u0dc0\u0dba\u0dc3 \u0d85\u0dc0\u0dd4\u0dbb\u0dd4\u0daf\u0dd4 \u0dc3\u0db3\u0dc4\u0dcf \u0daf\u0dd4\u0db8\u0dca\u0dbb\u0dd2\u0dba </p>\n",
|
||||
"<p>Transformer encoder </p>\n": "<p>\u0da7\u0dca\u0dbb\u0dcf\u0db1\u0dca\u0dc3\u0dca\u0dc6\u0ddd\u0db8\u0dbb\u0dca\u0d91\u0db1\u0dca\u0d9a\u0ddd\u0da9\u0dbb\u0dba </p>\n",
|
||||
"<p>Transformer with <span translate=no>_^_0_^_</span> layers </p>\n": "<p><span translate=no>_^_0_^_</span> \u0dc3\u0dca\u0dae\u0dbb \u0dc3\u0dc4\u0dd2\u0dad \u0da7\u0dca\u0dbb\u0dcf\u0db1\u0dca\u0dc3\u0dca\u0dc6\u0ddd\u0db8\u0dbb\u0dca </p>\n",
|
||||
"<p>Use Tiny Shakespeare dataset </p>\n": "<p>\u0d9a\u0dd4\u0da9\u0dcf\u0dc2\u0dda\u0d9a\u0dca\u0dc3\u0dca\u0db4\u0dd2\u0dba\u0dbb\u0dca \u0daf\u0dad\u0dca\u0dad \u0d9a\u0da7\u0dca\u0da7\u0dbd\u0dba \u0db7\u0dcf\u0dc0\u0dd2\u0dad\u0dcf \u0d9a\u0dbb\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<p>Use a context size of <span translate=no>_^_0_^_</span> </p>\n": "<p>\u0d9a\u0dc3\u0db1\u0dca\u0daf\u0dbb\u0dca\u0db7\u0dba \u0db4\u0dca\u0dbb\u0db8\u0dcf\u0dab\u0dba \u0db7\u0dcf\u0dc0\u0dd2\u0dad\u0dcf <span translate=no>_^_0_^_</span> </p>\n",
|
||||
"<p>Use character level tokenizer </p>\n": "<p>\u0d85\u0d9a\u0dca\u0dc2\u0dbb\u0db8\u0da7\u0dca\u0da7\u0db8\u0dda \u0da7\u0ddd\u0d9a\u0db1\u0dba\u0dd2\u0dc3\u0dbb\u0dca \u0db7\u0dcf\u0dc0\u0dd2\u0dad\u0dcf \u0d9a\u0dbb\u0db1\u0dca\u0db1 </p>\n",
|
||||
"<ul><li><span translate=no>_^_0_^_</span> are the input tokens of shape <span translate=no>_^_1_^_</span></li></ul>\n": "<ul><li><span translate=no>_^_0_^_</span> \u0dc4\u0dd0\u0da9\u0dba\u0dda \u0d86\u0daf\u0dcf\u0db1 \u0da7\u0ddd\u0d9a\u0db1 \u0dc0\u0dda <span translate=no>_^_1_^_</span></li></ul>\n",
|
||||
"<ul><li><span translate=no>_^_0_^_</span> is the number of tokens in the vocabulary </li>\n<li><span translate=no>_^_1_^_</span> is the embedding size </li>\n<li><span translate=no>_^_2_^_</span> is the number of transformer layers </li>\n<li><span translate=no>_^_3_^_</span> is the layer. We use <span translate=no>_^_4_^_</span> copies of this for the transformer.</li></ul>\n": "<ul><li><span translate=no>_^_0_^_</span> \u0dba\u0db1\u0dd4 \u0dc0\u0da0\u0db1 \u0db8\u0dcf\u0dbd\u0dcf\u0dc0\u0dda \u0da7\u0ddd\u0d9a\u0db1 \u0d9c\u0dab\u0db1 </li>\n<li><span translate=no>_^_1_^_</span> \u0d9a\u0dcf\u0dc0\u0dd0\u0daf\u0dca\u0daf\u0dd3\u0db8 \u0db4\u0dca\u0dbb\u0db8\u0dcf\u0dab\u0dba \u0dc0\u0dda </li>\n<li><span translate=no>_^_2_^_</span> \u0da7\u0dca\u0dbb\u0dcf\u0db1\u0dca\u0dc3\u0dca\u0dc6\u0ddd\u0db8\u0dbb\u0dca \u0dc3\u0dca\u0dae\u0dbb \u0d9c\u0dab\u0db1 </li>\n<li><span translate=no>_^_3_^_</span> \u0dc3\u0dca\u0dad\u0dbb\u0dba \u0dc0\u0dda. \u0d85\u0db4\u0dd2 \u0da7\u0dca\u0dbb\u0dcf\u0db1\u0dca\u0dc3\u0dca\u0dc6\u0ddd\u0db8\u0dbb\u0dca \u0dc3\u0db3\u0dc4\u0dcf \u0db8\u0dd9\u0dc4\u0dd2 <span translate=no>_^_4_^_</span> \u0db4\u0dd2\u0da7\u0db4\u0dad\u0dca \u0db7\u0dcf\u0dc0\u0dd2\u0dad\u0dcf \u0d9a\u0dbb\u0db8\u0dd4. </li></ul>\n",
|
||||
"<ul><li><span translate=no>_^_0_^_</span> is the number of features in a token embedding </li>\n<li><span translate=no>_^_1_^_</span> is the number of features in the hidden layer of the FFN </li>\n<li><span translate=no>_^_2_^_</span> is FTA activation module </li>\n<li><span translate=no>_^_3_^_</span> is dropout probability for the hidden layer</li></ul>\n": "<ul><li><span translate=no>_^_0_^_</span> \u0dba\u0db1\u0dd4 \u0da7\u0ddd\u0d9a\u0db1 \u0d9a\u0dcf\u0dc0\u0dd0\u0daf\u0dca\u0daf\u0dd3\u0db8 \u0dad\u0dd4\u0dc5 \u0d87\u0dad\u0dd2 \u0dc0\u0dd2\u0dc1\u0dda\u0dc2\u0dcf\u0d82\u0d9c \u0d9c\u0dab\u0db1 </li>\n<li><span translate=no>_^_1_^_</span> \u0dba\u0db1\u0dd4 FFN \u0dc4\u0dd2 \u0dc3\u0dd0\u0d9f\u0dc0\u0dd4\u0dab\u0dd4 \u0dc3\u0dca\u0dae\u0dbb\u0dba\u0dda \u0d87\u0dad\u0dd2 \u0dbd\u0d9a\u0dca\u0dc2\u0dab \u0d9c\u0dab\u0db1 </li>\n<li><span translate=no>_^_2_^_</span> FTA \u0dc3\u0d9a\u0dca\u0dbb\u0dd2\u0dba \u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dda \u0db8\u0ddc\u0da9\u0dd2\u0dba\u0dd4\u0dbd\u0dba </li>\n<li><span translate=no>_^_3_^_</span> \u0dc3\u0dd0\u0d9f\u0dc0\u0dd4\u0dab\u0dd4 \u0dc3\u0dca\u0dad\u0dbb\u0dba \u0dc3\u0db3\u0dc4\u0dcf \u0d85\u0dad\u0dc4\u0dd0\u0dbb \u0daf\u0dd0\u0db8\u0dd3\u0db8\u0dda \u0dc3\u0db8\u0dca\u0db7\u0dcf\u0dc0\u0dd2\u0dad\u0dcf\u0dc0</li></ul>\n",
|
||||
"Fuzzy Tiling Activation Experiment": "\u0db1\u0ddc\u0db4\u0dd0\u0dc4\u0dd0\u0daf\u0dd2\u0dbd\u0dd2 \u0da7\u0dba\u0dd2\u0dbd\u0dca \u0dc3\u0d9a\u0dca\u0dbb\u0dd3\u0dba \u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dda \u0d85\u0dad\u0dca\u0dc4\u0daf\u0dcf \u0db6\u0dd0\u0dbd\u0dd3\u0db8",
|
||||
"Training a transformer with FTA in FFN on Tiny Shakespeare.": "\u0d9a\u0dd4\u0da9\u0dcf \u0dc2\u0dda\u0d9a\u0dca\u0dc3\u0dca\u0db4\u0dd2\u0dba\u0dbb\u0dca \u0db8\u0dad FFN \u0dc4\u0dd2 FTA \u0dc3\u0db8\u0d9f \u0da7\u0dca\u0dbb\u0dcf\u0db1\u0dca\u0dc3\u0dca\u0dc6\u0ddd\u0db8\u0dbb\u0dba\u0d9a\u0dca \u0db4\u0dd4\u0dc4\u0dd4\u0dab\u0dd4 \u0d9a\u0dd2\u0dbb\u0dd3\u0db8."
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"<h1><a href=\"index.html\">Fuzzy Tiling Activation</a> Experiment</h1>\n<p><a href=\"https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/activations/fta/experiment.ipynb\"><span translate=no>_^_0_^_</span></a></p>\n<p>Here we train a transformer that uses <a href=\"index.html\">Fuzzy Tiling Activation</a> in the <a href=\"../../transformers/feed_forward.html\">Feed-Forward Network</a>. We use it for a language model and train it on Tiny Shakespeare dataset for demonstration.</p>\n<p>However, this is probably not the ideal task for FTA, and we believe FTA is more suitable for modeling data with continuous variables.</p>\n": "<h1><a href=\"index.html\">\u6a21\u7cca\u62fc\u8d34\u6fc0\u6d3b</a>\u5b9e\u9a8c</h1>\n<p><a href=\"https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/activations/fta/experiment.ipynb\"><span translate=no>_^_0_^_</span></a></p>\n<p>\u5728\u8fd9\u91cc\uff0c\u6211\u4eec\u8bad\u7ec3\u4e00\u53f0\u5728<a href=\"../../transformers/feed_forward.html\">\u524d\u9988\u7f51\u7edc</a>\u4e2d\u4f7f\u7528<a href=\"index.html\">\u6a21\u7cca\u5207\u7247\u6fc0\u6d3b</a>\u7684\u53d8\u538b\u5668\u3002\u6211\u4eec\u5c06\u5176\u7528\u4f5c\u8bed\u8a00\u6a21\u578b\uff0c\u5e76\u5728\u5c0f\u838e\u58eb\u6bd4\u4e9a\u6570\u636e\u96c6\u4e0a\u5bf9\u5176\u8fdb\u884c\u8bad\u7ec3\u4ee5\u8fdb\u884c\u6f14\u793a\u3002</p>\n<p>\u4f46\u662f\uff0c\u5bf9\u4e8e FTA \u6765\u8bf4\uff0c\u8fd9\u53ef\u80fd\u4e0d\u662f\u7406\u60f3\u7684\u4efb\u52a1\uff0c\u6211\u4eec\u8ba4\u4e3a FTA \u66f4\u9002\u5408\u5bf9\u5177\u6709\u8fde\u7eed\u53d8\u91cf\u7684\u6570\u636e\u8fdb\u884c\u5efa\u6a21\u3002</p>\n",
|
||||
"<h2>Auto-Regressive model</h2>\n<p>This is an autoregressive transformer model that uses Feed-Forward Networks with (Fuzzy Tiling Activations)(index.html).</p>\n": "<h2>\u81ea\u56de\u5f52\u6a21\u578b</h2>\n<p>\u8fd9\u662f\u4e00\u4e2a\u81ea\u56de\u5f52\u53d8\u538b\u5668\u6a21\u578b\uff0c\u5b83\u4f7f\u7528\u524d\u9988\u7f51\u7edc\u548c\uff08\u6a21\u7cca\u5e73\u94fa\u6fc0\u6d3b\uff09\uff08index.html\uff09\u3002</p>\n",
|
||||
"<h2>Configurations</h2>\n<p>This inherits from <a href=\"../../experiments/nlp_autoregression.html#NLPAutoRegressionConfigs\"><span translate=no>_^_0_^_</span></a></p>\n": "<h2>\u914d\u7f6e</h2>\n<p>\u8fd9\u7ee7\u627f\u81ea <a href=\"../../experiments/nlp_autoregression.html#NLPAutoRegressionConfigs\"><span translate=no>_^_0_^_</span></a></p>\n",
|
||||
"<h2>FFN module with <a href=\"index.html\">FTA</a> activation</h2>\n": "<h2>\u5e26\u6709 F <a href=\"index.html\">TA \u6fc0\u6d3b\u529f\u80fd\u7684 FF</a> N \u6a21\u5757</h2>\n",
|
||||
"<h4>Create and run the experiment</h4>\n": "<h4>\u521b\u5efa\u5e76\u8fd0\u884c\u5b9e\u9a8c</h4>\n",
|
||||
"<h4>Initialize the model</h4>\n": "<h4>\u521d\u59cb\u5316\u6a21\u578b</h4>\n",
|
||||
"<p> </p>\n": "<p></p>\n",
|
||||
"<p><span translate=no>_^_0_^_</span> </p>\n": "<p><span translate=no>_^_0_^_</span></p>\n",
|
||||
"<p><span translate=no>_^_0_^_</span> and <span translate=no>_^_1_^_</span> for DeepNorm </p>\n": "<p><span translate=no>_^_0_^_</span>\u5bf9<span translate=no>_^_1_^_</span>\u4e8e DeepNorm</p>\n",
|
||||
"<p>Activation function <span translate=no>_^_0_^_</span> </p>\n": "<p>\u6fc0\u6d3b\u529f\u80fd<span translate=no>_^_0_^_</span></p>\n",
|
||||
"<p>Adam optimizer with no warmup </p>\n": "<p>\u6ca1\u6709\u9884\u70ed\u7684 Adam \u4f18\u5316\u5668</p>\n",
|
||||
"<p>Apply dropout </p>\n": "<p>\u7533\u8bf7\u9000\u5b66</p>\n",
|
||||
"<p>Batch size <span translate=no>_^_0_^_</span> </p>\n": "<p>\u6279\u91cf\u5927\u5c0f<span translate=no>_^_0_^_</span></p>\n",
|
||||
"<p>Create FTA activation module </p>\n": "<p>\u521b\u5efa FTA \u6fc0\u6d3b\u6a21\u5757</p>\n",
|
||||
"<p>Create auto-regressive mask </p>\n": "<p>\u521b\u5efa\u81ea\u52a8\u56de\u5f52\u906e\u7f69</p>\n",
|
||||
"<p>Create configs </p>\n": "<p>\u521b\u5efa\u914d\u7f6e</p>\n",
|
||||
"<p>Create experiment </p>\n": "<p>\u521b\u5efa\u5b9e\u9a8c</p>\n",
|
||||
"<p>Create the transformer. We re-use <a href=\"../../transformers/models.html#TransformerLayer\"><span translate=no>_^_0_^_</span></a> and <a href=\"../../transformers/mha.html\"><span translate=no>_^_1_^_</span></a> implementations. </p>\n": "<p>\u521b\u5efa\u53d8\u538b\u5668\u3002\u6211\u4eec\u91cd\u590d\u4f7f\u7528<a href=\"../../transformers/models.html#TransformerLayer\"><span translate=no>_^_0_^_</span></a>\u548c<a href=\"../../transformers/mha.html\"><span translate=no>_^_1_^_</span></a>\u5b9e\u73b0\u3002</p>\n",
|
||||
"<p>Embedding size </p>\n": "<p>\u5d4c\u5165\u5927\u5c0f</p>\n",
|
||||
"<p>FTA </p>\n": "<p>\u81ea\u8d38\u533a</p>\n",
|
||||
"<p>Feed forward layer size </p>\n": "<p>\u524d\u9988\u56fe\u5c42\u5927\u5c0f</p>\n",
|
||||
"<p>Get logits </p>\n": "<p>\u83b7\u53d6\u65e5\u5fd7</p>\n",
|
||||
"<p>Get the token embeddings </p>\n": "<p>\u83b7\u53d6\u4ee4\u724c\u5d4c\u5165</p>\n",
|
||||
"<p>Hidden layer dropout </p>\n": "<p>\u9690\u85cf\u56fe\u5c42\u4e22\u5931</p>\n",
|
||||
"<p>Layer one parameterized by weight <span translate=no>_^_0_^_</span> and bias <span translate=no>_^_1_^_</span> </p>\n": "<p>\u7b2c\u4e00\u5c42\u6309\u6743\u91cd<span translate=no>_^_0_^_</span>\u548c\u504f\u5dee\u8fdb\u884c\u53c2\u6570\u5316<span translate=no>_^_1_^_</span></p>\n",
|
||||
"<p>Layer two parameterized by weight <span translate=no>_^_0_^_</span> and bias <span translate=no>_^_1_^_</span> </p>\n": "<p>\u7b2c\u4e8c\u5c42\u6309\u6743\u91cd<span translate=no>_^_0_^_</span>\u548c\u504f\u5dee\u8fdb\u884c\u53c2\u6570\u5316<span translate=no>_^_1_^_</span></p>\n",
|
||||
"<p>Model </p>\n": "<p>\u578b\u53f7</p>\n",
|
||||
"<p>Move to the device </p>\n": "<p>\u79fb\u5230\u8bbe\u5907</p>\n",
|
||||
"<p>Number of heads in the attention </p>\n": "<p>\u5173\u6ce8\u7684\u5934\u90e8\u6570\u91cf</p>\n",
|
||||
"<p>Number of layers </p>\n": "<p>\u5c42\u6570</p>\n",
|
||||
"<p>Override configurations </p>\n": "<p>\u8986\u76d6\u914d\u7f6e</p>\n",
|
||||
"<p>Prompt separator is blank </p>\n": "<p>\u63d0\u793a\u5206\u9694\u7b26\u4e3a\u7a7a</p>\n",
|
||||
"<p>Readout layer </p>\n": "<p>\u8bfb\u51fa\u5c42</p>\n",
|
||||
"<p>Return results </p>\n": "<p>\u8fd4\u56de\u7ed3\u679c</p>\n",
|
||||
"<p>Run training </p>\n": "<p>\u8dd1\u6b65\u8bad\u7ec3</p>\n",
|
||||
"<p>Set model(s) for saving and loading </p>\n": "<p>\u8bbe\u7f6e\u7528\u4e8e\u4fdd\u5b58\u548c\u52a0\u8f7d\u7684\u6a21\u578b</p>\n",
|
||||
"<p>Size of each attention head </p>\n": "<p>\u6bcf\u4e2a\u6ce8\u610f\u5934\u7684\u5927\u5c0f</p>\n",
|
||||
"<p>Start the experiment </p>\n": "<p>\u5f00\u59cb\u5b9e\u9a8c</p>\n",
|
||||
"<p>Starting prompt for sampling </p>\n": "<p>\u5f00\u59cb\u91c7\u6837\u63d0\u793a</p>\n",
|
||||
"<p>Subsequent mask, will mask out tokens from seeing future tokens </p>\n": "<p>\u540e\u7eed\u7684\u63a9\u7801\uff0c\u5c06\u63a9\u76d6\u4ee4\u724c\u4ee5\u514d\u770b\u5230\u672a\u6765\u7684\u4ee3\u5e01</p>\n",
|
||||
"<p>Switch between training and validation for <span translate=no>_^_0_^_</span> times per epoch </p>\n": "<p>\u5728\u8bad\u7ec3\u548c\u9a8c\u8bc1\u4e4b\u95f4\u5207\u6362\u6bcf\u4e2a\u7eaa\u5143\u7684<span translate=no>_^_0_^_</span>\u6b21\u6570</p>\n",
|
||||
"<p>The mask will be initialized on the first call </p>\n": "<p>\u63a9\u7801\u5c06\u5728\u7b2c\u4e00\u6b21\u8c03\u7528\u65f6\u521d\u59cb\u5316</p>\n",
|
||||
"<p>Token embedding layer </p>\n": "<p>\u4ee4\u724c\u5d4c\u5165\u5c42</p>\n",
|
||||
"<p>Train for 32 epochs </p>\n": "<p>\u8bad\u7ec3 32 \u4e2a\u65f6\u4ee3</p>\n",
|
||||
"<p>Transformer encoder </p>\n": "<p>\u53d8\u538b\u5668\u7f16\u7801</p>\n",
|
||||
"<p>Transformer with <span translate=no>_^_0_^_</span> layers </p>\n": "<p>\u5e26<span translate=no>_^_0_^_</span>\u5c42\u7684\u53d8\u538b\u5668</p>\n",
|
||||
"<p>Use Tiny Shakespeare dataset </p>\n": "<p>\u4f7f\u7528\u5c0f\u838e\u58eb\u6bd4\u4e9a\u6570\u636e\u96c6</p>\n",
|
||||
"<p>Use a context size of <span translate=no>_^_0_^_</span> </p>\n": "<p>\u4f7f\u7528\u4e0a\u4e0b\u6587\u5927\u5c0f\u4e3a<span translate=no>_^_0_^_</span></p>\n",
|
||||
"<p>Use character level tokenizer </p>\n": "<p>\u4f7f\u7528\u89d2\u8272\u7b49\u7ea7\u5206\u8bcd\u5668</p>\n",
|
||||
"<ul><li><span translate=no>_^_0_^_</span> are the input tokens of shape <span translate=no>_^_1_^_</span></li></ul>\n": "<ul><li><span translate=no>_^_0_^_</span>\u662f\u5f62\u72b6\u7684\u8f93\u5165\u6807\u8bb0<span translate=no>_^_1_^_</span></li></ul>\n",
|
||||
"<ul><li><span translate=no>_^_0_^_</span> is the number of tokens in the vocabulary </li>\n<li><span translate=no>_^_1_^_</span> is the embedding size </li>\n<li><span translate=no>_^_2_^_</span> is the number of transformer layers </li>\n<li><span translate=no>_^_3_^_</span> is the layer. We use <span translate=no>_^_4_^_</span> copies of this for the transformer.</li></ul>\n": "<ul><li><span translate=no>_^_0_^_</span>\u662f\u8bcd\u6c47\u8868\u4e2d\u4ee3\u5e01\u7684\u6570\u91cf</li>\n<li><span translate=no>_^_1_^_</span>\u662f\u5d4c\u5165\u7684\u5927\u5c0f</li>\n<li><span translate=no>_^_2_^_</span>\u662f\u53d8\u538b\u5668\u5c42\u7684\u6570\u91cf</li>\n<li><span translate=no>_^_3_^_</span>\u662f\u5c42\u3002\u6211\u4eec\u5728\u53d8\u538b\u5668\u4e0a\u4f7f\u7528\u8fd9\u4e2a<span translate=no>_^_4_^_</span>\u526f\u672c\u3002</li></ul>\n",
|
||||
"<ul><li><span translate=no>_^_0_^_</span> is the number of features in a token embedding </li>\n<li><span translate=no>_^_1_^_</span> is the number of features in the hidden layer of the FFN </li>\n<li><span translate=no>_^_2_^_</span> is FTA activation module </li>\n<li><span translate=no>_^_3_^_</span> is dropout probability for the hidden layer</li></ul>\n": "<ul><li><span translate=no>_^_0_^_</span>\u662f\u4ee4\u724c\u5d4c\u5165\u4e2d\u7684\u8981\u7d20\u6570\u91cf</li>\n<li><span translate=no>_^_1_^_</span>\u662f FFN \u9690\u85cf\u5c42\u4e2d\u7684\u8981\u7d20\u6570\u91cf</li>\n<li><span translate=no>_^_2_^_</span>\u662f FTA \u6fc0\u6d3b\u6a21\u5757</li>\n<li><span translate=no>_^_3_^_</span>\u662f\u9690\u85cf\u5c42\u7684\u4e22\u5931\u6982\u7387</li></ul>\n",
|
||||
"Fuzzy Tiling Activation Experiment": "\u6a21\u7cca\u5e73\u94fa\u6fc0\u6d3b\u5b9e\u9a8c",
|
||||
"Training a transformer with FTA in FFN on Tiny Shakespeare.": "\u5728 Tiny Shakespeare \u7684 FFN \u4e2d\u4f7f\u7528\u81ea\u7531\u8d38\u6613\u534f\u5b9a\u8bad\u7ec3\u53d8\u538b\u5668\u3002"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"swish.py": "swish.py"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"swish.py": "swish.py"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"swish.py": "swish.py"
|
||||
}
|
||||
Reference in New Issue
Block a user