chore: import upstream snapshot with attribution

2026-07-13 12:19:01 +08:00
commit 3b90d1192f
2172 changed files with 594509 additions and 0 deletions
@@ -0,0 +1,12 @@
+{
+ "<h1>Utilities</h1>\n": "<h1>\u30e6\u30fc\u30c6\u30a3\u30ea\u30c6\u30a3</h1>\n",
+ "<h2>Clone Module</h2>\n<p>Make a <span translate=no>_^_0_^_</span> with clones of a given module</p>\n": "<h2>\u30af\u30ed\u30fc\u30f3\u30fb\u30e2\u30b8\u30e5\u30fc\u30eb</h2>\n<p><span translate=no>_^_0_^_</span>\u4e0e\u3048\u3089\u308c\u305f\u30e2\u30b8\u30e5\u30fc\u30eb\u306e\u30af\u30ed\u30fc\u30f3\u3092\u4f7f\u3063\u3066\u4f5c\u308b</p>\n",
+ "<p> <a id=\"cycle_dataloader\"></a></p>\n<h2>Cycle Data Loader</h2>\n<p>Infinite loader that recycles the data loader after each epoch</p>\n": "<p><a id=\"cycle_dataloader\"></a></p>\n<h2>\u30b5\u30a4\u30af\u30eb\u30c7\u30fc\u30bf\u30ed\u30fc\u30c0\u30fc</h2>\n<p>\u30a8\u30dd\u30c3\u30af\u3054\u3068\u306b\u30c7\u30fc\u30bf\u30ed\u30fc\u30c0\u30fc\u3092\u30ea\u30b5\u30a4\u30af\u30eb\u3059\u308b\u7121\u9650\u30ed\u30fc\u30c0\u30fc</p>\n",
+ "<p> <a id=\"map_style_dataset\"></a></p>\n<h2>Map Style Dataset</h2>\n<p>This converts an <a href=\"https://pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset\"><span translate=no>_^_0_^_</span></a> to a <a href=\"https://pytorch.org/docs/stable/data.html#map-style-datasets\">map-style dataset</a> so that we can shuffle the dataset.</p>\n<p><em>This only works when the dataset size is small and can be held in memory.</em></p>\n": "<p><a id=\"map_style_dataset\"></a></p>\n<h2>\u30de\u30c3\u30d7\u30b9\u30bf\u30a4\u30eb\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8</h2>\n<p>\u3053\u308c\u306b\u3088\u308a\u3001<a href=\"https://pytorch.org/docs/stable/data.html#map-style-datasets\">\u304c\u30de\u30c3\u30d7\u30b9\u30bf\u30a4\u30eb\u306e\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u306b\u5909\u63db\u3055\u308c\u3001<a href=\"https://pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset\"><span translate=no>_^_0_^_</span></a>\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u3092\u30b7\u30e3\u30c3\u30d5\u30eb\u3067\u304d\u308b\u3088\u3046\u306b\u306a\u308a\u307e\u3059</a>\u3002</p>\n<p><em>\u3053\u308c\u306f\u3001\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u306e\u30b5\u30a4\u30ba\u304c\u5c0f\u3055\u304f\u3001\u30e1\u30e2\u30ea\u306b\u4fdd\u6301\u3067\u304d\u308b\u5834\u5408\u306b\u306e\u307f\u6a5f\u80fd\u3057\u307e\u3059\u3002</em></p>\n",
+ "<p>Create an iterator </p>\n": "<p>\u30a4\u30c6\u30ec\u30fc\u30bf\u306e\u4f5c\u6210</p>\n",
+ "<p>Get a sample by index </p>\n": "<p>\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3067\u30b5\u30f3\u30d7\u30eb\u3092\u53d6\u5f97</p>\n",
+ "<p>Load the data to memory </p>\n": "<p>\u30c7\u30fc\u30bf\u3092\u30e1\u30e2\u30ea\u306b\u8aad\u307f\u8fbc\u3080</p>\n",
+ "<p>Size of the dataset </p>\n": "<p>\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u306e\u30b5\u30a4\u30ba</p>\n",
+ "A bunch of utility functions and classes": "\u305f\u304f\u3055\u3093\u306e\u30e6\u30fc\u30c6\u30a3\u30ea\u30c6\u30a3\u95a2\u6570\u3068\u30af\u30e9\u30b9",
+ "Utilities": "\u30e6\u30fc\u30c6\u30a3\u30ea\u30c6\u30a3"
+}
@@ -0,0 +1,12 @@
+{
+ "<h1>Utilities</h1>\n": "<h1>\u0d8b\u0db4\u0dba\u0ddd\u0d9c\u0dd2\u0dad\u0dcf</h1>\n",
+ "<h2>Clone Module</h2>\n<p>Make a <span translate=no>_^_0_^_</span> with clones of a given module</p>\n": "<h2>\u0d9a\u0dca\u0dbd\u0ddd\u0db1\u0db8\u0ddc\u0da9\u0dd2\u0dba\u0dd4\u0dbd\u0dba</h2>\n<p>\u0daf\u0dd3\u0d87\u0dad\u0dd2 \u0db8\u0ddc\u0da9\u0dd2\u0dba\u0dd4\u0dbd\u0dba\u0d9a \u0d9a\u0dca\u0dbd\u0ddd\u0db1 <span translate=no>_^_0_^_</span> \u0dc3\u0db8\u0d9f \u0dc3\u0dcf\u0daf\u0db1\u0dca\u0db1</p>\n",
+ "<p> <a id=\"cycle_dataloader\"></a></p>\n<h2>Cycle Data Loader</h2>\n<p>Infinite loader that recycles the data loader after each epoch</p>\n": "<p> <a id=\"cycle_dataloader\"></a></p>\n<h2>\u0da0\u0d9a\u0dca\u0dbb\u0dd3\u0dba\u0daf\u0dad\u0dca\u0dad \u0d9a\u0dcf\u0dbb\u0d9a\u0dba</h2>\n<p>\u0d91\u0d9a\u0dca\u0d91\u0d9a\u0dca \u0d91\u0db4\u0ddd\u0da0\u0dca \u0db4\u0dc3\u0dd4 \u0daf\u0dad\u0dca\u0dad \u0d9a\u0dcf\u0dbb\u0d9a\u0dba \u0db4\u0dca\u0dbb\u0dad\u0dd2\u0da0\u0d9a\u0dca\u0dbb\u0dd3\u0d9a\u0dbb\u0dab\u0dba \u0d9a\u0dbb\u0db1 \u0d85\u0db1\u0db1\u0dca\u0dad \u0d9a\u0dcf\u0dbb\u0d9a\u0dba</p>\n",
+ "<p> <a id=\"map_style_dataset\"></a></p>\n<h2>Map Style Dataset</h2>\n<p>This converts an <a href=\"https://pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset\"><span translate=no>_^_0_^_</span></a> to a <a href=\"https://pytorch.org/docs/stable/data.html#map-style-datasets\">map-style dataset</a> so that we can shuffle the dataset.</p>\n<p><em>This only works when the dataset size is small and can be held in memory.</em></p>\n": "<p> <a id=\"map_style_dataset\"></a></p>\n<h2>\u0dc3\u0dd2\u0dad\u0dd2\u0dba\u0db8\u0dc3\u0dca\u0da7\u0dba\u0dd2\u0dbd\u0dca \u0daf\u0dad\u0dca\u0dad \u0dc3\u0db8\u0dd4\u0daf\u0dcf\u0dba</h2>\n<p>\u0db8\u0dd9\u0dba <a href=\"https://pytorch.org/docs/stable/data.html#map-style-datasets\">\u0dc3\u0dd2\u0dad\u0dd2\u0dba\u0db8\u0dca \u0dc0\u0dd2\u0dbd\u0dcf\u0dc3\u0dd2\u0dad\u0dcf\u0dc0\u0dda \u0daf\u0dad\u0dca\u0dad \u0d9a\u0da7\u0dca\u0da7\u0dbd\u0dba\u0d9a\u0dca</a> \u0db6\u0dc0\u0da7 <a href=\"https://pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset\"><span translate=no>_^_0_^_</span></a> \u0db4\u0dbb\u0dd2\u0dc0\u0dbb\u0dca\u0dad\u0db1\u0dba \u0d9a\u0dbb\u0db1 \u0d85\u0dad\u0dbb \u0d91\u0db8\u0d9f\u0dd2\u0db1\u0dca \u0d85\u0db4\u0da7 \u0daf\u0dad\u0dca\u0dad \u0d9a\u0da7\u0dca\u0da7\u0dbd\u0dba \u0db8\u0dcf\u0dbb\u0dd4 \u0d9a\u0dc5 \u0dc4\u0dd0\u0d9a\u0dd2\u0dba. </p>\n<p><em>\u0db8\u0dd9\u0dba\u0d9a\u0dca\u0dbb\u0dd2\u0dba\u0dcf\u0dad\u0dca\u0db8\u0d9a \u0dc0\u0db1\u0dca\u0db1\u0dda \u0daf\u0dad\u0dca\u0dad \u0dc3\u0db8\u0dd4\u0daf\u0dcf\u0dba \u0db4\u0dca\u0dbb\u0db8\u0dcf\u0dab\u0dba \u0d9a\u0dd4\u0da9\u0dcf \u0dc0\u0db1 \u0d85\u0dad\u0dbb \u0db8\u0dad\u0d9a\u0dba\u0dda \u0dbb\u0db3\u0dc0\u0dcf \u0d9c\u0dad \u0dc4\u0dd0\u0d9a\u0dd2 \u0dc0\u0dd2\u0da7 \u0db4\u0db8\u0dab\u0dd2. </em></p>\n",
+ "<p>Create an iterator </p>\n": "<p>\u0d9a\u0d85\u0db1\u0dd4\u0d9a\u0dcf\u0dbb\u0d9a\u0dba\u0d9a\u0dca \u0dc3\u0dcf\u0daf\u0db1\u0dca\u0db1 </p>\n",
+ "<p>Get a sample by index </p>\n": "<p>\u0daf\u0dbb\u0dca\u0dc1\u0d9a\u0dba\u0d85\u0db1\u0dd4\u0dc0 \u0db1\u0dd2\u0dba\u0dd0\u0daf\u0dd2\u0dba\u0d9a\u0dca \u0dbd\u0db6\u0dcf \u0d9c\u0db1\u0dca\u0db1 </p>\n",
+ "<p>Load the data to memory </p>\n": "<p>\u0daf\u0dad\u0dca\u0dad\u0db8\u0dad\u0d9a\u0dba \u0dc0\u0dd9\u0dad \u0db4\u0dd6\u0dbb\u0dab\u0dba \u0d9a\u0dbb\u0db1\u0dca\u0db1 </p>\n",
+ "<p>Size of the dataset </p>\n": "<p>\u0daf\u0dad\u0dca\u0dad\u0dc3\u0db8\u0dd4\u0daf\u0dcf\u0dba \u0db4\u0dca\u0dbb\u0db8\u0dcf\u0dab\u0dba </p>\n",
+ "A bunch of utility functions and classes": "\u0d8b\u0db4\u0dba\u0ddd\u0d9c\u0dd2\u0dad\u0dcf \u0d9a\u0dcf\u0dbb\u0dca\u0dba\u0dba\u0db1\u0dca \u0dc3\u0dc4 \u0db4\u0db1\u0dca\u0dad\u0dd2 \u0db4\u0ddc\u0d9a\u0dd4\u0dbb\u0d9a\u0dca",
+ "Utilities": "\u0d8b\u0db4\u0dba\u0ddd\u0d9c\u0dd2\u0dad\u0dcf"
+}
@@ -0,0 +1,12 @@
+{
+ "<h1>Utilities</h1>\n": "<h1>\u516c\u5171\u4e8b\u4e1a</h1>\n",
+ "<h2>Clone Module</h2>\n<p>Make a <span translate=no>_^_0_^_</span> with clones of a given module</p>\n": "<h2>\u514b\u9686\u6a21\u5757</h2>\n<p><span translate=no>_^_0_^_</span>\u4f7f\u7528\u7ed9\u5b9a\u6a21\u5757\u7684\u514b\u9686\u5236\u4f5c\u4e00\u4e2a</p>\n",
+ "<p> <a id=\"cycle_dataloader\"></a></p>\n<h2>Cycle Data Loader</h2>\n<p>Infinite loader that recycles the data loader after each epoch</p>\n": "<p><a id=\"cycle_dataloader\"></a></p>\n<h2>\u5faa\u73af\u6570\u636e\u52a0\u8f7d\u5668</h2>\n<p>\u65e0\u9650\u52a0\u8f7d\u5668\uff0c\u5728\u6bcf\u4e2a\u7eaa\u5143\u4e4b\u540e\u56de\u6536\u6570\u636e\u52a0\u8f7d\u5668</p>\n",
+ "<p> <a id=\"map_style_dataset\"></a></p>\n<h2>Map Style Dataset</h2>\n<p>This converts an <a href=\"https://pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset\"><span translate=no>_^_0_^_</span></a> to a <a href=\"https://pytorch.org/docs/stable/data.html#map-style-datasets\">map-style dataset</a> so that we can shuffle the dataset.</p>\n<p><em>This only works when the dataset size is small and can be held in memory.</em></p>\n": "<p><a id=\"map_style_dataset\"></a></p>\n<h2>\u5730\u56fe\u6837\u5f0f\u6570\u636e\u96c6</h2>\n<p>\u8fd9\u4f1a\u5c06\u8f6c\u6362<a href=\"https://pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset\"><span translate=no>_^_0_^_</span></a>\u4e3a<a href=\"https://pytorch.org/docs/stable/data.html#map-style-datasets\">\u5730\u56fe\u6837\u5f0f\u7684\u6570\u636e\u96c6</a>\uff0c\u4ee5\u4fbf\u6211\u4eec\u53ef\u4ee5\u968f\u673a\u6392\u5217\u6570\u636e\u96c6\u3002</p>\n<p><em>\u8fd9\u4ec5\u5728\u6570\u636e\u96c6\u5927\u5c0f\u8f83\u5c0f\u4e14\u53ef\u4ee5\u4fdd\u5b58\u5728\u5185\u5b58\u4e2d\u65f6\u624d\u6709\u6548\u3002</em></p>\n",
+ "<p>Create an iterator </p>\n": "<p>\u521b\u5efa\u8fed\u4ee3\u5668</p>\n",
+ "<p>Get a sample by index </p>\n": "<p>\u6309\u7d22\u5f15\u83b7\u53d6\u6837\u672c</p>\n",
+ "<p>Load the data to memory </p>\n": "<p>\u5c06\u6570\u636e\u52a0\u8f7d\u5230\u5185\u5b58</p>\n",
+ "<p>Size of the dataset </p>\n": "<p>\u6570\u636e\u96c6\u7684\u5927\u5c0f</p>\n",
+ "A bunch of utility functions and classes": "\u4e00\u5806\u5b9e\u7528\u51fd\u6570\u548c\u7c7b",
+ "Utilities": "\u516c\u5171\u4e8b\u4e1a"
+}
@@ -0,0 +1,7 @@
+{
+ "<h3>Basic english tokenizer</h3>\n<p>We use character level tokenizer in this experiment. You can switch by setting,</p>\n<span translate=no>_^_0_^_</span><p>in the configurations dictionary when starting the experiment.</p>\n": "<h3>\u30d9\u30fc\u30b7\u30c3\u30af\u30fb\u30a4\u30f3\u30b0\u30ea\u30c3\u30b7\u30e5\u30fb\u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc</h3>\n<p>\u3053\u306e\u5b9f\u9a13\u3067\u306f\u3001\u30ad\u30e3\u30e9\u30af\u30bf\u30fc\u30ec\u30d9\u30eb\u306e\u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc\u3092\u4f7f\u7528\u3057\u307e\u3059\u3002\u8a2d\u5b9a\u3067\u5207\u308a\u66ff\u3048\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u304c\u3001</p>\n<span translate=no>_^_0_^_</span><p>\u5b9f\u9a13\u3092\u958b\u59cb\u3059\u308b\u3068\u304d\u306b\u69cb\u6210\u8f9e\u66f8\u306b\u3042\u308a\u307e\u3059\u3002</p>\n",
+ "<h3>Character level tokenizer</h3>\n": "<h3>\u30ad\u30e3\u30e9\u30af\u30bf\u30fc\u30ec\u30d9\u30eb\u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc</h3>\n",
+ "<p> <a id=\"TokenizerConfigs\"></a></p>\n<h2>Tokenizer Configurations</h2>\n": "<p><a id=\"TokenizerConfigs\"></a></p>\n<h2>\u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc\u8a2d\u5b9a</h2>\n",
+ "<p> Character level tokenizer configuration</p>\n": "<p>\u30ad\u30e3\u30e9\u30af\u30bf\u30fc\u30ec\u30d9\u30eb\u306e\u30c8\u30fc\u30af\u30ca\u30a4\u30b6\u30fc\u8a2d\u5b9a</p>\n",
+ "tokenizer.py": "tokenizer.py"
+}
@@ -0,0 +1,7 @@
+{
+ "<h3>Basic english tokenizer</h3>\n<p>We use character level tokenizer in this experiment. You can switch by setting,</p>\n<span translate=no>_^_0_^_</span><p>in the configurations dictionary when starting the experiment.</p>\n": "<h3>\u0db8\u0dd6\u0dbd\u0dd2\u0d9a\u0d89\u0d82\u0d9c\u0dca\u0dbb\u0dd3\u0dc3\u0dd2 \u0da7\u0ddd\u0d9a\u0db1\u0dba\u0dd2\u0dc3\u0dbb\u0dca</h3>\n<p>\u0db8\u0dd9\u0db8\u0d85\u0dad\u0dca\u0dc4\u0daf\u0dcf \u0db6\u0dd0\u0dbd\u0dd3\u0db8\u0dda\u0daf\u0dd3 \u0d85\u0db4\u0dd2 \u0da0\u0dbb\u0dd2\u0dad \u0db8\u0da7\u0dca\u0da7\u0db8\u0dda \u0da7\u0ddd\u0d9a\u0db1\u0dba\u0dd2\u0dc3\u0dbb\u0dca \u0db7\u0dcf\u0dc0\u0dd2\u0dad\u0dcf \u0d9a\u0dbb\u0db8\u0dd4. \u0dc3\u0dd0\u0d9a\u0dc3\u0dd3\u0db8\u0dd9\u0db1\u0dca \u0d94\u0db6\u0da7 \u0db8\u0dcf\u0dbb\u0dd4 \u0dc0\u0dd2\u0dba \u0dc4\u0dd0\u0d9a\u0dd2\u0dba,</p>\n<span translate=no>_^_0_^_</span><p>\u0d85\u0dad\u0dca\u0dc4\u0daf\u0dcf\u0db6\u0dd0\u0dbd\u0dd3\u0db8 \u0d86\u0dbb\u0db8\u0dca\u0db7 \u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dda\u0daf\u0dd3 \u0dc0\u0dd2\u0db1\u0dca\u0dba\u0dcf\u0dc3 \u0d9a\u0dd2\u0dbb\u0dd3\u0db8\u0dda \u0dc1\u0db6\u0dca\u0daf\u0d9a\u0ddd\u0dc2\u0dba\u0dda. </p>\n",
+ "<h3>Character level tokenizer</h3>\n": "<h3>\u0d85\u0d9a\u0dca\u0dc2\u0dbb\u0db8\u0da7\u0dca\u0da7\u0db8\u0dda \u0da7\u0ddd\u0d9a\u0db1\u0dba\u0dd2\u0dc3\u0dbb\u0dca</h3>\n",
+ "<p> <a id=\"TokenizerConfigs\"></a></p>\n<h2>Tokenizer Configurations</h2>\n": "<p> <a id=\"TokenizerConfigs\"></a></p>\n<h2>\u0da7\u0ddd\u0d9a\u0db1\u0dba\u0dd2\u0dc3\u0dbb\u0dca\u0dc0\u0dd2\u0db1\u0dca\u0dba\u0dcf\u0dc3\u0dba\u0db1\u0dca</h2>\n",
+ "<p> Character level tokenizer configuration</p>\n": "<p> \u0d85\u0d9a\u0dca\u0dc2\u0dbb\u0db8\u0da7\u0dca\u0da7\u0db8\u0dda \u0da7\u0ddd\u0d9a\u0db1\u0dba\u0dd2\u0dc3\u0dbb\u0dca \u0dc0\u0dd2\u0db1\u0dca\u0dba\u0dcf\u0dc3\u0dba</p>\n",
+ "tokenizer.py": "tokenizer.py"
+}
@@ -0,0 +1,7 @@
+{
+ "<h3>Basic english tokenizer</h3>\n<p>We use character level tokenizer in this experiment. You can switch by setting,</p>\n<span translate=no>_^_0_^_</span><p>in the configurations dictionary when starting the experiment.</p>\n": "<h3>\u57fa\u7840\u82f1\u8bed\u5206\u8bcd\u5668</h3>\n<p>\u6211\u4eec\u5728\u8fd9\u4e2a\u5b9e\u9a8c\u4e2d\u4f7f\u7528\u89d2\u8272\u7b49\u7ea7\u5206\u8bcd\u5668\u3002\u4f60\u53ef\u4ee5\u901a\u8fc7\u8bbe\u7f6e\u8fdb\u884c\u5207\u6362\uff0c</p>\n<span translate=no>_^_0_^_</span><p>\u5f00\u59cb\u5b9e\u9a8c\u65f6\u5728\u914d\u7f6e\u5b57\u5178\u4e2d\u3002</p>\n",
+ "<h3>Character level tokenizer</h3>\n": "<h3>\u89d2\u8272\u7b49\u7ea7\u5206\u8bcd\u5668</h3>\n",
+ "<p> <a id=\"TokenizerConfigs\"></a></p>\n<h2>Tokenizer Configurations</h2>\n": "<p><a id=\"TokenizerConfigs\"></a></p>\n<h2>\u5206\u8bcd\u5668\u914d\u7f6e</h2>\n",
+ "<p> Character level tokenizer configuration</p>\n": "<p>\u89d2\u8272\u7ea7\u522b\u5206\u8bcd\u5668\u914d\u7f6e</p>\n",
+ "tokenizer.py": "tokenizer.py"
+}