wehub-resource-sync
461bf6fd40
CI / lint (3.11) (push) Has been cancelled
CI / lint (3.12) (push) Has been cancelled
CI / lint (3.13) (push) Has been cancelled
CI / shellcheck (push) Has been cancelled
CI / shfmt (push) Has been cancelled
CI / setup (3.11) (push) Has been cancelled
CI / setup (3.12) (push) Has been cancelled
CI / setup (3.13) (push) Has been cancelled
CI / check-licenses (3.12) (push) Has been cancelled
CI / test_unit (3.11) (push) Has been cancelled
CI / test_unit (3.12) (push) Has been cancelled
CI / test_unit (3.13) (push) Has been cancelled
CI / test_unit_no_extras (3.11) (push) Has been cancelled
CI / test_unit_no_extras (3.12) (push) Has been cancelled
CI / test_json_to_html (3.12) (push) Has been cancelled
CI / test_unit_no_extras (3.13) (push) Has been cancelled
CI / test_unit_dependency_extras (csv, 3.12, --extra csv) (push) Has been cancelled
CI / test_unit_dependency_extras (xlsx, 3.11, --extra xlsx) (push) Has been cancelled
CI / test_unit_dependency_extras (xlsx, 3.12, --extra xlsx) (push) Has been cancelled
CI / test_unit_dependency_extras (csv, 3.11, --extra csv) (push) Has been cancelled
CI / test_unit_dependency_extras (csv, 3.13, --extra csv) (push) Has been cancelled
CI / test_unit_dependency_extras (docx, 3.11, --extra docx) (push) Has been cancelled
CI / test_unit_dependency_extras (docx, 3.12, --extra docx) (push) Has been cancelled
CI / test_unit_dependency_extras (docx, 3.13, --extra docx) (push) Has been cancelled
CI / test_unit_dependency_extras (markdown, 3.11, --extra md) (push) Has been cancelled
CI / test_unit_dependency_extras (markdown, 3.12, --extra md) (push) Has been cancelled
CI / test_unit_dependency_extras (markdown, 3.13, --extra md) (push) Has been cancelled
CI / test_unit_dependency_extras (odt, 3.11, --extra odt) (push) Has been cancelled
CI / test_unit_dependency_extras (odt, 3.12, --extra odt) (push) Has been cancelled
CI / test_unit_dependency_extras (odt, 3.13, --extra odt) (push) Has been cancelled
CI / test_unit_dependency_extras (pdf-image, 3.11, --extra pdf --extra image --extra paddleocr) (push) Has been cancelled
CI / test_unit_dependency_extras (pdf-image, 3.12, --extra pdf --extra image --extra paddleocr) (push) Has been cancelled
CI / test_unit_dependency_extras (pdf-image, 3.13, --extra pdf --extra image --extra paddleocr) (push) Has been cancelled
CI / test_unit_dependency_extras (pptx, 3.11, --extra pptx) (push) Has been cancelled
CI / test_unit_dependency_extras (pptx, 3.12, --extra pptx) (push) Has been cancelled
CI / test_unit_dependency_extras (pptx, 3.13, --extra pptx) (push) Has been cancelled
CI / test_unit_dependency_extras (pypandoc, 3.11, --extra epub --extra org --extra rtf --extra rst) (push) Has been cancelled
CI / test_unit_dependency_extras (pypandoc, 3.12, --extra epub --extra org --extra rtf --extra rst) (push) Has been cancelled
CI / test_unit_dependency_extras (pypandoc, 3.13, --extra epub --extra org --extra rtf --extra rst) (push) Has been cancelled
Build And Push Docker Image / set-short-sha (push) Has been cancelled
Partition Benchmark / setup (push) Has been cancelled
Partition Benchmark / Measure and compare partition() runtime (push) Has been cancelled
CI / test_unit_dependency_extras (xlsx, 3.13, --extra xlsx) (push) Has been cancelled
CI / test_ingest_src (3.12) (push) Has been cancelled
CI / test_json_to_markdown (3.12) (push) Has been cancelled
CI / changelog (push) Has been cancelled
CI / test_dockerfile (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Build And Push Docker Image / build-images (linux/amd64, opensource-linux-8core) (push) Has been cancelled
Build And Push Docker Image / build-images (linux/arm64, ubuntu-24.04-arm) (push) Has been cancelled
Build And Push Docker Image / publish-images (push) Has been cancelled
784 lines
23 KiB
JSON
784 lines
23 KiB
JSON
[
|
|
{
|
|
"type": "NarrativeText",
|
|
"element_id": "7581b3e14a56c276896da707704c221e",
|
|
"text": "output values. These are concatenated and once again projected, resulting in the final values, as depicted in Figure 2.",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
293.6572265625,
|
|
205.97867111111094
|
|
],
|
|
[
|
|
293.6572265625,
|
|
263.95533777777774
|
|
],
|
|
[
|
|
1406.7781982421875,
|
|
263.95533777777774
|
|
],
|
|
[
|
|
1406.7781982421875,
|
|
205.97867111111094
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "NarrativeText",
|
|
"element_id": "5f0b9e258d134a12434aaa080638e9de",
|
|
"text": "Multi-head attention allows the model to jointly attend to information from different representation subspaces at different positions. With a single attention head, averaging inhibits this.",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
291.65728759765625,
|
|
281.803671111111
|
|
],
|
|
[
|
|
291.65728759765625,
|
|
339.78033777777785
|
|
],
|
|
[
|
|
1411.1898193359375,
|
|
339.78033777777785
|
|
],
|
|
[
|
|
1411.1898193359375,
|
|
281.803671111111
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Formula",
|
|
"element_id": "2f5b0b2ffa8872dde498f34cd4af6bd9",
|
|
"text": "MultiHead(Q, K, V ) = Concat(head1, ..., headh)W O where headi = Attention(QW Q i , KW K i , V W V i )",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
519.2777777777777,
|
|
404.578103333333
|
|
],
|
|
[
|
|
519.2777777777777,
|
|
489.7858811111105
|
|
],
|
|
[
|
|
1210.03466796875,
|
|
489.7858811111105
|
|
],
|
|
[
|
|
1210.03466796875,
|
|
404.578103333333
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "NarrativeText",
|
|
"element_id": "703f1d4e9204c8b7ea94191f87138425",
|
|
"text": "Where the projections are parameter matrices W Q and W O \u2208 Rhdv\u00d7dmodel. i \u2208 Rdmodel\u00d7dk , W K i \u2208 Rdmodel\u00d7dk , W V i \u2208 Rdmodel\u00d7dv",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
297.09197998046875,
|
|
563.6114366666665
|
|
],
|
|
[
|
|
297.09197998046875,
|
|
632.4775599999999
|
|
],
|
|
[
|
|
1403.7974853515625,
|
|
632.4775599999999
|
|
],
|
|
[
|
|
1403.7974853515625,
|
|
563.6114366666665
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "NarrativeText",
|
|
"element_id": "e3e4737377b1614b02426ccc77bdcfc3",
|
|
"text": "In this work we employ h = 8 parallel attention layers, or heads. For each of these we use dk = dv = dmodel/h = 64. Due to the reduced dimension of each head, the total computational cost is similar to that of single-head attention with full dimensionality.",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
298.486328125,
|
|
649.7170677777777
|
|
],
|
|
[
|
|
298.486328125,
|
|
738.6081155555555
|
|
],
|
|
[
|
|
1411.319091796875,
|
|
738.6081155555555
|
|
],
|
|
[
|
|
1411.319091796875,
|
|
649.7170677777777
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Title",
|
|
"element_id": "31e28cc49f5625cec5e262fbb4b7e5f0",
|
|
"text": "3.2.3 Applications of Attention in our Model",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
297.9392395019531,
|
|
777.8849538888888
|
|
],
|
|
[
|
|
297.9392395019531,
|
|
805.5588427777777
|
|
],
|
|
[
|
|
841.6689453125,
|
|
805.5588427777777
|
|
],
|
|
[
|
|
841.6689453125,
|
|
777.8849538888888
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "NarrativeText",
|
|
"element_id": "f84e983da98f26bd5c141846aeffd0aa",
|
|
"text": "The Transformer uses multi-head attention in three different ways:",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
299.14166666666665,
|
|
831.0064488888887
|
|
],
|
|
[
|
|
299.14166666666665,
|
|
858.6803377777776
|
|
],
|
|
[
|
|
1042.12109375,
|
|
858.6803377777776
|
|
],
|
|
[
|
|
1042.12109375,
|
|
831.0064488888887
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "ListItem",
|
|
"element_id": "fd24bf7bf21b4aab2a36021f9ebb253b",
|
|
"text": "\u2022 In \"encoder-decoder attention\" layers, the queries come from the previous decoder layer, and the memory keys and values come from the output of the encoder. This allows every position in the decoder to attend over all positions in the input sequence. This mimics the typical encoder-decoder attention mechanisms in sequence-to-sequence models such as [38, 2, 9].",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
364.7121887207031,
|
|
888.9064488888888
|
|
],
|
|
[
|
|
364.7121887207031,
|
|
1037.7942266666666
|
|
],
|
|
[
|
|
1414.6971435546875,
|
|
1037.7942266666666
|
|
],
|
|
[
|
|
1414.6971435546875,
|
|
888.9064488888888
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "ListItem",
|
|
"element_id": "77762865993fd26c55c87cb45d75cad8",
|
|
"text": "\u2022 The encoder contains self-attention layers. In a self-attention layer all of the keys, values and queries come from the same place, in this case, the output of the previous layer in the encoder. Each position in the encoder can attend to all positions in the previous layer of the encoder.",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
357.6517333984375,
|
|
1054.1092266666667
|
|
],
|
|
[
|
|
357.6517333984375,
|
|
1172.6914488888888
|
|
],
|
|
[
|
|
1408.7703857421875,
|
|
1172.6914488888888
|
|
],
|
|
[
|
|
1408.7703857421875,
|
|
1054.1092266666667
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "ListItem",
|
|
"element_id": "41b9b9d2a4329a8f6075f4776403c2de",
|
|
"text": "\u2022 Similarly, self-attention layers in the decoder allow each position in the decoder to attend to all positions in the decoder up to and including that position. We need to prevent leftward information flow in the decoder to preserve the auto-regressive property. We implement this inside of scaled dot-product attention by masking out (setting to \u2212\u221e) all values in the input of the softmax which correspond to illegal connections. See Figure 2.",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
366.0421447753906,
|
|
1189.006448888889
|
|
],
|
|
[
|
|
366.0421447753906,
|
|
1337.8914488888888
|
|
],
|
|
[
|
|
1416.1551513671875,
|
|
1337.8914488888888
|
|
],
|
|
[
|
|
1416.1551513671875,
|
|
1189.006448888889
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Title",
|
|
"element_id": "3b1f6da814e3826309b614d8b8dc9266",
|
|
"text": "3.3 Position-wise Feed-Forward Networks",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
297.3927917480469,
|
|
1380.9071761111109
|
|
],
|
|
[
|
|
297.3927917480469,
|
|
1408.5810649999999
|
|
],
|
|
[
|
|
816.5335693359375,
|
|
1408.5810649999999
|
|
],
|
|
[
|
|
816.5335693359375,
|
|
1380.9071761111109
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "NarrativeText",
|
|
"element_id": "46bb05e8d9c19147942fb75345ae3dbb",
|
|
"text": "In addition to attention sub-layers, each of the layers in our encoder and decoder contains a fully connected feed-forward network, which is applied to each position separately and identically. This consists of two linear transformations with a ReLU activation in between.",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
299.68072509765625,
|
|
1437.7647822222223
|
|
],
|
|
[
|
|
299.68072509765625,
|
|
1526.0442266666664
|
|
],
|
|
[
|
|
1413.704345703125,
|
|
1526.0442266666664
|
|
],
|
|
[
|
|
1413.704345703125,
|
|
1437.7647822222223
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Formula",
|
|
"element_id": "eda9b46d50730928c8437d6149e01a2b",
|
|
"text": "FFN(x) = max(0, xW1 + b1)W2 + b2 (2)",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
621.480712890625,
|
|
1577.511512222222
|
|
],
|
|
[
|
|
621.480712890625,
|
|
1607.7247699999998
|
|
],
|
|
[
|
|
1401.8538655555558,
|
|
1607.7247699999998
|
|
],
|
|
[
|
|
1401.8538655555558,
|
|
1577.511512222222
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "NarrativeText",
|
|
"element_id": "43c1741dc91b5b67a03a726873df3be5",
|
|
"text": "While the linear transformations are the same across different positions, they use different parameters from layer to layer. Another way of describing this is as two convolutions with kernel size 1. The dimensionality of input and output is dmodel = 512, and the inner-layer has dimensionality df f = 2048.",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
296.0164489746094,
|
|
1638.6370044444443
|
|
],
|
|
[
|
|
296.0164489746094,
|
|
1759.1525477777777
|
|
],
|
|
[
|
|
1410.78564453125,
|
|
1759.1525477777777
|
|
],
|
|
[
|
|
1410.78564453125,
|
|
1638.6370044444443
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Title",
|
|
"element_id": "63fc763509dec0fa03ba8296e4b0616e",
|
|
"text": "3.4 Embeddings and Softmax",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
297.77691650390625,
|
|
1800.2349538888889
|
|
],
|
|
[
|
|
297.77691650390625,
|
|
1827.9088427777776
|
|
],
|
|
[
|
|
669.7491455078125,
|
|
1827.9088427777776
|
|
],
|
|
[
|
|
669.7491455078125,
|
|
1800.2349538888889
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "NarrativeText",
|
|
"element_id": "ab8cefbb53c308302ee0e3c0c7ecfd25",
|
|
"text": "Similarly to other sequence transduction models, we use learned embeddings to convert the input tokens and output tokens to vectors of dimension dmodel. We also use the usual learned linear transfor- mation and softmax function to convert the decoder output to predicted next-token probabilities. In our model, we share the same weight matrix between the two embedding layers and the pre-softmax dmodel. linear transformation, similar to [30]. In the embedding layers, we multiply those weights by",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
300.0,
|
|
1857.09256
|
|
],
|
|
[
|
|
300.0,
|
|
2008.33428
|
|
],
|
|
[
|
|
1408.764892578125,
|
|
2008.33428
|
|
],
|
|
[
|
|
1408.764892578125,
|
|
1857.09256
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Footer",
|
|
"element_id": "b45e24bb89196d4b50d76df531acfaf2",
|
|
"text": "5",
|
|
"metadata": {
|
|
"coordinates": {
|
|
"points": [
|
|
[
|
|
842.5238037109375,
|
|
2061.325893333333
|
|
],
|
|
[
|
|
842.5238037109375,
|
|
2088.999782222222
|
|
],
|
|
[
|
|
857.5114135742188,
|
|
2088.999782222222
|
|
],
|
|
[
|
|
857.5114135742188,
|
|
2061.325893333333
|
|
]
|
|
],
|
|
"system": "PixelSpace",
|
|
"layout_width": 1700,
|
|
"layout_height": 2200
|
|
},
|
|
"filetype": "application/pdf",
|
|
"languages": [
|
|
"eng"
|
|
],
|
|
"page_number": 1,
|
|
"filename": "page-with-formula.pdf",
|
|
"data_source": {
|
|
"url": "https://unstructuredio.sharepoint.com/Shared%20Documents/nested/page-with-formula.pdf",
|
|
"version": "1.0",
|
|
"record_locator": {
|
|
"server_path": "/Shared Documents/nested/page-with-formula.pdf",
|
|
"site_url": "https://unstructuredio.sharepoint.com"
|
|
},
|
|
"date_created": "1719424780.0",
|
|
"date_modified": "1719424780.0"
|
|
}
|
|
}
|
|
}
|
|
] |