chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,449 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="si">
|
||||
<head>
|
||||
<meta http-equiv="content-type" content="text/html;charset=utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<meta name="description" content="මෙය ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරයක් සඳහා සටහන් සහිත පුහුණු කේතයකි."/>
|
||||
|
||||
<meta name="twitter:card" content="summary"/>
|
||||
<meta name="twitter:image:src" content="https://avatars1.githubusercontent.com/u/64068543?s=400&v=4"/>
|
||||
<meta name="twitter:title" content="දුම්රිය ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්"/>
|
||||
<meta name="twitter:description" content="මෙය ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරයක් සඳහා සටහන් සහිත පුහුණු කේතයකි."/>
|
||||
<meta name="twitter:site" content="@labmlai"/>
|
||||
<meta name="twitter:creator" content="@labmlai"/>
|
||||
|
||||
<meta property="og:url" content="https://nn.labml.ai/transformers/feedback/experiment.html"/>
|
||||
<meta property="og:title" content="දුම්රිය ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්"/>
|
||||
<meta property="og:image" content="https://avatars1.githubusercontent.com/u/64068543?s=400&v=4"/>
|
||||
<meta property="og:site_name" content="දුම්රිය ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්"/>
|
||||
<meta property="og:type" content="object"/>
|
||||
<meta property="og:title" content="දුම්රිය ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්"/>
|
||||
<meta property="og:description" content="මෙය ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරයක් සඳහා සටහන් සහිත පුහුණු කේතයකි."/>
|
||||
|
||||
<title>දුම්රිය ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්</title>
|
||||
<link rel="shortcut icon" href="/icon.png"/>
|
||||
<link rel="stylesheet" href="../../pylit.css?v=1">
|
||||
<link rel="canonical" href="https://nn.labml.ai/transformers/feedback/experiment.html"/>
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.18/dist/katex.min.css" integrity="sha384-zTROYFVGOfTw7JV7KUu8udsvW2fx4lWOsCEDqhBreBwlHI4ioVRtmIvEThzJHGET" crossorigin="anonymous">
|
||||
|
||||
<!-- Global site tag (gtag.js) - Google Analytics -->
|
||||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-4V3HC8HBLH"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
|
||||
function gtag() {
|
||||
dataLayer.push(arguments);
|
||||
}
|
||||
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-4V3HC8HBLH');
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<div id='container'>
|
||||
<div id="background"></div>
|
||||
<div class='section'>
|
||||
<div class='docs'>
|
||||
<p>
|
||||
<a class="parent" href="/">home</a>
|
||||
<a class="parent" href="../index.html">transformers</a>
|
||||
<a class="parent" href="index.html">feedback</a>
|
||||
</p>
|
||||
<p>
|
||||
<a href="https://github.com/labmlai/annotated_deep_learning_paper_implementations" target="_blank">
|
||||
<img alt="Github"
|
||||
src="https://img.shields.io/github/stars/labmlai/annotated_deep_learning_paper_implementations?style=social"
|
||||
style="max-width:100%;"/></a>
|
||||
<a href="https://twitter.com/labmlai" rel="nofollow" target="_blank">
|
||||
<img alt="Twitter"
|
||||
src="https://img.shields.io/twitter/follow/labmlai?style=social"
|
||||
style="max-width:100%;"/></a>
|
||||
</p>
|
||||
<p>
|
||||
<a href="https://github.com/labmlai/annotated_deep_learning_paper_implementations/tree/master/labml_nn/transformers/feedback/experiment.py" target="_blank">
|
||||
View code on Github</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-0'>
|
||||
<div class='docs doc-strings'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-0'>#</a>
|
||||
</div>
|
||||
<h1>දුම්රියප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්</h1>
|
||||
<p>මෙයස්වයංක්රීයව ප්රතිගාමී කිරීම සඳහා <a href="index.html">ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්</a> ආකෘතියක් පුහුණු කරයි. යතුරු සහ අගයන් පූර්ව ගණනය කරනු ලබන මුල් ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරය හෝ නව අනුවාදය ඔබට තෝරා ගත හැකිය. </p>
|
||||
<p>කුඩාෂේක්ස්පියර් දත්ත කට්ටලය පිළිබඳ ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරයක් පුහුණු කිරීම සඳහා කොලබ් සටහන් පොතක් මෙන්න. </p>
|
||||
<p><a href="https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/transformers/feedback/experiment.ipynb"><img alt="Open In Colab" src="https://colab.research.google.com/assets/colab-badge.svg"></a> <a href="https://app.labml.ai/run/d8eb9416530a11eb8fb50242ac1c0002"> <img alt="View Run" src="https://img.shields.io/badge/labml-experiment-brightgreen"></a></p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">19</span><span></span><span class="kn">import</span> <span class="nn">torch</span>
|
||||
<span class="lineno">20</span><span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">nn</span>
|
||||
<span class="lineno">21</span>
|
||||
<span class="lineno">22</span><span class="kn">from</span> <span class="nn">labml</span> <span class="kn">import</span> <span class="n">experiment</span>
|
||||
<span class="lineno">23</span><span class="kn">from</span> <span class="nn">labml.configs</span> <span class="kn">import</span> <span class="n">option</span>
|
||||
<span class="lineno">24</span><span class="kn">from</span> <span class="nn">labml.utils.pytorch</span> <span class="kn">import</span> <span class="n">get_modules</span>
|
||||
<span class="lineno">25</span><span class="kn">from</span> <span class="nn">labml_helpers.module</span> <span class="kn">import</span> <span class="n">Module</span>
|
||||
<span class="lineno">26</span>
|
||||
<span class="lineno">27</span><span class="kn">from</span> <span class="nn">labml_nn.experiments.nlp_autoregression</span> <span class="kn">import</span> <span class="n">NLPAutoRegressionConfigs</span>
|
||||
<span class="lineno">28</span><span class="kn">from</span> <span class="nn">labml_nn.transformers</span> <span class="kn">import</span> <span class="n">Encoder</span><span class="p">,</span> <span class="n">Generator</span><span class="p">,</span> <span class="n">TransformerConfigs</span>
|
||||
<span class="lineno">29</span><span class="kn">from</span> <span class="nn">labml_nn.transformers.utils</span> <span class="kn">import</span> <span class="n">subsequent_mask</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-1'>
|
||||
<div class='docs doc-strings'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-1'>#</a>
|
||||
</div>
|
||||
<h2>ස්වයංක්රීයප්රතිගාමී ආකෘතිය</h2>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">32</span><span class="k">class</span> <span class="nc">AutoregressiveModel</span><span class="p">(</span><span class="n">Module</span><span class="p">):</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-2'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-2'>#</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">37</span> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n_vocab</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">d_model</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">transformer</span><span class="p">:</span> <span class="n">Module</span><span class="p">):</span>
|
||||
<span class="lineno">38</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-3'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-3'>#</a>
|
||||
</div>
|
||||
<p>ටෝකන්කාවැද්දීම මොඩියුලය </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">40</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_embed</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">n_vocab</span><span class="p">,</span> <span class="n">d_model</span><span class="p">)</span>
|
||||
<span class="lineno">41</span> <span class="bp">self</span><span class="o">.</span><span class="n">transformer</span> <span class="o">=</span> <span class="n">transformer</span>
|
||||
<span class="lineno">42</span> <span class="bp">self</span><span class="o">.</span><span class="n">generator</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">d_model</span><span class="p">,</span> <span class="n">n_vocab</span><span class="p">)</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-4'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-4'>#</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">44</span> <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-5'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-5'>#</a>
|
||||
</div>
|
||||
<p>ටෝකනකාවැද්දීම </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">46</span> <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">src_embed</span><span class="p">(</span><span class="n">x</span><span class="p">)</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-6'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-6'>#</a>
|
||||
</div>
|
||||
<p>ට්රාන්ස්ෆෝමරයහරහා එය ධාවනය කරන්න </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">48</span> <span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">transformer</span><span class="p">(</span><span class="n">x</span><span class="p">)</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-7'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-7'>#</a>
|
||||
</div>
|
||||
<p>ඊළඟටෝකනයේ පිවිසුම් ජනනය කරන්න </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">50</span> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">generator</span><span class="p">(</span><span class="n">res</span><span class="p">),</span> <span class="kc">None</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-8'>
|
||||
<div class='docs doc-strings'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-8'>#</a>
|
||||
</div>
|
||||
<h2>වින්යාසකිරීම්</h2>
|
||||
<p>අපිඅත්හදා බැලීම ආරම්භ කරන විට පෙරනිමි වින්යාස කළ හැකි අතර එය අධික ලෙස ධාවනය වනු ඇත</p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">53</span><span class="k">class</span> <span class="nc">Configs</span><span class="p">(</span><span class="n">NLPAutoRegressionConfigs</span><span class="p">):</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-9'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-9'>#</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">60</span> <span class="n">model</span><span class="p">:</span> <span class="n">AutoregressiveModel</span>
|
||||
<span class="lineno">61</span>
|
||||
<span class="lineno">62</span> <span class="n">d_model</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">512</span>
|
||||
<span class="lineno">63</span> <span class="n">heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span>
|
||||
<span class="lineno">64</span> <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||||
<span class="lineno">65</span> <span class="n">d_ff</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2048</span>
|
||||
<span class="lineno">66</span> <span class="n">n_layers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">6</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-10'>
|
||||
<div class='docs doc-strings'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-10'>#</a>
|
||||
</div>
|
||||
<p> <a href="index.html">මුල් ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරය</a>සාදන්න. </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">69</span><span class="nd">@option</span><span class="p">(</span><span class="n">Configs</span><span class="o">.</span><span class="n">model</span><span class="p">)</span>
|
||||
<span class="lineno">70</span><span class="k">def</span> <span class="nf">feedback_transformer</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">Configs</span><span class="p">):</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-11'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-11'>#</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">74</span> <span class="kn">from</span> <span class="nn">labml_nn.transformers.feedback</span> <span class="kn">import</span> <span class="n">FeedbackTransformer</span><span class="p">,</span> <span class="n">FeedbackTransformerLayer</span><span class="p">,</span> \
|
||||
<span class="lineno">75</span> <span class="n">FeedbackAttention</span><span class="p">,</span> <span class="n">FeedForward</span>
|
||||
<span class="lineno">76</span>
|
||||
<span class="lineno">77</span> <span class="k">return</span> <span class="n">AutoregressiveModel</span><span class="p">(</span>
|
||||
<span class="lineno">78</span> <span class="n">c</span><span class="o">.</span><span class="n">n_tokens</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span>
|
||||
<span class="lineno">79</span> <span class="n">FeedbackTransformer</span><span class="p">(</span>
|
||||
<span class="lineno">80</span> <span class="n">FeedbackTransformerLayer</span><span class="p">(</span><span class="n">d_model</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span>
|
||||
<span class="lineno">81</span> <span class="n">attn</span><span class="o">=</span><span class="n">FeedbackAttention</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">heads</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">),</span>
|
||||
<span class="lineno">82</span> <span class="n">feed_forward</span><span class="o">=</span><span class="n">FeedForward</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">),</span>
|
||||
<span class="lineno">83</span> <span class="n">dropout_prob</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">),</span>
|
||||
<span class="lineno">84</span> <span class="n">c</span><span class="o">.</span><span class="n">n_layers</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-12'>
|
||||
<div class='docs doc-strings'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-12'>#</a>
|
||||
</div>
|
||||
<p> පූර්වගණනය කළ යතුරු සහ අගයන් සමඟ <a href="index.html#kv_shared">යාවත්කාලීන කරන ලද ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරයක්</a>සාදන්න. </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">87</span><span class="nd">@option</span><span class="p">(</span><span class="n">Configs</span><span class="o">.</span><span class="n">model</span><span class="p">)</span>
|
||||
<span class="lineno">88</span><span class="k">def</span> <span class="nf">feedback_transformer_kv</span><span class="p">(</span><span class="n">c</span><span class="p">:</span> <span class="n">Configs</span><span class="p">):</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-13'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-13'>#</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">92</span> <span class="kn">from</span> <span class="nn">labml_nn.transformers.feedback</span> <span class="kn">import</span> <span class="n">FeedbackTransformerKV</span><span class="p">,</span> <span class="n">FeedbackTransformerLayer</span><span class="p">,</span> \
|
||||
<span class="lineno">93</span> <span class="n">FeedbackAttention</span><span class="p">,</span> <span class="n">FeedForward</span>
|
||||
<span class="lineno">94</span>
|
||||
<span class="lineno">95</span> <span class="k">return</span> <span class="n">AutoregressiveModel</span><span class="p">(</span>
|
||||
<span class="lineno">96</span> <span class="n">c</span><span class="o">.</span><span class="n">n_tokens</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span>
|
||||
<span class="lineno">97</span> <span class="n">FeedbackTransformerKV</span><span class="p">(</span>
|
||||
<span class="lineno">98</span> <span class="n">FeedbackTransformerLayer</span><span class="p">(</span><span class="n">d_model</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span>
|
||||
<span class="lineno">99</span> <span class="n">attn</span><span class="o">=</span><span class="n">FeedbackAttention</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">heads</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">,</span>
|
||||
<span class="lineno">100</span> <span class="n">is_kv_precomputed</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
|
||||
<span class="lineno">101</span> <span class="n">feed_forward</span><span class="o">=</span><span class="n">FeedForward</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_ff</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">),</span>
|
||||
<span class="lineno">102</span> <span class="n">dropout_prob</span><span class="o">=</span><span class="n">c</span><span class="o">.</span><span class="n">dropout</span><span class="p">),</span>
|
||||
<span class="lineno">103</span> <span class="n">c</span><span class="o">.</span><span class="n">n_layers</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">d_model</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">heads</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">device</span><span class="p">)</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-14'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-14'>#</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">106</span><span class="k">def</span> <span class="nf">main</span><span class="p">():</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-15'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-15'>#</a>
|
||||
</div>
|
||||
<p>අත්හදාබැලීම සාදන්න </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">108</span> <span class="n">experiment</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"feedback_transformer"</span><span class="p">)</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-16'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-16'>#</a>
|
||||
</div>
|
||||
<p>වින්යාසසාදන්න </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">110</span> <span class="n">conf</span> <span class="o">=</span> <span class="n">Configs</span><span class="p">()</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-17'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-17'>#</a>
|
||||
</div>
|
||||
<p>වින්යාසයන්පූරණය කරන්න </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">112</span> <span class="n">experiment</span><span class="o">.</span><span class="n">configs</span><span class="p">(</span><span class="n">conf</span><span class="p">,</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-18'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-18'>#</a>
|
||||
</div>
|
||||
<p>අභිබවායාම සඳහා වින්යාසයන් පිළිබඳ ශබ්දකෝෂයක් </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">114</span> <span class="p">{</span><span class="s1">'tokenizer'</span><span class="p">:</span> <span class="s1">'character'</span><span class="p">,</span>
|
||||
<span class="lineno">115</span> <span class="s1">'text'</span><span class="p">:</span> <span class="s1">'tiny_shakespeare'</span><span class="p">,</span>
|
||||
<span class="lineno">116</span> <span class="s1">'optimizer.learning_rate'</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span>
|
||||
<span class="lineno">117</span> <span class="s1">'optimizer.optimizer'</span><span class="p">:</span> <span class="s1">'Noam'</span><span class="p">,</span>
|
||||
<span class="lineno">118</span> <span class="s1">'prompt'</span><span class="p">:</span> <span class="s1">'It is'</span><span class="p">,</span>
|
||||
<span class="lineno">119</span> <span class="s1">'prompt_separator'</span><span class="p">:</span> <span class="s1">''</span><span class="p">,</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-19'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-19'>#</a>
|
||||
</div>
|
||||
<p>මුල්ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර් <code class="highlight"><span></span><span class="n">feedback_transformer</span></code>
|
||||
සඳහා භාවිතා කරන්න </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">122</span> <span class="s1">'model'</span><span class="p">:</span> <span class="s1">'feedback_transformer_kv'</span><span class="p">,</span>
|
||||
<span class="lineno">123</span>
|
||||
<span class="lineno">124</span> <span class="s1">'train_loader'</span><span class="p">:</span> <span class="s1">'shuffled_train_loader'</span><span class="p">,</span>
|
||||
<span class="lineno">125</span> <span class="s1">'valid_loader'</span><span class="p">:</span> <span class="s1">'shuffled_valid_loader'</span><span class="p">,</span>
|
||||
<span class="lineno">126</span>
|
||||
<span class="lineno">127</span> <span class="s1">'seq_len'</span><span class="p">:</span> <span class="mi">128</span><span class="p">,</span>
|
||||
<span class="lineno">128</span> <span class="s1">'epochs'</span><span class="p">:</span> <span class="mi">128</span><span class="p">,</span>
|
||||
<span class="lineno">129</span> <span class="s1">'batch_size'</span><span class="p">:</span> <span class="mi">64</span><span class="p">,</span>
|
||||
<span class="lineno">130</span> <span class="s1">'inner_iterations'</span><span class="p">:</span> <span class="mi">25</span><span class="p">})</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-20'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-20'>#</a>
|
||||
</div>
|
||||
<p>ඉතිරිකිරීම සහ පැටවීම සඳහා ආකෘති සකසන්න </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">133</span> <span class="n">experiment</span><span class="o">.</span><span class="n">add_pytorch_models</span><span class="p">(</span><span class="n">get_modules</span><span class="p">(</span><span class="n">conf</span><span class="p">))</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-21'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-21'>#</a>
|
||||
</div>
|
||||
<p>අත්හදාබැලීම ආරම්භ කරන්න </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">136</span> <span class="k">with</span> <span class="n">experiment</span><span class="o">.</span><span class="n">start</span><span class="p">():</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-22'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-22'>#</a>
|
||||
</div>
|
||||
<p>පුහුණුලූපය ධාවනය කරන්න </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
<div class="highlight"><pre><span class="lineno">138</span> <span class="n">conf</span><span class="o">.</span><span class="n">run</span><span class="p">()</span>
|
||||
<span class="lineno">139</span>
|
||||
<span class="lineno">140</span>
|
||||
<span class="lineno">141</span><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
|
||||
<span class="lineno">142</span> <span class="n">main</span><span class="p">()</span></pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class='footer'>
|
||||
<a href="https://papers.labml.ai">Trending Research Papers</a>
|
||||
<a href="https://labml.ai">labml.ai</a>
|
||||
</div>
|
||||
</div>
|
||||
<script src=../../interactive.js?v=1"></script>
|
||||
<script>
|
||||
function handleImages() {
|
||||
var images = document.querySelectorAll('p>img')
|
||||
|
||||
for (var i = 0; i < images.length; ++i) {
|
||||
handleImage(images[i])
|
||||
}
|
||||
}
|
||||
|
||||
function handleImage(img) {
|
||||
img.parentElement.style.textAlign = 'center'
|
||||
|
||||
var modal = document.createElement('div')
|
||||
modal.id = 'modal'
|
||||
|
||||
var modalContent = document.createElement('div')
|
||||
modal.appendChild(modalContent)
|
||||
|
||||
var modalImage = document.createElement('img')
|
||||
modalContent.appendChild(modalImage)
|
||||
|
||||
var span = document.createElement('span')
|
||||
span.classList.add('close')
|
||||
span.textContent = 'x'
|
||||
modal.appendChild(span)
|
||||
|
||||
img.onclick = function () {
|
||||
console.log('clicked')
|
||||
document.body.appendChild(modal)
|
||||
modalImage.src = img.src
|
||||
}
|
||||
|
||||
span.onclick = function () {
|
||||
document.body.removeChild(modal)
|
||||
}
|
||||
}
|
||||
|
||||
handleImages()
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,135 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="si">
|
||||
<head>
|
||||
<meta http-equiv="content-type" content="text/html;charset=utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<meta name="description" content=""/>
|
||||
|
||||
<meta name="twitter:card" content="summary"/>
|
||||
<meta name="twitter:image:src" content="https://avatars1.githubusercontent.com/u/64068543?s=400&v=4"/>
|
||||
<meta name="twitter:title" content="ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්"/>
|
||||
<meta name="twitter:description" content=""/>
|
||||
<meta name="twitter:site" content="@labmlai"/>
|
||||
<meta name="twitter:creator" content="@labmlai"/>
|
||||
|
||||
<meta property="og:url" content="https://nn.labml.ai/transformers/feedback/readme.html"/>
|
||||
<meta property="og:title" content="ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්"/>
|
||||
<meta property="og:image" content="https://avatars1.githubusercontent.com/u/64068543?s=400&v=4"/>
|
||||
<meta property="og:site_name" content="ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්"/>
|
||||
<meta property="og:type" content="object"/>
|
||||
<meta property="og:title" content="ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්"/>
|
||||
<meta property="og:description" content=""/>
|
||||
|
||||
<title>ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්</title>
|
||||
<link rel="shortcut icon" href="/icon.png"/>
|
||||
<link rel="stylesheet" href="../../pylit.css?v=1">
|
||||
<link rel="canonical" href="https://nn.labml.ai/transformers/feedback/readme.html"/>
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.18/dist/katex.min.css" integrity="sha384-zTROYFVGOfTw7JV7KUu8udsvW2fx4lWOsCEDqhBreBwlHI4ioVRtmIvEThzJHGET" crossorigin="anonymous">
|
||||
|
||||
<!-- Global site tag (gtag.js) - Google Analytics -->
|
||||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-4V3HC8HBLH"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
|
||||
function gtag() {
|
||||
dataLayer.push(arguments);
|
||||
}
|
||||
|
||||
gtag('js', new Date());
|
||||
|
||||
gtag('config', 'G-4V3HC8HBLH');
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<div id='container'>
|
||||
<div id="background"></div>
|
||||
<div class='section'>
|
||||
<div class='docs'>
|
||||
<p>
|
||||
<a class="parent" href="/">home</a>
|
||||
<a class="parent" href="../index.html">transformers</a>
|
||||
<a class="parent" href="index.html">feedback</a>
|
||||
</p>
|
||||
<p>
|
||||
<a href="https://github.com/labmlai/annotated_deep_learning_paper_implementations" target="_blank">
|
||||
<img alt="Github"
|
||||
src="https://img.shields.io/github/stars/labmlai/annotated_deep_learning_paper_implementations?style=social"
|
||||
style="max-width:100%;"/></a>
|
||||
<a href="https://twitter.com/labmlai" rel="nofollow" target="_blank">
|
||||
<img alt="Twitter"
|
||||
src="https://img.shields.io/twitter/follow/labmlai?style=social"
|
||||
style="max-width:100%;"/></a>
|
||||
</p>
|
||||
<p>
|
||||
<a href="https://github.com/labmlai/annotated_deep_learning_paper_implementations/tree/master/labml_nn/transformers/feedback/readme.md" target="_blank">
|
||||
View code on Github</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class='section' id='section-0'>
|
||||
<div class='docs'>
|
||||
<div class='section-link'>
|
||||
<a href='#section-0'>#</a>
|
||||
</div>
|
||||
<h1><a href="https://nn.labml.ai/transformers/feedback/index.html">ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර්</a></h1>
|
||||
<p>මෙය <a href="https://pytorch.org">PyTorch</a> ක්රියාත්මක කිරීම කඩදාසි <a href="https://papers.labml.ai/paper/2002.09402">ප්රතිපෝෂණ මතකය සමඟ අනුක්රමික ට්රාන්ස්ෆෝමර්වල ඉහළ මට්ටමේ නිරූපණයන් වෙත ප්රවේශ වීම</a> . </p>
|
||||
<p>සාමාන්යට්රාන්ස්ෆෝමර් සමාන්තරව ටෝකන සකසනවා. සෑම ට්රාන්ස්ෆෝමර් ස්ථරයක්ම පෙර ස්ථරයේ ප්රතිදානයන් කෙරෙහි අවධානය යොමු කරයි. ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරය පෙර පියවරයන්හි සියලුම ස්ථරවල ප්රතිදානය කෙරෙහි අවධානය යොමු කරයි. එබැවින් මෙය පුනරාවර්තනය එකතු කරන අතර, අපි ටෝකන්-විසින්-ටෝකන් සැකසිය යුතුය. මෙය පුහුණුව සැලකිය යුතු ලෙස මන්දගාමී වේ (අනුක්රමයේ දිග අනුව 5X - 10X පමණ). කෙසේ වෙතත්, ප්රතිපෝෂණ ට්රාන්ස්ෆෝමර් පුරෝකථනය කිරීමේදී වේගවත් වන්නේ ඔබ මතක දෛශික හැඹිලි කළහොත් ඊළඟ ටෝකනය පුරෝකථනය කළ හැකි බැවිනි. </p>
|
||||
<p>පුහුණුවවේගවත් කිරීම සඳහා පත්රිකාව සාකච්ඡා කරන්නේ කෙටි අනුක්රමික දිගකින් ආරම්භ කර එය ක්රමයෙන් වැඩි කිරීමයි. ආරම්භක ස්ථානය ලෙස පෙර පුහුණු සමාන්තර ට්රාන්ස්ෆෝමරයක් භාවිතා කිරීම ද ඔවුහු සාකච්ඡා කරති. </p>
|
||||
<p>මුල්ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරය සියලු ස්ථරවල ප්රතිදානයන් තබා නොගනී. ඒ වෙනුවට එය සියලු ස්ථරවල නිමැවුමේ බර තැබූ එකතුව තබා ගනී. මෙය අනාවැකිය තුළ හැඹිලි සඳහා භාවිතා කරන මතකය අඩු කරයි. මෙම ගොනුවේ පළමු භාගය මෙය ක්රියාත්මක කරයි. </p>
|
||||
<p>යාවත්කාලීනකරන ලද ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරය ස්ථර අතර යතුරු සහ අගයන් ගණනය කිරීමට භාවිතා කරන බර බෙදා ගනී. ඉන්පසු අපි එක් එක් පියවර සඳහා යතුරු සහ අගයන් එක් වරක් පමණක් ගණනය කර ඒවා හැඹිලි කර තබමු. මෙම ගොනුවේ <a href="#shared_kv">දෙවන භාගය</a> මෙය ක්රියාත්මක කරයි. කාර්ය සාධනය වැඩි දියුණු කිරීම සඳහා අපි අභිරුචි PyTorch ශ්රිතයක් ක්රියාත්මක කළෙමු. </p>
|
||||
<p>කුඩාෂේක්ස්පියර් දත්ත කට්ටලය පිළිබඳ ප්රතිපෝෂණ ට්රාන්ස්ෆෝමරයක් පුහුණු කිරීම සඳහා පුහුණු <a href="experiment.html">කේතය</a> සහ සටහන් පොතක් මෙන්න. </p>
|
||||
<p><a href="https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/transformers/feedback/experiment.ipynb">කොලැබ් නෝට්බුක්</a></p>
|
||||
<p><a href="https://colab.research.google.com/github/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/transformers/feedback/experiment.ipynb"><img alt="Open In Colab" src="https://colab.research.google.com/assets/colab-badge.svg"></a> <a href="https://app.labml.ai/run/d8eb9416530a11eb8fb50242ac1c0002"> <img alt="View Run" src="https://img.shields.io/badge/labml-experiment-brightgreen"></a> </p>
|
||||
|
||||
</div>
|
||||
<div class='code'>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class='footer'>
|
||||
<a href="https://papers.labml.ai">Trending Research Papers</a>
|
||||
<a href="https://labml.ai">labml.ai</a>
|
||||
</div>
|
||||
</div>
|
||||
<script src=../../interactive.js?v=1"></script>
|
||||
<script>
|
||||
function handleImages() {
|
||||
var images = document.querySelectorAll('p>img')
|
||||
|
||||
for (var i = 0; i < images.length; ++i) {
|
||||
handleImage(images[i])
|
||||
}
|
||||
}
|
||||
|
||||
function handleImage(img) {
|
||||
img.parentElement.style.textAlign = 'center'
|
||||
|
||||
var modal = document.createElement('div')
|
||||
modal.id = 'modal'
|
||||
|
||||
var modalContent = document.createElement('div')
|
||||
modal.appendChild(modalContent)
|
||||
|
||||
var modalImage = document.createElement('img')
|
||||
modalContent.appendChild(modalImage)
|
||||
|
||||
var span = document.createElement('span')
|
||||
span.classList.add('close')
|
||||
span.textContent = 'x'
|
||||
modal.appendChild(span)
|
||||
|
||||
img.onclick = function () {
|
||||
console.log('clicked')
|
||||
document.body.appendChild(modal)
|
||||
modalImage.src = img.src
|
||||
}
|
||||
|
||||
span.onclick = function () {
|
||||
document.body.removeChild(modal)
|
||||
}
|
||||
}
|
||||
|
||||
handleImages()
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user