chore: import upstream snapshot with attribution

2026-07-13 12:45:52 +08:00
commit aa6c9c0720
43 changed files with 5178 additions and 0 deletions
@@ -0,0 +1,115 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "# code by Tae Hwan Jung @graykode\n",
+        "import numpy as np\n",
+        "import torch\n",
+        "import torch.nn as nn\n",
+        "import torch.optim as optim\n",
+        "import matplotlib.pyplot as plt\n",
+        "\n",
+        "def random_batch():\n",
+        "    random_inputs = []\n",
+        "    random_labels = []\n",
+        "    random_index = np.random.choice(range(len(skip_grams)), batch_size, replace=False)\n",
+        "\n",
+        "    for i in random_index:\n",
+        "        random_inputs.append(np.eye(voc_size)[skip_grams[i][0]])  # target\n",
+        "        random_labels.append(skip_grams[i][1])  # context word\n",
+        "\n",
+        "    return random_inputs, random_labels\n",
+        "\n",
+        "# Model\n",
+        "class Word2Vec(nn.Module):\n",
+        "    def __init__(self):\n",
+        "        super(Word2Vec, self).__init__()\n",
+        "        # W and WT is not Traspose relationship\n",
+        "        self.W = nn.Linear(voc_size, embedding_size, bias=False) # voc_size > embedding_size Weight\n",
+        "        self.WT = nn.Linear(embedding_size, voc_size, bias=False) # embedding_size > voc_size Weight\n",
+        "\n",
+        "    def forward(self, X):\n",
+        "        # X : [batch_size, voc_size]\n",
+        "        hidden_layer = self.W(X) # hidden_layer : [batch_size, embedding_size]\n",
+        "        output_layer = self.WT(hidden_layer) # output_layer : [batch_size, voc_size]\n",
+        "        return output_layer\n",
+        "\n",
+        "if __name__ == '__main__':\n",
+        "    batch_size = 2 # mini-batch size\n",
+        "    embedding_size = 2 # embedding size\n",
+        "\n",
+        "    sentences = [\"apple banana fruit\", \"banana orange fruit\", \"orange banana fruit\",\n",
+        "                 \"dog cat animal\", \"cat monkey animal\", \"monkey dog animal\"]\n",
+        "\n",
+        "    word_sequence = \" \".join(sentences).split()\n",
+        "    word_list = \" \".join(sentences).split()\n",
+        "    word_list = list(set(word_list))\n",
+        "    word_dict = {w: i for i, w in enumerate(word_list)}\n",
+        "    voc_size = len(word_list)\n",
+        "\n",
+        "    # Make skip gram of one size window\n",
+        "    skip_grams = []\n",
+        "    for i in range(1, len(word_sequence) - 1):\n",
+        "        target = word_dict[word_sequence[i]]\n",
+        "        context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]]\n",
+        "        for w in context:\n",
+        "            skip_grams.append([target, w])\n",
+        "\n",
+        "    model = Word2Vec()\n",
+        "\n",
+        "    criterion = nn.CrossEntropyLoss()\n",
+        "    optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
+        "\n",
+        "    # Training\n",
+        "    for epoch in range(5000):\n",
+        "        input_batch, target_batch = random_batch()\n",
+        "        input_batch = torch.Tensor(input_batch)\n",
+        "        target_batch = torch.LongTensor(target_batch)\n",
+        "\n",
+        "        optimizer.zero_grad()\n",
+        "        output = model(input_batch)\n",
+        "\n",
+        "        # output : [batch_size, voc_size], target_batch : [batch_size] (LongTensor, not one-hot)\n",
+        "        loss = criterion(output, target_batch)\n",
+        "        if (epoch + 1) % 1000 == 0:\n",
+        "            print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
+        "\n",
+        "        loss.backward()\n",
+        "        optimizer.step()\n",
+        "\n",
+        "    for i, label in enumerate(word_list):\n",
+        "        W, WT = model.parameters()\n",
+        "        x, y = W[0][i].item(), W[1][i].item()\n",
+        "        plt.scatter(x, y)\n",
+        "        plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n",
+        "    plt.show()\n"
+      ],
+      "outputs": [],
+      "execution_count": null
+    }
+  ],
+  "metadata": {
+    "anaconda-cloud": {},
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.1"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 4
+}
@@ -0,0 +1,82 @@
+# %%
+# code by Tae Hwan Jung @graykode
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import matplotlib.pyplot as plt
+
+def random_batch():
+    random_inputs = []
+    random_labels = []
+    random_index = np.random.choice(range(len(skip_grams)), batch_size, replace=False)
+
+    for i in random_index:
+        random_inputs.append(np.eye(voc_size)[skip_grams[i][0]])  # target
+        random_labels.append(skip_grams[i][1])  # context word
+
+    return random_inputs, random_labels
+
+# Model
+class Word2Vec(nn.Module):
+    def __init__(self):
+        super(Word2Vec, self).__init__()
+        # W and WT is not Traspose relationship
+        self.W = nn.Linear(voc_size, embedding_size, bias=False) # voc_size > embedding_size Weight
+        self.WT = nn.Linear(embedding_size, voc_size, bias=False) # embedding_size > voc_size Weight
+
+    def forward(self, X):
+        # X : [batch_size, voc_size]
+        hidden_layer = self.W(X) # hidden_layer : [batch_size, embedding_size]
+        output_layer = self.WT(hidden_layer) # output_layer : [batch_size, voc_size]
+        return output_layer
+
+if __name__ == '__main__':
+    batch_size = 2 # mini-batch size
+    embedding_size = 2 # embedding size
+
+    sentences = ["apple banana fruit", "banana orange fruit", "orange banana fruit",
+                 "dog cat animal", "cat monkey animal", "monkey dog animal"]
+
+    word_sequence = " ".join(sentences).split()
+    word_list = " ".join(sentences).split()
+    word_list = list(set(word_list))
+    word_dict = {w: i for i, w in enumerate(word_list)}
+    voc_size = len(word_list)
+
+    # Make skip gram of one size window
+    skip_grams = []
+    for i in range(1, len(word_sequence) - 1):
+        target = word_dict[word_sequence[i]]
+        context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]]
+        for w in context:
+            skip_grams.append([target, w])
+
+    model = Word2Vec()
+
+    criterion = nn.CrossEntropyLoss()
+    optimizer = optim.Adam(model.parameters(), lr=0.001)
+
+    # Training
+    for epoch in range(5000):
+        input_batch, target_batch = random_batch()
+        input_batch = torch.Tensor(input_batch)
+        target_batch = torch.LongTensor(target_batch)
+
+        optimizer.zero_grad()
+        output = model(input_batch)
+
+        # output : [batch_size, voc_size], target_batch : [batch_size] (LongTensor, not one-hot)
+        loss = criterion(output, target_batch)
+        if (epoch + 1) % 1000 == 0:
+            print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))
+
+        loss.backward()
+        optimizer.step()
+
+    for i, label in enumerate(word_list):
+        W, WT = model.parameters()
+        x, y = W[0][i].item(), W[1][i].item()
+        plt.scatter(x, y)
+        plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')
+    plt.show()