wehub/rasbt--deeplearning-models
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
b2987b63e61509e359c0065927a2f789c861533e
rasbt--deeplearning-models/pytorch-lightning_ipynb/transformer/distilbert-finetuning-ii-amp/distilbert-finetuning-ii-ampb16.ipynb
T
wehub-resource-sync b2987b63e6 chore: import upstream snapshot with attribution
2026-07-13 13:29:39 +08:00

1162 lines
34 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "markdown",
"id": "3c5d72f4",
"metadata": {},
"source": [
"# Finetuning a DistilBERT Classifier in Lightning"
]
},
{
"cell_type": "markdown",
"id": "0873756d-262a-4525-b809-4e0b3a1e63dd",
"metadata": {},
"source": [
"![](figures/finetuning-ii.png)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "6fd9cda8",
"metadata": {},
"outputs": [],
"source": [
"# pip install transformers"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "92ea5612",
"metadata": {},
"outputs": [],
"source": [
"# pip install datasets"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "fe7191cf-62ed-4793-8358-bee70b233d05",
"metadata": {},
"outputs": [],
"source": [
"# pip install lightning"
]
},
{
"cell_type": "markdown",
"id": "4cfd724d",
"metadata": {},
"source": [
"# 1 Loading the Dataset"
]
},
{
"cell_type": "markdown",
"id": "fd06d930",
"metadata": {},
"source": [
"The IMDB movie review dataset consists of 50k movie reviews with sentiment label (0: negative, 1: positive)."
]
},
{
"cell_type": "markdown",
"id": "60fe0b76",
"metadata": {},
"source": [
"## 1a) Load from `datasets` Hub"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "447e24bb",
"metadata": {},
"outputs": [],
"source": [
"from datasets import list_datasets, load_dataset"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "2baf2f16",
"metadata": {},
"outputs": [],
"source": [
"# list_datasets()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "6310d5bf",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Found cached dataset imdb (/home/sebastian/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1b7cb09fb60242bc802cd825c40aec72",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/3 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"DatasetDict({\n",
" train: Dataset({\n",
" features: ['text', 'label'],\n",
" num_rows: 25000\n",
" })\n",
" test: Dataset({\n",
" features: ['text', 'label'],\n",
" num_rows: 25000\n",
" })\n",
" unsupervised: Dataset({\n",
" features: ['text', 'label'],\n",
" num_rows: 50000\n",
" })\n",
"})\n"
]
}
],
"source": [
"imdb_data = load_dataset(\"imdb\")\n",
"print(imdb_data)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "552bbb2e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'text': \"This film is terrible. You don't really need to read this review further. If you are planning on watching it, suffice to say - don't (unless you are studying how not to make a good movie).<br /><br />The acting is horrendous... serious amateur hour. Throughout the movie I thought that it was interesting that they found someone who speaks and looks like Michael Madsen, only to find out that it is actually him! A new low even for him!!<br /><br />The plot is terrible. People who claim that it is original or good have probably never seen a decent movie before. Even by the standard of Hollywood action flicks, this is a terrible movie.<br /><br />Don't watch it!!! Go for a jog instead - at least you won't feel like killing yourself.\",\n",
" 'label': 0}"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"imdb_data[\"train\"][99]"
]
},
{
"cell_type": "markdown",
"id": "40bdb9c5",
"metadata": {},
"source": [
"## 1b) Load from local directory"
]
},
{
"cell_type": "markdown",
"id": "9103ec2d",
"metadata": {},
"source": [
"The IMDB movie review set can be downloaded from http://ai.stanford.edu/~amaas/data/sentiment/. After downloading the dataset, decompress the files.\n",
"\n",
"A) If you are working with Linux or MacOS X, open a new terminal window cd into the download directory and execute\n",
"\n",
" tar -zxf aclImdb_v1.tar.gz\n",
"\n",
"B) If you are working with Windows, download an archiver such as 7Zip to extract the files from the download archive."
]
},
{
"cell_type": "markdown",
"id": "ac508bb8",
"metadata": {},
"source": [
"C) Use the following code to download and unzip the dataset via Python"
]
},
{
"cell_type": "markdown",
"id": "241ecc96",
"metadata": {},
"source": [
"**Download the movie reviews**"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "02aeade4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"100% | 80.23 MB | 2.64 MB/s | 30.34 sec elapsed"
]
}
],
"source": [
"import os\n",
"import sys\n",
"import tarfile\n",
"import time\n",
"import urllib.request\n",
"\n",
"source = \"http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\"\n",
"target = \"aclImdb_v1.tar.gz\"\n",
"\n",
"if os.path.exists(target):\n",
" os.remove(target)\n",
"\n",
"\n",
"def reporthook(count, block_size, total_size):\n",
" global start_time\n",
" if count == 0:\n",
" start_time = time.time()\n",
" return\n",
" duration = time.time() - start_time\n",
" progress_size = int(count * block_size)\n",
" speed = progress_size / (1024.0**2 * duration)\n",
" percent = count * block_size * 100.0 / total_size\n",
"\n",
" sys.stdout.write(\n",
" f\"\\r{int(percent)}% | {progress_size / (1024.**2):.2f} MB \"\n",
" f\"| {speed:.2f} MB/s | {duration:.2f} sec elapsed\"\n",
" )\n",
" sys.stdout.flush()\n",
"\n",
"\n",
"if not os.path.isdir(\"aclImdb\") and not os.path.isfile(\"aclImdb_v1.tar.gz\"):\n",
" urllib.request.urlretrieve(source, target, reporthook)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "2a867dcc",
"metadata": {},
"outputs": [],
"source": [
"if not os.path.isdir(\"aclImdb\"):\n",
"\n",
" with tarfile.open(target, \"r:gz\") as tar:\n",
" tar.extractall()"
]
},
{
"cell_type": "markdown",
"id": "9318d4d0",
"metadata": {},
"source": [
"**Convert them to a pandas DataFrame and save them as CSV**"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "464e587c",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|███████████████████████████████████████████████████████████████████████████████████| 50000/50000 [00:27<00:00, 1809.68it/s]\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"\n",
"import numpy as np\n",
"import pandas as pd\n",
"from packaging import version\n",
"from tqdm import tqdm\n",
"\n",
"# change the `basepath` to the directory of the\n",
"# unzipped movie dataset\n",
"\n",
"basepath = \"aclImdb\"\n",
"\n",
"labels = {\"pos\": 1, \"neg\": 0}\n",
"\n",
"df = pd.DataFrame()\n",
"\n",
"with tqdm(total=50000) as pbar:\n",
" for s in (\"test\", \"train\"):\n",
" for l in (\"pos\", \"neg\"):\n",
" path = os.path.join(basepath, s, l)\n",
" for file in sorted(os.listdir(path)):\n",
" with open(os.path.join(path, file), \"r\", encoding=\"utf-8\") as infile:\n",
" txt = infile.read()\n",
"\n",
" if version.parse(pd.__version__) >= version.parse(\"1.3.2\"):\n",
" x = pd.DataFrame(\n",
" [[txt, labels[l]]], columns=[\"review\", \"sentiment\"]\n",
" )\n",
" df = pd.concat([df, x], ignore_index=False)\n",
"\n",
" else:\n",
" df = df.append([[txt, labels[l]]], ignore_index=True)\n",
" pbar.update()\n",
"df.columns = [\"text\", \"label\"]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "02649593",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"\n",
"np.random.seed(0)\n",
"df = df.reindex(np.random.permutation(df.index))"
]
},
{
"cell_type": "markdown",
"id": "59ca0386",
"metadata": {},
"source": [
"**Basic datasets analysis and sanity checks**"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "c2db547a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Class distribution:\n"
]
},
{
"data": {
"text/plain": [
"array([25000, 25000])"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"print(\"Class distribution:\")\n",
"np.bincount(df[\"label\"].values)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "a007e612",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(4, 173.0, 2470)"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"text_len = df[\"text\"].apply(lambda x: len(x.split()))\n",
"text_len.min(), text_len.median(), text_len.max() "
]
},
{
"cell_type": "markdown",
"id": "00f4b04d",
"metadata": {},
"source": [
"**Split data into training, validation, and test sets**"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "ff703901",
"metadata": {},
"outputs": [],
"source": [
"df_shuffled = df.sample(frac=1, random_state=1).reset_index()\n",
"\n",
"df_train = df_shuffled.iloc[:35_000]\n",
"df_val = df_shuffled.iloc[35_000:40_000]\n",
"df_test = df_shuffled.iloc[40_000:]\n",
"\n",
"df_train.to_csv(\"train.csv\", index=False, encoding=\"utf-8\")\n",
"df_val.to_csv(\"validation.csv\", index=False, encoding=\"utf-8\")\n",
"df_test.to_csv(\"test.csv\", index=False, encoding=\"utf-8\")"
]
},
{
"cell_type": "markdown",
"id": "846d83b1",
"metadata": {},
"source": [
"# 2 Tokenization and Numericalization"
]
},
{
"cell_type": "markdown",
"id": "2bd5f770",
"metadata": {},
"source": [
"**Load the dataset via `load_dataset`**"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "a1aa66c7",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using custom data configuration default-2352009af2284f6d\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Downloading and preparing dataset csv/default to /home/sebastian/.cache/huggingface/datasets/csv/default-2352009af2284f6d/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317...\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "acea2a6e77cd4cde9e9fd5d112a35f69",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Downloading data files: 0%| | 0/3 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c1c067de4083435fafd7627871abc92c",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Extracting data files: 0%| | 0/3 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Generating train split: 0 examples [00:00, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/sebastian/miniforge3/envs/lightning2/lib/python3.9/site-packages/datasets/download/streaming_download_manager.py:776: FutureWarning: the 'mangle_dupe_cols' keyword is deprecated and will be removed in a future version. Please take steps to stop the use of 'mangle_dupe_cols'\n",
" return pd.read_csv(xopen(filepath_or_buffer, \"rb\", use_auth_token=use_auth_token), **kwargs)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Generating validation split: 0 examples [00:00, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/sebastian/miniforge3/envs/lightning2/lib/python3.9/site-packages/datasets/download/streaming_download_manager.py:776: FutureWarning: the 'mangle_dupe_cols' keyword is deprecated and will be removed in a future version. Please take steps to stop the use of 'mangle_dupe_cols'\n",
" return pd.read_csv(xopen(filepath_or_buffer, \"rb\", use_auth_token=use_auth_token), **kwargs)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Generating test split: 0 examples [00:00, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dataset csv downloaded and prepared to /home/sebastian/.cache/huggingface/datasets/csv/default-2352009af2284f6d/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317. Subsequent calls will reuse this data.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/sebastian/miniforge3/envs/lightning2/lib/python3.9/site-packages/datasets/download/streaming_download_manager.py:776: FutureWarning: the 'mangle_dupe_cols' keyword is deprecated and will be removed in a future version. Please take steps to stop the use of 'mangle_dupe_cols'\n",
" return pd.read_csv(xopen(filepath_or_buffer, \"rb\", use_auth_token=use_auth_token), **kwargs)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a6bf5b53e94c47e4b40c8647b78fb44c",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/3 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"DatasetDict({\n",
" train: Dataset({\n",
" features: ['index', 'text', 'label'],\n",
" num_rows: 35000\n",
" })\n",
" validation: Dataset({\n",
" features: ['index', 'text', 'label'],\n",
" num_rows: 5000\n",
" })\n",
" test: Dataset({\n",
" features: ['index', 'text', 'label'],\n",
" num_rows: 10000\n",
" })\n",
"})\n"
]
}
],
"source": [
"imdb_dataset = load_dataset(\n",
" \"csv\",\n",
" data_files={\n",
" \"train\": \"train.csv\",\n",
" \"validation\": \"validation.csv\",\n",
" \"test\": \"test.csv\",\n",
" },\n",
")\n",
"\n",
"print(imdb_dataset)"
]
},
{
"cell_type": "markdown",
"id": "029adc8f-cdfe-4386-9552-a1120f49adee",
"metadata": {},
"source": [
"**Tokenize the dataset**"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "5ea762ba",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tokenizer input max length: 512\n",
"Tokenizer vocabulary size: 30522\n"
]
}
],
"source": [
"from transformers import AutoTokenizer\n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
"print(\"Tokenizer input max length:\", tokenizer.model_max_length)\n",
"print(\"Tokenizer vocabulary size:\", tokenizer.vocab_size)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "8432c15c",
"metadata": {},
"outputs": [],
"source": [
"def tokenize_text(batch):\n",
" return tokenizer(batch[\"text\"], truncation=True, padding=True)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "0bb392cf",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ed0b8624e0ea4400ab0967bba8998e4b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/1 [00:00<?, ?ba/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c8d4c4c04a6b45bea53800338047c761",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/1 [00:00<?, ?ba/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "94972c84d1504fab964c388114017010",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/1 [00:00<?, ?ba/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"imdb_tokenized = imdb_dataset.map(tokenize_text, batched=True, batch_size=None)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "6d4103c3",
"metadata": {},
"outputs": [],
"source": [
"del imdb_dataset"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "89ef894c-978f-47f2-9d61-cb6a9f38e745",
"metadata": {},
"outputs": [],
"source": [
"imdb_tokenized.set_format(\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "0ea67091-aeb7-46c1-871f-638ce58d8a0e",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
]
},
{
"cell_type": "markdown",
"id": "4c4f8cd8-e641-45fb-9893-70677631917a",
"metadata": {},
"source": [
"# 3 Set Up DataLoaders"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "0807b068-7d8f-4055-a26a-177e07dea4c7",
"metadata": {},
"outputs": [],
"source": [
"from torch.utils.data import DataLoader, Dataset\n",
"\n",
"\n",
"class IMDBDataset(Dataset):\n",
" def __init__(self, dataset_dict, partition_key=\"train\"):\n",
" self.partition = dataset_dict[partition_key]\n",
"\n",
" def __getitem__(self, index):\n",
" return self.partition[index]\n",
"\n",
" def __len__(self):\n",
" return self.partition.num_rows"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "90cb08f3-ef77-4351-8b19-42d99dd24f98",
"metadata": {},
"outputs": [],
"source": [
"train_dataset = IMDBDataset(imdb_tokenized, partition_key=\"train\")\n",
"val_dataset = IMDBDataset(imdb_tokenized, partition_key=\"validation\")\n",
"test_dataset = IMDBDataset(imdb_tokenized, partition_key=\"test\")\n",
"\n",
"train_loader = DataLoader(\n",
" dataset=train_dataset,\n",
" batch_size=64,\n",
" shuffle=True, \n",
" num_workers=4\n",
")\n",
"\n",
"val_loader = DataLoader(\n",
" dataset=val_dataset,\n",
" batch_size=64,\n",
" num_workers=4\n",
")\n",
"\n",
"test_loader = DataLoader(\n",
" dataset=test_dataset,\n",
" batch_size=64,\n",
" num_workers=4\n",
")"
]
},
{
"cell_type": "markdown",
"id": "78e774ab-45a0-4c48-ad61-a3d0e1927ef4",
"metadata": {},
"source": [
"# 4 Initializing DistilBERT"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "dc28ddbe-1a96-4c24-9f5c-40ffdca4a572",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.bias', 'vocab_projector.bias', 'vocab_layer_norm.weight', 'vocab_projector.weight', 'vocab_transform.bias', 'vocab_transform.weight']\n",
"- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
"- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
"Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'pre_classifier.weight', 'classifier.weight', 'pre_classifier.bias']\n",
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
]
}
],
"source": [
"from transformers import AutoModelForSequenceClassification\n",
"\n",
"model = AutoModelForSequenceClassification.from_pretrained(\n",
" \"distilbert-base-uncased\", num_labels=2)"
]
},
{
"cell_type": "markdown",
"id": "def1cf25-0a7d-4bb2-9419-b7a8fe1c1eab",
"metadata": {},
"source": [
"## 5 Finetuning"
]
},
{
"cell_type": "markdown",
"id": "534f7a59-2c86-4895-ad7c-2cdd675b003a",
"metadata": {},
"source": [
"**Wrap in LightningModule for Training**"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "9f2c474d",
"metadata": {},
"outputs": [],
"source": [
"import lightning as L\n",
"import torch\n",
"import torchmetrics\n",
"\n",
"\n",
"class LightningModel(L.LightningModule):\n",
" def __init__(self, model, learning_rate=5e-5):\n",
" super().__init__()\n",
"\n",
" self.learning_rate = learning_rate\n",
" self.model = model\n",
"\n",
" self.val_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=2)\n",
" self.test_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=2)\n",
"\n",
" def forward(self, input_ids, attention_mask, labels):\n",
" return self.model(input_ids, attention_mask=attention_mask, labels=labels)\n",
" \n",
" def training_step(self, batch, batch_idx):\n",
" outputs = self(batch[\"input_ids\"], attention_mask=batch[\"attention_mask\"],\n",
" labels=batch[\"label\"]) \n",
" self.log(\"train_loss\", outputs[\"loss\"])\n",
" return outputs[\"loss\"] # this is passed to the optimizer for training\n",
"\n",
" def validation_step(self, batch, batch_idx):\n",
" outputs = self(batch[\"input_ids\"], attention_mask=batch[\"attention_mask\"],\n",
" labels=batch[\"label\"]) \n",
" self.log(\"val_loss\", outputs[\"loss\"], prog_bar=True)\n",
" \n",
" logits = outputs[\"logits\"]\n",
" predicted_labels = torch.argmax(logits, 1)\n",
" self.val_acc(predicted_labels, batch[\"label\"])\n",
" self.log(\"val_acc\", self.val_acc, prog_bar=True)\n",
" \n",
" def test_step(self, batch, batch_idx):\n",
" outputs = self(batch[\"input_ids\"], attention_mask=batch[\"attention_mask\"],\n",
" labels=batch[\"label\"]) \n",
" \n",
" logits = outputs[\"logits\"]\n",
" predicted_labels = torch.argmax(logits, 1)\n",
" self.test_acc(predicted_labels, batch[\"label\"])\n",
" self.log(\"accuracy\", self.test_acc, prog_bar=True)\n",
"\n",
" def configure_optimizers(self):\n",
" optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)\n",
" return optimizer\n",
" \n",
"\n",
"lightning_model = LightningModel(model)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "e6dab813-e1fc-47cd-87a1-5eb8070699c6",
"metadata": {},
"outputs": [],
"source": [
"from lightning.pytorch.callbacks import ModelCheckpoint\n",
"from lightning.pytorch.loggers import CSVLogger\n",
"\n",
"\n",
"callbacks = [\n",
" ModelCheckpoint(\n",
" save_top_k=1, mode=\"max\", monitor=\"val_acc\"\n",
" ) # save top 1 model\n",
"]\n",
"logger = CSVLogger(save_dir=\"logs/\", name=\"my-model\")"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "492aa043-02da-459e-a266-091b34254ac6",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using bfloat16 Automatic Mixed Precision (AMP)\n",
"GPU available: True (cuda), used: True\n",
"TPU available: False, using: 0 TPU cores\n",
"IPU available: False, using: 0 IPUs\n",
"HPU available: False, using: 0 HPUs\n",
"You are using a CUDA device ('NVIDIA A100-SXM4-40GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
"Missing logger folder: logs/my-model\n",
"LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]\n",
"\n",
" | Name | Type | Params\n",
"-----------------------------------------------------------------\n",
"0 | model | DistilBertForSequenceClassification | 67.0 M\n",
"1 | val_acc | MulticlassAccuracy | 0 \n",
"2 | test_acc | MulticlassAccuracy | 0 \n",
"-----------------------------------------------------------------\n",
"67.0 M Trainable params\n",
"0 Non-trainable params\n",
"67.0 M Total params\n",
"133.910 Total estimated model params size (MB)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Sanity Checking: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "26fa7b1ba00849a597fdd5e03b7872bf",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Training: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Validation: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Validation: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Validation: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"`Trainer.fit` stopped: `max_epochs=3` reached.\n"
]
}
],
"source": [
"import time\n",
"\n",
"trainer = L.Trainer(\n",
" max_epochs=3,\n",
" callbacks=callbacks,\n",
" precision=\"bf16\",\n",
" accelerator=\"gpu\",\n",
" devices=[1],\n",
" logger=logger,\n",
" log_every_n_steps=10,\n",
" deterministic=True\n",
")\n",
"\n",
"start = time.time()\n",
"trainer.fit(model=lightning_model,\n",
" train_dataloaders=train_loader,\n",
" val_dataloaders=val_loader)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "b6116cbc-f2c5-43c0-bb76-737be4164752",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time elapsed 6.12 min\n"
]
}
],
"source": [
"end = time.time()\n",
"elapsed = end-start\n",
"print(f\"Time elapsed {elapsed/60:.2f} min\")"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "eeb92de4-d483-4627-b9f3-f0bba0cddd9c",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"You are using a CUDA device ('NVIDIA A100-SXM4-40GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
"Restoring states from the checkpoint path at logs/my-model/version_0/checkpoints/epoch=2-step=1641.ckpt\n",
"Lightning automatically upgraded your loaded checkpoint from v2.0.0dev to v2.0.0dev. To apply the upgrade to your files permanently, run `python -m lightning.pytorch.utilities.upgrade_checkpoint --file logs/my-model/version_0/checkpoints/epoch=2-step=1641.ckpt`\n",
"LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]\n",
"Loaded model weights from the checkpoint at logs/my-model/version_0/checkpoints/epoch=2-step=1641.ckpt\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "cbeb91cb6ee34932bb268f9fab708fdd",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Testing: 0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
"┃<span style=\"font-weight: bold\"> Test metric </span>┃<span style=\"font-weight: bold\"> DataLoader 0 </span>┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
"│<span style=\"color: #008080; text-decoration-color: #008080\"> accuracy </span>│<span style=\"color: #800080; text-decoration-color: #800080\"> 0.9277999997138977 </span>│\n",
"└───────────────────────────┴───────────────────────────┘\n",
"</pre>\n"
],
"text/plain": [
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1m Test metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
"│\u001b[36m \u001b[0m\u001b[36m accuracy \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.9277999997138977 \u001b[0m\u001b[35m \u001b[0m│\n",
"└───────────────────────────┴───────────────────────────┘\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"[{'accuracy': 0.9277999997138977}]"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"trainer.test(lightning_model, dataloaders=test_loader, ckpt_path=\"best\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.15"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink