chore: import upstream snapshot with attribution
Build documentation / build (push) Failing after 0s
Deploy "method_comparison" Gradio to Spaces / deploy (push) Has been cancelled
Deploy "PEFT shop" Gradio app to Spaces / deploy (push) Has been cancelled
tests on transformers main / tests (push) Has been cancelled
tests / check_code_quality (push) Has been cancelled
tests / tests (ubuntu-latest, 3.10) (push) Has been cancelled
tests / tests (ubuntu-latest, 3.11) (push) Has been cancelled
tests / tests (ubuntu-latest, 3.12) (push) Has been cancelled
tests / tests (ubuntu-latest, 3.13) (push) Has been cancelled
tests / tests (windows-latest, 3.10) (push) Has been cancelled
tests / tests (windows-latest, 3.11) (push) Has been cancelled
tests / tests (windows-latest, 3.12) (push) Has been cancelled
tests / tests (windows-latest, 3.13) (push) Has been cancelled
Secret Leaks / trufflehog (push) Has been cancelled
CI security linting / zizmor latest via Cargo (push) Has been cancelled
Build documentation / build (push) Failing after 0s
Deploy "method_comparison" Gradio to Spaces / deploy (push) Has been cancelled
Deploy "PEFT shop" Gradio app to Spaces / deploy (push) Has been cancelled
tests on transformers main / tests (push) Has been cancelled
tests / check_code_quality (push) Has been cancelled
tests / tests (ubuntu-latest, 3.10) (push) Has been cancelled
tests / tests (ubuntu-latest, 3.11) (push) Has been cancelled
tests / tests (ubuntu-latest, 3.12) (push) Has been cancelled
tests / tests (ubuntu-latest, 3.13) (push) Has been cancelled
tests / tests (windows-latest, 3.10) (push) Has been cancelled
tests / tests (windows-latest, 3.11) (push) Has been cancelled
tests / tests (windows-latest, 3.12) (push) Has been cancelled
tests / tests (windows-latest, 3.13) (push) Has been cancelled
Secret Leaks / trufflehog (push) Has been cancelled
CI security linting / zizmor latest via Cargo (push) Has been cancelled
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,512 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d36e1e93-ae93-4a4e-93c6-68fd868d2882",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Using C3A for sequence classification"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ddfc0610-55f6-4343-a950-125ccf0f45ac",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"In this example, we fine-tune Roberta (base) on a sequence classification task using C3A."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "45addd81-d4f3-4dfd-960d-3920d347f0a6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Imports"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a9935ae2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# To run this notebook, please run `pip install evaluate` to install additional dependencies not covered by PEFT.\n",
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_model,\n",
|
||||
" C3AConfig,\n",
|
||||
" PeftType,\n",
|
||||
")\n",
|
||||
"from peft.utils import infer_device\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed, AutoConfig\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "62c959bf-7cc2-49e0-b97e-4c10ec3b9bf3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Parameters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e3b13308",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_size = 32\n",
|
||||
"model_name_or_path = \"roberta-base\"\n",
|
||||
"task = \"mrpc\"\n",
|
||||
"peft_type = PeftType.C3A\n",
|
||||
"device = infer_device()\n",
|
||||
"num_epochs = 5 # for better results, increase this number\n",
|
||||
"block_size = 768 # for better results, increase this number\n",
|
||||
"max_length = 512\n",
|
||||
"torch.manual_seed(0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0526f571",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = C3AConfig(\n",
|
||||
" task_type=\"SEQ_CLS\", \n",
|
||||
" block_size=block_size,\n",
|
||||
" target_modules=[\"query\", \"value\"],\n",
|
||||
")\n",
|
||||
"head_lr = 4e-6 # the learning rate for the classification head for NLU tasks\n",
|
||||
"ft_lr = 3e-1 # the learning rate for C3A parameters, a much larger LR than that is usually used, at least 1e-1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c075c5d2-a457-4f37-a7f1-94fd0d277972",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Loading data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "7bb52cb4-d1c3-4b04-8bf0-f39ca88af139",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "e69c5e1f-d27b-4264-a41e-fc9b99d025e6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "0209f778-c93b-40eb-a4e0-24c25db03980",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=max_length)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
" remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "7453954e-982c-46f0-b09c-589776e6d6cb",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f3b9b2e8-f415-4d0f-9fb4-436f1a3585ea",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Preparing the C3A model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "2ed5ac74",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"trainable params: 610,562 || all params: 125,257,732 || trainable%: 0.4874\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True, max_length=None)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "0d2d0381",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"head_param = list(map(id, model.classifier.parameters()))\n",
|
||||
"\n",
|
||||
"others_param = filter(lambda p: id(p) not in head_param, model.parameters()) \n",
|
||||
"\n",
|
||||
"optimizer = AdamW([\n",
|
||||
" {\"params\": model.classifier.parameters(), \"lr\": head_lr},\n",
|
||||
" {\"params\": others_param, \"lr\": ft_lr}\n",
|
||||
"],weight_decay=0.)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c0dd5aa8-977b-4ac0-8b96-884b17bcdd00",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Training"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "fa0e73be",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████| 115/115 [00:04<00:00, 24.62it/s]\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 49.02it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 0: {'accuracy': 0.7990196078431373, 'f1': 0.8614864864864865}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:04<00:00, 26.18it/s]\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 49.86it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 1: {'accuracy': 0.8651960784313726, 'f1': 0.897196261682243}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:04<00:00, 26.21it/s]\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 49.86it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 2: {'accuracy': 0.8676470588235294, 'f1': 0.9018181818181819}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:04<00:00, 26.08it/s]\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 50.27it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 3: {'accuracy': 0.8725490196078431, 'f1': 0.9084507042253521}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:04<00:00, 26.15it/s]\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 49.68it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 4: {'accuracy': 0.8799019607843137, 'f1': 0.9126559714795008}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.to(device)\n",
|
||||
"for epoch in range(num_epochs):\n",
|
||||
" model.train()\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
"\n",
|
||||
" model.eval()\n",
|
||||
" for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_metric = metric.compute()\n",
|
||||
" print(f\"epoch {epoch}:\", eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f2b2caca",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Share adapters on the 🤗 Hub"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7b23af6f-cf6e-486f-9d10-0eada95b631f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"account_id = \"Your-Hugging-Face-Hub-Account\"\n",
|
||||
"token = \"Your-Hugging-Face-Hub-Token\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "990b3c93",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model.push_to_hub(f\"{account_id}/roberta-base-mrpc-peft-c3a\", token=token)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9d140b26",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load adapters from the Hub\n",
|
||||
"\n",
|
||||
"You can also directly load adapters from the Hub using the commands below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "c283e028-b349-46b0-a20e-cde0ee5fbd7b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"from transformers import AutoTokenizer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "320b10a0-4ea8-4786-9f3c-4670019c6b18",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"peft_model_id = f\"{account_id}/roberta-base-mrpc-peft-c3a\"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "b3a94049-bc01-4f2e-8cf9-66daf24a4402",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load the FourierFT model\n",
|
||||
"inference_model = PeftModel.from_pretrained(inference_model, peft_model_id, config=config)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "bd919fef-4e9a-4dc5-a957-7b879cfc5d38",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 51.18it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'accuracy': 0.8799019607843137, 'f1': 0.9126559714795008}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"inference_model.to(device)\n",
|
||||
"inference_model.eval()\n",
|
||||
"for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = inference_model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "peft",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.12"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d36e1e93-ae93-4a4e-93c6-68fd868d2882",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Using FourierFT for sequence classification"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ddfc0610-55f6-4343-a950-125ccf0f45ac",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"In this example, we fine-tune Roberta (base) on a sequence classification task using FourierFT."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "45addd81-d4f3-4dfd-960d-3920d347f0a6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Imports"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "a9935ae2",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/zgaoat/anaconda3/envs/pr2/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||||
" from .autonotebook import tqdm as notebook_tqdm\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# To run this notebook, please run `pip install evaluate` to install additional dependencies not covered by PEFT.\n",
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_model,\n",
|
||||
" FourierFTConfig,\n",
|
||||
" PeftType,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed, AutoConfig\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "62c959bf-7cc2-49e0-b97e-4c10ec3b9bf3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Parameters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e3b13308",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<torch._C.Generator at 0x78e2a49744b0>"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"batch_size = 32\n",
|
||||
"model_name_or_path = \"roberta-base\"\n",
|
||||
"task = \"mrpc\"\n",
|
||||
"peft_type = PeftType.FOURIERFT\n",
|
||||
"device = torch.accelerator.current_accelerator().type if hasattr(torch, \"accelerator\") else \"cuda\"\n",
|
||||
"num_epochs = 5 # for better results, increase this number\n",
|
||||
"n_frequency = 1000 # for better results, increase this number\n",
|
||||
"scaling = 150.0\n",
|
||||
"max_length = 512\n",
|
||||
"torch.manual_seed(0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "0526f571",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = FourierFTConfig(\n",
|
||||
" task_type=\"SEQ_CLS\", \n",
|
||||
" n_frequency=n_frequency,\n",
|
||||
" target_modules=[\"query\", \"value\"],\n",
|
||||
" scaling = scaling,\n",
|
||||
")\n",
|
||||
"head_lr = 6e-3 # the learning rate for the classification head for NLU tasks\n",
|
||||
"fft_lr = 6e-2 # the learning rate for the parameters other than the classification head (q,v in this case)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c075c5d2-a457-4f37-a7f1-94fd0d277972",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Loading data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "7bb52cb4-d1c3-4b04-8bf0-f39ca88af139",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "e69c5e1f-d27b-4264-a41e-fc9b99d025e6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "0209f778-c93b-40eb-a4e0-24c25db03980",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=max_length)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
" remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "7453954e-982c-46f0-b09c-589776e6d6cb",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f3b9b2e8-f415-4d0f-9fb4-436f1a3585ea",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Preparing the FourierFT model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "2ed5ac74",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"trainable params: 616,130 || all params: 125,263,300 || trainable%: 0.4919\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True, max_length=None)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "0d2d0381",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"head_param = list(map(id, model.classifier.parameters()))\n",
|
||||
"\n",
|
||||
"others_param = filter(lambda p: id(p) not in head_param, model.parameters()) \n",
|
||||
"\n",
|
||||
"optimizer = AdamW([\n",
|
||||
" {\"params\": model.classifier.parameters(), \"lr\": head_lr},\n",
|
||||
" {\"params\": others_param, \"lr\": fft_lr}\n",
|
||||
"],weight_decay=0.)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c0dd5aa8-977b-4ac0-8b96-884b17bcdd00",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Training"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "fa0e73be",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████| 115/115 [00:06<00:00, 19.03it/s]\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 41.72it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 0: {'accuracy': 0.8161764705882353, 'f1': 0.8709122203098106}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:05<00:00, 20.61it/s]\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 42.91it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 1: {'accuracy': 0.8480392156862745, 'f1': 0.8966666666666666}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:05<00:00, 20.63it/s]\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 42.65it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 2: {'accuracy': 0.8676470588235294, 'f1': 0.9075342465753424}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:05<00:00, 20.56it/s]\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 42.11it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 3: {'accuracy': 0.8504901960784313, 'f1': 0.8988391376451078}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:05<00:00, 20.50it/s]\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 43.15it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 4: {'accuracy': 0.8725490196078431, 'f1': 0.9103448275862069}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.to(device)\n",
|
||||
"for epoch in range(num_epochs):\n",
|
||||
" model.train()\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
"\n",
|
||||
" model.eval()\n",
|
||||
" for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_metric = metric.compute()\n",
|
||||
" print(f\"epoch {epoch}:\", eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f2b2caca",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Share adapters on the 🤗 Hub"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "7b23af6f-cf6e-486f-9d10-0eada95b631f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"account_id = ... # your Hugging Face Hub account ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "990b3c93",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/zgaoat/anaconda3/envs/pr2/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
|
||||
" warnings.warn(\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"CommitInfo(commit_url='https://huggingface.co/zgaoat/roberta-base-mrpc-peft-fourierft/commit/064eb35cbb7a1073b4d8fafbeccee43a0a4e37c9', commit_message='Upload model', commit_description='', oid='064eb35cbb7a1073b4d8fafbeccee43a0a4e37c9', pr_url=None, pr_revision=None, pr_num=None)"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.push_to_hub(f\"{account_id}/roberta-base-mrpc-peft-fourierft\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9d140b26",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load adapters from the Hub\n",
|
||||
"\n",
|
||||
"You can also directly load adapters from the Hub using the commands below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "c283e028-b349-46b0-a20e-cde0ee5fbd7b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"from transformers import AutoTokenizer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "320b10a0-4ea8-4786-9f3c-4670019c6b18",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"peft_model_id = f\"{account_id}/roberta-base-mrpc-peft-fourierft\"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "b3a94049-bc01-4f2e-8cf9-66daf24a4402",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load the FourierFT model\n",
|
||||
"inference_model = PeftModel.from_pretrained(inference_model, peft_model_id, config=config)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "bd919fef-4e9a-4dc5-a957-7b879cfc5d38",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████| 13/13 [00:00<00:00, 43.06it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'accuracy': 0.8725490196078431, 'f1': 0.9103448275862069}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"inference_model.to(device)\n",
|
||||
"inference_model.eval()\n",
|
||||
"for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = inference_model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,527 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "900b542d-0249-453c-a915-a061b80af69f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# PyTorch AO (torchao) with int8_dynamic_activation_int8_weight"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "10e1acc3-50b8-4d40-bdf3-0133c113cc4b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Imports"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "a9935ae2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import argparse\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from torchao.quantization import Int8DynamicActivationInt8WeightConfig\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_config,\n",
|
||||
" get_peft_model,\n",
|
||||
" get_peft_model_state_dict,\n",
|
||||
" set_peft_model_state_dict,\n",
|
||||
" LoraConfig,\n",
|
||||
" PeftType,\n",
|
||||
" PrefixTuningConfig,\n",
|
||||
" PromptEncoderConfig,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, TorchAoConfig, get_linear_schedule_with_warmup, set_seed\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "eafdd532-b1eb-4aac-8077-3386a84c7cdb",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Parameters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e3b13308",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_size = 16\n",
|
||||
"model_name_or_path = \"google/gemma-2-2b\"\n",
|
||||
"task = \"mrpc\"\n",
|
||||
"device = torch.accelerator.current_accelerator().type if hasattr(torch, \"accelerator\") else \"cuda\"\n",
|
||||
"num_epochs = 5\n",
|
||||
"lr = 2e-5\n",
|
||||
"\n",
|
||||
"lora_rank = 16\n",
|
||||
"lora_alpha = 32\n",
|
||||
"lora_dropout = 0.1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c7fb69bf-0182-4111-b715-e2e659b42b1d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "d2f4d25e-30b9-431f-95c3-adb390dc6fcd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
"\n",
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "1ea852bc-a040-4244-8fd3-516307cecd14",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",
|
||||
" return outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "cf5ef289-f42f-4582-bd5e-9852ad8beff2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
" remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "739b3655-9db0-48bc-8542-308c6d5e0b8b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "0288f311-8475-4a0e-99af-e4b909d10e01",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"train\"],\n",
|
||||
" shuffle=True,\n",
|
||||
" collate_fn=collate_fn,\n",
|
||||
" batch_size=batch_size,\n",
|
||||
")\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"],\n",
|
||||
" shuffle=False,\n",
|
||||
" collate_fn=collate_fn,\n",
|
||||
" batch_size=batch_size,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fcaf6f9e-c9d1-445a-9f08-18ef462f67ce",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "e5dfff56-ea80-4561-aeaf-43216bbb9af7",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "2ac42f98e60d412496fe77ed7eb5c6df",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Loading checkpoint shards: 0%| | 0/3 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of Gemma2ForSequenceClassification were not initialized from the model checkpoint at google/gemma-2-2b and are newly initialized: ['score.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"quant_config = TorchAoConfig(quant_type=Int8DynamicActivationInt8WeightConfig())\n",
|
||||
"model = AutoModelForSequenceClassification.from_pretrained(\n",
|
||||
" model_name_or_path, return_dict=True, device_map=0, dtype=torch.bfloat16, quantization_config=quant_config\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "0526f571",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = LoraConfig(\n",
|
||||
" task_type=\"SEQ_CLS\",\n",
|
||||
" r=lora_rank,\n",
|
||||
" lora_alpha=lora_alpha,\n",
|
||||
" lora_dropout=lora_dropout,\n",
|
||||
" target_modules=[\"q_proj\", \"v_proj\"],\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "ceeae329-e931-4d52-8a28-9c87e5cdb4cf",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"trainable params: 3,199,488 || all params: 2,617,545,984 || trainable%: 0.1222\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1b3d2544-3028-4e2a-9c56-d4d7d9d674de",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Training"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "0d2d0381",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"optimizer = AdamW(params=model.parameters(), lr=lr)\n",
|
||||
"\n",
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "f04c88ca-84eb-4184-afe6-3869b6f96b76",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"PeftModelForSequenceClassification(\n",
|
||||
" (base_model): LoraModel(\n",
|
||||
" (model): Gemma2ForSequenceClassification(\n",
|
||||
" (model): Gemma2Model(\n",
|
||||
" (embed_tokens): Embedding(256000, 2304, padding_idx=0)\n",
|
||||
" (layers): ModuleList(\n",
|
||||
" (0-25): 26 x Gemma2DecoderLayer(\n",
|
||||
" (self_attn): Gemma2Attention(\n",
|
||||
" (q_proj): lora.TorchaoLoraLinear(\n",
|
||||
" (base_layer): Linear(in_features=2304, out_features=2048, weight=LinearActivationQuantizedTensor(activation=<function _int8_symm_per_token_reduced_range_quant at 0x7a846f516520>, weight=AffineQuantizedTensor(shape=torch.Size([2048, 2304]), block_size=(1, 2304), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None)))\n",
|
||||
" (lora_dropout): ModuleDict(\n",
|
||||
" (default): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (lora_A): ModuleDict(\n",
|
||||
" (default): Linear(in_features=2304, out_features=16, bias=False)\n",
|
||||
" )\n",
|
||||
" (lora_B): ModuleDict(\n",
|
||||
" (default): Linear(in_features=16, out_features=2048, bias=False)\n",
|
||||
" )\n",
|
||||
" (lora_embedding_A): ParameterDict()\n",
|
||||
" (lora_embedding_B): ParameterDict()\n",
|
||||
" (lora_magnitude_vector): ModuleDict()\n",
|
||||
" )\n",
|
||||
" (k_proj): Linear(in_features=2304, out_features=1024, weight=LinearActivationQuantizedTensor(activation=<function _int8_symm_per_token_reduced_range_quant at 0x7a846f516520>, weight=AffineQuantizedTensor(shape=torch.Size([1024, 2304]), block_size=(1, 2304), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None)))\n",
|
||||
" (v_proj): lora.TorchaoLoraLinear(\n",
|
||||
" (base_layer): Linear(in_features=2304, out_features=1024, weight=LinearActivationQuantizedTensor(activation=<function _int8_symm_per_token_reduced_range_quant at 0x7a846f516520>, weight=AffineQuantizedTensor(shape=torch.Size([1024, 2304]), block_size=(1, 2304), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None)))\n",
|
||||
" (lora_dropout): ModuleDict(\n",
|
||||
" (default): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (lora_A): ModuleDict(\n",
|
||||
" (default): Linear(in_features=2304, out_features=16, bias=False)\n",
|
||||
" )\n",
|
||||
" (lora_B): ModuleDict(\n",
|
||||
" (default): Linear(in_features=16, out_features=1024, bias=False)\n",
|
||||
" )\n",
|
||||
" (lora_embedding_A): ParameterDict()\n",
|
||||
" (lora_embedding_B): ParameterDict()\n",
|
||||
" (lora_magnitude_vector): ModuleDict()\n",
|
||||
" )\n",
|
||||
" (o_proj): Linear(in_features=2048, out_features=2304, weight=LinearActivationQuantizedTensor(activation=<function _int8_symm_per_token_reduced_range_quant at 0x7a846f516520>, weight=AffineQuantizedTensor(shape=torch.Size([2304, 2048]), block_size=(1, 2048), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None)))\n",
|
||||
" (rotary_emb): Gemma2RotaryEmbedding()\n",
|
||||
" )\n",
|
||||
" (mlp): Gemma2MLP(\n",
|
||||
" (gate_proj): Linear(in_features=2304, out_features=9216, weight=LinearActivationQuantizedTensor(activation=<function _int8_symm_per_token_reduced_range_quant at 0x7a846f516520>, weight=AffineQuantizedTensor(shape=torch.Size([9216, 2304]), block_size=(1, 2304), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None)))\n",
|
||||
" (up_proj): Linear(in_features=2304, out_features=9216, weight=LinearActivationQuantizedTensor(activation=<function _int8_symm_per_token_reduced_range_quant at 0x7a846f516520>, weight=AffineQuantizedTensor(shape=torch.Size([9216, 2304]), block_size=(1, 2304), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None)))\n",
|
||||
" (down_proj): Linear(in_features=9216, out_features=2304, weight=LinearActivationQuantizedTensor(activation=<function _int8_symm_per_token_reduced_range_quant at 0x7a846f516520>, weight=AffineQuantizedTensor(shape=torch.Size([2304, 9216]), block_size=(1, 9216), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None)))\n",
|
||||
" (act_fn): PytorchGELUTanh()\n",
|
||||
" )\n",
|
||||
" (input_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
|
||||
" (post_attention_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
|
||||
" (pre_feedforward_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
|
||||
" (post_feedforward_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (norm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
|
||||
" )\n",
|
||||
" (score): ModulesToSaveWrapper(\n",
|
||||
" (original_module): Linear(in_features=2304, out_features=2, bias=False)\n",
|
||||
" (modules_to_save): ModuleDict(\n",
|
||||
" (default): Linear(in_features=2304, out_features=2, bias=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.config.use_cache = False\n",
|
||||
"model.to(device)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "fa0e73be",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/230 [00:00<?, ?it/s]You're using a GemmaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 230/230 [00:43<00:00, 5.27it/s]\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [00:04<00:00, 5.33it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 1 | train loss 1.7618 | {'accuracy': 0.46568627450980393, 'f1': 0.5458333333333333}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 230/230 [00:43<00:00, 5.29it/s]\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [00:04<00:00, 5.47it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 2 | train loss 1.1905 | {'accuracy': 0.5245098039215687, 'f1': 0.6325757575757576}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 230/230 [00:43<00:00, 5.32it/s]\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [00:04<00:00, 5.34it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 3 | train loss 1.1478 | {'accuracy': 0.5318627450980392, 'f1': 0.6456400742115028}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 230/230 [00:43<00:00, 5.29it/s]\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [00:04<00:00, 5.36it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 4 | train loss 1.1384 | {'accuracy': 0.5367647058823529, 'f1': 0.6506469500924215}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 230/230 [00:44<00:00, 5.21it/s]\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [00:04<00:00, 5.43it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 5 | train loss 1.1365 | {'accuracy': 0.5367647058823529, 'f1': 0.6506469500924215}\n",
|
||||
"CPU times: user 4min 2s, sys: 399 ms, total: 4min 2s\n",
|
||||
"Wall time: 4min 2s\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"for epoch in range(1, num_epochs + 1):\n",
|
||||
" model.train()\n",
|
||||
" train_losses = []\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" if not torch.isfinite(loss):\n",
|
||||
" raise ValueError(\"non-finite loss encountered\")\n",
|
||||
"\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" train_losses.append(loss.item())\n",
|
||||
"\n",
|
||||
" model.eval()\n",
|
||||
" for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_metric = metric.compute()\n",
|
||||
" train_loss = sum(train_losses) / len(train_losses)\n",
|
||||
" print(f\"epoch {epoch} | train loss {train_loss:.4f} |\", eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "6a1f937b-a0a5-40ec-8e41-5a5a18c6bff6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# memory: 4122MiB"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "412e41ee-72d3-4e71-bd3a-703b37429c57",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# PyTorch AO (torchao) with int8_weight_only"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "10e1acc3-50b8-4d40-bdf3-0133c113cc4b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Imports"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "a9935ae2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import argparse\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from torchao.quantization import Int8WeightOnlyConfig\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_config,\n",
|
||||
" get_peft_model,\n",
|
||||
" get_peft_model_state_dict,\n",
|
||||
" set_peft_model_state_dict,\n",
|
||||
" LoraConfig,\n",
|
||||
" PeftType,\n",
|
||||
" PrefixTuningConfig,\n",
|
||||
" PromptEncoderConfig,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, TorchAoConfig, get_linear_schedule_with_warmup, set_seed\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "eafdd532-b1eb-4aac-8077-3386a84c7cdb",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Parameters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e3b13308",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_size = 16\n",
|
||||
"model_name_or_path = \"google/gemma-2-2b\"\n",
|
||||
"task = \"mrpc\"\n",
|
||||
"device = torch.accelerator.current_accelerator().type if hasattr(torch, \"accelerator\") else \"cuda\"\n",
|
||||
"num_epochs = 5\n",
|
||||
"lr = 2e-5\n",
|
||||
"\n",
|
||||
"lora_rank = 16\n",
|
||||
"lora_alpha = 32\n",
|
||||
"lora_dropout = 0.1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c7fb69bf-0182-4111-b715-e2e659b42b1d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "d2f4d25e-30b9-431f-95c3-adb390dc6fcd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
"\n",
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "1ea852bc-a040-4244-8fd3-516307cecd14",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",
|
||||
" return outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "cf5ef289-f42f-4582-bd5e-9852ad8beff2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
" remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "739b3655-9db0-48bc-8542-308c6d5e0b8b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "0288f311-8475-4a0e-99af-e4b909d10e01",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"train\"],\n",
|
||||
" shuffle=True,\n",
|
||||
" collate_fn=collate_fn,\n",
|
||||
" batch_size=batch_size,\n",
|
||||
")\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"],\n",
|
||||
" shuffle=False,\n",
|
||||
" collate_fn=collate_fn,\n",
|
||||
" batch_size=batch_size,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fcaf6f9e-c9d1-445a-9f08-18ef462f67ce",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "e5dfff56-ea80-4561-aeaf-43216bbb9af7",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "512d9dc10a4d4ecc88b9440575b0973a",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Loading checkpoint shards: 0%| | 0/3 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of Gemma2ForSequenceClassification were not initialized from the model checkpoint at google/gemma-2-2b and are newly initialized: ['score.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"quant_config = TorchAoConfig(quant_type=Int8WeightOnlyConfig())\n",
|
||||
"model = AutoModelForSequenceClassification.from_pretrained(\n",
|
||||
" model_name_or_path, return_dict=True, device_map=0, dtype=torch.bfloat16, quantization_config=quant_config\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "0526f571",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = LoraConfig(\n",
|
||||
" task_type=\"SEQ_CLS\",\n",
|
||||
" r=lora_rank,\n",
|
||||
" lora_alpha=lora_alpha,\n",
|
||||
" lora_dropout=lora_dropout,\n",
|
||||
" target_modules=[\"q_proj\", \"v_proj\"],\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "ceeae329-e931-4d52-8a28-9c87e5cdb4cf",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"trainable params: 3,199,488 || all params: 2,617,545,984 || trainable%: 0.1222\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1b3d2544-3028-4e2a-9c56-d4d7d9d674de",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Training"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "0d2d0381",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"optimizer = AdamW(params=model.parameters(), lr=lr)\n",
|
||||
"\n",
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "f04c88ca-84eb-4184-afe6-3869b6f96b76",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"PeftModelForSequenceClassification(\n",
|
||||
" (base_model): LoraModel(\n",
|
||||
" (model): Gemma2ForSequenceClassification(\n",
|
||||
" (model): Gemma2Model(\n",
|
||||
" (embed_tokens): Embedding(256000, 2304, padding_idx=0)\n",
|
||||
" (layers): ModuleList(\n",
|
||||
" (0-25): 26 x Gemma2DecoderLayer(\n",
|
||||
" (self_attn): Gemma2Attention(\n",
|
||||
" (q_proj): lora.TorchaoLoraLinear(\n",
|
||||
" (base_layer): Linear(in_features=2304, out_features=2048, weight=AffineQuantizedTensor(shape=torch.Size([2048, 2304]), block_size=(1, 2304), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None))\n",
|
||||
" (lora_dropout): ModuleDict(\n",
|
||||
" (default): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (lora_A): ModuleDict(\n",
|
||||
" (default): Linear(in_features=2304, out_features=16, bias=False)\n",
|
||||
" )\n",
|
||||
" (lora_B): ModuleDict(\n",
|
||||
" (default): Linear(in_features=16, out_features=2048, bias=False)\n",
|
||||
" )\n",
|
||||
" (lora_embedding_A): ParameterDict()\n",
|
||||
" (lora_embedding_B): ParameterDict()\n",
|
||||
" (lora_magnitude_vector): ModuleDict()\n",
|
||||
" )\n",
|
||||
" (k_proj): Linear(in_features=2304, out_features=1024, weight=AffineQuantizedTensor(shape=torch.Size([1024, 2304]), block_size=(1, 2304), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None))\n",
|
||||
" (v_proj): lora.TorchaoLoraLinear(\n",
|
||||
" (base_layer): Linear(in_features=2304, out_features=1024, weight=AffineQuantizedTensor(shape=torch.Size([1024, 2304]), block_size=(1, 2304), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None))\n",
|
||||
" (lora_dropout): ModuleDict(\n",
|
||||
" (default): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (lora_A): ModuleDict(\n",
|
||||
" (default): Linear(in_features=2304, out_features=16, bias=False)\n",
|
||||
" )\n",
|
||||
" (lora_B): ModuleDict(\n",
|
||||
" (default): Linear(in_features=16, out_features=1024, bias=False)\n",
|
||||
" )\n",
|
||||
" (lora_embedding_A): ParameterDict()\n",
|
||||
" (lora_embedding_B): ParameterDict()\n",
|
||||
" (lora_magnitude_vector): ModuleDict()\n",
|
||||
" )\n",
|
||||
" (o_proj): Linear(in_features=2048, out_features=2304, weight=AffineQuantizedTensor(shape=torch.Size([2304, 2048]), block_size=(1, 2048), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None))\n",
|
||||
" (rotary_emb): Gemma2RotaryEmbedding()\n",
|
||||
" )\n",
|
||||
" (mlp): Gemma2MLP(\n",
|
||||
" (gate_proj): Linear(in_features=2304, out_features=9216, weight=AffineQuantizedTensor(shape=torch.Size([9216, 2304]), block_size=(1, 2304), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None))\n",
|
||||
" (up_proj): Linear(in_features=2304, out_features=9216, weight=AffineQuantizedTensor(shape=torch.Size([9216, 2304]), block_size=(1, 2304), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None))\n",
|
||||
" (down_proj): Linear(in_features=9216, out_features=2304, weight=AffineQuantizedTensor(shape=torch.Size([2304, 9216]), block_size=(1, 9216), device=cuda:0, layout_type=PlainLayoutType(), layout_tensor_dtype=torch.int8, quant_min=None, quant_max=None))\n",
|
||||
" (act_fn): PytorchGELUTanh()\n",
|
||||
" )\n",
|
||||
" (input_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
|
||||
" (post_attention_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
|
||||
" (pre_feedforward_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
|
||||
" (post_feedforward_layernorm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (norm): Gemma2RMSNorm((2304,), eps=1e-06)\n",
|
||||
" )\n",
|
||||
" (score): ModulesToSaveWrapper(\n",
|
||||
" (original_module): Linear(in_features=2304, out_features=2, bias=False)\n",
|
||||
" (modules_to_save): ModuleDict(\n",
|
||||
" (default): Linear(in_features=2304, out_features=2, bias=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.config.use_cache = False\n",
|
||||
"model.to(device)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "fa0e73be",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/230 [00:00<?, ?it/s]You're using a GemmaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 230/230 [00:31<00:00, 7.19it/s]\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [00:01<00:00, 16.19it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 1 | train loss 1.0672 | {'accuracy': 0.6715686274509803, 'f1': 0.7751677852348994}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 230/230 [00:31<00:00, 7.26it/s]\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [00:01<00:00, 16.19it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 2 | train loss 0.6261 | {'accuracy': 0.7377450980392157, 'f1': 0.8201680672268907}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 230/230 [00:31<00:00, 7.25it/s]\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [00:01<00:00, 16.15it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 3 | train loss 0.4743 | {'accuracy': 0.7867647058823529, 'f1': 0.8502581755593803}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 230/230 [00:31<00:00, 7.30it/s]\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [00:01<00:00, 16.17it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 4 | train loss 0.4006 | {'accuracy': 0.803921568627451, 'f1': 0.8586572438162544}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 230/230 [00:31<00:00, 7.26it/s]\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [00:01<00:00, 16.10it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 5 | train loss 0.3585 | {'accuracy': 0.8235294117647058, 'f1': 0.8791946308724832}\n",
|
||||
"CPU times: user 2min 8s, sys: 38 s, total: 2min 46s\n",
|
||||
"Wall time: 2min 46s\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"for epoch in range(1, num_epochs + 1):\n",
|
||||
" model.train()\n",
|
||||
" train_losses = []\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" if not torch.isfinite(loss):\n",
|
||||
" raise ValueError(\"non-finite loss encountered\")\n",
|
||||
"\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" train_losses.append(loss.item())\n",
|
||||
"\n",
|
||||
" model.eval()\n",
|
||||
" for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_metric = metric.compute()\n",
|
||||
" train_loss = sum(train_losses) / len(train_losses)\n",
|
||||
" print(f\"epoch {epoch} | train loss {train_loss:.4f} |\", eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "6a1f937b-a0a5-40ec-8e41-5a5a18c6bff6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# memory: 18098MiB"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,713 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "a9935ae2",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"===================================BUG REPORT===================================\n",
|
||||
"Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
|
||||
"For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n",
|
||||
"================================================================================\n",
|
||||
"CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n",
|
||||
"CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
|
||||
"CUDA SETUP: Detected CUDA version 117\n",
|
||||
"CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import argparse\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_config,\n",
|
||||
" get_peft_model,\n",
|
||||
" get_peft_model_state_dict,\n",
|
||||
" set_peft_model_state_dict,\n",
|
||||
" LoraConfig,\n",
|
||||
" PeftType,\n",
|
||||
" PrefixTuningConfig,\n",
|
||||
" PromptEncoderConfig,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e3b13308",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_size = 32\n",
|
||||
"model_name_or_path = \"roberta-large\"\n",
|
||||
"task = \"mrpc\"\n",
|
||||
"peft_type = PeftType.LORA\n",
|
||||
"device = torch.accelerator.current_accelerator().type if hasattr(torch, \"accelerator\") else \"cuda\"\n",
|
||||
"num_epochs = 20"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "0526f571",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = LoraConfig(task_type=\"SEQ_CLS\", inference_mode=False, r=8, lora_alpha=16, lora_dropout=0.1)\n",
|
||||
"lr = 3e-4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "c2697d07",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "0f74797387a941cbb0709487b8808eba",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading readme: 0%| | 0.00/27.9k [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Found cached dataset glue (/home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "1a9ecc2f624343c3af8d1824afb66ac5",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/3 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "33b071c0e5794cb48b38bbf68f22b49b",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/4 [00:00<?, ?ba/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "a977694036394d5c99adfb13c023e258",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/1 [00:00<?, ?ba/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "facc8d9092dc4abe9e553fc8e5b795b8",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/2 [00:00<?, ?ba/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
"\n",
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
" remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2ed5ac74",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()\n",
|
||||
"model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "0d2d0381",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"optimizer = AdamW(params=model.parameters(), lr=lr)\n",
|
||||
"\n",
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "fa0e73be",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:28<00:00, 4.08it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.68it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 0: {'accuracy': 0.7009803921568627, 'f1': 0.8189910979228486}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.18it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.64it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 1: {'accuracy': 0.7622549019607843, 'f1': 0.8482003129890453}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.20it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.63it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 2: {'accuracy': 0.8651960784313726, 'f1': 0.9005424954792043}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.21it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.62it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 3: {'accuracy': 0.8921568627450981, 'f1': 0.9228070175438596}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.20it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.62it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 4: {'accuracy': 0.8970588235294118, 'f1': 0.9257950530035336}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.16it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.01it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 5: {'accuracy': 0.8823529411764706, 'f1': 0.9169550173010381}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:30<00:00, 3.81it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.62it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 6: {'accuracy': 0.8799019607843137, 'f1': 0.9170896785109983}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.16it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.61it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 7: {'accuracy': 0.8799019607843137, 'f1': 0.9150779896013865}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.18it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.61it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 8: {'accuracy': 0.8921568627450981, 'f1': 0.9233449477351917}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.18it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.59it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 9: {'accuracy': 0.8872549019607843, 'f1': 0.9217687074829931}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.16it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.61it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 10: {'accuracy': 0.8774509803921569, 'f1': 0.9137931034482758}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:29<00:00, 3.90it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.81it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 11: {'accuracy': 0.9068627450980392, 'f1': 0.9321428571428573}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:28<00:00, 4.05it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.59it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 12: {'accuracy': 0.8946078431372549, 'f1': 0.925476603119584}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.17it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.58it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 13: {'accuracy': 0.8897058823529411, 'f1': 0.922279792746114}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.18it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.61it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 14: {'accuracy': 0.8970588235294118, 'f1': 0.9265734265734265}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.18it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.60it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 15: {'accuracy': 0.8970588235294118, 'f1': 0.9263157894736843}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.17it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.59it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 16: {'accuracy': 0.8921568627450981, 'f1': 0.9233449477351917}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.18it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.58it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 17: {'accuracy': 0.8897058823529411, 'f1': 0.9220103986135182}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:30<00:00, 3.78it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.58it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 18: {'accuracy': 0.8921568627450981, 'f1': 0.9233449477351917}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00, 4.16it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.60it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 19: {'accuracy': 0.8946078431372549, 'f1': 0.924693520140105}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.to(device)\n",
|
||||
"for epoch in range(num_epochs):\n",
|
||||
" model.train()\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
"\n",
|
||||
" model.eval()\n",
|
||||
" for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_metric = metric.compute()\n",
|
||||
" print(f\"epoch {epoch}:\", eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f2b2caca",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Share adapters on the 🤗 Hub"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "990b3c93",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"CommitInfo(commit_url='https://huggingface.co/smangrul/roberta-large-peft-lora/commit/c2c661898b8b6a0c68ecd068931e598d0a79686b', commit_message='Upload model', commit_description='', oid='c2c661898b8b6a0c68ecd068931e598d0a79686b', pr_url=None, pr_revision=None, pr_num=None)"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.push_to_hub(\"smangrul/roberta-large-peft-lora\", use_auth_token=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9d140b26",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load adapters from the Hub\n",
|
||||
"\n",
|
||||
"You can also directly load adapters from the Hub using the commands below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "4d55c87d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.bias', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias']\n",
|
||||
"- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
|
||||
"- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.out_proj.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
|
||||
" 0%| | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.45it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'accuracy': 0.8946078431372549, 'f1': 0.924693520140105}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
|
||||
"\n",
|
||||
"peft_model_id = \"smangrul/roberta-large-peft-lora\"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"\n",
|
||||
"# Load the Lora model\n",
|
||||
"inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)\n",
|
||||
"\n",
|
||||
"inference_model.to(device)\n",
|
||||
"inference_model.eval()\n",
|
||||
"for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = inference_model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "27c43da1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,685 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "a825ba6b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"===================================BUG REPORT===================================\n",
|
||||
"Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
|
||||
"For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n",
|
||||
"================================================================================\n",
|
||||
"CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n",
|
||||
"CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
|
||||
"CUDA SETUP: Detected CUDA version 117\n",
|
||||
"CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import argparse\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_config,\n",
|
||||
" get_peft_model,\n",
|
||||
" get_peft_model_state_dict,\n",
|
||||
" set_peft_model_state_dict,\n",
|
||||
" PeftType,\n",
|
||||
" PrefixTuningConfig,\n",
|
||||
" PromptEncoderConfig,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2bd7cbb2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_size = 32\n",
|
||||
"model_name_or_path = \"roberta-large\"\n",
|
||||
"task = \"mrpc\"\n",
|
||||
"peft_type = PeftType.P_TUNING\n",
|
||||
"device = torch.accelerator.current_accelerator().type if hasattr(torch, \"accelerator\") else \"cuda\"\n",
|
||||
"num_epochs = 20"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "33d9b62e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = PromptEncoderConfig(task_type=\"SEQ_CLS\", num_virtual_tokens=20, encoder_hidden_size=128)\n",
|
||||
"lr = 1e-3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "152b6177",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Found cached dataset glue (/home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "a451b90675e0451489cc6426465afa32",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/3 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-9fa7887f9eaa03ae.arrow\n",
|
||||
"Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-dc593149bbeafe80.arrow\n",
|
||||
"Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-140ebe5b70e09817.arrow\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
"\n",
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
" remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f6bc8144",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()\n",
|
||||
"model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "af41c571",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"optimizer = AdamW(params=model.parameters(), lr=lr)\n",
|
||||
"\n",
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0, # 0.06*(len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "90993c93",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00, 3.54it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.91it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 0: {'accuracy': 0.6985294117647058, 'f1': 0.8172362555720655}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.61it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.87it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 1: {'accuracy': 0.6936274509803921, 'f1': 0.806201550387597}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.61it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.88it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 2: {'accuracy': 0.7132352941176471, 'f1': 0.8224582701062216}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.61it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.87it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 3: {'accuracy': 0.7083333333333334, 'f1': 0.8199697428139183}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.61it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.90it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 4: {'accuracy': 0.7205882352941176, 'f1': 0.8246153846153846}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.62it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.90it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 5: {'accuracy': 0.7009803921568627, 'f1': 0.8200589970501474}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00, 3.59it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.89it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 6: {'accuracy': 0.7254901960784313, 'f1': 0.8292682926829268}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.60it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.86it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 7: {'accuracy': 0.7230392156862745, 'f1': 0.8269525267993874}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:34<00:00, 3.34it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.88it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 8: {'accuracy': 0.7254901960784313, 'f1': 0.8297872340425533}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.60it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.77it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 9: {'accuracy': 0.7230392156862745, 'f1': 0.828006088280061}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00, 3.58it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.88it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 10: {'accuracy': 0.7181372549019608, 'f1': 0.8183254344391785}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.60it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.87it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 11: {'accuracy': 0.7132352941176471, 'f1': 0.803361344537815}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.59it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.85it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 12: {'accuracy': 0.7107843137254902, 'f1': 0.8206686930091186}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00, 3.59it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.85it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 13: {'accuracy': 0.7181372549019608, 'f1': 0.8254931714719272}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00, 3.59it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.87it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 14: {'accuracy': 0.7156862745098039, 'f1': 0.8253012048192772}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00, 3.59it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.84it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 15: {'accuracy': 0.7230392156862745, 'f1': 0.8242612752721618}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00, 3.49it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:02<00:00, 5.84it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 16: {'accuracy': 0.7181372549019608, 'f1': 0.8200312989045383}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00, 3.49it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.84it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 17: {'accuracy': 0.7107843137254902, 'f1': 0.8217522658610272}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.60it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.88it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 18: {'accuracy': 0.7254901960784313, 'f1': 0.8292682926829268}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00, 3.61it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 6.89it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 19: {'accuracy': 0.7107843137254902, 'f1': 0.8206686930091186}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.to(device)\n",
|
||||
"for epoch in range(num_epochs):\n",
|
||||
" model.train()\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
"\n",
|
||||
" model.eval()\n",
|
||||
" for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_metric = metric.compute()\n",
|
||||
" print(f\"epoch {epoch}:\", eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a43bd9fb",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Share adapters on the 🤗 Hub"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "871b75aa",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"CommitInfo(commit_url='https://huggingface.co/smangrul/roberta-large-peft-p-tuning/commit/fa7abe613f498c76df5e16c85d9c19c3019587a7', commit_message='Upload model', commit_description='', oid='fa7abe613f498c76df5e16c85d9c19c3019587a7', pr_url=None, pr_revision=None, pr_num=None)"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.push_to_hub(\"smangrul/roberta-large-peft-p-tuning\", use_auth_token=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1c6a9036",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load adapters from the Hub\n",
|
||||
"\n",
|
||||
"You can also directly load adapters from the Hub using the commands below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "91b0b8f5",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.bias', 'lm_head.dense.weight', 'lm_head.bias']\n",
|
||||
"- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
|
||||
"- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "e650799d58ec4bd1b21b6bc28ddf2069",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading: 0%| | 0.00/4.29M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 7.18it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'accuracy': 0.7107843137254902, 'f1': 0.8206686930091186}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
|
||||
"\n",
|
||||
"peft_model_id = \"smangrul/roberta-large-peft-p-tuning\"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"\n",
|
||||
"# Load the Lora model\n",
|
||||
"inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)\n",
|
||||
"\n",
|
||||
"inference_model.to(device)\n",
|
||||
"inference_model.eval()\n",
|
||||
"for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = inference_model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1a8d69d1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,692 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "9ff5004e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"===================================BUG REPORT===================================\n",
|
||||
"Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
|
||||
"For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n",
|
||||
"================================================================================\n",
|
||||
"CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n",
|
||||
"CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
|
||||
"CUDA SETUP: Detected CUDA version 117\n",
|
||||
"CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import argparse\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_config,\n",
|
||||
" get_peft_model,\n",
|
||||
" get_peft_model_state_dict,\n",
|
||||
" set_peft_model_state_dict,\n",
|
||||
" PeftType,\n",
|
||||
" PrefixTuningConfig,\n",
|
||||
" PromptEncoderConfig,\n",
|
||||
" PromptTuningConfig,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e32c4a9e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_size = 32\n",
|
||||
"model_name_or_path = \"roberta-large\"\n",
|
||||
"task = \"mrpc\"\n",
|
||||
"peft_type = PeftType.PROMPT_TUNING\n",
|
||||
"device = torch.accelerator.current_accelerator().type if hasattr(torch, \"accelerator\") else \"cuda\"\n",
|
||||
"num_epochs = 20"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "622fe9c8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = PromptTuningConfig(task_type=\"SEQ_CLS\", num_virtual_tokens=10)\n",
|
||||
"lr = 1e-3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "74e9efe0",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Found cached dataset glue (/home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "76198cec552441818ff107910275e5be",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/3 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-9fa7887f9eaa03ae.arrow\n",
|
||||
"Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-dc593149bbeafe80.arrow\n",
|
||||
"Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-140ebe5b70e09817.arrow\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
"\n",
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
" remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a3c15af0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()\n",
|
||||
"model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "6d3c5edb",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"optimizer = AdamW(params=model.parameters(), lr=lr)\n",
|
||||
"\n",
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "4d279225",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [02:09<00:00, 1.13s/it]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:08<00:00, 1.62it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 0: {'accuracy': 0.678921568627451, 'f1': 0.7956318252730109}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:50<00:00, 1.04it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.22it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 1: {'accuracy': 0.696078431372549, 'f1': 0.8171091445427728}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:36<00:00, 1.19it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00, 2.00it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 2: {'accuracy': 0.6985294117647058, 'f1': 0.8161434977578476}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:37<00:00, 1.18it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00, 2.09it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 3: {'accuracy': 0.7058823529411765, 'f1': 0.7979797979797979}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [02:03<00:00, 1.07s/it]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:07<00:00, 1.71it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 4: {'accuracy': 0.696078431372549, 'f1': 0.8132530120481929}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:53<00:00, 1.01it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.19it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 5: {'accuracy': 0.7107843137254902, 'f1': 0.8121019108280254}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:35<00:00, 1.20it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.20it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 6: {'accuracy': 0.6911764705882353, 'f1': 0.7692307692307693}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:36<00:00, 1.20it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.18it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 7: {'accuracy': 0.7156862745098039, 'f1': 0.8209876543209876}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:35<00:00, 1.20it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.22it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 8: {'accuracy': 0.7205882352941176, 'f1': 0.8240740740740742}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:36<00:00, 1.19it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.21it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 9: {'accuracy': 0.7205882352941176, 'f1': 0.8229813664596273}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:36<00:00, 1.20it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.35it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 10: {'accuracy': 0.7156862745098039, 'f1': 0.8164556962025317}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:35<00:00, 1.20it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.22it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 11: {'accuracy': 0.7058823529411765, 'f1': 0.8113207547169811}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:32<00:00, 1.24it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.48it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 12: {'accuracy': 0.7009803921568627, 'f1': 0.7946127946127945}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:32<00:00, 1.24it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.38it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 13: {'accuracy': 0.7230392156862745, 'f1': 0.8186195826645265}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:29<00:00, 1.29it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.31it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 14: {'accuracy': 0.7058823529411765, 'f1': 0.8130841121495327}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:30<00:00, 1.27it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.39it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 15: {'accuracy': 0.7181372549019608, 'f1': 0.8194662480376768}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:28<00:00, 1.29it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.35it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 16: {'accuracy': 0.7254901960784313, 'f1': 0.8181818181818181}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:30<00:00, 1.27it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.30it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 17: {'accuracy': 0.7205882352941176, 'f1': 0.820754716981132}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:30<00:00, 1.27it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.36it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 18: {'accuracy': 0.7254901960784313, 'f1': 0.821656050955414}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:28<00:00, 1.29it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.43it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 19: {'accuracy': 0.7303921568627451, 'f1': 0.8242811501597445}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.to(device)\n",
|
||||
"for epoch in range(num_epochs):\n",
|
||||
" model.train()\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
"\n",
|
||||
" model.eval()\n",
|
||||
" for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_metric = metric.compute()\n",
|
||||
" print(f\"epoch {epoch}:\", eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e1ff3f44",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Share adapters on the 🤗 Hub"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "0bf79cb5",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"CommitInfo(commit_url='https://huggingface.co/smangrul/roberta-large-peft-prompt-tuning/commit/893a909d8499aa8778d58c781d43c3a8d9360de8', commit_message='Upload model', commit_description='', oid='893a909d8499aa8778d58c781d43c3a8d9360de8', pr_url=None, pr_revision=None, pr_num=None)"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.push_to_hub(\"smangrul/roberta-large-peft-prompt-tuning\", use_auth_token=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "73870ad7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load adapters from the Hub\n",
|
||||
"\n",
|
||||
"You can also directly load adapters from the Hub using the commands below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "0654a552",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "24581bb98582444ca6114b9fa267847f",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading: 0%| | 0.00/368 [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.weight', 'lm_head.decoder.weight', 'lm_head.dense.bias']\n",
|
||||
"- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
|
||||
"- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "f1584da4d1c54cc3873a515182674980",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading: 0%| | 0.00/4.25M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.58it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'accuracy': 0.7303921568627451, 'f1': 0.8242811501597445}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
|
||||
"\n",
|
||||
"peft_model_id = \"smangrul/roberta-large-peft-prompt-tuning\"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"\n",
|
||||
"# Load the Lora model\n",
|
||||
"inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)\n",
|
||||
"\n",
|
||||
"inference_model.to(device)\n",
|
||||
"inference_model.eval()\n",
|
||||
"for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = inference_model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.4"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,787 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d36e1e93-ae93-4a4e-93c6-68fd868d2882",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Using VB-LoRA for sequence classification"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ddfc0610-55f6-4343-a950-125ccf0f45ac",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"In this example, we fine-tune Roberta on a sequence classification task using VB-LoRA.\n",
|
||||
"\n",
|
||||
"This notebook is adapted from `examples/sequence_classification/VeRA.ipynb`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "45addd81-d4f3-4dfd-960d-3920d347f0a6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Imports"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a9935ae2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_model,\n",
|
||||
" VBLoRAConfig,\n",
|
||||
" PeftType,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "62c959bf-7cc2-49e0-b97e-4c10ec3b9bf3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Parameters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e3b13308",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<torch._C.Generator at 0x7f4fc7c3c750>"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"batch_size = 32\n",
|
||||
"model_name_or_path = \"roberta-large\"\n",
|
||||
"task = \"mrpc\"\n",
|
||||
"peft_type = PeftType.VBLORA\n",
|
||||
"device = torch.accelerator.current_accelerator().type if hasattr(torch, \"accelerator\") else \"cuda\"\n",
|
||||
"num_epochs = 20\n",
|
||||
"rank = 4\n",
|
||||
"max_length = 128\n",
|
||||
"num_vectors = 90\n",
|
||||
"vector_length = 256\n",
|
||||
"torch.manual_seed(0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "0526f571",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = VBLoRAConfig(\n",
|
||||
" task_type=\"SEQ_CLS\", \n",
|
||||
" r=rank,\n",
|
||||
" topk=2,\n",
|
||||
" target_modules=['key', 'value', 'query', 'output.dense', 'intermediate.dense'],\n",
|
||||
" num_vectors=num_vectors,\n",
|
||||
" vector_length=vector_length,\n",
|
||||
" save_only_topk_weights=True, # Set to True to reduce storage space. Note that the saved parameters cannot be used to resume training from checkpoints.\n",
|
||||
" vblora_dropout=0.,\n",
|
||||
")\n",
|
||||
"head_lr = 4e-3\n",
|
||||
"vector_bank_lr = 1e-3\n",
|
||||
"logits_lr = 1e-2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c075c5d2-a457-4f37-a7f1-94fd0d277972",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Loading data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "7bb52cb4-d1c3-4b04-8bf0-f39ca88af139",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "e69c5e1f-d27b-4264-a41e-fc9b99d025e6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "0209f778-c93b-40eb-a4e0-24c25db03980",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=max_length)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
" remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "7453954e-982c-46f0-b09c-589776e6d6cb",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f3b9b2e8-f415-4d0f-9fb4-436f1a3585ea",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Preparing the VB-LoRA model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "2ed5ac74",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"trainable params: 1,696,770 || all params: 357,058,564 || trainable%: 0.4752\n",
|
||||
"VB-LoRA params to-be-saved (float32-equivalent): 33,408 || total params to-be-saved: 1,085,058\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True, max_length=None)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()\n",
|
||||
"model.print_savable_parameters()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "0d2d0381",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS\n",
|
||||
"from transformers.trainer_pt_utils import get_parameter_names\n",
|
||||
"\n",
|
||||
"decay_parameters = get_parameter_names(model, ALL_LAYERNORM_LAYERS)\n",
|
||||
"decay_parameters = [name for name in decay_parameters if \"bias\" not in name]\n",
|
||||
"vector_bank_parameters = [name for name, _ in model.named_parameters() if \"vector_bank\" in name]\n",
|
||||
"logits_parameters = [name for name, _ in model.named_parameters() if \"logits\" in name ]\n",
|
||||
"\n",
|
||||
"optimizer_grouped_parameters = [\n",
|
||||
" {\n",
|
||||
" \"params\": [p for n, p in model.named_parameters() if n in decay_parameters and \\\n",
|
||||
" n not in logits_parameters and n not in vector_bank_parameters],\n",
|
||||
" \"weight_decay\": 0.1,\n",
|
||||
" \"lr\": head_lr,\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"params\": [p for n, p in model.named_parameters() if n not in decay_parameters and \\\n",
|
||||
" n not in logits_parameters and n not in vector_bank_parameters],\n",
|
||||
" \"weight_decay\": 0.0,\n",
|
||||
" \"lr\": head_lr,\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"params\": [p for n, p in model.named_parameters() if n in vector_bank_parameters],\n",
|
||||
" \"lr\": vector_bank_lr,\n",
|
||||
" \"weight_decay\": 0.0,\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"params\": [p for n, p in model.named_parameters() if n in logits_parameters],\n",
|
||||
" \"lr\": logits_lr,\n",
|
||||
" \"weight_decay\": 0.0,\n",
|
||||
" },\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"optimizer = AdamW(optimizer_grouped_parameters)\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c0dd5aa8-977b-4ac0-8b96-884b17bcdd00",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Training"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "fa0e73be",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.33it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.84it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 0: {'accuracy': 0.6691176470588235, 'f1': 0.786053882725832}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.37it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.83it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 1: {'accuracy': 0.5833333333333334, 'f1': 0.6136363636363636}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.34it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.82it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 2: {'accuracy': 0.7107843137254902, 'f1': 0.8238805970149253}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.34it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.80it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 3: {'accuracy': 0.8284313725490197, 'f1': 0.8833333333333333}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.34it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.79it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 4: {'accuracy': 0.8480392156862745, 'f1': 0.8847583643122676}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.30it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.78it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 5: {'accuracy': 0.8676470588235294, 'f1': 0.898876404494382}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.31it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.76it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 6: {'accuracy': 0.8602941176470589, 'f1': 0.9035532994923858}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.32it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.76it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 7: {'accuracy': 0.8774509803921569, 'f1': 0.911660777385159}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.33it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.79it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 8: {'accuracy': 0.8872549019607843, 'f1': 0.9172661870503597}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.32it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.78it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 9: {'accuracy': 0.875, 'f1': 0.9113043478260869}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.32it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.76it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 10: {'accuracy': 0.8823529411764706, 'f1': 0.9166666666666666}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.33it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.76it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 11: {'accuracy': 0.8970588235294118, 'f1': 0.9252669039145908}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.32it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.75it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 12: {'accuracy': 0.8946078431372549, 'f1': 0.9246935201401051}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.33it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.76it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 13: {'accuracy': 0.9068627450980392, 'f1': 0.9316546762589928}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.33it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.76it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 14: {'accuracy': 0.8946078431372549, 'f1': 0.9225225225225225}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.33it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.76it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 15: {'accuracy': 0.8995098039215687, 'f1': 0.926391382405745}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.30it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.76it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 16: {'accuracy': 0.9068627450980392, 'f1': 0.9316546762589928}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.31it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.77it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 17: {'accuracy': 0.8921568627450981, 'f1': 0.9217081850533808}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.33it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.77it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 18: {'accuracy': 0.8995098039215687, 'f1': 0.9266547406082289}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 115/115 [00:34<00:00, 3.33it/s]\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.77it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 19: {'accuracy': 0.9044117647058824, 'f1': 0.9297297297297298}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.to(device)\n",
|
||||
"\n",
|
||||
"for epoch in range(num_epochs):\n",
|
||||
" model.train()\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
"\n",
|
||||
" model.eval()\n",
|
||||
" for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_metric = metric.compute()\n",
|
||||
" print(f\"epoch {epoch}:\", eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f2b2caca",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Share adapters on the 🤗 Hub"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "7b23af6f-cf6e-486f-9d10-0eada95b631f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"account_id = ... # your Hugging Face Hub account ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "990b3c93",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model.push_to_hub(f\"{account_id}/roberta-large-peft-vblora\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9d140b26",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load adapters from the Hub\n",
|
||||
"\n",
|
||||
"You can also directly load adapters from the Hub using the commands below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "c283e028-b349-46b0-a20e-cde0ee5fbd7b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"from transformers import AutoTokenizer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "320b10a0-4ea8-4786-9f3c-4670019c6b18",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"peft_model_id = f\"{account_id}/roberta-large-peft-vblora\"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "b3a94049-bc01-4f2e-8cf9-66daf24a4402",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load the model\n",
|
||||
"inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "bd919fef-4e9a-4dc5-a957-7b879cfc5d38",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████| 13/13 [00:01<00:00, 7.81it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'accuracy': 0.9044117647058824, 'f1': 0.9297297297297298}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"inference_model.to(device)\n",
|
||||
"inference_model.eval()\n",
|
||||
"for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = inference_model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.14"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,531 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d36e1e93-ae93-4a4e-93c6-68fd868d2882",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Using VeRA for sequence classification"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ddfc0610-55f6-4343-a950-125ccf0f45ac",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"In this example, we fine-tune Roberta on a sequence classification task using VeRA."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "45addd81-d4f3-4dfd-960d-3920d347f0a6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Imports"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "a9935ae2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_model,\n",
|
||||
" VeraConfig,\n",
|
||||
" PeftType,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed, AutoConfig\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "62c959bf-7cc2-49e0-b97e-4c10ec3b9bf3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Parameters"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "e3b13308",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<torch._C.Generator at 0x7b8246b7c850>"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"batch_size = 128\n",
|
||||
"model_name_or_path = \"roberta-base\"\n",
|
||||
"task = \"mrpc\"\n",
|
||||
"peft_type = PeftType.VERA\n",
|
||||
"device = torch.accelerator.current_accelerator().type if hasattr(torch, \"accelerator\") else \"cuda\"\n",
|
||||
"num_epochs = 5 # for best results, increase this number\n",
|
||||
"rank = 8 # for best results, increase this number\n",
|
||||
"max_length = 128\n",
|
||||
"torch.manual_seed(0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "0526f571",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = VeraConfig(\n",
|
||||
" task_type=\"SEQ_CLS\", \n",
|
||||
" r=rank,\n",
|
||||
" d_initial=0.1,\n",
|
||||
" target_modules=[\"query\", \"value\", \"intermediate.dense\"],\n",
|
||||
" save_projection=True,\n",
|
||||
")\n",
|
||||
"head_lr = 1e-2\n",
|
||||
"vera_lr = 2e-2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c075c5d2-a457-4f37-a7f1-94fd0d277972",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Loading data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "7bb52cb4-d1c3-4b04-8bf0-f39ca88af139",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "e69c5e1f-d27b-4264-a41e-fc9b99d025e6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "0209f778-c93b-40eb-a4e0-24c25db03980",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=max_length)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
" remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "7453954e-982c-46f0-b09c-589776e6d6cb",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f3b9b2e8-f415-4d0f-9fb4-436f1a3585ea",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Preparing the VeRA model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "2ed5ac74",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"trainable params: 647,714 || all params: 125,294,884 || trainable%: 0.5170\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True, max_length=None)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "0d2d0381",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"optimizer = AdamW(\n",
|
||||
" [\n",
|
||||
" {\"params\": [p for n, p in model.named_parameters() if \"vera_lambda_\" in n], \"lr\": vera_lr},\n",
|
||||
" {\"params\": [p for n, p in model.named_parameters() if \"classifier\" in n], \"lr\": head_lr},\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c0dd5aa8-977b-4ac0-8b96-884b17bcdd00",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Training"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "fa0e73be",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/29 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████| 29/29 [00:18<00:00, 1.58it/s]\n",
|
||||
"100%|██████████| 4/4 [00:01<00:00, 3.52it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 0: {'accuracy': 0.7475490196078431, 'f1': 0.8367670364500792}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 29/29 [00:17<00:00, 1.68it/s]\n",
|
||||
"100%|██████████| 4/4 [00:01<00:00, 3.37it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 1: {'accuracy': 0.7671568627450981, 'f1': 0.8536209553158706}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 29/29 [00:17<00:00, 1.66it/s]\n",
|
||||
"100%|██████████| 4/4 [00:01<00:00, 3.33it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 2: {'accuracy': 0.8553921568627451, 'f1': 0.8959435626102292}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 29/29 [00:17<00:00, 1.64it/s]\n",
|
||||
"100%|██████████| 4/4 [00:01<00:00, 3.35it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 3: {'accuracy': 0.8823529411764706, 'f1': 0.9133574007220215}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 29/29 [00:17<00:00, 1.63it/s]\n",
|
||||
"100%|██████████| 4/4 [00:01<00:00, 3.17it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 4: {'accuracy': 0.8897058823529411, 'f1': 0.9183303085299456}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.to(device)\n",
|
||||
"for epoch in range(num_epochs):\n",
|
||||
" model.train()\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
"\n",
|
||||
" model.eval()\n",
|
||||
" for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_metric = metric.compute()\n",
|
||||
" print(f\"epoch {epoch}:\", eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f2b2caca",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Share adapters on the 🤗 Hub"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "7b23af6f-cf6e-486f-9d10-0eada95b631f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"account_id = ... # your Hugging Face Hub account ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "990b3c93",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"CommitInfo(commit_url='https://huggingface.co/BenjaminB/roberta-large-peft-vera/commit/d720cdc67b97df9cd1453b14d3e0fb17efc8779b', commit_message='Upload model', commit_description='', oid='d720cdc67b97df9cd1453b14d3e0fb17efc8779b', pr_url=None, pr_revision=None, pr_num=None)"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.push_to_hub(f\"{account_id}/roberta-large-peft-vera\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9d140b26",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load adapters from the Hub\n",
|
||||
"\n",
|
||||
"You can also directly load adapters from the Hub using the commands below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "c283e028-b349-46b0-a20e-cde0ee5fbd7b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"from transformers import AutoTokenizer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "320b10a0-4ea8-4786-9f3c-4670019c6b18",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"peft_model_id = f\"{account_id}/roberta-large-peft-vera\"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "b3a94049-bc01-4f2e-8cf9-66daf24a4402",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load the Vera model\n",
|
||||
"inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "bd919fef-4e9a-4dc5-a957-7b879cfc5d38",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/4 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:01<00:00, 3.14it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'accuracy': 0.8480392156862745, 'f1': 0.8938356164383561}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"inference_model.to(device)\n",
|
||||
"inference_model.eval()\n",
|
||||
"for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = inference_model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "13cc0125-a052-4e26-b7ff-af0f763be34e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
import argparse
|
||||
|
||||
import evaluate
|
||||
import torch
|
||||
from accelerate import Accelerator, DistributedDataParallelKwargs
|
||||
from datasets import load_dataset
|
||||
from torch.optim import AdamW
|
||||
from torch.utils.data import DataLoader
|
||||
from tqdm import tqdm
|
||||
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed
|
||||
|
||||
from peft import (
|
||||
PrefixTuningConfig,
|
||||
PromptEncoderConfig,
|
||||
PromptTuningConfig,
|
||||
get_peft_model,
|
||||
)
|
||||
from peft.utils.other import fsdp_auto_wrap_policy
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="PEFT a transformers model on a sequence classification task")
|
||||
parser.add_argument(
|
||||
"--num_virtual_tokens",
|
||||
type=int,
|
||||
default=20,
|
||||
help="num_virtual_tokens if the number of virtual tokens used in prompt/prefix/P tuning.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--encoder_hidden_size",
|
||||
type=int,
|
||||
default=128,
|
||||
help="encoder_hidden_size if the encoder hidden size used in P tuninig/Prefix tuning.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model_name_or_path",
|
||||
type=str,
|
||||
help="Path to pretrained model or model identifier from huggingface.co/models.",
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--per_device_train_batch_size",
|
||||
type=int,
|
||||
default=8,
|
||||
help="Batch size (per device) for the training dataloader.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--per_device_eval_batch_size",
|
||||
type=int,
|
||||
default=8,
|
||||
help="Batch size (per device) for the evaluation dataloader.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--learning_rate",
|
||||
type=float,
|
||||
default=1e-3,
|
||||
help="Initial learning rate (after the potential warmup period) to use.",
|
||||
)
|
||||
parser.add_argument("--num_train_epochs", type=int, default=3, help="Total number of training epochs to perform.")
|
||||
parser.add_argument(
|
||||
"--num_warmup_steps", type=int, default=0, help="Number of steps for the warmup in the lr scheduler."
|
||||
)
|
||||
parser.add_argument("--output_dir", type=str, default=None, help="Where to store the final model.")
|
||||
parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.")
|
||||
parser.add_argument(
|
||||
"--peft_type",
|
||||
type=str,
|
||||
default="p_tuning",
|
||||
help="The PEFT type to use.",
|
||||
choices=["p_tuning", "prefix_tuning", "prompt_tuning"],
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
assert args.output_dir is not None, "Need an `output_dir` to store the finetune model and verify."
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
ddp_scaler = DistributedDataParallelKwargs(find_unused_parameters=True)
|
||||
accelerator = Accelerator(kwargs_handlers=[ddp_scaler])
|
||||
|
||||
task = "mrpc"
|
||||
|
||||
# If passed along, set the training seed now.
|
||||
if args.seed is not None:
|
||||
set_seed(args.seed)
|
||||
|
||||
if args.peft_type == "p_tuning":
|
||||
peft_config = PromptEncoderConfig(
|
||||
task_type="SEQ_CLS",
|
||||
num_virtual_tokens=args.num_virtual_tokens,
|
||||
encoder_hidden_size=args.encoder_hidden_size,
|
||||
)
|
||||
elif args.peft_type == "prefix_tuning":
|
||||
peft_config = PrefixTuningConfig(
|
||||
task_type="SEQ_CLS",
|
||||
num_virtual_tokens=args.num_virtual_tokens,
|
||||
encoder_hidden_size=args.encoder_hidden_size,
|
||||
)
|
||||
else:
|
||||
peft_config = PromptTuningConfig(task_type="SEQ_CLS", num_virtual_tokens=args.num_virtual_tokens)
|
||||
|
||||
tokenizer_kwargs = {}
|
||||
|
||||
if any(k in args.model_name_or_path for k in ("gpt", "opt", "bloom")):
|
||||
tokenizer_kwargs["padding_side"] = "left"
|
||||
else:
|
||||
tokenizer_kwargs["padding_side"] = "right"
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, **tokenizer_kwargs)
|
||||
if tokenizer.pad_token_id is None:
|
||||
tokenizer.pad_token_id = tokenizer.eos_token_id
|
||||
|
||||
datasets = load_dataset("glue", task)
|
||||
metric = evaluate.load("glue", task)
|
||||
|
||||
def tokenize_function(examples):
|
||||
# max_length=None => use the model max length (it's actually the default)
|
||||
outputs = tokenizer(examples["sentence1"], examples["sentence2"], truncation=True, max_length=None)
|
||||
return outputs
|
||||
|
||||
def collate_fn(examples):
|
||||
return tokenizer.pad(examples, padding="longest", return_tensors="pt")
|
||||
|
||||
with accelerator.main_process_first():
|
||||
tokenized_datasets = datasets.map(
|
||||
tokenize_function,
|
||||
batched=True,
|
||||
remove_columns=["idx", "sentence1", "sentence2"],
|
||||
)
|
||||
|
||||
# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
|
||||
# transformers library
|
||||
tokenized_datasets = tokenized_datasets.rename_column("label", "labels")
|
||||
|
||||
# Instantiate dataloaders.
|
||||
train_dataloader = DataLoader(
|
||||
tokenized_datasets["train"], shuffle=True, collate_fn=collate_fn, batch_size=args.per_device_train_batch_size
|
||||
)
|
||||
eval_dataloader = DataLoader(
|
||||
tokenized_datasets["validation"],
|
||||
shuffle=False,
|
||||
collate_fn=collate_fn,
|
||||
batch_size=args.per_device_eval_batch_size,
|
||||
)
|
||||
|
||||
model = AutoModelForSequenceClassification.from_pretrained(args.model_name_or_path)
|
||||
model = get_peft_model(model, peft_config)
|
||||
model.print_trainable_parameters()
|
||||
|
||||
if getattr(accelerator.state, "fsdp_plugin", None) is not None:
|
||||
accelerator.state.fsdp_plugin.auto_wrap_policy = fsdp_auto_wrap_policy(model)
|
||||
model = accelerator.prepare(model)
|
||||
|
||||
optimizer = AdamW(params=model.parameters(), lr=args.learning_rate)
|
||||
|
||||
# Instantiate scheduler
|
||||
lr_scheduler = get_linear_schedule_with_warmup(
|
||||
optimizer=optimizer,
|
||||
num_warmup_steps=args.num_warmup_steps,
|
||||
num_training_steps=(len(train_dataloader) * args.num_train_epochs),
|
||||
)
|
||||
|
||||
if getattr(accelerator.state, "fsdp_plugin", None) is not None:
|
||||
train_dataloader, eval_dataloader, optimizer, lr_scheduler = accelerator.prepare(
|
||||
train_dataloader, eval_dataloader, optimizer, lr_scheduler
|
||||
)
|
||||
else:
|
||||
model, train_dataloader, eval_dataloader, optimizer, lr_scheduler = accelerator.prepare(
|
||||
model, train_dataloader, eval_dataloader, optimizer, lr_scheduler
|
||||
)
|
||||
|
||||
for epoch in range(args.num_train_epochs):
|
||||
model.train()
|
||||
for step, batch in enumerate(tqdm(train_dataloader)):
|
||||
outputs = model(**batch)
|
||||
loss = outputs.loss
|
||||
accelerator.backward(loss)
|
||||
optimizer.step()
|
||||
lr_scheduler.step()
|
||||
optimizer.zero_grad()
|
||||
|
||||
model.eval()
|
||||
samples_seen = 0
|
||||
for step, batch in enumerate(tqdm(eval_dataloader)):
|
||||
with torch.no_grad():
|
||||
outputs = model(**batch)
|
||||
predictions = outputs.logits.argmax(dim=-1)
|
||||
predictions, references = accelerator.gather((predictions, batch["labels"]))
|
||||
# If we are in a multiprocess environment, the last batch has duplicates
|
||||
if accelerator.num_processes > 1:
|
||||
if step == len(eval_dataloader) - 1:
|
||||
predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
|
||||
references = references[: len(eval_dataloader.dataset) - samples_seen]
|
||||
else:
|
||||
samples_seen += references.shape[0]
|
||||
metric.add_batch(
|
||||
predictions=predictions,
|
||||
references=references,
|
||||
)
|
||||
eval_metric = metric.compute()
|
||||
accelerator.print(f"epoch {epoch}:", eval_metric)
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
unwrapped_model = accelerator.unwrap_model(model)
|
||||
unwrapped_model.save_pretrained(args.output_dir, state_dict=accelerator.get_state_dict(model))
|
||||
if accelerator.is_main_process:
|
||||
tokenizer.save_pretrained(args.output_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,710 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "a825ba6b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"===================================BUG REPORT===================================\n",
|
||||
"Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
|
||||
"For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n",
|
||||
"================================================================================\n",
|
||||
"CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n",
|
||||
"CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
|
||||
"CUDA SETUP: Detected CUDA version 117\n",
|
||||
"CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import argparse\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_config,\n",
|
||||
" get_peft_model,\n",
|
||||
" get_peft_model_state_dict,\n",
|
||||
" set_peft_model_state_dict,\n",
|
||||
" PeftType,\n",
|
||||
" PrefixTuningConfig,\n",
|
||||
" PromptEncoderConfig,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed\n",
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2bd7cbb2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_size = 32\n",
|
||||
"model_name_or_path = \"roberta-large\"\n",
|
||||
"task = \"mrpc\"\n",
|
||||
"peft_type = PeftType.PREFIX_TUNING\n",
|
||||
"device = torch.accelerator.current_accelerator().type if hasattr(torch, \"accelerator\") else \"cuda\"\n",
|
||||
"num_epochs = 20"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "33d9b62e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = PrefixTuningConfig(task_type=\"SEQ_CLS\", num_virtual_tokens=20)\n",
|
||||
"lr = 1e-2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "152b6177",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Found cached dataset glue (/home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "be1eddbb9a7d4e6dae32fd026e167f96",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/3 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-9fa7887f9eaa03ae.arrow\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b61574844b6c499b8960fd4d78c5e549",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/1 [00:00<?, ?ba/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-7e7eacaa5160936d.arrow\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"if any(k in model_name_or_path for k in (\"gpt\", \"opt\", \"bloom\")):\n",
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
"\n",
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
" remove_columns=[\"idx\", \"sentence1\", \"sentence2\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the\n",
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f6bc8144",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()\n",
|
||||
"model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "af41c571",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"optimizer = AdamW(params=model.parameters(), lr=lr)\n",
|
||||
"\n",
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "90993c93",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:29<00:00, 3.87it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.32it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 0: {'accuracy': 0.7132352941176471, 'f1': 0.7876588021778584}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:26<00:00, 4.42it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.36it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 1: {'accuracy': 0.6838235294117647, 'f1': 0.8122270742358079}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:26<00:00, 4.41it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.35it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 2: {'accuracy': 0.8088235294117647, 'f1': 0.8717105263157895}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:26<00:00, 4.39it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.34it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 3: {'accuracy': 0.7549019607843137, 'f1': 0.8475609756097561}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:26<00:00, 4.37it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00, 8.34it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 4: {'accuracy': 0.8480392156862745, 'f1': 0.8938356164383561}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:40<00:00, 2.87it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00, 1.93it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 5: {'accuracy': 0.8651960784313726, 'f1': 0.9053356282271946}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:53<00:00, 1.01it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:07<00:00, 1.79it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 6: {'accuracy': 0.8700980392156863, 'f1': 0.9065255731922399}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:42<00:00, 1.12it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.43it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 7: {'accuracy': 0.8676470588235294, 'f1': 0.9042553191489361}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00, 1.31it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.45it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 8: {'accuracy': 0.875, 'f1': 0.9103690685413005}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:29<00:00, 1.29it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.48it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 9: {'accuracy': 0.8799019607843137, 'f1': 0.913884007029877}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:43<00:00, 1.11it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00, 1.88it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 10: {'accuracy': 0.8725490196078431, 'f1': 0.902621722846442}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:53<00:00, 1.02it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00, 2.02it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 11: {'accuracy': 0.875, 'f1': 0.9090909090909091}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:29<00:00, 1.28it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:04<00:00, 2.65it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 12: {'accuracy': 0.8823529411764706, 'f1': 0.9139784946236559}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00, 1.31it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.46it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 13: {'accuracy': 0.8602941176470589, 'f1': 0.9018932874354562}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00, 1.31it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.47it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 14: {'accuracy': 0.8700980392156863, 'f1': 0.9075043630017452}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00, 1.31it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.49it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 15: {'accuracy': 0.875, 'f1': 0.9087656529516995}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00, 1.32it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.49it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 16: {'accuracy': 0.8578431372549019, 'f1': 0.9003436426116839}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00, 1.31it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.22it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 17: {'accuracy': 0.8627450980392157, 'f1': 0.903448275862069}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:28<00:00, 1.31it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:04<00:00, 2.65it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 18: {'accuracy': 0.8700980392156863, 'f1': 0.9078260869565218}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00, 1.32it/s]\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00, 2.45it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"epoch 19: {'accuracy': 0.8774509803921569, 'f1': 0.9125874125874125}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.to(device)\n",
|
||||
"for epoch in range(num_epochs):\n",
|
||||
" model.train()\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
"\n",
|
||||
" model.eval()\n",
|
||||
" for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_metric = metric.compute()\n",
|
||||
" print(f\"epoch {epoch}:\", eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7734299c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Share adapters on the 🤗 Hub"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "afaf42dd",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"CommitInfo(commit_url='https://huggingface.co/smangrul/roberta-large-peft-prefix-tuning/commit/a00e05a4c9a68e700221784f8e073c2e194637c3', commit_message='Upload model', commit_description='', oid='a00e05a4c9a68e700221784f8e073c2e194637c3', pr_url=None, pr_revision=None, pr_num=None)"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.push_to_hub(\"smangrul/roberta-large-peft-prefix-tuning\", use_auth_token=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "42b20e77",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load adapters from the Hub\n",
|
||||
"\n",
|
||||
"You can also directly load adapters from the Hub using the commands below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "868e7580",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "2ce57b4de8ae4f868115733abc2fb883",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading: 0%| | 0.00/373 [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.bias', 'lm_head.decoder.weight', 'lm_head.dense.bias']\n",
|
||||
"- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
|
||||
"- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
|
||||
"Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "ace158c926a44b31a9b0ea80411bd7a9",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading: 0%| | 0.00/8.14M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00, 2.04it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'accuracy': 0.8774509803921569, 'f1': 0.9125874125874125}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
|
||||
"\n",
|
||||
"peft_model_id = \"smangrul/roberta-large-peft-prefix-tuning\"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"\n",
|
||||
"# Load the Lora model\n",
|
||||
"inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)\n",
|
||||
"\n",
|
||||
"inference_model.to(device)\n",
|
||||
"inference_model.eval()\n",
|
||||
"for step, batch in enumerate(tqdm(eval_dataloader)):\n",
|
||||
" batch.to(device)\n",
|
||||
" with torch.no_grad():\n",
|
||||
" outputs = inference_model(**batch)\n",
|
||||
" predictions = outputs.logits.argmax(dim=-1)\n",
|
||||
" predictions, references = predictions, batch[\"labels\"]\n",
|
||||
" metric.add_batch(\n",
|
||||
" predictions=predictions,\n",
|
||||
" references=references,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
transformers
|
||||
accelerate
|
||||
evaluate
|
||||
tqdm
|
||||
datasets
|
||||
torchao
|
||||
Reference in New Issue
Block a user