wehub-resource-sync
468fc26eb0
CodeQL Advanced / Analyze (actions) (push) Has been cancelled
CodeQL Advanced / Analyze (javascript-typescript) (push) Has been cancelled
CodeQL Advanced / Analyze (python) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
116 lines
4.2 KiB
Python
116 lines
4.2 KiB
Python
import builtins
|
|
import torch
|
|
import torchtext
|
|
import collections
|
|
import os
|
|
|
|
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
|
|
vocab = None
|
|
tokenizer = torchtext.data.utils.get_tokenizer('basic_english')
|
|
|
|
def load_dataset(ngrams=1,min_freq=1):
|
|
global vocab, tokenizer
|
|
print("Loading dataset...")
|
|
train_dataset, test_dataset = torchtext.datasets.AG_NEWS(root='./data')
|
|
train_dataset = list(train_dataset)
|
|
test_dataset = list(test_dataset)
|
|
classes = ['World', 'Sports', 'Business', 'Sci/Tech']
|
|
print('Building vocab...')
|
|
counter = collections.Counter()
|
|
for (label, line) in train_dataset:
|
|
counter.update(torchtext.data.utils.ngrams_iterator(tokenizer(line),ngrams=ngrams))
|
|
vocab = torchtext.vocab.vocab(counter, min_freq=min_freq)
|
|
return train_dataset,test_dataset,classes,vocab
|
|
|
|
stoi_hash = {}
|
|
def encode(x,voc=None,unk=0,tokenizer=tokenizer):
|
|
global stoi_hash
|
|
v = vocab if voc is None else voc
|
|
if v in stoi_hash.keys():
|
|
stoi = stoi_hash[v]
|
|
else:
|
|
# Handle both vocab.vocab objects (with get_stoi() method) and GloVe objects (with stoi attribute)
|
|
if hasattr(v, 'get_stoi'):
|
|
stoi = v.get_stoi()
|
|
else:
|
|
stoi = v.stoi
|
|
stoi_hash[v]=stoi
|
|
return [stoi.get(s,unk) for s in tokenizer(x)]
|
|
|
|
def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200):
|
|
optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)
|
|
loss_fn = loss_fn.to(device)
|
|
net.train()
|
|
total_loss,acc,count,i = 0,0,0,0
|
|
for labels,features in dataloader:
|
|
optimizer.zero_grad()
|
|
features, labels = features.to(device), labels.to(device)
|
|
out = net(features)
|
|
loss = loss_fn(out,labels) #cross_entropy(out,labels)
|
|
loss.backward()
|
|
optimizer.step()
|
|
total_loss+=loss
|
|
_,predicted = torch.max(out,1)
|
|
acc+=(predicted==labels).sum()
|
|
count+=len(labels)
|
|
i+=1
|
|
if i%report_freq==0:
|
|
print(f"{count}: acc={acc.item()/count}")
|
|
if epoch_size and count>epoch_size:
|
|
break
|
|
return total_loss.item()/count, acc.item()/count
|
|
|
|
def padify(b,voc=None,tokenizer=tokenizer):
|
|
# b is the list of tuples of length batch_size
|
|
# - first element of a tuple = label,
|
|
# - second = feature (text sequence)
|
|
# build vectorized sequence
|
|
v = [encode(x[1],voc=voc,tokenizer=tokenizer) for x in b]
|
|
# compute max length of a sequence in this minibatch
|
|
l = max(map(len,v))
|
|
return ( # tuple of two tensors - labels and features
|
|
torch.LongTensor([t[0]-1 for t in b]),
|
|
torch.stack([torch.nn.functional.pad(torch.tensor(t),(0,l-len(t)),mode='constant',value=0) for t in v])
|
|
)
|
|
|
|
def offsetify(b,voc=None):
|
|
# first, compute data tensor from all sequences
|
|
x = [torch.tensor(encode(t[1],voc=voc)) for t in b]
|
|
# now, compute the offsets by accumulating the tensor of sequence lengths
|
|
o = [0] + [len(t) for t in x]
|
|
o = torch.tensor(o[:-1]).cumsum(dim=0)
|
|
return (
|
|
torch.LongTensor([t[0]-1 for t in b]), # labels
|
|
torch.cat(x), # text
|
|
o
|
|
)
|
|
|
|
def train_epoch_emb(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200,use_pack_sequence=False):
|
|
optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)
|
|
loss_fn = loss_fn.to(device)
|
|
net.train()
|
|
total_loss,acc,count,i = 0,0,0,0
|
|
for labels,text,off in dataloader:
|
|
optimizer.zero_grad()
|
|
labels,text = labels.to(device), text.to(device)
|
|
if use_pack_sequence:
|
|
off = off.to('cpu')
|
|
else:
|
|
off = off.to(device)
|
|
out = net(text, off)
|
|
loss = loss_fn(out,labels) #cross_entropy(out,labels)
|
|
loss.backward()
|
|
optimizer.step()
|
|
total_loss+=loss
|
|
_,predicted = torch.max(out,1)
|
|
acc+=(predicted==labels).sum()
|
|
count+=len(labels)
|
|
i+=1
|
|
if i%report_freq==0:
|
|
print(f"{count}: acc={acc.item()/count}")
|
|
if epoch_size and count>epoch_size:
|
|
break
|
|
return total_loss.item()/count, acc.item()/count
|
|
|