Fix torchtext issue
This commit is contained in:
Dmitri Soshnikov
parent
eab705b0c1
commit
7b3d15da18
|
|
@ -8,6 +8,8 @@
|
|||
}
|
||||
},
|
||||
|
||||
"hostRequirements": { "cpus": 4, "memory": "8gb", "storage": "100gb"},
|
||||
|
||||
// Configure tool-specific properties.
|
||||
"customizations": {
|
||||
// Configure properties specific to VS Code.
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ tokenizers==0.10.3
|
|||
torch==1.11.0
|
||||
torchaudio
|
||||
torchinfo
|
||||
torchtext
|
||||
torchvision
|
||||
torchtext==0.12.0
|
||||
torchvision==0.12.0
|
||||
torchdata
|
||||
tqdm==4.62.3
|
||||
transformers==4.3.3
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
[](https://GitHub.com/microsoft/AI-For-Beginners/watchers/)
|
||||
[](https://GitHub.com/microsoft/AI-For-Beginners/network/)
|
||||
[](https://GitHub.com/microsoft/AI-For-Beginners/stargazers/)
|
||||
[](https://mybinder.org/v2/gh/microsoft/ai-for-beginners/HEAD)
|
||||
|
||||
# Artificial Intelligence for Beginners - A Curriculum
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ tokenizers==0.10.3
|
|||
torch==1.11.0
|
||||
torchaudio
|
||||
torchinfo
|
||||
torchtext
|
||||
torchvision
|
||||
torchtext==0.12.0
|
||||
torchvision==0.12.0
|
||||
torchdata
|
||||
tqdm==4.62.3
|
||||
transformers==4.3.3
|
||||
|
|
|
|||
|
|
@ -20,12 +20,12 @@ def load_dataset(ngrams=1,min_freq=1):
|
|||
counter = collections.Counter()
|
||||
for (label, line) in train_dataset:
|
||||
counter.update(torchtext.data.utils.ngrams_iterator(tokenizer(line),ngrams=ngrams))
|
||||
vocab = torchtext.vocab.Vocab(counter, min_freq=min_freq)
|
||||
vocab = torchtext.vocab.vocab(counter, min_freq=min_freq)
|
||||
return train_dataset,test_dataset,classes,vocab
|
||||
|
||||
def encode(x,voc=None,unk=0,tokenizer=tokenizer):
|
||||
v = vocab if voc is None else voc
|
||||
return [v.stoi.get(s,unk) for s in tokenizer(x)]
|
||||
return [v.get_stoi().get(s,unk) for s in tokenizer(x)]
|
||||
|
||||
def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200):
|
||||
optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)
|
||||
|
|
|
|||
|
|
@ -20,12 +20,12 @@ def load_dataset(ngrams=1,min_freq=1):
|
|||
counter = collections.Counter()
|
||||
for (label, line) in train_dataset:
|
||||
counter.update(torchtext.data.utils.ngrams_iterator(tokenizer(line),ngrams=ngrams))
|
||||
vocab = torchtext.vocab.Vocab(counter, min_freq=min_freq)
|
||||
vocab = torchtext.vocab.vocab(counter, min_freq=min_freq)
|
||||
return train_dataset,test_dataset,classes,vocab
|
||||
|
||||
def encode(x,voc=None,unk=0,tokenizer=tokenizer):
|
||||
v = vocab if voc is None else voc
|
||||
return [v.stoi.get(s,unk) for s in tokenizer(x)]
|
||||
return [v.get_stoi().get(s,unk) for s in tokenizer(x)]
|
||||
|
||||
def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200):
|
||||
optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)
|
||||
|
|
|
|||
|
|
@ -53,9 +53,9 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Vocabulary size = 84\n",
|
||||
"Encoding of 'a' is 4\n",
|
||||
"Character with code 13 is h\n"
|
||||
"Vocabulary size = 82\n",
|
||||
"Encoding of 'a' is 1\n",
|
||||
"Character with code 13 is c\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -66,12 +66,12 @@
|
|||
"counter = collections.Counter()\n",
|
||||
"for (label, line) in train_dataset:\n",
|
||||
" counter.update(char_tokenizer(line))\n",
|
||||
"vocab = torchtext.vocab.Vocab(counter)\n",
|
||||
"vocab = torchtext.vocab.vocab(counter)\n",
|
||||
"\n",
|
||||
"vocab_size = len(vocab)\n",
|
||||
"print(f\"Vocabulary size = {vocab_size}\")\n",
|
||||
"print(f\"Encoding of 'a' is {vocab.stoi['a']}\")\n",
|
||||
"print(f\"Character with code 13 is {vocab.itos[13]}\")"
|
||||
"print(f\"Encoding of 'a' is {vocab.get_stoi()['a']}\")\n",
|
||||
"print(f\"Character with code 13 is {vocab.get_itos()[13]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -83,23 +83,23 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"tensor([43, 4, 11, 11, 2, 26, 5, 23, 2, 38, 3, 4, 10, 9, 2, 31, 11, 4,\n",
|
||||
" 21, 2, 38, 4, 14, 25, 2, 34, 8, 5, 6, 2, 5, 13, 3, 2, 38, 11,\n",
|
||||
" 4, 14, 25, 2, 55, 37, 3, 15, 5, 3, 10, 9, 56, 2, 37, 3, 15, 5,\n",
|
||||
" 3, 10, 9, 2, 29, 2, 26, 13, 6, 10, 5, 29, 9, 3, 11, 11, 3, 10,\n",
|
||||
" 9, 27, 2, 43, 4, 11, 11, 2, 26, 5, 10, 3, 3, 5, 58, 9, 2, 12,\n",
|
||||
" 21, 7, 8, 12, 11, 7, 8, 18, 61, 22, 4, 8, 12, 2, 6, 19, 2, 15,\n",
|
||||
" 11, 5, 10, 4, 29, 14, 20, 8, 7, 14, 9, 27, 2, 4, 10, 3, 2, 9,\n",
|
||||
" 3, 3, 7, 8, 18, 2, 18, 10, 3, 3, 8, 2, 4, 18, 4, 7, 8, 23])"
|
||||
"tensor([ 0, 1, 2, 2, 3, 4, 5, 6, 3, 7, 8, 1, 9, 10, 3, 11, 2, 1,\n",
|
||||
" 12, 3, 7, 1, 13, 14, 3, 15, 16, 5, 17, 3, 5, 18, 8, 3, 7, 2,\n",
|
||||
" 1, 13, 14, 3, 19, 20, 8, 21, 5, 8, 9, 10, 22, 3, 20, 8, 21, 5,\n",
|
||||
" 8, 9, 10, 3, 23, 3, 4, 18, 17, 9, 5, 23, 10, 8, 2, 2, 8, 9,\n",
|
||||
" 10, 24, 3, 0, 1, 2, 2, 3, 4, 5, 9, 8, 8, 5, 25, 10, 3, 26,\n",
|
||||
" 12, 27, 16, 26, 2, 27, 16, 28, 29, 30, 1, 16, 26, 3, 17, 31, 3, 21,\n",
|
||||
" 2, 5, 9, 1, 23, 13, 32, 16, 27, 13, 10, 24, 3, 1, 9, 8, 3, 10,\n",
|
||||
" 8, 8, 27, 16, 28, 3, 28, 9, 8, 8, 16, 3, 1, 28, 1, 27, 16, 6])"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
|
|
@ -128,29 +128,29 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(tensor([[43, 4, 11, ..., 18, 61, 22],\n",
|
||||
" [ 4, 11, 11, ..., 61, 22, 4],\n",
|
||||
" [11, 11, 2, ..., 22, 4, 8],\n",
|
||||
"(tensor([[ 0, 1, 2, ..., 28, 29, 30],\n",
|
||||
" [ 1, 2, 2, ..., 29, 30, 1],\n",
|
||||
" [ 2, 2, 3, ..., 30, 1, 16],\n",
|
||||
" ...,\n",
|
||||
" [37, 3, 15, ..., 4, 18, 4],\n",
|
||||
" [ 3, 15, 5, ..., 18, 4, 7],\n",
|
||||
" [15, 5, 3, ..., 4, 7, 8]], device='cuda:0'),\n",
|
||||
" tensor([[ 4, 11, 11, ..., 61, 22, 4],\n",
|
||||
" [11, 11, 2, ..., 22, 4, 8],\n",
|
||||
" [11, 2, 26, ..., 4, 8, 12],\n",
|
||||
" [20, 8, 21, ..., 1, 28, 1],\n",
|
||||
" [ 8, 21, 5, ..., 28, 1, 27],\n",
|
||||
" [21, 5, 8, ..., 1, 27, 16]]),\n",
|
||||
" tensor([[ 1, 2, 2, ..., 29, 30, 1],\n",
|
||||
" [ 2, 2, 3, ..., 30, 1, 16],\n",
|
||||
" [ 2, 3, 4, ..., 1, 16, 26],\n",
|
||||
" ...,\n",
|
||||
" [ 3, 15, 5, ..., 18, 4, 7],\n",
|
||||
" [15, 5, 3, ..., 4, 7, 8],\n",
|
||||
" [ 5, 3, 10, ..., 7, 8, 23]], device='cuda:0'))"
|
||||
" [ 8, 21, 5, ..., 28, 1, 27],\n",
|
||||
" [21, 5, 8, ..., 1, 27, 16],\n",
|
||||
" [ 5, 8, 9, ..., 27, 16, 6]]))"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
|
|
@ -180,7 +180,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -207,7 +207,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -216,7 +216,7 @@
|
|||
" out, s = net(enc(chars).view(1,-1).to(device))\n",
|
||||
" for i in range(size):\n",
|
||||
" nc = torch.argmax(out[0][-1])\n",
|
||||
" chars.append(vocab.itos[nc])\n",
|
||||
" chars.append(vocab.get_itos()[nc])\n",
|
||||
" out, s = net(nc.view(1,-1),s)\n",
|
||||
" return ''.join(chars)"
|
||||
]
|
||||
|
|
@ -234,35 +234,15 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Current loss = 4.442246913909912\n",
|
||||
"today ggrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrg\n",
|
||||
"Current loss = 2.1178359985351562\n",
|
||||
"today and a could a the to the to the to the to the to the to the to the to the to the to the to the to th\n",
|
||||
"Current loss = 1.6465336084365845\n",
|
||||
"today on Tuesday the company to the United States and a policing to the United States and a policing to th\n",
|
||||
"Current loss = 2.3716814517974854\n",
|
||||
"today to the United States and a new men to the United States and a new men to the United States and a new\n",
|
||||
"Current loss = 1.6844098567962646\n",
|
||||
"today of the first the first the first the first the first the first the first the first the first the fir\n",
|
||||
"Current loss = 1.702707052230835\n",
|
||||
"today of the United States a said the United States a said the United States a said the United States a sa\n",
|
||||
"Current loss = 1.9633255004882812\n",
|
||||
"today of the first the first the first the first the first the first the first the first the first the fir\n",
|
||||
"Current loss = 1.8642014265060425\n",
|
||||
"today of the second a second a second a second a second a second a second a second a second a second a sec\n",
|
||||
"Current loss = 1.7720613479614258\n",
|
||||
"today and and and the company of the company of the company of the company of the company of the company o\n",
|
||||
"Current loss = 1.52818763256073\n",
|
||||
"today and the company of the company of the company of the company of the company of the company of the co\n",
|
||||
"Current loss = 1.5444810390472412\n",
|
||||
"today and the counters to the first the counters to the first the counters to the first the counters to th\n"
|
||||
"Current loss = 4.398899078369141\n",
|
||||
"today sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -355,7 +335,7 @@
|
|||
" #nc = torch.argmax(out[0][-1])\n",
|
||||
" out_dist = out[0][-1].div(temperature).exp()\n",
|
||||
" nc = torch.multinomial(out_dist,1)[0]\n",
|
||||
" chars.append(vocab.itos[nc])\n",
|
||||
" chars.append(vocab.get_itos()[nc])\n",
|
||||
" out, s = net(nc.view(1,-1),s)\n",
|
||||
" return ''.join(chars)\n",
|
||||
" \n",
|
||||
|
|
@ -372,10 +352,13 @@
|
|||
}
|
||||
],
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "16af2a8bbb083ea23e5e41c7f5787656b2ce26968575d8763f2c4b17f9cd711f"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "py37_pytorch",
|
||||
"display_name": "Python 3.8.12 ('py38')",
|
||||
"language": "python",
|
||||
"name": "conda-env-py37_pytorch-py"
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
|
|
@ -387,7 +370,7 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.7"
|
||||
"version": "3.8.12"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
|
|
|||
|
|
@ -455,10 +455,13 @@
|
|||
}
|
||||
],
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "16af2a8bbb083ea23e5e41c7f5787656b2ce26968575d8763f2c4b17f9cd711f"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "py38_tensorflow",
|
||||
"display_name": "Python 3.8.12 ('py38')",
|
||||
"language": "python",
|
||||
"name": "conda-env-py38_tensorflow-py"
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
|
|
@ -470,7 +473,7 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.10"
|
||||
"version": "3.8.12"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,104 @@
|
|||
import builtins
|
||||
import torch
|
||||
import torchtext
|
||||
import collections
|
||||
import os
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
vocab = None
|
||||
tokenizer = torchtext.data.utils.get_tokenizer('basic_english')
|
||||
|
||||
def load_dataset(ngrams=1,min_freq=1):
|
||||
global vocab, tokenizer
|
||||
print("Loading dataset...")
|
||||
train_dataset, test_dataset = torchtext.datasets.AG_NEWS(root='./data')
|
||||
train_dataset = list(train_dataset)
|
||||
test_dataset = list(test_dataset)
|
||||
classes = ['World', 'Sports', 'Business', 'Sci/Tech']
|
||||
print('Building vocab...')
|
||||
counter = collections.Counter()
|
||||
for (label, line) in train_dataset:
|
||||
counter.update(torchtext.data.utils.ngrams_iterator(tokenizer(line),ngrams=ngrams))
|
||||
vocab = torchtext.vocab.vocab(counter, min_freq=min_freq)
|
||||
return train_dataset,test_dataset,classes,vocab
|
||||
|
||||
def encode(x,voc=None,unk=0,tokenizer=tokenizer):
|
||||
v = vocab if voc is None else voc
|
||||
return [v.get_stoi().get(s,unk) for s in tokenizer(x)]
|
||||
|
||||
def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200):
|
||||
optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)
|
||||
loss_fn = loss_fn.to(device)
|
||||
net.train()
|
||||
total_loss,acc,count,i = 0,0,0,0
|
||||
for labels,features in dataloader:
|
||||
optimizer.zero_grad()
|
||||
features, labels = features.to(device), labels.to(device)
|
||||
out = net(features)
|
||||
loss = loss_fn(out,labels) #cross_entropy(out,labels)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
total_loss+=loss
|
||||
_,predicted = torch.max(out,1)
|
||||
acc+=(predicted==labels).sum()
|
||||
count+=len(labels)
|
||||
i+=1
|
||||
if i%report_freq==0:
|
||||
print(f"{count}: acc={acc.item()/count}")
|
||||
if epoch_size and count>epoch_size:
|
||||
break
|
||||
return total_loss.item()/count, acc.item()/count
|
||||
|
||||
def padify(b,voc=None,tokenizer=tokenizer):
|
||||
# b is the list of tuples of length batch_size
|
||||
# - first element of a tuple = label,
|
||||
# - second = feature (text sequence)
|
||||
# build vectorized sequence
|
||||
v = [encode(x[1],voc=voc,tokenizer=tokenizer) for x in b]
|
||||
# compute max length of a sequence in this minibatch
|
||||
l = max(map(len,v))
|
||||
return ( # tuple of two tensors - labels and features
|
||||
torch.LongTensor([t[0]-1 for t in b]),
|
||||
torch.stack([torch.nn.functional.pad(torch.tensor(t),(0,l-len(t)),mode='constant',value=0) for t in v])
|
||||
)
|
||||
|
||||
def offsetify(b,voc=None):
|
||||
# first, compute data tensor from all sequences
|
||||
x = [torch.tensor(encode(t[1],voc=voc)) for t in b]
|
||||
# now, compute the offsets by accumulating the tensor of sequence lengths
|
||||
o = [0] + [len(t) for t in x]
|
||||
o = torch.tensor(o[:-1]).cumsum(dim=0)
|
||||
return (
|
||||
torch.LongTensor([t[0]-1 for t in b]), # labels
|
||||
torch.cat(x), # text
|
||||
o
|
||||
)
|
||||
|
||||
def train_epoch_emb(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200,use_pack_sequence=False):
|
||||
optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)
|
||||
loss_fn = loss_fn.to(device)
|
||||
net.train()
|
||||
total_loss,acc,count,i = 0,0,0,0
|
||||
for labels,text,off in dataloader:
|
||||
optimizer.zero_grad()
|
||||
labels,text = labels.to(device), text.to(device)
|
||||
if use_pack_sequence:
|
||||
off = off.to('cpu')
|
||||
else:
|
||||
off = off.to(device)
|
||||
out = net(text, off)
|
||||
loss = loss_fn(out,labels) #cross_entropy(out,labels)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
total_loss+=loss
|
||||
_,predicted = torch.max(out,1)
|
||||
acc+=(predicted==labels).sum()
|
||||
count+=len(labels)
|
||||
i+=1
|
||||
if i%report_freq==0:
|
||||
print(f"{count}: acc={acc.item()/count}")
|
||||
if epoch_size and count>epoch_size:
|
||||
break
|
||||
return total_loss.item()/count, acc.item()/count
|
||||
|
||||
Loading…
Reference in New Issue