RomanNum3ral
/
AI-For-Beginners
mirror of https://github.com/microsoft/AI-For-Beginners.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix torchtext issue

This commit is contained in:
Dmitri Soshnikov 2022-05-25 00:40:51 +03:00
parent eab705b0c1
commit 7b3d15da18
9 changed files with 166 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.devcontainer/devcontainer.json
View File

@ -8,6 +8,8 @@
} }
}, },
"hostRequirements": { "cpus": 4, "memory": "8gb", "storage": "100gb"},
// Configure tool-specific properties. // Configure tool-specific properties.
"customizations": { "customizations": {
// Configure properties specific to VS Code. // Configure properties specific to VS Code.

5
.devcontainer/requirements.txt
View File

@ -21,7 +21,8 @@ tokenizers==0.10.3
torch==1.11.0 torch==1.11.0
torchaudio torchaudio
torchinfo torchinfo
torchtext torchtext==0.12.0
torchvision torchvision==0.12.0
torchdata
tqdm==4.62.3 tqdm==4.62.3
transformers==4.3.3 transformers==4.3.3

1
README.md
View File

@ -7,6 +7,7 @@
[![GitHub watchers](https://img.shields.io/github/watchers/microsoft/AI-For-Beginners.svg?style=social&label=Watch)](https://GitHub.com/microsoft/AI-For-Beginners/watchers/) [![GitHub watchers](https://img.shields.io/github/watchers/microsoft/AI-For-Beginners.svg?style=social&label=Watch)](https://GitHub.com/microsoft/AI-For-Beginners/watchers/)
[![GitHub forks](https://img.shields.io/github/forks/microsoft/AI-For-Beginners.svg?style=social&label=Fork)](https://GitHub.com/microsoft/AI-For-Beginners/network/) [![GitHub forks](https://img.shields.io/github/forks/microsoft/AI-For-Beginners.svg?style=social&label=Fork)](https://GitHub.com/microsoft/AI-For-Beginners/network/)
[![GitHub stars](https://img.shields.io/github/stars/microsoft/AI-For-Beginners.svg?style=social&label=Star)](https://GitHub.com/microsoft/AI-For-Beginners/stargazers/) [![GitHub stars](https://img.shields.io/github/stars/microsoft/AI-For-Beginners.svg?style=social&label=Star)](https://GitHub.com/microsoft/AI-For-Beginners/stargazers/)
[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/microsoft/ai-for-beginners/HEAD)
# Artificial Intelligence for Beginners - A Curriculum # Artificial Intelligence for Beginners - A Curriculum

5
binder/requirements.txt
View File

@ -21,7 +21,8 @@ tokenizers==0.10.3
torch==1.11.0 torch==1.11.0
torchaudio torchaudio
torchinfo torchinfo
torchtext torchtext==0.12.0
torchvision torchvision==0.12.0
torchdata
tqdm==4.62.3 tqdm==4.62.3
transformers==4.3.3 transformers==4.3.3

4
lessons/5-NLP/14-Embeddings/torchnlp.py
View File

@ -20,12 +20,12 @@ def load_dataset(ngrams=1,min_freq=1):
counter = collections.Counter() counter = collections.Counter()
for (label, line) in train_dataset: for (label, line) in train_dataset:
counter.update(torchtext.data.utils.ngrams_iterator(tokenizer(line),ngrams=ngrams)) counter.update(torchtext.data.utils.ngrams_iterator(tokenizer(line),ngrams=ngrams))
vocab = torchtext.vocab.Vocab(counter, min_freq=min_freq) vocab = torchtext.vocab.vocab(counter, min_freq=min_freq)
return train_dataset,test_dataset,classes,vocab return train_dataset,test_dataset,classes,vocab
def encode(x,voc=None,unk=0,tokenizer=tokenizer): def encode(x,voc=None,unk=0,tokenizer=tokenizer):
v = vocab if voc is None else voc v = vocab if voc is None else voc
return [v.stoi.get(s,unk) for s in tokenizer(x)] return [v.get_stoi().get(s,unk) for s in tokenizer(x)]
def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200): def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200):
optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr) optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)

4
lessons/5-NLP/16-RNN/torchnlp.py
View File

@ -20,12 +20,12 @@ def load_dataset(ngrams=1,min_freq=1):
counter = collections.Counter() counter = collections.Counter()
for (label, line) in train_dataset: for (label, line) in train_dataset:
counter.update(torchtext.data.utils.ngrams_iterator(tokenizer(line),ngrams=ngrams)) counter.update(torchtext.data.utils.ngrams_iterator(tokenizer(line),ngrams=ngrams))
vocab = torchtext.vocab.Vocab(counter, min_freq=min_freq) vocab = torchtext.vocab.vocab(counter, min_freq=min_freq)
return train_dataset,test_dataset,classes,vocab return train_dataset,test_dataset,classes,vocab
def encode(x,voc=None,unk=0,tokenizer=tokenizer): def encode(x,voc=None,unk=0,tokenizer=tokenizer):
v = vocab if voc is None else voc v = vocab if voc is None else voc
return [v.stoi.get(s,unk) for s in tokenizer(x)] return [v.get_stoi().get(s,unk) for s in tokenizer(x)]
def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200): def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200):
optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr) optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)

103
lessons/5-NLP/17-GenerativeNetworks/GenerativePyTorch.ipynb
View File

@ -53,9 +53,9 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Vocabulary size = 84\n", "Vocabulary size = 82\n",
"Encoding of 'a' is 4\n", "Encoding of 'a' is 1\n",
"Character with code 13 is h\n" "Character with code 13 is c\n"
] ]
} }
], ],
@ -66,12 +66,12 @@
"counter = collections.Counter()\n", "counter = collections.Counter()\n",
"for (label, line) in train_dataset:\n", "for (label, line) in train_dataset:\n",
" counter.update(char_tokenizer(line))\n", " counter.update(char_tokenizer(line))\n",
"vocab = torchtext.vocab.Vocab(counter)\n", "vocab = torchtext.vocab.vocab(counter)\n",
"\n", "\n",
"vocab_size = len(vocab)\n", "vocab_size = len(vocab)\n",
"print(f\"Vocabulary size = {vocab_size}\")\n", "print(f\"Vocabulary size = {vocab_size}\")\n",
"print(f\"Encoding of 'a' is {vocab.stoi['a']}\")\n", "print(f\"Encoding of 'a' is {vocab.get_stoi()['a']}\")\n",
"print(f\"Character with code 13 is {vocab.itos[13]}\")" "print(f\"Character with code 13 is {vocab.get_itos()[13]}\")"
] ]
}, },
{ {
@ -83,23 +83,23 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"tensor([43, 4, 11, 11, 2, 26, 5, 23, 2, 38, 3, 4, 10, 9, 2, 31, 11, 4,\n", "tensor([ 0, 1, 2, 2, 3, 4, 5, 6, 3, 7, 8, 1, 9, 10, 3, 11, 2, 1,\n",
" 21, 2, 38, 4, 14, 25, 2, 34, 8, 5, 6, 2, 5, 13, 3, 2, 38, 11,\n", " 12, 3, 7, 1, 13, 14, 3, 15, 16, 5, 17, 3, 5, 18, 8, 3, 7, 2,\n",
" 4, 14, 25, 2, 55, 37, 3, 15, 5, 3, 10, 9, 56, 2, 37, 3, 15, 5,\n", " 1, 13, 14, 3, 19, 20, 8, 21, 5, 8, 9, 10, 22, 3, 20, 8, 21, 5,\n",
" 3, 10, 9, 2, 29, 2, 26, 13, 6, 10, 5, 29, 9, 3, 11, 11, 3, 10,\n", " 8, 9, 10, 3, 23, 3, 4, 18, 17, 9, 5, 23, 10, 8, 2, 2, 8, 9,\n",
" 9, 27, 2, 43, 4, 11, 11, 2, 26, 5, 10, 3, 3, 5, 58, 9, 2, 12,\n", " 10, 24, 3, 0, 1, 2, 2, 3, 4, 5, 9, 8, 8, 5, 25, 10, 3, 26,\n",
" 21, 7, 8, 12, 11, 7, 8, 18, 61, 22, 4, 8, 12, 2, 6, 19, 2, 15,\n", " 12, 27, 16, 26, 2, 27, 16, 28, 29, 30, 1, 16, 26, 3, 17, 31, 3, 21,\n",
" 11, 5, 10, 4, 29, 14, 20, 8, 7, 14, 9, 27, 2, 4, 10, 3, 2, 9,\n", " 2, 5, 9, 1, 23, 13, 32, 16, 27, 13, 10, 24, 3, 1, 9, 8, 3, 10,\n",
" 3, 3, 7, 8, 18, 2, 18, 10, 3, 3, 8, 2, 4, 18, 4, 7, 8, 23])" " 8, 8, 27, 16, 28, 3, 28, 9, 8, 8, 16, 3, 1, 28, 1, 27, 16, 6])"
] ]
}, },
"execution_count": 5, "execution_count": 3,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -128,29 +128,29 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"(tensor([[43, 4, 11, ..., 18, 61, 22],\n", "(tensor([[ 0, 1, 2, ..., 28, 29, 30],\n",
" [ 4, 11, 11, ..., 61, 22, 4],\n", " [ 1, 2, 2, ..., 29, 30, 1],\n",
" [11, 11, 2, ..., 22, 4, 8],\n", " [ 2, 2, 3, ..., 30, 1, 16],\n",
" ...,\n", " ...,\n",
" [37, 3, 15, ..., 4, 18, 4],\n", " [20, 8, 21, ..., 1, 28, 1],\n",
" [ 3, 15, 5, ..., 18, 4, 7],\n", " [ 8, 21, 5, ..., 28, 1, 27],\n",
" [15, 5, 3, ..., 4, 7, 8]], device='cuda:0'),\n", " [21, 5, 8, ..., 1, 27, 16]]),\n",
" tensor([[ 4, 11, 11, ..., 61, 22, 4],\n", " tensor([[ 1, 2, 2, ..., 29, 30, 1],\n",
" [11, 11, 2, ..., 22, 4, 8],\n", " [ 2, 2, 3, ..., 30, 1, 16],\n",
" [11, 2, 26, ..., 4, 8, 12],\n", " [ 2, 3, 4, ..., 1, 16, 26],\n",
" ...,\n", " ...,\n",
" [ 3, 15, 5, ..., 18, 4, 7],\n", " [ 8, 21, 5, ..., 28, 1, 27],\n",
" [15, 5, 3, ..., 4, 7, 8],\n", " [21, 5, 8, ..., 1, 27, 16],\n",
" [ 5, 3, 10, ..., 7, 8, 23]], device='cuda:0'))" " [ 5, 8, 9, ..., 27, 16, 6]]))"
] ]
}, },
"execution_count": 8, "execution_count": 4,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -180,7 +180,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": 5,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -207,7 +207,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 13, "execution_count": 8,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -216,7 +216,7 @@
" out, s = net(enc(chars).view(1,-1).to(device))\n", " out, s = net(enc(chars).view(1,-1).to(device))\n",
" for i in range(size):\n", " for i in range(size):\n",
" nc = torch.argmax(out[0][-1])\n", " nc = torch.argmax(out[0][-1])\n",
" chars.append(vocab.itos[nc])\n", " chars.append(vocab.get_itos()[nc])\n",
" out, s = net(nc.view(1,-1),s)\n", " out, s = net(nc.view(1,-1),s)\n",
" return ''.join(chars)" " return ''.join(chars)"
] ]
@ -234,35 +234,15 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 9,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Current loss = 4.442246913909912\n", "Current loss = 4.398899078369141\n",
"today ggrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrg\n", "today sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr sr s\n"
"Current loss = 2.1178359985351562\n",
"today and a could a the to the to the to the to the to the to the to the to the to the to the to the to th\n",
"Current loss = 1.6465336084365845\n",
"today on Tuesday the company to the United States and a policing to the United States and a policing to th\n",
"Current loss = 2.3716814517974854\n",
"today to the United States and a new men to the United States and a new men to the United States and a new\n",
"Current loss = 1.6844098567962646\n",
"today of the first the first the first the first the first the first the first the first the first the fir\n",
"Current loss = 1.702707052230835\n",
"today of the United States a said the United States a said the United States a said the United States a sa\n",
"Current loss = 1.9633255004882812\n",
"today of the first the first the first the first the first the first the first the first the first the fir\n",
"Current loss = 1.8642014265060425\n",
"today of the second a second a second a second a second a second a second a second a second a second a sec\n",
"Current loss = 1.7720613479614258\n",
"today and and and the company of the company of the company of the company of the company of the company o\n",
"Current loss = 1.52818763256073\n",
"today and the company of the company of the company of the company of the company of the company of the co\n",
"Current loss = 1.5444810390472412\n",
"today and the counters to the first the counters to the first the counters to the first the counters to th\n"
] ]
} }
], ],
@ -355,7 +335,7 @@
" #nc = torch.argmax(out[0][-1])\n", " #nc = torch.argmax(out[0][-1])\n",
" out_dist = out[0][-1].div(temperature).exp()\n", " out_dist = out[0][-1].div(temperature).exp()\n",
" nc = torch.multinomial(out_dist,1)[0]\n", " nc = torch.multinomial(out_dist,1)[0]\n",
" chars.append(vocab.itos[nc])\n", " chars.append(vocab.get_itos()[nc])\n",
" out, s = net(nc.view(1,-1),s)\n", " out, s = net(nc.view(1,-1),s)\n",
" return ''.join(chars)\n", " return ''.join(chars)\n",
" \n", " \n",
@ -372,10 +352,13 @@
} }
], ],
"metadata": { "metadata": {
"interpreter": {
"hash": "16af2a8bbb083ea23e5e41c7f5787656b2ce26968575d8763f2c4b17f9cd711f"
},
"kernelspec": { "kernelspec": {
"display_name": "py37_pytorch", "display_name": "Python 3.8.12 ('py38')",
"language": "python", "language": "python",
"name": "conda-env-py37_pytorch-py" "name": "python3"
}, },
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
@ -387,7 +370,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.7.7" "version": "3.8.12"
} }
}, },
"nbformat": 4, "nbformat": 4,

9
lessons/5-NLP/17-GenerativeNetworks/GenerativeTF.ipynb
View File

@ -455,10 +455,13 @@
} }
], ],
"metadata": { "metadata": {
"interpreter": {
"hash": "16af2a8bbb083ea23e5e41c7f5787656b2ce26968575d8763f2c4b17f9cd711f"
},
"kernelspec": { "kernelspec": {
"display_name": "py38_tensorflow", "display_name": "Python 3.8.12 ('py38')",
"language": "python", "language": "python",
"name": "conda-env-py38_tensorflow-py" "name": "python3"
}, },
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
@ -470,7 +473,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.10" "version": "3.8.12"
} }
}, },
"nbformat": 4, "nbformat": 4,

104
lessons/5-NLP/17-GenerativeNetworks/torchnlp.py Normal file
View File

@ -0,0 +1,104 @@
import builtins
import torch
import torchtext
import collections
import os
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
vocab = None
tokenizer = torchtext.data.utils.get_tokenizer('basic_english')
def load_dataset(ngrams=1,min_freq=1):
global vocab, tokenizer
print("Loading dataset...")
train_dataset, test_dataset = torchtext.datasets.AG_NEWS(root='./data')
train_dataset = list(train_dataset)
test_dataset = list(test_dataset)
classes = ['World', 'Sports', 'Business', 'Sci/Tech']
print('Building vocab...')
counter = collections.Counter()
for (label, line) in train_dataset:
counter.update(torchtext.data.utils.ngrams_iterator(tokenizer(line),ngrams=ngrams))
vocab = torchtext.vocab.vocab(counter, min_freq=min_freq)
return train_dataset,test_dataset,classes,vocab
def encode(x,voc=None,unk=0,tokenizer=tokenizer):
v = vocab if voc is None else voc
return [v.get_stoi().get(s,unk) for s in tokenizer(x)]
def train_epoch(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200):
optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)
loss_fn = loss_fn.to(device)
net.train()
total_loss,acc,count,i = 0,0,0,0
for labels,features in dataloader:
optimizer.zero_grad()
features, labels = features.to(device), labels.to(device)
out = net(features)
loss = loss_fn(out,labels) #cross_entropy(out,labels)
loss.backward()
optimizer.step()
total_loss+=loss
_,predicted = torch.max(out,1)
acc+=(predicted==labels).sum()
count+=len(labels)
i+=1
if i%report_freq==0:
print(f"{count}: acc={acc.item()/count}")
if epoch_size and count>epoch_size:
break
return total_loss.item()/count, acc.item()/count
def padify(b,voc=None,tokenizer=tokenizer):
# b is the list of tuples of length batch_size
# - first element of a tuple = label,
# - second = feature (text sequence)
# build vectorized sequence
v = [encode(x[1],voc=voc,tokenizer=tokenizer) for x in b]
# compute max length of a sequence in this minibatch
l = max(map(len,v))
return ( # tuple of two tensors - labels and features
torch.LongTensor([t[0]-1 for t in b]),
torch.stack([torch.nn.functional.pad(torch.tensor(t),(0,l-len(t)),mode='constant',value=0) for t in v])
)
def offsetify(b,voc=None):
# first, compute data tensor from all sequences
x = [torch.tensor(encode(t[1],voc=voc)) for t in b]
# now, compute the offsets by accumulating the tensor of sequence lengths
o = [0] + [len(t) for t in x]
o = torch.tensor(o[:-1]).cumsum(dim=0)
return (
torch.LongTensor([t[0]-1 for t in b]), # labels
torch.cat(x), # text
o
)
def train_epoch_emb(net,dataloader,lr=0.01,optimizer=None,loss_fn = torch.nn.CrossEntropyLoss(),epoch_size=None, report_freq=200,use_pack_sequence=False):
optimizer = optimizer or torch.optim.Adam(net.parameters(),lr=lr)
loss_fn = loss_fn.to(device)
net.train()
total_loss,acc,count,i = 0,0,0,0
for labels,text,off in dataloader:
optimizer.zero_grad()
labels,text = labels.to(device), text.to(device)
if use_pack_sequence:
off = off.to('cpu')
else:
off = off.to(device)
out = net(text, off)
loss = loss_fn(out,labels) #cross_entropy(out,labels)
loss.backward()
optimizer.step()
total_loss+=loss
_,predicted = torch.max(out,1)
acc+=(predicted==labels).sum()
count+=len(labels)
i+=1
if i%report_freq==0:
print(f"{count}: acc={acc.item()/count}")
if epoch_size and count>epoch_size:
break
return total_loss.item()/count, acc.item()/count