From f2c33d1f85352c3e96cdfe5443a4df1033943412 Mon Sep 17 00:00:00 2001 From: Evgenii Pishchik <57502208+Pe4enIks@users.noreply.github.com> Date: Fri, 8 Apr 2022 03:10:47 +0300 Subject: [PATCH] pytorch autoencoders --- .../09-Autoencoders/AutoEncodersPytorch.ipynb | 1 + 4-ComputerVision/09-Autoencoders/images/aae.png | Bin 0 -> 50083 bytes 2 files changed, 1 insertion(+) create mode 100644 4-ComputerVision/09-Autoencoders/AutoEncodersPytorch.ipynb create mode 100644 4-ComputerVision/09-Autoencoders/images/aae.png diff --git a/4-ComputerVision/09-Autoencoders/AutoEncodersPytorch.ipynb b/4-ComputerVision/09-Autoencoders/AutoEncodersPytorch.ipynb new file mode 100644 index 00000000..397aec83 --- /dev/null +++ b/4-ComputerVision/09-Autoencoders/AutoEncodersPytorch.ipynb @@ -0,0 +1 @@ +{"cells":[{"cell_type":"markdown","metadata":{"id":"u2U7TIRdKNGu"},"source":["# [Autoencoders](https://arxiv.org/abs/2201.03898)"]},{"cell_type":"markdown","metadata":{"id":"it5nmu_-c5-E"},"source":["When training CNNs, one of the problems is that we need a lot of labeled data. In the case of image classification, we need to separate images into different classes, which is a manual effort.\n","\n","However, we might want to use raw (unlabeled) data for training CNN feature extractors, which is called **self-supervised learning**. Instead of labels, we will use training images as both network input and output. The main idea of **autoencoder** is that we will have an **encoder network** that converts input image into some **latent space** (normally it is just a vector of some smaller size), the then **decoder network**, whose goal would be to reconstruct the original image.\n","\n","Since we are training autoencoder to capture as much of the information from the original image as possible for accurate reconstruction, the network tries to find the best **embedding** of input images to capture the meaning.\n","\n","![AutoEncoder Diagram](images/autoencoder_schema.jpg)\n","\n","Let's create simplest autoencoder for MNIST!"]},{"cell_type":"code","execution_count":125,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:29:06.172141Z","iopub.status.busy":"2022-04-07T23:29:06.171876Z","iopub.status.idle":"2022-04-07T23:29:06.179468Z","shell.execute_reply":"2022-04-07T23:29:06.178621Z","shell.execute_reply.started":"2022-04-07T23:29:06.172113Z"},"id":"6n8fzzN-2T1h","trusted":true},"outputs":[],"source":["import torch\n","import torchvision\n","import matplotlib.pyplot as plt\n","from torchvision import transforms\n","from torch import nn\n","from torch import optim\n","from tqdm import tqdm\n","import numpy as np\n","import torch.nn.functional as F\n","torch.manual_seed(42)\n","np.random.seed(42)"]},{"cell_type":"code","execution_count":56,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:08:18.678524Z","iopub.status.busy":"2022-04-07T22:08:18.677978Z","iopub.status.idle":"2022-04-07T22:08:18.682963Z","shell.execute_reply":"2022-04-07T22:08:18.682107Z","shell.execute_reply.started":"2022-04-07T22:08:18.678485Z"},"id":"bjL-jOgi3gmG","trusted":true},"outputs":[],"source":["device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n","train_size = 0.9\n","lr = 1e-3\n","eps = 1e-8\n","batch_size = 256\n","epochs = 30"]},{"cell_type":"code","execution_count":57,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:08:19.781260Z","iopub.status.busy":"2022-04-07T22:08:19.781001Z","iopub.status.idle":"2022-04-07T22:08:19.786750Z","shell.execute_reply":"2022-04-07T22:08:19.785852Z","shell.execute_reply.started":"2022-04-07T22:08:19.781233Z"},"id":"g2Eo43713Sxb","trusted":true},"outputs":[],"source":["def mnist(train_part, transform=None):\n"," dataset = torchvision.datasets.MNIST('.', download=True, transform=transform)\n"," train_part = int(train_part * len(dataset))\n"," train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_part, len(dataset) - train_part])\n"," return train_dataset, test_dataset"]},{"cell_type":"code","execution_count":87,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:34:10.615629Z","iopub.status.busy":"2022-04-07T22:34:10.615306Z","iopub.status.idle":"2022-04-07T22:34:10.619693Z","shell.execute_reply":"2022-04-07T22:34:10.619011Z","shell.execute_reply.started":"2022-04-07T22:34:10.615594Z"},"id":"K7FaPscQ4kd4","trusted":true},"outputs":[],"source":["transform = transforms.Compose([\n"," transforms.ToTensor(),\n","])"]},{"cell_type":"code","execution_count":88,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:34:10.622595Z","iopub.status.busy":"2022-04-07T22:34:10.621696Z","iopub.status.idle":"2022-04-07T22:34:10.659892Z","shell.execute_reply":"2022-04-07T22:34:10.659237Z","shell.execute_reply.started":"2022-04-07T22:34:10.622558Z"},"id":"jPaRjZEK18lW","trusted":true},"outputs":[],"source":["train_dataset, test_dataset = mnist(train_size, transform)"]},{"cell_type":"code","execution_count":89,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:34:10.661312Z","iopub.status.busy":"2022-04-07T22:34:10.660992Z","iopub.status.idle":"2022-04-07T22:34:10.669417Z","shell.execute_reply":"2022-04-07T22:34:10.668728Z","shell.execute_reply.started":"2022-04-07T22:34:10.661275Z"},"id":"jAI3uK86_zHM","trusted":true},"outputs":[],"source":["train_dataloader = torch.utils.data.DataLoader(train_dataset, drop_last=True, batch_size=batch_size, shuffle=True)\n","test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False)\n","dataloaders = (train_dataloader, test_dataloader)"]},{"cell_type":"code","execution_count":61,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:08:22.882317Z","iopub.status.busy":"2022-04-07T22:08:22.880010Z","iopub.status.idle":"2022-04-07T22:08:22.894982Z","shell.execute_reply":"2022-04-07T22:08:22.893986Z","shell.execute_reply.started":"2022-04-07T22:08:22.882274Z"},"id":"LdyQz4092fRV","trusted":true},"outputs":[],"source":["def plotn(n, data, noisy=False, super_res=None):\n"," fig, ax = plt.subplots(1, n)\n"," for i, z in enumerate(data):\n"," if i == n:\n"," break\n"," preprocess = z[0].reshape(1, 28, 28) if z[0].shape[1] == 28 else z[0].reshape(1, 14, 14) if z[0].shape[1] == 14 else z[0]\n"," if super_res is not None:\n"," _transform = transforms.Resize((int(preprocess.shape[1] / super_res), int(preprocess.shape[2] / super_res)))\n"," preprocess = _transform(preprocess)\n","\n"," if noisy:\n"," shapes = list(preprocess.shape)\n"," preprocess += noisify(shapes)\n","\n"," ax[i].imshow(preprocess[0])\n"," plt.show()"]},{"cell_type":"code","execution_count":74,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:16:44.659930Z","iopub.status.busy":"2022-04-07T22:16:44.659661Z","iopub.status.idle":"2022-04-07T22:16:44.665194Z","shell.execute_reply":"2022-04-07T22:16:44.664502Z","shell.execute_reply.started":"2022-04-07T22:16:44.659902Z"},"id":"FjpCEs-oWu6_","trusted":true},"outputs":[],"source":["def noisify(shapes):\n"," return np.random.normal(loc=0.25, scale=0.25, size=shapes)"]},{"cell_type":"code","execution_count":90,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:34:10.671163Z","iopub.status.busy":"2022-04-07T22:34:10.670898Z","iopub.status.idle":"2022-04-07T22:34:11.195124Z","shell.execute_reply":"2022-04-07T22:34:11.194396Z","shell.execute_reply.started":"2022-04-07T22:34:10.671129Z"},"id":"NeWJoiFC4A6J","outputId":"e680eb07-bf94-4301-8742-852103885624","trusted":true},"outputs":[],"source":["plotn(5, train_dataset)"]},{"cell_type":"code","execution_count":91,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:34:11.197716Z","iopub.status.busy":"2022-04-07T22:34:11.196275Z","iopub.status.idle":"2022-04-07T22:34:11.206490Z","shell.execute_reply":"2022-04-07T22:34:11.205773Z","shell.execute_reply.started":"2022-04-07T22:34:11.197674Z"},"id":"NnI2YvOg4DbT","trusted":true},"outputs":[],"source":["class Encoder(nn.Module):\n"," def __init__(self):\n"," super().__init__()\n"," self.conv1 = nn.Conv2d(1, 16, kernel_size=(3, 3), padding='same')\n"," self.maxpool1 = nn.MaxPool2d(kernel_size=(2, 2))\n"," self.conv2 = nn.Conv2d(16, 8, kernel_size=(3, 3), padding='same')\n"," self.maxpool2 = nn.MaxPool2d(kernel_size=(2, 2))\n"," self.conv3 = nn.Conv2d(8, 8, kernel_size=(3, 3), padding='same')\n"," self.maxpool3 = nn.MaxPool2d(kernel_size=(2, 2), padding=(1, 1))\n"," self.relu = nn.ReLU()\n","\n"," def forward(self, input):\n"," hidden1 = self.maxpool1(self.relu(self.conv1(input)))\n"," hidden2 = self.maxpool2(self.relu(self.conv2(hidden1)))\n"," encoded = self.maxpool3(self.relu(self.conv3(hidden2)))\n"," return encoded"]},{"cell_type":"code","execution_count":92,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:34:11.208243Z","iopub.status.busy":"2022-04-07T22:34:11.207603Z","iopub.status.idle":"2022-04-07T22:34:11.218984Z","shell.execute_reply":"2022-04-07T22:34:11.218304Z","shell.execute_reply.started":"2022-04-07T22:34:11.208207Z"},"id":"mZGB4Vr47478","trusted":true},"outputs":[],"source":["class Decoder(nn.Module):\n"," def __init__(self):\n"," super().__init__()\n"," self.conv1 = nn.Conv2d(8, 8, kernel_size=(3, 3), padding='same')\n"," self.upsample1 = nn.Upsample(scale_factor=(2, 2))\n"," self.conv2 = nn.Conv2d(8, 8, kernel_size=(3, 3), padding='same')\n"," self.upsample2 = nn.Upsample(scale_factor=(2, 2))\n"," self.conv3 = nn.Conv2d(8, 16, kernel_size=(3, 3))\n"," self.upsample3 = nn.Upsample(scale_factor=(2, 2))\n"," self.conv4 = nn.Conv2d(16, 1, kernel_size=(3, 3), padding='same')\n"," self.relu = nn.ReLU()\n"," self.sigmoid = nn.Sigmoid()\n","\n"," def forward(self, input):\n"," hidden1 = self.upsample1(self.relu(self.conv1(input)))\n"," hidden2 = self.upsample2(self.relu(self.conv2(hidden1)))\n"," hidden3 = self.upsample3(self.relu(self.conv3(hidden2)))\n"," decoded = self.sigmoid(self.conv4(hidden3))\n"," return decoded"]},{"cell_type":"code","execution_count":93,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:34:11.220928Z","iopub.status.busy":"2022-04-07T22:34:11.220360Z","iopub.status.idle":"2022-04-07T22:34:11.228411Z","shell.execute_reply":"2022-04-07T22:34:11.227831Z","shell.execute_reply.started":"2022-04-07T22:34:11.220891Z"},"id":"SDGiAPhbBBLY","trusted":true},"outputs":[],"source":["class AutoEncoder(nn.Module):\n"," def __init__(self, super_resolution=False):\n"," super().__init__()\n"," if not super_resolution:\n"," self.encoder = Encoder()\n"," else:\n"," self.encoder = SuperResolutionEncoder()\n"," self.decoder = Decoder()\n","\n"," def forward(self, input):\n"," encoded = self.encoder(input)\n"," decoded = self.decoder(encoded)\n"," return decoded"]},{"cell_type":"code","execution_count":94,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:34:11.231510Z","iopub.status.busy":"2022-04-07T22:34:11.230756Z","iopub.status.idle":"2022-04-07T22:34:11.242940Z","shell.execute_reply":"2022-04-07T22:34:11.242317Z","shell.execute_reply.started":"2022-04-07T22:34:11.231476Z"},"id":"nZyG_mNu_Pnc","trusted":true},"outputs":[],"source":["model = AutoEncoder().to(device)\n","optimizer = optim.Adam(model.parameters(), lr=lr, eps=eps)\n","loss_fn = nn.BCELoss()"]},{"cell_type":"code","execution_count":68,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:08:36.364247Z","iopub.status.busy":"2022-04-07T22:08:36.363981Z","iopub.status.idle":"2022-04-07T22:08:36.381955Z","shell.execute_reply":"2022-04-07T22:08:36.380758Z","shell.execute_reply.started":"2022-04-07T22:08:36.364218Z"},"id":"iiIy87v2_rUr","trusted":true},"outputs":[],"source":["def train(dataloaders, model, loss_fn, optimizer, epochs, device, noisy=False, super_res=None):\n"," tqdm_iter = tqdm(range(epochs))\n"," train_dataloader, test_dataloader = dataloaders[0], dataloaders[1]\n"," noisy_tensor = None\n"," test_noisy_tensor = None\n","\n"," for epoch in tqdm_iter:\n"," model.train()\n"," train_loss = 0.0\n"," test_loss = 0.0\n","\n"," for batch in train_dataloader:\n"," imgs, labels = batch\n"," shapes = list(imgs.shape)\n","\n"," if super_res is not None:\n"," shapes[2], shapes[3] = int(shapes[2] / super_res), int(shapes[3] / super_res)\n"," _transform = transforms.Resize((shapes[2], shapes[3]))\n"," imgs_transformed = _transform(imgs)\n"," imgs_transformed = imgs_transformed.to(device)\n","\n"," imgs = imgs.to(device)\n"," labels = labels.to(device)\n","\n"," if noisy and noisy_tensor is None:\n"," noisy_tensor = torch.FloatTensor(noisify(shapes)).to(device)\n"," else:\n"," noisy_tensor = torch.zeros(tuple(shapes)).to(device)\n","\n"," if super_res is None:\n"," imgs_noisy = imgs + noisy_tensor\n"," else:\n"," imgs_noisy = imgs_transformed + noisy_tensor\n","\n"," imgs_noisy = torch.clamp(imgs_noisy, 0., 1.)\n","\n"," preds = model(imgs_noisy)\n"," loss = loss_fn(preds, imgs)\n","\n"," optimizer.zero_grad()\n"," loss.backward()\n"," optimizer.step()\n","\n"," train_loss += loss.item()\n","\n"," model.eval()\n"," with torch.no_grad():\n"," for batch in test_dataloader:\n"," imgs, labels = batch\n"," shapes = list(imgs.shape)\n","\n"," if super_res is not None:\n"," shapes[2], shapes[3] = int(shapes[2] / super_res), int(shapes[3] / super_res)\n"," _transform = transforms.Resize((shapes[2], shapes[3]))\n"," imgs_transformed = _transform(imgs)\n"," imgs_transformed = imgs_transformed.to(device)\n","\n","\n"," imgs = imgs.to(device)\n"," labels = labels.to(device)\n","\n"," if noisy and test_noisy_tensor is None:\n"," test_noisy_tensor = torch.FloatTensor(noisify(shapes)).to(device)\n"," else:\n"," test_noisy_tensor = torch.zeros(tuple(shapes)).to(device)\n","\n"," if super_res is None:\n"," imgs_noisy = imgs + test_noisy_tensor\n"," else:\n"," imgs_noisy = imgs_transformed + test_noisy_tensor\n","\n"," imgs_noisy = torch.clamp(imgs_noisy, 0., 1.)\n","\n"," preds = model(imgs_noisy)\n"," loss = loss_fn(preds, imgs)\n","\n"," test_loss += loss.item()\n","\n"," train_loss /= len(train_dataloader)\n"," test_loss /= len(test_dataloader)\n","\n"," tqdm_dct = {'train loss:': train_loss, 'test loss:': test_loss}\n"," tqdm_iter.set_postfix(tqdm_dct, refresh=True)\n"," tqdm_iter.refresh()"]},{"cell_type":"code","execution_count":95,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:34:11.246183Z","iopub.status.busy":"2022-04-07T22:34:11.245971Z","iopub.status.idle":"2022-04-07T22:41:00.353349Z","shell.execute_reply":"2022-04-07T22:41:00.352609Z","shell.execute_reply.started":"2022-04-07T22:34:11.246159Z"},"id":"PMqO8eOxCemz","trusted":true},"outputs":[],"source":["train(dataloaders, model, loss_fn, optimizer, epochs, device)"]},{"cell_type":"code","execution_count":96,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:41:00.355694Z","iopub.status.busy":"2022-04-07T22:41:00.354744Z","iopub.status.idle":"2022-04-07T22:41:01.345106Z","shell.execute_reply":"2022-04-07T22:41:01.344413Z","shell.execute_reply.started":"2022-04-07T22:41:00.355654Z"},"id":"kR3n0EnjOts0","trusted":true},"outputs":[],"source":["model.eval()\n","predictions = []\n","plots = 5\n","for i, data in enumerate(test_dataset):\n"," if i == plots:\n"," break\n"," predictions.append(model(data[0].to(device).unsqueeze(0)).detach().cpu())\n","plotn(plots, test_dataset)\n","plotn(plots, predictions)"]},{"cell_type":"markdown","metadata":{"id":"1JUmf9i1dg2i"},"source":["> **Task 1**: Try to train autoencoder with very small latent vector size, eg. 2, and plot the dots corresponding to different digits. *Hint: Use fully-connected dense layer after the convoluitonal part to reduce the vector size to the required value.*\n","\n","> **Task 2**: Starting from different digits, obtain their latent space representations, and see what effect adding some noise to the latent space has on the resulting digits."]},{"cell_type":"markdown","metadata":{"id":"tjdFt03rULX-"},"source":["## Denoising\n","\n","Autoencoders can be effectively used to remove noise from images. In order to train denoiser, we will start with noise-free images, and add artificial noise to them. Then, we will feed autoencoder with noisy images as input, and noise-free images as output.\n","\n","Let's see how this works for MNISТ:"]},{"cell_type":"code","execution_count":75,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:17:04.340030Z","iopub.status.busy":"2022-04-07T22:17:04.339488Z","iopub.status.idle":"2022-04-07T22:17:04.722138Z","shell.execute_reply":"2022-04-07T22:17:04.721381Z","shell.execute_reply.started":"2022-04-07T22:17:04.339991Z"},"id":"1Yj9ZRDmUPxX","outputId":"7e3a1624-4326-41a2-8f94-3d41e11cd411","trusted":true},"outputs":[],"source":["plotn(5, train_dataset, noisy=True)"]},{"cell_type":"code","execution_count":76,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:17:19.192439Z","iopub.status.busy":"2022-04-07T22:17:19.192168Z","iopub.status.idle":"2022-04-07T22:17:19.201586Z","shell.execute_reply":"2022-04-07T22:17:19.200834Z","shell.execute_reply.started":"2022-04-07T22:17:19.192408Z"},"id":"qxo8NDLLUvut","trusted":true},"outputs":[],"source":["model = AutoEncoder().to(device)\n","optimizer = optim.Adam(model.parameters(), lr=lr, eps=eps)\n","loss_fn = nn.BCELoss()"]},{"cell_type":"code","execution_count":77,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:17:25.541867Z","iopub.status.busy":"2022-04-07T22:17:25.540974Z","iopub.status.idle":"2022-04-07T22:26:30.268786Z","shell.execute_reply":"2022-04-07T22:26:30.268076Z","shell.execute_reply.started":"2022-04-07T22:17:25.541820Z"},"id":"JAYzgfoTUBHM","trusted":true},"outputs":[],"source":["train(dataloaders, model, loss_fn, optimizer, 40, device, noisy=True)"]},{"cell_type":"code","execution_count":81,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:27:12.268888Z","iopub.status.busy":"2022-04-07T22:27:12.268406Z","iopub.status.idle":"2022-04-07T22:27:13.034069Z","shell.execute_reply":"2022-04-07T22:27:13.033393Z","shell.execute_reply.started":"2022-04-07T22:27:12.268848Z"},"id":"IaPfyJ0SV7XY","trusted":true},"outputs":[],"source":["model.eval()\n","predictions = []\n","noise = []\n","plots = 5\n","for i, data in enumerate(test_dataset):\n"," if i == plots:\n"," break\n"," shapes = data[0].shape\n"," noisy_data = data[0] + torch.FloatTensor(noisify(shapes))\n"," noise.append(noisy_data)\n"," predictions.append(model(noisy_data.to(device).unsqueeze(0)).detach().cpu())\n","plotn(plots, noise)\n","plotn(plots, predictions)"]},{"cell_type":"markdown","metadata":{"id":"MyvmZEHzdoJT"},"source":["> **Exercise:** See how denoiser trained on MNIST digits works for different images. As an example, you can take [Fashion MNIST](https://pytorch.org/vision/stable/generated/torchvision.datasets.FashionMNIST.html#torchvision.datasets.FashionMNIST) dataset, which has the same image size. Note that denoiser works well only on the same image type that is was trained on (i.e. for the same probability distribution of input data)."]},{"cell_type":"markdown","metadata":{"id":"yxFKm_OxdqfO"},"source":["## Super-Resolution\n","\n","Similarly to denoiser, we can train autoencoders to increase the resolution of the image. To train super-resolution network, we will start with high-res images, and automatically downscale them to produce network inputs. We will then feed autoencoder with small images as inputs and high-res images as outputs.\n","\n","For that let's downscale image to 14x14 at train."]},{"cell_type":"code","execution_count":82,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:27:26.148045Z","iopub.status.busy":"2022-04-07T22:27:26.147775Z","iopub.status.idle":"2022-04-07T22:27:26.528086Z","shell.execute_reply":"2022-04-07T22:27:26.527399Z","shell.execute_reply.started":"2022-04-07T22:27:26.148016Z"},"id":"trida5guu9js","outputId":"3dd13ba4-0351-4982-c162-578735af23f8","trusted":true},"outputs":[],"source":["super_res_koeff = 2.0\n","plotn(5, train_dataset, super_res=super_res_koeff)"]},{"cell_type":"code","execution_count":83,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:27:26.613377Z","iopub.status.busy":"2022-04-07T22:27:26.613175Z","iopub.status.idle":"2022-04-07T22:27:26.620037Z","shell.execute_reply":"2022-04-07T22:27:26.619230Z","shell.execute_reply.started":"2022-04-07T22:27:26.613352Z"},"id":"9vC57e-rei4p","trusted":true},"outputs":[],"source":["class SuperResolutionEncoder(nn.Module):\n"," def __init__(self):\n"," super().__init__()\n"," self.conv1 = nn.Conv2d(1, 16, kernel_size=(3, 3), padding='same')\n"," self.maxpool1 = nn.MaxPool2d(kernel_size=(2, 2))\n"," self.conv2 = nn.Conv2d(16, 8, kernel_size=(3, 3), padding='same')\n"," self.maxpool2 = nn.MaxPool2d(kernel_size=(2, 2), padding=(1, 1))\n"," self.relu = nn.ReLU()\n","\n"," def forward(self, input):\n"," hidden1 = self.maxpool1(self.relu(self.conv1(input)))\n"," encoded = self.maxpool2(self.relu(self.conv2(hidden1)))\n"," return encoded"]},{"cell_type":"code","execution_count":84,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:27:27.713826Z","iopub.status.busy":"2022-04-07T22:27:27.713074Z","iopub.status.idle":"2022-04-07T22:27:27.722103Z","shell.execute_reply":"2022-04-07T22:27:27.721327Z","shell.execute_reply.started":"2022-04-07T22:27:27.713789Z"},"id":"d78J288qe5qJ","trusted":true},"outputs":[],"source":["model = AutoEncoder(super_resolution=True).to(device)\n","optimizer = optim.Adam(model.parameters(), lr=lr, eps=eps)\n","loss_fn = nn.BCELoss()"]},{"cell_type":"code","execution_count":85,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:27:28.959163Z","iopub.status.busy":"2022-04-07T22:27:28.958701Z","iopub.status.idle":"2022-04-07T22:34:09.817260Z","shell.execute_reply":"2022-04-07T22:34:09.816576Z","shell.execute_reply.started":"2022-04-07T22:27:28.959126Z"},"id":"CJ_zcN5Je6I-","outputId":"2311bd2a-55c6-4fdf-c4d2-37f0426fb61c","trusted":true},"outputs":[],"source":["train(dataloaders, model, loss_fn, optimizer, epochs, device, super_res=2.0)"]},{"cell_type":"code","execution_count":86,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:34:09.819386Z","iopub.status.busy":"2022-04-07T22:34:09.818889Z","iopub.status.idle":"2022-04-07T22:34:10.614124Z","shell.execute_reply":"2022-04-07T22:34:10.613402Z","shell.execute_reply.started":"2022-04-07T22:34:09.819345Z"},"id":"YsVfcCKKfjv1","outputId":"e7e029b0-b403-4feb-f869-d6367957741a","trusted":true},"outputs":[],"source":["model.eval()\n","predictions = []\n","plots = 5\n","shapes = test_dataset[0][0].shape\n","\n","for i, data in enumerate(test_dataset):\n"," if i == plots:\n"," break\n"," _transform = transforms.Resize((int(shapes[1] / super_res_koeff), int(shapes[2] / super_res_koeff)))\n"," predictions.append(model(_transform(data[0]).to(device).unsqueeze(0)).detach().cpu())\n","plotn(plots, test_dataset, super_res=super_res_koeff)\n","plotn(plots, predictions)"]},{"cell_type":"markdown","metadata":{"id":"-aZqFJthfu9-"},"source":["> **Exercise**: Try to train super-resolution network on [CIFAR-10](https://pytorch.org/vision/stable/generated/torchvision.datasets.CIFAR10.html) for 2x and 4x upscaling. Use noise as input to 4x upscaling model and observe the result."]},{"cell_type":"markdown","metadata":{"id":"A3DOJU1-gTJV"},"source":["# [Variational Auto-Encoders (VAE)](https://arxiv.org/abs/1906.02691)\n","\n","Traditional autoencoders reduce the dimension of the input data somehow, figuring out the important features of input images. However, latent vectors ofter do not make much sense. In other words, taking MNIST dataset as an example, figuring out which digits correspond to different latent vectors is not an easy task, because close latent vectors would not necessarily correspond to the same digits. \n","\n","On the other hand, to train *generative* models it is better to have some understanding of the latent space. This idea leads us to **variational autoencoder** (VAE).\n","\n","VAE is the autoencoder that learns to predict *statistical distribution* of the latent parameters, so-called **latent distribution**. For example, we can assume that latent vectors would be distributed as $N(\\mathrm{z\\_mean},e^{\\mathrm{z\\_log}})$, where $\\mathrm{z\\_mean}, \\mathrm{z\\_log} \\in\\mathbb{R}^d$. Encoder in VAE learns to predict those parameters, and then decoder takes a random vector from this distribution to reconstruct the object.\n","\n","To summarize:\n","\n"," * From input vector, we predict `z_mean` and `z_log` (instead of predicting the standard deviation itself, we predict it's logarithm)\n"," * We sample a vector `sample(z_val in code)` from the distribution $N(\\mathrm{z\\_mean},e^{\\mathrm{z\\_log\\_sigma}})$\n"," * Decoder tries to decode the original image using `sample` as an input vector\n","\n"," "]},{"cell_type":"code","execution_count":35,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T21:58:49.985541Z","iopub.status.busy":"2022-04-07T21:58:49.985283Z","iopub.status.idle":"2022-04-07T21:58:49.996198Z","shell.execute_reply":"2022-04-07T21:58:49.995415Z","shell.execute_reply.started":"2022-04-07T21:58:49.985513Z"},"id":"aT_GEWeU409I","trusted":true},"outputs":[],"source":["class VAEEncoder(nn.Module):\n"," def __init__(self, device):\n"," super().__init__()\n"," self.intermediate_dim = 512\n"," self.latent_dim = 2\n"," self.linear = nn.Linear(784, self.intermediate_dim)\n"," self.z_mean = nn.Linear(self.intermediate_dim, self.latent_dim)\n"," self.z_log = nn.Linear(self.intermediate_dim, self.latent_dim)\n"," self.relu = nn.ReLU()\n"," self.device = device\n","\n"," def forward(self, input):\n"," bs = input.shape[0]\n","\n"," hidden = self.relu(self.linear(input))\n"," z_mean = self.z_mean(hidden)\n"," z_log = self.z_log(hidden)\n","\n"," eps = torch.FloatTensor(np.random.normal(size=(bs, self.latent_dim))).to(device)\n"," z_val = z_mean + torch.exp(z_log) * eps\n"," return z_mean, z_log, z_val"]},{"cell_type":"code","execution_count":36,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T21:58:51.180232Z","iopub.status.busy":"2022-04-07T21:58:51.179476Z","iopub.status.idle":"2022-04-07T21:58:51.187153Z","shell.execute_reply":"2022-04-07T21:58:51.185336Z","shell.execute_reply.started":"2022-04-07T21:58:51.180193Z"},"id":"XWi4oCcq409p","trusted":true},"outputs":[],"source":["class VAEDecoder(nn.Module):\n"," def __init__(self):\n"," super().__init__()\n"," self.intermediate_dim = 512\n"," self.latent_dim = 2\n"," self.linear = nn.Linear(self.latent_dim, self.intermediate_dim)\n"," self.output = nn.Linear(self.intermediate_dim, 784)\n"," self.relu = nn.ReLU()\n"," self.sigmoid = nn.Sigmoid()\n","\n"," def forward(self, input):\n"," hidden = self.relu(self.linear(input))\n"," decoded = self.sigmoid(self.output(hidden))\n"," return decoded"]},{"cell_type":"code","execution_count":37,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T21:58:58.410064Z","iopub.status.busy":"2022-04-07T21:58:58.409811Z","iopub.status.idle":"2022-04-07T21:58:58.416646Z","shell.execute_reply":"2022-04-07T21:58:58.415810Z","shell.execute_reply.started":"2022-04-07T21:58:58.410038Z"},"id":"WukDYQ9f409p","trusted":true},"outputs":[],"source":["class VAEAutoEncoder(nn.Module):\n"," def __init__(self, device):\n"," super().__init__()\n"," self.encoder = VAEEncoder(device)\n"," self.decoder = VAEDecoder()\n"," self.z_vals = None\n","\n"," def forward(self, input):\n"," bs, c, h, w = input.shape[0], input.shape[1], input.shape[2], input.shape[3]\n"," input = input.view(bs, -1)\n"," encoded = self.encoder(input)\n"," self.z_vals = encoded\n"," decoded = self.decoder(encoded[2])\n"," return decoded\n"," \n"," def get_zvals(self):\n"," return self.z_vals"]},{"cell_type":"markdown","metadata":{},"source":["Variational auto-encoders use complex loss function that consists of two parts:\n","* **Reconstruction loss** is the loss function that shows how close reconstructed image is to the target (can be MSE). It is the same loss function as in normal autoencoders.\n","* **KL loss**, which ensures that latent variable distributions stays close to normal distribution. It is based on the notion of [Kullback-Leibler divergence](https://www.countbayesie.com/blog/2017/5/9/kullback-leibler-divergence-explained) - a metric to estimate how similar two statistical distributions are."]},{"cell_type":"code","execution_count":41,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T21:59:39.022178Z","iopub.status.busy":"2022-04-07T21:59:39.021647Z","iopub.status.idle":"2022-04-07T21:59:39.029102Z","shell.execute_reply":"2022-04-07T21:59:39.028326Z","shell.execute_reply.started":"2022-04-07T21:59:39.022140Z"},"trusted":true},"outputs":[],"source":["def vae_loss(preds, targets, z_vals):\n"," mse = nn.MSELoss()\n"," reconstruction_loss = mse(preds, targets.view(targets.shape[0], -1)) * 784.0\n"," temp = 1.0 + z_vals[1] - torch.square(z_vals[0]) - torch.exp(z_vals[1])\n"," kl_loss = -0.5 * torch.sum(temp, axis=-1)\n"," return torch.mean(reconstruction_loss + kl_loss)"]},{"cell_type":"code","execution_count":42,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T21:59:39.390807Z","iopub.status.busy":"2022-04-07T21:59:39.390555Z","iopub.status.idle":"2022-04-07T21:59:39.405174Z","shell.execute_reply":"2022-04-07T21:59:39.404462Z","shell.execute_reply.started":"2022-04-07T21:59:39.390779Z"},"trusted":true},"outputs":[],"source":["model = VAEAutoEncoder(device).to(device)\n","optimizer = optim.RMSprop(model.parameters(), lr=lr, eps=eps)"]},{"cell_type":"code","execution_count":43,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T21:59:39.718068Z","iopub.status.busy":"2022-04-07T21:59:39.717856Z","iopub.status.idle":"2022-04-07T21:59:39.727781Z","shell.execute_reply":"2022-04-07T21:59:39.726792Z","shell.execute_reply.started":"2022-04-07T21:59:39.718043Z"},"trusted":true},"outputs":[],"source":["def train_vae(dataloaders, model, optimizer, epochs, device):\n"," tqdm_iter = tqdm(range(epochs))\n"," train_dataloader, test_dataloader = dataloaders[0], dataloaders[1]\n","\n"," for epoch in tqdm_iter:\n"," model.train()\n"," train_loss = 0.0\n"," test_loss = 0.0\n","\n"," for batch in train_dataloader:\n"," imgs, labels = batch\n"," imgs = imgs.to(device)\n"," labels = labels.to(device)\n","\n"," preds = model(imgs)\n"," z_vals = model.get_zvals()\n"," loss = vae_loss(preds, imgs, z_vals)\n","\n"," optimizer.zero_grad()\n"," loss.backward()\n"," optimizer.step()\n","\n"," train_loss += loss.item()\n","\n"," model.eval()\n"," with torch.no_grad():\n"," for batch in test_dataloader:\n"," imgs, labels = batch\n"," imgs = imgs.to(device)\n"," labels = labels.to(device)\n","\n"," preds = model(imgs)\n"," z_vals = model.get_zvals()\n"," loss = vae_loss(preds, imgs, z_vals)\n","\n"," test_loss += loss.item()\n","\n"," train_loss /= len(train_dataloader)\n"," test_loss /= len(test_dataloader)\n","\n"," tqdm_dct = {'train loss:': train_loss, 'test loss:': test_loss}\n"," tqdm_iter.set_postfix(tqdm_dct, refresh=True)\n"," tqdm_iter.refresh()"]},{"cell_type":"code","execution_count":44,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T21:59:40.174195Z","iopub.status.busy":"2022-04-07T21:59:40.173962Z","iopub.status.idle":"2022-04-07T22:04:29.373590Z","shell.execute_reply":"2022-04-07T22:04:29.372894Z","shell.execute_reply.started":"2022-04-07T21:59:40.174168Z"},"trusted":true},"outputs":[],"source":["train_vae(dataloaders, model, optimizer, epochs, device)"]},{"cell_type":"code","execution_count":50,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T22:06:02.195473Z","iopub.status.busy":"2022-04-07T22:06:02.195214Z","iopub.status.idle":"2022-04-07T22:06:03.098105Z","shell.execute_reply":"2022-04-07T22:06:03.097411Z","shell.execute_reply.started":"2022-04-07T22:06:02.195445Z"},"trusted":true},"outputs":[],"source":["model.eval()\n","predictions = []\n","plots = 5\n","for i, data in enumerate(test_dataset):\n"," if i == plots:\n"," break\n"," predictions.append(model(data[0].to(device).unsqueeze(0)).view(1, 28, 28).detach().cpu())\n","plotn(plots, test_dataset)\n","plotn(plots, predictions)"]},{"cell_type":"markdown","metadata":{},"source":["> **Task**: In our sample, we have trained fully-connected VAE. Now take the CNN from traditional auto-encoder above and create CNN-based VAE."]},{"cell_type":"markdown","metadata":{},"source":["# [Adversarial Auto-Encoders (AAE)](https://arxiv.org/abs/1511.05644)"]},{"cell_type":"markdown","metadata":{},"source":["Adversarial Auto-Encoders is a **combination** of Generative Adversarial Networks and Variational Auto-Encoders. \n","\n","Encoder will be the generator, discriminator will learn to distinguish the real images encoder output from generated ones. Encoder output is a distribution, from this output decoder will try decode image.\n","\n","In this approach we have **three loss functions**: generator loss, discriminator loss from GAN's and reconstruction loss from VAE.\n","\n"," "]},{"cell_type":"code","execution_count":169,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:42:29.048789Z","iopub.status.busy":"2022-04-07T23:42:29.048510Z","iopub.status.idle":"2022-04-07T23:42:29.057368Z","shell.execute_reply":"2022-04-07T23:42:29.056579Z","shell.execute_reply.started":"2022-04-07T23:42:29.048761Z"},"trusted":true},"outputs":[],"source":["class AAEEncoder(nn.Module):\n"," def __init__(self, input_dim, inter_dim, latent_dim):\n"," super().__init__()\n"," self.linear1 = nn.Linear(input_dim, inter_dim)\n"," self.linear2 = nn.Linear(inter_dim, inter_dim)\n"," self.linear3 = nn.Linear(inter_dim, inter_dim)\n"," self.linear4 = nn.Linear(inter_dim, latent_dim)\n"," self.relu = nn.ReLU()\n"," \n"," def forward(self, input):\n"," hidden1 = self.relu(self.linear1(input))\n"," hidden2 = self.relu(self.linear2(hidden1))\n"," hidden3 = self.relu(self.linear3(hidden2))\n"," encoded = self.linear4(hidden3)\n"," return encoded"]},{"cell_type":"code","execution_count":170,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:42:29.868218Z","iopub.status.busy":"2022-04-07T23:42:29.867956Z","iopub.status.idle":"2022-04-07T23:42:29.877232Z","shell.execute_reply":"2022-04-07T23:42:29.875007Z","shell.execute_reply.started":"2022-04-07T23:42:29.868183Z"},"trusted":true},"outputs":[],"source":["class AAEDecoder(nn.Module):\n"," def __init__(self, latent_dim, inter_dim, output_dim):\n"," super().__init__()\n"," self.linear1 = nn.Linear(latent_dim, inter_dim)\n"," self.linear2 = nn.Linear(inter_dim, inter_dim)\n"," self.linear3 = nn.Linear(inter_dim, inter_dim)\n"," self.linear4 = nn.Linear(inter_dim, output_dim)\n"," self.relu = nn.ReLU()\n"," self.sigmoid = nn.Sigmoid()\n"," \n"," def forward(self, input):\n"," hidden1 = self.relu(self.linear1(input))\n"," hidden2 = self.relu(self.linear2(hidden1))\n"," hidden3 = self.relu(self.linear3(hidden2))\n"," decoded = self.sigmoid(self.linear4(hidden3))\n"," return decoded"]},{"cell_type":"code","execution_count":171,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:42:30.613336Z","iopub.status.busy":"2022-04-07T23:42:30.613083Z","iopub.status.idle":"2022-04-07T23:42:30.622002Z","shell.execute_reply":"2022-04-07T23:42:30.621267Z","shell.execute_reply.started":"2022-04-07T23:42:30.613308Z"},"trusted":true},"outputs":[],"source":["class AAEDiscriminator(nn.Module):\n"," def __init__(self, latent_dim, inter_dim):\n"," super().__init__()\n"," self.latent_dim = latent_dim\n"," self.inter_dim = inter_dim\n"," self.linear1 = nn.Linear(latent_dim, inter_dim)\n"," self.linear2 = nn.Linear(inter_dim, inter_dim)\n"," self.linear3 = nn.Linear(inter_dim, inter_dim)\n"," self.linear4 = nn.Linear(inter_dim, inter_dim)\n"," self.linear5 = nn.Linear(inter_dim, 1)\n"," self.relu = nn.ReLU()\n"," self.sigmoid = nn.Sigmoid()\n"," \n"," def forward(self, input):\n"," hidden1 = self.relu(self.linear1(input))\n"," hidden2 = self.relu(self.linear2(hidden1))\n"," hidden3 = self.relu(self.linear3(hidden2))\n"," hidden4 = self.relu(self.linear4(hidden3))\n"," decoded = self.sigmoid(self.linear4(hidden4))\n"," return decoded\n"," \n"," def get_dims(self):\n"," return self.latent_dim, self.inter_dim\n"," "]},{"cell_type":"code","execution_count":173,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:42:35.303062Z","iopub.status.busy":"2022-04-07T23:42:35.302489Z","iopub.status.idle":"2022-04-07T23:42:35.307122Z","shell.execute_reply":"2022-04-07T23:42:35.306088Z","shell.execute_reply.started":"2022-04-07T23:42:35.303023Z"},"trusted":true},"outputs":[],"source":["input_dims = 784\n","inter_dims = 1000\n","latent_dims = 150"]},{"cell_type":"code","execution_count":174,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:42:36.281582Z","iopub.status.busy":"2022-04-07T23:42:36.280891Z","iopub.status.idle":"2022-04-07T23:42:36.341758Z","shell.execute_reply":"2022-04-07T23:42:36.341059Z","shell.execute_reply.started":"2022-04-07T23:42:36.281520Z"},"trusted":true},"outputs":[],"source":["aae_encoder = AAEEncoder(input_dims, inter_dims, latent_dims).to(device)\n","aae_decoder = AAEDecoder(latent_dims, inter_dims, input_dims).to(device)\n","aae_discriminator = AAEDiscriminator(latent_dims, int(inter_dims / 2)).to(device)"]},{"cell_type":"code","execution_count":175,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:42:36.966785Z","iopub.status.busy":"2022-04-07T23:42:36.966433Z","iopub.status.idle":"2022-04-07T23:42:36.973600Z","shell.execute_reply":"2022-04-07T23:42:36.972878Z","shell.execute_reply.started":"2022-04-07T23:42:36.966750Z"},"trusted":true},"outputs":[],"source":["lr = 1e-4\n","regularization_lr = 5e-5"]},{"cell_type":"code","execution_count":176,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:42:37.935003Z","iopub.status.busy":"2022-04-07T23:42:37.934685Z","iopub.status.idle":"2022-04-07T23:42:37.941516Z","shell.execute_reply":"2022-04-07T23:42:37.940603Z","shell.execute_reply.started":"2022-04-07T23:42:37.934966Z"},"trusted":true},"outputs":[],"source":["optim_encoder = optim.Adam(aae_encoder.parameters(), lr=lr)\n","optim_encoder_regularization = optim.Adam(aae_encoder.parameters(), lr=regularization_lr)\n","optim_decoder = optim.Adam(aae_decoder.parameters(), lr=lr)\n","optim_discriminator = optim.Adam(aae_discriminator.parameters(), lr=regularization_lr)"]},{"cell_type":"code","execution_count":177,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:42:38.761534Z","iopub.status.busy":"2022-04-07T23:42:38.760934Z","iopub.status.idle":"2022-04-07T23:42:38.783970Z","shell.execute_reply":"2022-04-07T23:42:38.782836Z","shell.execute_reply.started":"2022-04-07T23:42:38.761498Z"},"trusted":true},"outputs":[],"source":["def train_aae(dataloaders, models, optimizers, epochs, device):\n"," tqdm_iter = tqdm(range(epochs))\n"," train_dataloader, test_dataloader = dataloaders[0], dataloaders[1]\n"," \n"," enc, dec, disc = models[0], models[1], models[2]\n"," optim_enc, optim_enc_reg, optim_dec, optim_disc = optimizers[0], optimizers[1], optimizers[2], optimizers[3]\n"," \n"," eps = 1e-9\n","\n"," for epoch in tqdm_iter:\n"," enc.train()\n"," dec.train()\n"," disc.train()\n","\n"," train_reconst_loss = 0.0\n"," train_disc_loss = 0.0\n"," train_enc_loss = 0.0\n"," \n"," test_reconst_loss = 0.0\n"," test_disc_loss = 0.0\n"," test_enc_loss = 0.0\n","\n"," for batch in train_dataloader:\n"," imgs, labels = batch\n"," imgs = imgs.view(imgs.shape[0], -1).to(device)\n"," labels = labels.to(device)\n"," \n"," enc.zero_grad()\n"," dec.zero_grad()\n"," disc.zero_grad()\n"," \n"," encoded = enc(imgs)\n"," decoded = dec(encoded)\n"," \n"," reconstruction_loss = F.binary_cross_entropy(decoded, imgs)\n"," reconstruction_loss.backward()\n"," \n"," optim_enc.step()\n"," optim_dec.step()\n"," enc.eval()\n","\n"," latent_dim, disc_inter_dim = disc.get_dims()\n"," real = torch.randn(imgs.shape[0], latent_dim).to(device)\n"," \n"," disc_real = disc(real)\n"," disc_fake = disc(enc(imgs))\n"," \n"," disc_loss = -torch.mean(torch.log(disc_real + eps) + torch.log(1.0 - disc_fake + eps))\n"," disc_loss.backward()\n"," \n"," optim_dec.step()\n"," enc.train()\n"," \n"," disc_fake = disc(enc(imgs))\n"," enc_loss = -torch.mean(torch.log(disc_fake + eps))\n"," enc_loss.backward()\n"," \n"," optim_enc_reg.step()\n","\n"," train_reconst_loss += reconstruction_loss.item()\n"," train_disc_loss += disc_loss.item()\n"," train_enc_loss += enc_loss.item()\n","\n"," enc.eval()\n"," dec.eval()\n"," disc.eval()\n","\n"," with torch.no_grad():\n"," for batch in test_dataloader:\n"," imgs, labels = batch\n"," imgs = imgs.view(imgs.shape[0], -1).to(device)\n"," labels = labels.to(device)\n","\n"," encoded = enc(imgs)\n"," decoded = dec(encoded)\n","\n"," reconstruction_loss = F.binary_cross_entropy(decoded, imgs)\n","\n"," latent_dim, disc_inter_dim = disc.get_dims()\n"," real = torch.randn(imgs.shape[0], latent_dim).to(device)\n","\n"," disc_real = disc(real)\n"," disc_fake = disc(enc(imgs))\n"," disc_loss = -torch.mean(torch.log(disc_real + eps) + torch.log(1.0 - disc_fake + eps))\n","\n"," disc_fake = disc(enc(imgs))\n"," enc_loss = -torch.mean(torch.log(disc_fake + eps))\n","\n"," test_reconst_loss += reconstruction_loss.item()\n"," test_disc_loss += disc_loss.item()\n"," test_enc_loss += enc_loss.item()\n","\n"," train_reconst_loss /= len(train_dataloader)\n"," train_disc_loss /= len(train_dataloader)\n"," train_enc_loss /= len(train_dataloader)\n"," \n"," test_reconst_loss /= len(test_dataloader)\n"," test_disc_loss /= len(test_dataloader)\n"," test_enc_loss /= len(test_dataloader)\n","\n"," tqdm_dct = {'train reconst loss:': train_reconst_loss, 'train disc loss:': train_disc_loss, 'train enc loss': train_enc_loss, \\\n"," 'test reconst loss:': test_reconst_loss, 'test disc loss:': test_disc_loss, 'test enc loss': test_enc_loss}\n"," tqdm_iter.set_postfix(tqdm_dct, refresh=True)\n"," tqdm_iter.refresh()"]},{"cell_type":"code","execution_count":178,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:42:39.548719Z","iopub.status.busy":"2022-04-07T23:42:39.548152Z","iopub.status.idle":"2022-04-07T23:42:39.554618Z","shell.execute_reply":"2022-04-07T23:42:39.552145Z","shell.execute_reply.started":"2022-04-07T23:42:39.548682Z"},"trusted":true},"outputs":[],"source":["models = (aae_encoder, aae_decoder, aae_discriminator)\n","optimizers = (optim_encoder, optim_encoder_regularization, optim_decoder, optim_discriminator)"]},{"cell_type":"code","execution_count":179,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:42:40.199497Z","iopub.status.busy":"2022-04-07T23:42:40.198669Z","iopub.status.idle":"2022-04-07T23:51:58.463843Z","shell.execute_reply":"2022-04-07T23:51:58.463128Z","shell.execute_reply.started":"2022-04-07T23:42:40.199453Z"},"trusted":true},"outputs":[],"source":["train_aae(dataloaders, models, optimizers, epochs, device)"]},{"cell_type":"code","execution_count":181,"metadata":{"execution":{"iopub.execute_input":"2022-04-07T23:53:14.587179Z","iopub.status.busy":"2022-04-07T23:53:14.586912Z","iopub.status.idle":"2022-04-07T23:53:16.255114Z","shell.execute_reply":"2022-04-07T23:53:16.254415Z","shell.execute_reply.started":"2022-04-07T23:53:14.587150Z"},"trusted":true},"outputs":[],"source":["aae_encoder.eval()\n","aae_decoder.eval()\n","predictions = []\n","plots = 10\n","for i, data in enumerate(test_dataset):\n"," if i == plots:\n"," break\n"," pred = aae_decoder(aae_encoder(data[0].to(device).unsqueeze(0).view(1, 784)))\n"," predictions.append(pred.view(1, 28, 28).detach().cpu())\n","plotn(plots, test_dataset)\n","plotn(plots, predictions)"]},{"cell_type":"markdown","metadata":{},"source":["## Additional Materials\n","\n","* [Blog post on NeuroHive](https://neurohive.io/ru/osnovy-data-science/variacionnyj-avtojenkoder-vae/)\n","* [Variational Autoencoders Explained](https://kvfrans.com/variational-autoencoders-explained/)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"}},"nbformat":4,"nbformat_minor":4} diff --git a/4-ComputerVision/09-Autoencoders/images/aae.png b/4-ComputerVision/09-Autoencoders/images/aae.png new file mode 100644 index 0000000000000000000000000000000000000000..61609563f4cdf913826db13721771e9af960b6ee GIT binary patch literal 50083 zcmdSBWmuJ6*EYH=Fu?>x!XTuj1tb&!X=zxLbceLGiIPfpNy7rEMVAOjNrSMYyFt3Y zapCiP&;9N9efM$fWA8tk!~1rvYh7zzbIviwInHs8i65Uyi(WWOau$U`T@ZWvL=J^I zsfj|JFgc9}SMpE$H^N_Mte&dcpit+oApadhMMPakp>CnXo;*@;jG7;Cb_`mNK0H+Z z@#u7-_T6J{YDPZy&Rk4A5Tr9xX>>f#XU%r%iIsLt5ZoKuDu@vP;JfXMi_%|-RJ=1XU^5m{)SVQx;AOV|=VZhW_$)wI? zOs6w7mrZNg;^vgit|~?;1EvQ*!}hK|On+a-rQC5mcJy-j;%nl+*9dR_|M1c$)R%L} z^xMuUL@QSJ5>tvjI`N*4j`jEOYg85Q^wEl(Fs$ZU(YGqqLei>6`wV&ZMvwd6-mI$i2Ec#}_`@jbV}iUF%TOL%Fqub>UY$LARzc2ywS)vE z$gI4vh`vC|rKK7$+L^2Pl|!tiOtnf_hR-TdX`Z3yRo4SCRq+pgN_Cg8r6?ca<>RP; zhXW=H!5b!``REfUv}`~Dx{&`9otT$IOV8l=OdZjORta z@jl+>eW?sJyf=uPgN=qF_sn?Q4p9MjLC zIqvc3v?glJe(?nW!S>%bIJpJKw}F;FCIp?anpAFur8xB8DtK2+;`c|5_UScNEU}G^G3!N%HV+e z+cLriyF1hRHoVg$3^$h-Vlr~j0pymIK|5i@)W!HH{9_{E#)I9ZYM8u9Lv_o73bk5Z zibZ{`vSU?&40PNpuX63py8Sv2QwcY-o)uBbH)tm9B?$@06rRe#L@e!sObd^c6lQV7o9Z^zpYv9G9q;SECfpi3$3=4gp!acEf#*Ghx z64yJ+olxee^6S;-JT#Mf3(>8u&yuqW(E`?$J(8AVO&VP4b|2T%&~D}{_ONlIFShUP1kIm~{yC7o z5>Q985LB?m!7;#~iThI-(Cvh#r?$KQi^~ocl5|H!sg_xa!z0Eia9z@r#xv zlFvm_8U1H9D8*{l2s?EnnG7tucbi}0RY^dg4|6K&i>~!xxz_3{_b7tL!;?;n-M;Nb zvg;qQIxJwSy+`@3G2t#QwzrjdUn-JrqcQRVF_}q0$oLG_ke3;2*J*MAwRky8ID5(n zZC7BeDXwG^qPrH5qM;g~Lo{7*;V{{mW-z&`QI2dc+ZWxz;M34Gp46!@7e5kPf8g?C zM9yba2ghAy|4U)vVUg5!yHt%0mGRUfVS(Gu!`+F@P^~gI*Gz(_zKrnbfy9$y)d5%Y z2Xz$dK+|(^4HRXFYQhV9dUyb-dPvP^(f{phMbXaPho}p@a z6Y0)#&0@Z$B}~-avs)d^M=@lD=p{H#ZqrM_$v7(!<))Zy#t%kH5{<9e7Tg$#S39GYDu)Z~U5HU0S-Pp0JhF1G8m$Mt5`SuLrst>f zDS0oOY1Z~p76JLg)Z?ZYe%?<#zuv2Bratr}drh`*XU5A9i918Sh>4Ob z9ncpuq#^^lI#T|8O`{KTxK*ofTs-*8z9<-9AOq91*jc4UV{{Opzq4Pu?X3`XBaEp-d271ZjPp)=&KgB zzr-q79Z=^}iK)$KHNf-)X#Bj=`h?P=Gn=hv4Q;d#X`NVQ$8KwD%(jhnH# zR58it?Fc&HGb61?IY^MNyJ{9 zqcD4!S?o66{=P zXTq>gXi+*a^CzoJMt?@V0OcWlBU!job*9WH&Ni?KBaRhF*6-zF{fHZs>qtx{+!yoa zGIfyCbqXvndxZ^UKcwV}W#e|R{eB=QFfA!}Y5l8xv00>9hqY!0C6|i!y9u`=%wNq@(w@R@aA@I>O-6}^uNj!iW zvxSjU$4Ysrq`6y6eeM6rO6H{L%{owtDM%|F4e>nY!HJ~yAcsz_2c);3qPN(_c2q4A zJ25>X`Fs+8&}xPIImQKeD9dvcEYi~4@0$!?bdNabdu!+fOy>Vm>~PpSPo*)W5@fj8 ze#sxpEMl^+md(vfC(|3K#`i$I+xmM&O}rQ}w!p4?r_T6Mj9;wIiPKf9zzS?1f8_r) zbG`Jt$(|82IdRFGk`5E|C%8Py>;gMn8^ER}(J1R}%jUxlLNd-IQJcYnSc(UZM)ZW2 zlSgXIr}AvKuC%^C9b#gWr(YVfm$Bh!6tI!$epuZ3eM?Tcu;2uH+l+M-wZqsFGs=ko zwwKIs2VGAZ==0$PNq=-@VNp&u`i~1P&zB2Vf~;>TH)^*(Y4_X8o2|@0Lni&zEc$<@ zmB^{o$5PnsqTFn^5<3eXJz~nMdyqmvrV!_TIMYZJLTqxt%=Kg7v;WL8-folur^a9p zUNM1l*37bO3@VNdS>?d;h>5wDy8qZBc8c?5Dkf6N6m>sp`?s>S&$T<$ejT@Pr{q5$ zj0SP%$N1q7bX}zia@j8E)SHR_QqL=678{x!s2H8GIZz)EB+DeWA}*ym?4ob)6=KOa z{rf~=s$fH>I$`lbxH%(PEUN4kM2Ba%^dxV~QUaTy?ax%5vgUpBR`XR!H7;N~&+}(( zu1SZwKV0xjepa2tHkKUMLqlz2WS51uNv*tgjnWHc>a{b4T4bOdo$HJm_Z)9e!+?Q+5#&n-`qSh zhj+int)I@~)(@rqJ~f)x=&sr(bS2J=ko|H}@iW@a@2XpUF@qM!UisieKlmU{;DTF~ zwy0#xWunSdZ+zuc`rSL}er2in9Z~j6zpd1E-TK)X`wQd@Gb6iti2Cg6Wxs!?z`k@w zSrPMqEmfwc#eRC;?YZIlGAppWiHU(qRlG<)E`dg}ZT!p?b|$Byy*j4Ok-59t1cnTO zLscUrT-vXxCBx#>@HabiF`^dFvU$;%^4<-zu@nEzEiDAJ4i&h_4*(`MGB~E%s z$Bms5tncpTE}TtQFWqoUPt1`o-6%55W9Q=;*%?edom3W*MG|$K`fBO}p9g-KeJ7?n~_-WI1 zW%NobP1QEb?!Sxql@#13RoGWRDMmOLnz7~Q)|m)k_K-o%vQjHyW)G` z4fU38<$VJQhtX*Yo)>9t+AmZ6js;fqD~l9eI=(~TNzFy_Q|-^BX=A`u+-6KS?s+tYK;iqV19p&^H6M# zxZZTot7XT;GF|6qBAvFZ?)2r@nCuoYHV(Jhn*QxV7sgLP#rH72QvN9>nRA>=yhh{{qAB-BdFHsH$sFH%MR45KhZsYOMTno!cyNp!Tx(>R4U6RnA^ID&YNs1jDD%XOQ+C`s9~F^&IDz?49Bm=>FQJ`ZrQhtb<}=lo^wU^pnzGe?ZA>Q3 z_Ruy3*(C~6r|@!bHgW2UP@(s;o5s`ckEsrD>3h{lhbPd=CmYTGqK#mk;Fk(#RdUDUZ*N&ucs6v2i>u)(6w}u7l9yfQKZMkHR>+v;!#A`s|Bc>!0K^H8sS92C z-#Ho!G!<4>NXZ^p2|8~sb`PHD+nH|`o>-LZ+2WiMJkuS$$!@OO8f9X>t3=Zoo?Lz+ z=ZdA}guUzfa=+~^LEyK149{GrC8{(kVVrY2K`a zsdwau%SdN1ZoVSLZU@hQ!4b_??n~a8UgOlqulGQt)IG8*=`QmzK~aP%<#e0zmD6$#;RB&-A8#R>f{iVII+u&Z`35o7|+PJY&-VT!mASOzDcNHaduz99!}~+JifVsKZxicBO;4}E zmdy`zix&;f-1wZY7OBR2{0_?TQ*5_M0g1**5ArIVj*tHSvvW4xa?d{Do!@^Y;!DoF z>sOvGRATu&DuTfAM#j9CdLPdx##3t7_$(UM{lZkpQmr0bO`7?XeBRIWJhhITqU_f? z<3Clod`-4 zcS8*maQ&*Y3jub2!B4{6<$KVHMh$_^k^Ve1um0A$5O+Oo>F9Ib*riO)-(2dxqH&sW zd)~a=ji1w9wHoYweKos^TKpwC!{0QJ))IQ$M&tl;*BStO$=cS)l&vjCso3$0CHYf} zg+s~rc9S0NbGK^O1>B-7>UNGmI7`6j8lgk}*xXep*$}ALy2$V9o15L55)!)MLEis~ zDo{8#OxmbMh)pl@J;OxR67R)7=jH!Y>tK97Cu&5H?uPY~zMRYWS&?zlvu%R@F)<1c z1lBgbwdnM2g1~iyy&b}Z2!@UM(i!oFKpiFi{;u0LC-Tupn!|+C)zTgZV%x4Y4tzTo zhX|d!i5j$LNPf4^>VV0d3;7F?M-=QOels}_?>adRJ>2!OKl75t;f7yXUzKI zFL=4$l9G$!OnQLT8hWb5SGLWUR10AeBBhdkmyg?-wlAxXFp(q1UqXIx@}-IHjxb6P zU(q#U?0)mp0CC$a-xCXVU0W|38u>Ib*lJ^EwpPY?-1dsRTGhK|eq$nfH%319Urge_ zXEw3yKc049zVp3|<@!@m{-oJYi)4jQ`@CfP2U~5eIv@HyKUrmUq~6)P)|(m%SG=c| zRC0<@s4xx3BX-F$mm#p&c^s)6-;sO8X*GB%pMg(8F3O&GY3}=jojRpCubg&q=HE#T zIYR4wIZ`Hb9vAsP0&!msqw7?SZg!TCp-wDOm(LR3q?&f`VeDD^_=Q7n`tXZ(yv9J) zTKCt~9E*eVgo-YIq!pW8o=~{)Dy;8>ow4|gy)lSLUiG9=GxpmBiEFXE6ch34AR4;g)cTg6TOUk?; z5&S7vLN3#qX>#X4Q(%|y_4TBMfwXbQNn2Pz25Y!NuOIHzySEul7no6~Sdf^jY#0=I zb`RY#m99m~4H9yZOV@l<;s)HEq;2X%v3ye>bPFZT(&?{qLBJ#ZLUCFF- zRl3QCJTlklaqVx(u0H;B9InTSLYNG5y9!NUjaAI>2Dhk@yW2F@SV~Z+qIjN}o49Q@ zW!u^xXykg_iLkL#9)w?0hnMSll$BHBzRE`MxP`T!LM9?u3d;M@kK<_}-@mUOf;xTl zs!9mL;ppf8)g{4DGG2R0fBv0R5r1;br%xBRx3}#g<)1$NfV>x_a8;=HBO7y_!CN|q zT4Q$^T&+L}s_kVeWpq#9L2V^lR_-fR?sRA88@7^vkuWZNxjH%HP`Mk8cY$O)XKL8> zn8)C&*4>bm3X}&MA~XD{CDt;k^SQAzo009qlvHC;zOGchIp|EnZJmgth}=Ada>D0d zQmflNj%s1)IxXnwy1P&)t(AWK#EIa5fCL$VeTDAs?p!oxU{XOt1B1M$rKLT4{(KOn z>w33x1dHzbirq!tL1N}IW>j0}^2=5jiR@ARIS+zJ0r!e>*9Sa2X!pG_3J&{ul{XnB zIj>(I-=8-Wkbm(ahjOMGpQ4N6a7(n1j~`hsfy-c_4@?P-Mw9mB;%SEkoaP(Zf2#PqEO3iB3OA|E4SqJKNpU<3pjh*eY;6(erl>E((pM|^k!b{ zezV)bbo-Z`KTXV~dtu1xc$Z%evBI;A$bbY9aNA7!?C+Tu4j91wQz{O^!tW%h1Dg$O zLdgUF>*%nF8>{=fn_OQ5e%`$Q++*KGO3IgVW^a2{!%6c0oIm&*rACoe_4MTus(*gs zfOM3^l(W=XO5R$0{Kq4z$iOoN<%ERZv{Tti{Bx#@g3tv*eui(Kgos6c$RGbdymW-C zs9UPGqXU}?Xqr=Y%Q1F#k4nZ4!oAL}Kh=gIHI9ZXd1bBt@Kao~^dRq1C*XZ)!5#SU zXx6AYwtq|V+fyY|8}aEG8F|bXNbU>L^vKAIV^=lqUM@7g77u;6>0~6#JXT&OJ{R*+ zO&fhJdwt#36$Xy}J`Mq1!5+rN~SfXtH$O5#4eK0Sh_tW3WTCJp$??z!T9)Iu} z7|fN%c&OI9#9IE`0{y>4$;a=n!$3nvJ6ZCsB0iq+BrTD-yU%Up3l0!708MJiLanB< z)IvTm^QEJ<=s@7ahmiWD5(d2wmr$r>Mr8Zr)Eaj-V@*SwSdWI(5<)CdLe@EQLHx%( zTXfyUyI?49Wch+z@^RzFv`kIcbxxvCE?1C6Oe+m)L?=py>YHC+VZBD$6MvVAJR7bb zDNiWDLRq_UVZKixQ~C2ELOz2sk;z33*|V7`-Y4LzCjkmjZBifRB^6+(o9fI2M>i=VmJ{t7D#rxw&-R~`G0Re$85>~t!-6pnYzyzN0 zWB)ZtARZOh3+ln@blk37j5&F|* zEaDRrpFmo``jPXckB;g@^EyZy08ypBe(!x zQK<7avt6|210{;8s;YEMOcv|*!NI}lnVDS;fiwjUOPW+PG^SmtvJt!vSwzg5nV0U0 zIOdpuvuprdqEMQKtl{|<13cz^xx(w~>j8;N?uWZ2Q!&oA^<)l4ZE=sz5)vvqIu`A3 zPzQ<)x$R>3x5uwcjE|?Lral!G7Va;4@g;;wBi%6OWu2Qd;)1V{^*c%dig8-}PFWFL=sufy9Hyh+lwnR>S z{%DJ%zjNpD>SRk`pIJ{{E5E4<8HCH`5u{ouqS5J$YS??=6qQ@G%sxz_$X{1UNi%(X z&PVguHBMUtkT4&?>TfzWHhJ(%WWC?MeS>?owY8BqvvYI7;o-)%NEn759aphf-@sW) z1Ry?kU0p2j2AjcClk^L)Ahn}RgbNA^LY#K!@LCrd>l5b-9%cywwi#d)z<^s)wmh^2q;AmwvX2(X=sb|6s~vvK(S z!ZR_M1Y8u6^ntQUA=JniA_z~pLmeWBGA%Xr$FMjo&GIPG{+@(b4v2W{xA>h1FIvET z@b71rp94up#ZBFhYzZhaT-M2GPy*9WKaj7!UA|g9RdV@qJjJ1WS;AxAtkm735Jr$d zkM>u3^;ifyqa%y3;n^P3uoqO@C*LEpDLixL%xB7QN$NNB#y^z;F@R=mmx#*4u|HqR zYG)nv>~n4p5FD925@1GrHm14>LWo|BgjMfDbug02_?3TVS%2|f7CXuY<_TZEP;tB? z9r(z2o`l0B=;$72hEo*uop#ztu=O?%F45=i&`gp_-_ND}Gf~x?Jmbp9t=b>!hCymRy4(C&99)>)f@6bG2Fa|+Q4-eGB zuN)yjR*}#zjg8{Sn^&)1iAzedmykkgm)>r7;od0yo+J=2=6Zp?e(G;fkg({yMc^?a zf}|X%JA7AJ%T+sXG9>}lhOcOszX4bZkScQf_U-;W zL;uF6CQBE5LRx0#-WWb-BqOB6y7hnMfw0!l-`N^?yqi>%KM_G<@|>IhUe)yRl7jTh z`sIgdSFd(8UYJ!#i7ydwOP`e{?eI(rK5JcUt`TjE6KHPR(q7X(hH~;>Ubez$mZJgZ z&^bEq1|7#PtS=1>WpF4cD~skn@G<4ySBktxPcgflP}QHX}*x9-lx6ghboi!8e3a?0ci_N zy6MXI{xkt{@wQ-lvot!>6*Ck78C3HOW2hd@c%cN%{jSst z@neBO&i!R}CI&+I4S# zzd*O1p!WOsdo(nnH#yDP3QT(+9+V-^e1GAl?bfn3d}ajYn&vqdh%{sA$=D({Ce1YZ zFkBMbw4<@VwtT_rFz19Qvvown%m4NqM@^YJ4X{*6DE|guPLU0`ZVvPp+oV)%Pu}D^ z+*>8TX^&K0iWy2X%Of=;4;{ZEcfEW2)^WTTo~ZPYRq?%;=$||)v6aZ!(WfHOu%nu? zgmW@8su5pVn<`Qt+Ej2d6e~9be$(g4S2}aU_ks%jro~-Ais6zUKc4QdXS;XL_vHl# z27Y;Wj(9dHR8#KBlXDA+8qYK}vnHBCGxGBlSXfy4E8QzCiYJH)`5|`=1RJI_KJDsH zTv=o&RE}T5o~X-qR83>~*(67ln2|aA7I?3Cz#Sm;t}cn2z(3<|di(e&xVcpzNvoqn zt|gpJ)Y#Y^XV; zA_Hb#2lr{)KD%@uPnv=r!Hen3&)g>LPYL<*(9LQuh_`m=dEOS~H0EYznY@7V;N05X zHSJDUguw2Cyp;hNX@7*!y^Z2v8Q=AAcc?etxD#PU_Z~h}e}i`+m*RN89_tLwvNE_q zuLFYe2(*S=R(&aH@85rh$OAEMJH`YzVCe4(HwJuS3+|SNT4{tU`Ijpiza#Lzqcqz* zBc z648K|iaI(803+>bQ=uwL?JO>XLt873S{{p|nLU1FSqdpb7}}j#OM!0rUQu&6dm_K0 zBx3GJQY~Q^Y!=zN^(eED-LMh@$+BY#3^v`EoBuAt)_5hmw=4ECcEWr$C$j_uPHzh27w%dz~;9k$n z`dkN`fnzxR=;}=F(V|0>v!JvIZ|6IOAz|}A1m0I!7aEc>r>;2d#Q;~+*UAj1l zGW6~LeY+#vZp&#}K%~z}{+8Ztb=+PrMx7yLdHI&JsTQ`k-gs{cn zp!-p9AghN(QN&@?umFt_A#T?o8@w;vDfb6mM6J|EY{S9VHG(xwF+!V26q*MFXvD8~ zax3w;M8?L(?w2vUyF94yi;5x@>*z|8ANcdv>2d~RdF;f2MJht*dwF~F+72O(w)ndk zUFMkcU!5E^7Tw|`S(EMG|6#~Q-8s^?|3OB9{QIVD8na9Sg>e&V0o4H}Bp2qTH5BJk z7p`JWt@Sm=`kcVoXhGrW!KkGG3r(%4&^)TKfU94)!TO}Sx_Tp!vb(G6m&{aatn$m3 z#Sj1xbBREoO=dchoF>B!uaf^P1Pz?I_f;CJTs@&#roaXYDi;xPwbeUVymCO=1sc_?3 z|1L$P!hE#uwZXz|skRr*D^M1eeS^A-pI;NQdWp++u8xilyZeF5fL*VK`FkeJnRDkl zH5>+#wpXXn4k@hLiy9v(|1A=LW-cTU++uB@(&DFcPQ@ftapw# z1acTOU)MJ@G(9}nk9hbp7s-FZA{`7DK=Fgf-%UUYZ$|~FtK8PUP^5kR`s~uBOA%~_ zez?cPqVe(Z$jdu-?kK?66#LVnboBH#yQ}a}+mlDdO_)R|lW|X`Dr>kT37dh>=;-Lx z>({4qrp}!`E2pj=1rB`|>Y2VgL$-O)HLiOh1l{oo)Tl&E>Zz>)hw8}FU|OB=XujIk zRv9n})}ewyz33U#TIBACk`;2q==^^{!An5;AP!bI;Jj3>c}BwwsWg|$H)od)7MJqf z58cX_zn#-`wt(4*9{Iuo;%(Vl+5E0MMo^-WlWBHER8Ou$_$*jveLk<1LUGSOe2PDMpU zM@J_IOy{T`t7n1=`2^niw*R4o0;8SJN-S2hC>H*&wG+GEkBk2)OHh`lKP4hX7B@FH zUg+yVlm2|;o#`+BK1?Uz2sCV0K;rQ5^P~IaWo17jg~!8pL)iOrrj_O=C}|;Sg1SB@ zd@NubQYoV6aB({mv`cKc` zZ#4v(zL*@SyVb$$hJTB&f8A4fgU2=%8P>M`E)!@Gz-Nx?JzLv+-sJOjv; z0VPwP-11W~ z{G+Zxm7hMED7~6!Lz;XF?D9tpQHcgDrSM2!C`54c`0?XN%=Gu8zck63n`1zRK-}Z| z_a{hcgepIOG(-l7LPh{?h#rqrYhRk$Au%UpC}$&(hZHgG6ELnO98-nDc7Uz*;TPDQ z=!Y-m6crWUcgrd&CPBo|QBppE#zMZf9o#$kcT+T=@Kwsz#F+MG&9ub}ArI&^P0h%V zx3|Y;X_O|X#5jqMjErbz!(=>;qVoF{nTPRE1f=EW%2Bv&#Y4-aHwI@?Fk3??&;{g{ z-L!{+59)}2M%dl|aQxJ=GIeOhXr9r6FaB9wU0?6jLJ$r_rDw7HE)(#ax?wc{f#on* zLOFyB2wc5WvF=&(^Je}8yr5`zR=>et-=2!42adN?w(9rz@#BXZP#K`Oj_US@bgZm$ z6z+RD<*O4QAZ+}2TMD~?C`9F*6JQ@NA0OL)Vp9r zHu-C)sbv69F5jMvL_Yf$Z62_0V$@)UvNG;SMfOmB`}XZUdU^?{U7i7cA^M-{! zM;GDV%VUr+XW>nMxpN`d7txT+e|>!fa^QJpaPt8Xk9IJd$6(FSkXD8gMh-Y*jO&J~ zzMkH6pbY zwN6r25&&M_BdnVHR#S6C%-cmN6#q9tO@usljfe<4$AUD&;N%n?3ybVs5ufEF*)sY_ z**VV!$&e4u%OL=$ASajLy1Ow0+Xvv%0a*#qj*5do9#J7gd`Xf(NH75^fV8`+y8(@U zA2kG~i4qbL*2V*6NHhRGJ=9o=fmgzhhv)wp%>S25fpf?ITfOz?Osa^GCxIda@_woa zs0(fb z%1GcS+U-C+o$JALhA^rrXlXHvk?^||EtW3vONHHk^ah`>=hJN=cE{!CVu;-LGgcrQ zG(n`l*w@#GfGwaMBBmxugwBQ=#>hfUeP%{l2H~1F%TzE*DXZ8(U!Yc<4Mu(i1ke-aRASt-lO1xz-Ff0>@1Rw~s0{P{CDY7k{h44+!%c8jHgx_Ty9z!)a!1kJUz zow@9}w3L{RF@FIjcXxNo3IxPsh0E+0BH$sgoa~()Ck}S@>HY#! zh`2x?EYNmDUA1TM+QFc`JMeqNOl6A<&zAah>6awRw{EEUEx z{WIR+1Onji4Tx4-u08BJSvj3!4wVVe1_bM7XJ>7<|L6lg%)mNJK=92{FTMl&3AO+9 zWTa&XSnc)e*FaW`p0;4e|IBu0Yy_Iuo;!W|YphvX5_}qe_!(RfB0RZtsX`5;{|J89 z5>Cs(%ws3;iuPaLxqEm0C-2goDc-JPB zl<;Vj+DpM6$pQ|*grm?{fLvz<2_jTgp+8_j?~VGY=R65^+&D> z!z`{a8>j{t(o)4au-nAWd$$AC z42SEEt-mJ|=fT0jPK%}6mpsGPS;#_&ehig#m=!UqO?MgBQ@EBsJOR#I0$!S>S)NX> zob^k-`W!LSH1u_Mz)ZF)qrwBWol<``H+zw42+}D$<+@UyDV-yjx4X9&r!0-zdVr(7 zcke08T60_H9E2bevnTKg?m_D&lFM>q7TW-h2CpC!Cjg9<#WPX-DMAn+lz zI_=#V$`>#2_aH$)LACxPiiwLA%&Ko-KnIsv`8K0~lJ=T{l6_^%d``_tYI*bl z-p`P)Q1(p!nP`$u0k3ez$!lmtLxA_&MZJ)Dky%`DD2bIp;Lx~|EEigWmv z>QBUo)|(#30;o7`jd=c(sV)NL8tTk=Pg3khn~|256U#v44KaXl3%Yyve#{h&iX414 z=g=y)76w_AHL@igZlWMzfY00~X3vSA}` zYWOK-s>Jg-Z``|kSLoM2_a$TCVw`rST! zMH~)73Q2STvugtmLqK;VHgvHkP|x2sH%o$HO%X6>H~Q2*fi%MqO7#!EYb7{!M&`iN+?6B*FuCCKstiKUArO|bU7ZoLfXtZRKgauZJyZgPXB4bkAPc|wzMieEkC zM9^#k%OlYTT7;aSpa+y&U2Sb~@89Dgzy*7O976D)SzTASh#GbV<$y{IGAd_Lz@!-m z825%cR00|`NQN-w(M-JAx@TspJu5910pJQ4wm@$HLf3+4Crs9X$LAMJ6o9hZBGTeG z%Bcb*Hn9NEymYLnTKY^7KS8RFHV%Lg;sPM0@?ab|6GHIL>E*0!@(SsEcYf_Gu}7cFt*xD}m9s~;Qnc`aLJ zIV$L=uk|Jc*rMaMo@3FxXEsmRjK0B!dC^&U{#RvidddtT6D*jgLX<@{hF8wD1gi zKvB8PMaLWe>N&N!{hf7P# zX=$={b_L*XfN%>R<7#zeU0us5XObf=Iat5^o$J8upcIF#9m>m&8lbus6m>-!7siEK z=`*KKF}*w;aeAlU>kM?Kuzmr+DsQNZT~TASErnw;Y;L%XX7IH;fXYhGaKE^KUO<5X z2c;eXMMfkypb^MXG!UFMimhvDFOh|1GPVU)Az6PUBYP|NOprJJ{k=Xee`vtuf>rb5D>5S@Ljf|oih`|6h0kKA*YSj^)5 zP7)+FQI`sE3BDV?oOR|2i^nOZJEl*&SzZKtk=Tdc8%}6&I%!vZQf2#5&wcggk(2M5 z$2n)3*7Tir_Ac=cxs;~R=uxjOSZc-ir)RCLNjkXi?7D8QxoB>yw-rG;q>OD%fj*PT z=Hh?_muRWOQV$qzVsjTchfo-K0#&8mUT9uXVJ?4?Ir!XK*b^%wqo>Xr^X;H1r8o?^ zDlb*+X4x+dV)R(kT=<#A&YnE(_2iC;rKPg6awxQ=le4mz>}+jSYG>A+G+fO$dKHQt z18&GEVNf>nEl-CB?_VQ6_3gR=i{o#PROssYKiPO`Wf5)V++nO$-Cto{Kr1b#zrfxbnWO>>|B{j-3jOQSv+rJm6%rNb z6jwPZ1H~9Y%6s?t5z?f0a&j^mgvyNib%nXjhIu5lV@|6(tX-Ts5Gmp}=KH=tb!`fU z>o0Z4mP-o~5fK?*ce+O4dCk8bV~*wF;prM2%zAPs-iST@**!4y6q?Rx(6)QSv{jF_ zhe*xfmttPhYm`SF)~t`UC6Y@fyL=-et8|g)ns;pkpF@sH7ibdM2M2D*XQ19?^x2$1 z-B(>RkIb{gEmwQ*p9Dgu&w7I@Pz-W|F*wHV<`U8~1Oi)E`nBi=Ftia89vOhIZ25J*h{@3WSm6;g#CZ;2~&#l1hI|30>&Qp^4DVWu9d z@wLQ09r|!d6R1Hm@8fVd4)cEY^|iIs(o$T%0sYmps@?>}(YcC5DXQz-fek9o1(6tR zM9u@S3^Tbu{QDX)&nzvoD|QE#On{^NN%OLHdL!@B3k#KH9vhA-_$6oYH}R2>|xz+SyZL_{vVwl$VhMHpnkT!7=-_=6Sto(Rgikv%w&Z&v7 zZ6DA}gP&2{6RmJCOMV*1X*h(XeI2Tkld8^S$jE~n+Fj_&69uy<(*@JvdLLf?nEK_5 zP%La?%F|%bGCRYrt=Ih^IVwT*W1okXlpl~PF%Z$?4Cu-I)pjFF!_P+a>JQ<5n`P^7 zaB|_y!uMOGSa$Z^w&9Np|I6}0mQ%AUjuF;Bz~n3tB-3!7t0RC);^@5H?;=@Tp=B>* zg#!D9{<}^-ozlEgt5dDLkZ3tT^-;2*1ygpcYZiai$OR%2*-Bef_v+zT!IDlFe2LkH2sw$ewo&;^UDzBAt7l7Y^L z+ps~cAs~<7tvGNfs|T4+DRnd?wM&Y#i*3I?SWxo}zn0nJQenoKp_6kj?coJTDQDk3 zK8yE8|C9+u#3__u`P|%GCzRdu_P{Jt&+K3^orUJ9wRTc;MoN!=0_0xIjE}!fe?c-y zPXA*ZSgLhO{?^5>(uu_iM9Gmfxz=-EAwEo&2FnqA4`|TqYsnrsrC-8Ci&Q z3g9vOVvlw*hWe6^$d?`}Qi9NHDX!!o1%Wol4{0sB!d%GTF$Y|lxp`zn+f;&XSM0oJ zjebM8>#qQ@lLJvF44I|#Uat?9J0lvHDY?3SqcM>JWJjeJcyAt=vLrb0-dUwxgr4@+v;xf%~pO= z`irOUk@zAh4Sqm9lQT3-ft;Wi;~Oa zDh$aR8ymUYZm$I6MwDHF_Mkiro(y5vcd!@OMklONT_y* z^Ag%3eZpt-jwjI)=X3;G&5!vE-suGxpU{v z+_~?bGvBWs&wlo^pS_+HpY>U5zY$K95xRTzoX2C3UeM<6wYEt*HeNrf07@^yt2W?DS(ZdU9K}nIUr{_ELV>Ji7`@P zcnm@KgsC)b+DrxnsTiGWk6CeYy5$~oEq}#j>_hA^{B8j<3ySvk_Px$|xA6RNRlA|O zB?SqD0lMm)IL1sY1NvlFDe69bG-rM!4A^atnt64ykPjivmGcJek?bwAR%V>-3eNan z$l0uK^^}^n*O-oBGltNyD5bpJ`Nh_dN9)A^`Tz)q%&WfLUfA2lY~$3@DOuz_veD=g?WcZ@fSyW zO0!2MlLL84iiiw!UqoIvkct*a6IM2kxKpHJ0X?aR!b zAbNTntDB0SQuQXb|#s-3U(ms`*+qP`gpZ7cB-9lr-4q zO8Ae~15^uMeFP6N-Q0QlCPY=JqBf0xuU0jveF*Cs^4mn&i3%S?4lXM^xIrs^s$nY6 z_L}75f!2?#=0zxKIZ z<4$Y+07n>&a_|26m$2Y1LJm^+UaJ1id(?`k9z?NxyJ{&O?`qKQT-m5`O?6%S%B2i35pI(`SRx> z9jtD~;O{M0V9c+eb*o11TT{d~j5TdIR!^HTBChb5D|!{-{}tOT_K+&QGX3*VdF+)x zu2E(IzIoNUaA}l{GWeU7=1yF;x2nPup3OZ~r1qH8;bZcskJrf$2So}W_O*7C#|q{v z)@X@0{=UA-xHO^#M_e3qDkF^T>fvepa~^oJk^5yge1w(*P6-5pS+xaL;qqq@@@{b0 zM$P>1p~@ytmqxjB7n@J;AuJk1`LS)z-zN1+Z$S2z8K0u@ty2X=ak&28_GwkPb5auC4>_~eWPy6A+58q!b{Q=IiT_uf|&_szX zy2O_ES7Tbq#Uc!JD$8FxMSjsUYo^VHysfh$TlK{-kz59ShfyXUr#S8FK@EGu*_IH= z{W+r>Q5>c{z(#MvxWeV}I7j0l)d<*+xE5wf$@pi*@M~Z_QHsVv4}t6vE>sVr+rD^+ z7?sPCTwI2a;o0^LvhDHlH*ZcEyY9c{=HaoO_T%Kx)o(qNF+TP+EuqnUq_?jNiN#1x z^YJqFs~?}*0Kdr?f-EF%Q_NOXXzW0bSzNd!^ndM&(tL z`U*v~-W_u5{5USBJC*Hx$L(2%k03d3+pjM=08t%5oFvP_&cy)(lVn*@A}wdj{x$Yr zI79MNWn?;hx0g*(%+dKT8n87SW<#c3-4IquliQ()hSO@XB)WDK?mh6SNM?xhqPSLu zqtk;MJ;j5XnF#-_H&r^PXbz2EBi_&hil?+$TFHjs!QP?bJmo7b_IQS&aU`+&O4X0fd zF>8e!lgb&d-;&tqi3G-Cu+*dNrVmE?Owg ze#!G8VjMtfC0+*b4kQzn2XsE}^U%wB0v^d&Y4-ADV~*;J$QPpJ>K&vtuL3!&)7AS@ z$*-}YU012EvNc9Dzsi!&Q>IrP@dP`X$+emuPhy8H(Fd_FQisCXcuXci1y)KX;W_ap z&hJQeW1DvN!3V^bDtaH(y+HE5t_Qtwpu-mr3eXO zz^-7q;J*+M58TWce#EsIsrG`4Fj|f_WNXsnGRGlIk{u@5kD#s9+XHGa%}}#ijGfCA zuGRxI1!p`7LIucw%rn!YR@W$|3xiNf=v z32QY3m_~B_F-HhuA>cMZ$sT7QRmV!24$0dj7uX8HKY|G$UEabXBHI+l z>l7>Mo}s^X(HAr_Cate*#}h1lsxe)LC<`*c`0bvo=ZyGe$OD?hdf^c%U;zv&13`sg z!Y7D#gCStS_saf~Gm7hScf~0*YCgw~*?Oz(%M*B79v&V@dBpqo?*SwPM5DE2s35bX zU73}L3M<}DnGIN~!w`@lfzt)emHXdgFAbND6c7p0hV%d)0r4^nJ-hxoOCfZ6{q@jN zvmSW_Z~x6)y!X{i?P9%p-M2S$%fl6-0iynkKq1&5K$C#VsaQ|=+juTH4)Xy+1QPT! z3nSz`v?td|T9B&! zj2jxJgSa`aQ%x&Mhuu@He0URq`vAH?I0mm@D>v$_pV4wFPSDDhO+&8NMgKR*c=47f z=%PR8<&GO^yd-S=bhp%>wr8C+or?ze#Is?Fx$)unrw5z4`y_I&=knVw_N6?d^XVTA zZPyEtphju~=?DNkfB9Zf0bMZ+93dE(7zWmyD1m_&yY4>ai<5pMP&bSoxs=I*do@Dv zS!(aVbGo1(5lK19^ZvW;&J zDb;Y`ULGQ%Kq5FSHvGSlOxa}P^0ulciJvgwqVO!>5oUeK>*Q~J#iAIf&&XFujCg9D zWWV?E&{ZWZzq05hoWu-JI}G&_QB6)#=$kz?v!N4f6oppo$guG>0H|jJj*QI+2-x2+ zL-BF=x6T`}5vj1n5DZz-fCKlGr(i=M{pmw5@FdoaDI;5Y6XIzn+;<=})vsG(`QL@X>WBhbc~C%6%S z4i9d~J|b6w;H_ZDWJs(vg^QOwKp}ok3(b@zL7k{fuea)ix;6BOD{;7`b4SQ)#pkl$ zZZ3^{?_A3~dqXxLGa&N+@>n41{W@jQd*1`goseS>SHt<&d{KMFi;`H<*9u{usVJN1J7!<+9~w%By+~z~a$77p&{n6txzvq6tX1V>JgiaB&Mg|b`2HP^+)=C6 zqr;u@*K-r=v4fWyLCJeJ>L7Uq5fLwi=p9ZdXOyJ6(#0Ie@OtriT{ZOnWzzjuKnpny zZKgE1kz~Wb7O9%@_8Vy`$wO_NteFBz;(gQ=o(jc6?km>6X?^-M!*>)iE-OV(C%?^R zBErY&8fGETn%6J{DU zRCjpPYpZN1szmD%2Mt{x%erJLjVCD+?snfXu$@qO2Q#p$_9EhIVeS=E@BX7IuaTw9 z4R$ZVI}p^I^QbSt48q9Vt3*rl*AFM4%!>5XzcDV-Kwas96GT(IlHVTq+CO73_N$r10IW*5@C$VwiN2`I>uCr=R83KgGWnl1wRyJsX;G(CcR;~k|TvB_*!816lLTl&937;3u6=w!9spn`9jPB= ze*9@VK#Y;$AvXf%Rkxv}trIq`Wn8V5lolQ3U-%>Iue2n*B}#upwYl%Ci+k$a=~J*! z0Q(>rtA;p<(|~%R71cv*lK(+;SkXX zRU!lc5Tkpuvas+Ru2OtlhG7PygaD|Zl#LBSNeDw><=XYw!JpiApd)&gLFR~{Bv=NB zHxI@GAYqr{mrjE7o41^4Qry816f-@Coz6yD?7eA`9*=prl0lFvdTtf2bDZ#a zviV%O1$~170D%+}L=FaQstbRGvLn2B4pJcCQyHT)GTV>_ z0e`Wba)110BfyGP!f|u?*+eKprEU4weMYu0_ z9Od$w0%GemhTVFEoxi4J#@N=tn>*Awc0(V8y<~^wqfQ|KHFCp~w+~AEveP|?L$)_p;KLCzrMXWL8i9IiHNh4Ci7S1T5s`rd`l9|X5gZK5#%Jem$qDI+7e zdqCnNC*`uHgv@YhMFxG#N0V~h~y zsAYS@9T+W5Hb!r)OO1EO_BAlL%kMwh5iHS)w0%H;46SVhy2C{BRu%Dd%%LiQr;A#a zQP{omT9WU&6fHuQ$l3}l$D5CYM_Qj&xSmgx;xcPcs4$3QI(Tvg_Yu`{bMZetSSxhG zI3AA?(aPP_L;dAyY#Q{}n2JlVTdB=DR~q5oQHTmsDpydteFYIC_}sqofSu_m;GbN8 zk(^8w+#ATGo>MH0Na5FuAS`(QMIbJxneQH*}U8!PoH*!ki z+++fZCQNhFu)l`ExT^2XJq=8`mANhQCDzh8myxCOnnnv2=e8d=4zz8KoqX-*aWX)k zvaY3MlJ;dzEqvr_chv-Qry2Nlyy___wkc4OV@5(J=(q)i%SiK`*XgI6m?evTsG6Q9 zM1CL7*IgPz_BM<(vdvDdKL=SXtVzOylIsn2J+>xQY*(RJ=+l2|KkLpGJ5Gd7Ccmc+ zS{d>(EtUmPI=)_O;n#KkRMP>x!tK7g%!+p_u*OPo3E~ve{;t*2S_uSTc+AE+ZdddRK#f`}zADfs+rMxC*lxt>)$JjO^_8ilPmjr!bk| z^gZr1F*>-@#}jtH!j0tbrQ8N@tp&AnI$Hwk43Zyl2Xl;ISn){9foDTXyxz<+|Ck0B z+$ocxcEa=Ls4zJo=)(v80Dpx%5hY}1RfHBGT@3jc@N6Jy4C-m^6js-@VaC2rH4}U; zcx=8kt20-sxY};aL5=}57etPM%}MZq;1RSKphavD$l+D87;1yk8Xg$|1j{0MvzoSGLi#K04UPcPM8zufiI;H-J2p==-o>%*hkw4 zR4zyXQEZvs_nJ%=UICXhyW|mh06Z)BK;{darizLRh_c{_GCi| z_^9EL>8OpxE3-P{Mlff-)y((m7j z@Ek*~P(ucV7_|1Mi8b?dpuNHIuXDqncHPjs`#Eo}9vDl8P;Z3vc2Wq3DJs5r-|zy) zHGUA*G;J91e?HHp4e&|!c?L3kNaLqxX11!%4&Bg$2Te~X2Z>K6VtTqCiZl|)#ROau z;PIVh#4pUscYy0D0!F30J z8Ab*12JwaM2B98^Hv)OHF%kOEfY=aUDjALDM9vX#Nj$k6yY>$28W?glwJ2~Gyc@s3 z2EF_F1LR3U&PIMUNhXN>E7Qo{K(^PVCU9>d{2$260zzbgOK1D|JUkJAOCg*#;{=8Z z9HLvM0Z=iC2M%S*Gl=xSc`nAO7+fyC*b|E#8Fw;bo#|hR!XhIHWEc&@oG5O$!qQ#J-&bKEbSoVsoJhp1xjTc=@^ZG#Cf`=1Qd9VfUT-n(?< zT)uOsxl>l&aeH-CP9JqW&c~5u@Mg-Env3OF@l&Xd8)Dq-M?TIx;ld zse=-W-q$W_ag-PY$ktUuzQqc++A@_12uF&f;P_t;jnrfy$NvMf{Qt^OA39eN&-9tB zn_I0bv|=+xYBg4i2e$y}r-_gd3k2<_*A|{2C-qPW>hQaPybc8Bc`F}y#(XFBG8XMf zo(G|&!5eRoQ@>-%|B)I$z^u9DHc_~#mySDTO7U*Xx5x2o`4BSnKn}#{rxA$KJrSha zkdc{%#d9F2l_FGsKs25Of?2U?p-El%`7%J^ueNwZ9!818M6xvS-__A_IUwf`*SJ)c zm`~)O*J`y8IMJJcuCKRudaf15B=$IwObK8{VRk2SkrF^a+`; z&b<3GK?mRN^CjGmeJu#hW1|D-VLsvXB|-2Y{ecbV#Rs`0V`L1-%zY8^NujsNU;h1j z0RGrFIGAcWQnENraa;z)F3boQ35)6KNi7mH0B~JZ4x0$M1^U-5w9w_sGZEXY>_e`N zkX{-2&HxB6VVaKV#Z-WfxZgY}7rh566Un1upd2IM;NZXp8Yl!*zWyz5WNh4vET{2I zYrm3Kwod-S9(3Qpl0PZbBC|v?Ay5k>3(PH4?cqk`JCrb}VECgIJeVa|&%QI4A3ZOe zhnvbf=}04o2no^(9SA2dpUH)-FYDokBA|Go{tO6)+U+B-4kxYHQbvJwkFXq}sId35 zr%ix7JmZc>n-U;8!12+t&IUCdZ+oK5XttpiV+nFHGPkdEK`waS&e{eFiIJowP_`Wb zD<3rj2iAZYStM;SC`6Hh>>oFJrI5f+c+AWn3`GYY+61K5V64K{j1vcZJ7T0fizyk| zPvU`4f46ESj?Zk$Yzm8=MsV_PxhwX3pT_WatP-;^T2XEgQFXIs>qM&SU_YK2#|v%A#w32L|Uc z>41D)XBbKwpX2^ll_vMy*CxG5h6)f)0UsW3!}4m(u^sr6BEnPZViy(#k;6TGU(w^!<23M8N8TBcN9#WqNI` zR;M_KS$z`|jEm22!ZnRf+9F9r)YS0Bz>r8U5^QX2NUiCDs3HtU!_=4*C`cSy zor75r{+&fIiA;g%2uGNj)dxe|L=6y{?7yN=Y|wodd>N!|8L&=Q=O8z33|z_$GzO{e z8EJNvLerA{m$Hg%en5MtRe2E1 zmq^Q51_&>$VT|_wDUO4^y@}BJLjxF;NNFZ2EYsm8c=JdP5};4;{4RR}PA|JT;v%1v z*?xn351aWoP@FC;KxO~n!h%6NG+MKfKR#MJ&Vl^de-c8mex6kEioCAYM4AYI##2P} z{!U?BKczFtty}%TcwntQI#@>P)R6M>5{{!8xa;4Ox!@2^hQEH4gJ>P_W``?mGpBp~ zVP;5eJyPL^Xe^}6_VuIRKR4kV=MyVFnFcSESt_l?WV3JRt_~Ob)0b5O(=l74asV3a zLvE@a)LKg%>>BO6C;DU>$$&#*vO{%K7ukRZi>V<}sR0N1y^BRi+Y_Xz(3vx5DzkFZo>o*_-s z55F{l6GsYcU?>1Af_Z?W**I-EX4!O0wZw$PxB|qb;_-=b`o6rq6sl&{4IH}d;#?O= zAj}Wsx?zCSI#{a+2q&WSMlG9Yz{3Poi_k|j5I9t@b>P2tzem%Lj$DxrZ-Fzw(}ui1 zkT9be5>H_XjGee&?8x)>B6oZy9!lR`VY`g?V|s z^?_iKnK8qr09O=lb+9dxT^!MV(B=sCL~h*S%zfm+AT|;5_=x@H=8{DGb#CbVz^>mx zh71H%*afZGu;P2#$d-;rZEbHaf)cc=gd#62Iqwj}p1?u&Z+{GFLFS%lmoy>(3tdDi zj9eU!|M1ErkmC0s*qfB1oqXu5mz|bJ`}1mpaSn4NO|fvTIvQWZErjDg9Mu!A%)bSC zfGnmslEaFpfBOKM6FL4n&+oq~Ao$UL428%Ja{R}n5gmpc|9dE875*n{r7Z|ik1cPE zROQdWX5a)2Y(Jx9p$cfWJmo%6w}xKgLSZrS!Af4Y0ufgkgi1K_J|J|4yo{Y}*6-RYv zE-!$KV4Kf=eHXlrg*Q0lc{Mc)UwkMjV`aaY*zLDe$I^BNDWIOoD(vV1 z_lQyy60S<>mY$jpvjkykwy%)?GbrZ?z}JB`_2-Dva&x5vZ(PC-SjvBh!s;U3bJ4oh zbPntLJPl9{N^hQu&tDm@R--`wY$~#zml6$}SR0h&9803hDHFUOV$sBNgxpWrSn!4N$+x(0`N#tP3@AHDSqY$p#V^8kolt6mrcWzGm;k5i%m}4LI@H-xCEgPuxP=6R(4f+YV7c|8qG%`M3Wa{hchs z|GVKWnJ1Cv>&J+BLwvEI$C7S=s5Hv(fi-*A_@LzFZPB@-Xc z2Fq&L@Tz%fBlzfX_&y7I=B2 zwCbOdF1zhDASAu4Txf6dg3g4C6uDCh#6!fgcz3kPf81M9qg{U*(cOr?v~@w^e4teM zA25SrJW=uB;GKuSy0^l$NA!Lzqt3v# z1T}c?fC0$xQX}g1B&x?MDueA@KK8ESo8>7cf)fkvP6#>0Q8Pge(3RmAPYk+%5BN-G zoCzqBqz$~)ATI18Lmr}m zgU7+#xhBKe647y&9WlHr$1ul~B1t%{{SC6U1j=V*MjuAVnYE~bO72nzgM=l}y4 z$4cdCX``>jO~C5(vXmEzge1`!=3p}>wW%x(e@Y~CH)xNP{;`wjp~5nIl-{%l3o?B)A`sy zzh&cYB@XXql!!jm{fkjva|B12-Tt>a5?BrwrLX8ixe!vw35B~$fL=o2+_cQ>6c(&^ssqzh7?C)^W+qQ5@4+BJF;ocs&kN&MW0 zS2)l*l~)4V!hjNa8f4l2myfQJMZim!GG3U%dUQQ=Hsgd#9B?Gr+pQ=htZ>xo9BB-f zZ(5*qX_fs9FNBYwuQ4L^9#GAY)vlB*upSl;9%#JGjD;LHa zj=M%Ow?9rRDq#jpqp)o{J$dnqvK%607y$^}4PRqMxI=8DNaQgYZ^Ackcb!zWxLA{T z6-0*gDx!b6d}9ODlbdRf-5yXpsO7!#Gm$+^D^)Tu$$ER0!bN4QB@83MN6{rRN2ZpU z!VzMWcye;MD$mx!>Tw$5(c}}y*<$l$n>p;k3pkzPJ5&tZvl7+T`}Re1afF5`y}M2) z`qQh|x&np%x~NdYoOEQJ+nX}+jC=Q*M)f31@nD6iWJk@w35FnNxX_PxFmhQd-7ip& z@vwyZg2vL@>3OQYwHXlsI(8cYN$JY)g5HC zpr&cd$_qJrOG>pFxvtxpKsru=1jUFXmU~KX?Ydoz)qcIuCZlS(XuyPHLMy{t`Q^=P zJyiA>b?I5YcFfkxrp$W`*bCfm)UBf%I9fZ&@7^kYJ)NR4^2m>MlqmNyH|wW zoCj`98sm@>MSnR_w)1hOr{e~W&$j~9Z|mKvqUB+VN2#q~7Q1KNEMqj=xvi02ND8lt zw~0Fa%5BHBqL&2K+z)(DcRay*(w2sDN0V&ld|GP~-MGA5&M!Z%jt;9icO37zrsanA z(&`)Jq!{{Ae!U!|%AL6&*G#5y1EuB-Iex6y%wF_Yb+ zjJf?+rBq-a7@$1(`!_G5Chu4!`*?hJdrkUCkf*ddyOgS(ZxUkCM@n!%wk9SCN@`1kKMy*J{QOahmmU zhz{H3Dw@|Q?vm9T9N(+ax`5SkDXhk=f1OHnV7359%cfFBa{jV(_9GkTwXvB=R=luS zRfleLr>*niNGBfGo_g9jwA1zWCRbO093pxE_?@;3dY+&%?jfgPOhzOzZn1co(Tvsw zUE-NMR5G1wy~eQ0h!^H+Goq_IuiG{|dQ-e}m0~SHi(W$)Jv5zr;A*&+XX=n$^v9xe z`&3v+CtL4?%C?QB7}b1ddqsvQ>w<1 zGRNOXvKcl}C);RgCe-1MhSgx_w)}6ogVE~RZreC6amRJH9I}@Xo#TdnR97g0e^QTt@`V7b2P4#*Lq?Ny*cf^p|@gur)c#& zyzglcQ_+Dw^dBbEir#wpacJ3?n}ys)w1YoA>~a)|O~facqQ?YEis@xFB8^e@iyj(8 z!B0|o!#y7dhfVH%@$Ttfswxw_SWZ%Wd*r&+9X3_|=CkRI>!_hgjUfs|)Hx z?s30c`TL)aHl{q0))QuC&u<)YRaJ~~vB`MO&sAtnejI;Y-X=xp-k4i?2-@;G_oh(W zO%uKp1>*q&D4u<+1r7wvC!p*#N>={Kd|*#6h1_XkQu#D5>Aq#Zy+4U6noV z`h+5*R>s8&_w3KKO`W)u=}*BZn}>Z@Gy-SF>;Z20NaC6z`s^`I?vkK@T*QlZ?b(ZC zsKxOuzsE$o{S-Y%q{q}8&7S=SVmoayc_>xfJ6cH&%~9{1dUQ8qx;I$cyW(kQPGqJ# z)kpGW|BO}I>b1x1A~jhV`5&W$4%CB(&6DR`q&ySXoQy7RnFQUZU97uH7#H8y=5inw zA-&M3qck)wLG_(+Kx|-)yChY)JsKY?FHgc+BsN%s(n&z!(!5X3`jm6^;P~_0iJkz1 z)fN&yM!qMEmoF_?j=0eQiXC(Je^+4=2@8p`vK3 z$V83p*tFJiU4`cIlQJwH>1WTsCT&-7yI34W5<66=*&ckmv)VkCUQ|?R1BE&?rf9p_ z-!ucCuc!8pj`O-?>zc=PkPjuYk6g4_N*%lxS7#Uz#3*aCK|Pv~nL6y>Vf!YWrF4Cj zzo9^vy$WXlEN{A)5 z{gHhT+0uPhSz+sG&C#ZRz+#OP2Q3ZrV-J@YE4O1FS_V4~ zGN}CYJ_S4PS(~Xdy2tz2>>103bcqn!r>*c#;_{c?vmR(- zXJA&vP8r@7x;LReHgjn7(%}168w15^&JoGZgmJKnb4h;4O*txxIYSc@HcAqtB5NJc zgquhwMZw_?&fLUi-BQV&H83`2g>Gp&>W0Rw!#Os7j1;5`G0Jwy z?FPn`I$LOy(SGgShp~i-2Rgak6C=7?J0feof7-ReT6N3RbJCKj1{ij%Epk$5jmg4( z^5N)5)o$w48U8%XG0b{z{arkg``DOM#qWK$>4An_?P2srtr7JT2G>EWd%0u{1XH!% zB}-paw9t~fVkpCKwXcLmHN?U*n<@NEV+ra?UL|9{j&#pX=1v$YySPra&}QJJ_8LMJ ziEnK*Dr_q6+`bHo?85m=+q+&@h(Aza-jA{L$A$F&=oH0Y56EUR_>)|k@XfsX>(Z6C zs#kwBMX&`cW6ZdWQ_FGQKScV^K<2iA@mR3@yygQ=PC0B%Y)3qOQ%{*-ewptp9UO`E zd=K3X88SC<1Iw1ANU5erRQ!fRu@fKs+&5%kMft~?6G>xhrm22s{H;oR@xG~APZv*M z8zFiI<6kT0JHG!YoKFXnVtz+uWqwj!Qegn(&;`GP!)?CN#Ox1_`q7B;%5?3-=ZgAV zx&n!VxCh1`4r|}_n#F8Ad?=**y?IGBvE(3WDld$9zodCt0$o5zil;P3d`C5YUx%1_ z#&3eX>{2twrahizWh18beBH#)%ZbU~%2@ijpP=sP_guD~N!;-f9TW7|9^ha*z?_hJy-JuxCz?eVVq(i+gxpUWOr8#lU zI3@KH?(CdiMqO)NV^C1+fp2x7>ArAH`|%?>3cq;utRHb3U%NwtUMdHj>1<_r`OC(L z)%$xGj^-775?j7E-&y8=oEv&>9yMj9h>jK^Y>=5}QIyMBJ|+_%+QT~%@VR&U{M92o*V*( zMv2B5#vt~I$qB;`noCKVc2C}W2|##z7(`U{lnh$JeB1glhQIbl2dy( z{Yr+(804~;UPZUXse3f|reVJHDex@5@0L}*y44v!s6al`i5g1V!OgPS=Ulund*RKC zHgP_wgWgvD@^41m4406GN$WUne{dc%My6DpNDV7Hdj?(pUd&DW`KL^YIVL)4#u2^x znVJn|+jxD!Bt%Uk*xNUY`9?%p#Fi?)hds=ai_@<$yoFokOZ;cT?9|!#u&3>OS>}!JRPZ^{wH2LB7E}6>Br(tY384R_o=^=A+vh zx(Kl5g^CsA^^viV!%I#|TA z-naVLbG^y&oP_*%cbF{;zk{>GzBQwpZp-Eb+GKft7?&HP~m_d`}X{Z3H+^AgY9W$^t$RJ*z=a^ zZ++|@su|?wA68pVzu(8jyoG z7bz+Nu)*i%+GirUi|-`U1h1y8pZ*qZ;A_rL6#H8-hR^bula)O!!zv0X5f&S%(I|KF zww?KA$X;sS$}@&v<-+asDqz5bt}8uGGcNa4fS7FkMDSjxDis5T3wppdz>VSBvSaq; z^*c}5Ha>s0Au7pF{p2H87w#Qo6y}L?ovwB*E)JD0r@o@%gC=9U3_H_rb`2p-{r>&i z;b?0lwVG?a#7$cF_eaC1XNmBH;WuF`I++enwQa1 z%+kVwjI?wB_o!L_pnQv_Z2YY#j)juSlFYql$sO!kvT^!JFY2|mQacpLWrt0di$*l> zi2|iQGK5YM=jkIyG`l7Z=O1Z9ZL>ICJ3jo0NQYZbJ8#*KCgeKL(sfIHKRqc&;h?Z3 z`zG1Tnu*I_+~j8K1TZh)Rd2en{A1~Ehg0(HPlmUXMSNURaJTJ~NrDE%T)31J;}%nJ zb36CaB}H~JS>iF<5|_)lYwcyX0~H$?J)4xfn~9nat1N5(eCZ69jC18GVxefUJFDrg zJa5`hWV+)X!TR*Q7}`LXy5f|FF~?kuVcZmS0Ug6#f?R0vYK=)bi2j!uUcQqL+HlIBFwEd&we@g`}gmL zg-g()s0-eKg)~mP_0Z}SNBOOn&ZuZ?>i*=9G^j9fsP0+PWgF0n_NZ_Dt8y(icY&36 z{OFb~MYHVe?W?8+x5i^+HBiZh0*F3 zR$}H5pIRxe(BU~hRc z^1C`=}-v`OVs+w|sy|-R1(C7=Zt-_c2Y?rb7B6>`pG*e>A0g{GoQ4PEyF9$m8;LY^_B z>$(rO#iTp*gg;GWb!S7fHY0!DWT2^}{fiiH4XVQz7%OM+b?zLd*5$g zO*^C*9@@dfsNN%Bz`;&Z$TSPvGAg!7a z;B?S@{A>9FUc)6_CkO4k>FlYGnIg*!wuXX8XFOrAMeRtPjNm| zVJWw9%YS7q`(r+K6(@#BDmuTnR7J$ZJHndjD>yKwmNVu7ls+p20gG|ot5%i9GZtw6 zw*_-NHow-yL--o36ZiU>wk4)mr`wBbRjf6I42Oc0w$yu#gJ@sgr&GCgabHEdt#rg+ zu;$rjAm_Z6qoK~-R6*$^uN@oKnbk{J%q>altM}^uY2@zUufHUI|KigBXy~F>pIoua zU9ERy-d$91sWVi@?C3`|u5;2*=&x&BRY5E@bPApdD@EQGupTvWs5iXoJQ$Gj{@U_W zReIeZVRj=+313Qid8bD6A@@#)moyohYI8iNW&9gvYsVTte*PNhb0~5t4P3LN`%N!w zNMIJWB)0yC?$%pL(yBPV8_^$isS+1W^5$=P5)jSXIP0}?W~0JdEy~qd&EKi~4^E%Z zC*$jbpI#7ucekAV(h7ZbnV`nOw*ou0d*J4r=X3M1Jp)JFS;yDi$`|)pCO7u|2@^}` zHbxkJYYAh?89gXGz@&?@w2-EdKh3Uf*X2^nktl!k0Bd)T7=T=t7&cQ+2^X+*|g65X+?wsC_g|$@{^8 zElqv(#Z^x1_wRiZj(mhac0Lbm-X{=V`zgxbXDpA#HVrl}bG+|*_3(XqsIN0xG%-25 z-9j-;@=~W61M6vV2PKW>Bx~KJ!KYXHI0Y!1^L+8Iu2~4p`+nGts_}ef8SC+xRY7IC z?Ay*mu93qSB5M8{6|b%CUNN`+rdShcbYaG4t>Nae@%|6f6F=>&?i{!)3@iMht_?4? zHTWmW7VJNH`3CJbyY5dKT)@P&v^hts%ca`AXSgDI;*tGnxZJt)g)@?q^9+MOi3cgg zvGe+k)>$^6U2AKBl?oXN?$F?{RhX8o>ykg?9 zKVm79B0r)^4{~vA_?VOT+nY>A2AE3g)JH39#o;wEho=oiC@S9h%921YRl1{GbBXFU zCL7vsoz_KLnOm`HMzuR?PzGwejl)4pWthP+xZc8TA@=>cNT0)MDXG6&!ge~1*r%lW zySCfp=W9%LQaY-JavN3$*~m&Ab#8tuf*XqT1E}D~#Ou}U*FwjRwlOWyytLzoN#w*_ ziI;#}Lhrrbn_Z$Pw z9Z`6vo0zn89QDGldv`?B;V+wb@syMud8S`vdwVdYk0qwbF8L#~)DjhwHJm00QKn26 z9?8P#|LtcNc?J4t@T87b;tC5>7eDOlS0)|yXjL&2nLPL5&Rl?b?KevCkV*;i`SJ;m-bSz5U4!@QGJiW< zUkwX=m7Xdh>*Z~4(xsHWKfL)-QkCy#ira(Sb&}!XjHMFn*<>k=`akfp!|a(1%)ODV z;N_5=L|U;ZBbUw1Zl}en4Cj=~H+Ubd9@IHkjPo-Kpf}1wWF#(NY4hP}+BGk7rpgVP zKPVy`ej(mJ_5GA7_fC>kYevz_vD_KbvO15n+d3#M?5M5Dl+(69?uv|&gv_0Fu^}yA z6~6lVluqL#U-R0ct!D*4G}ptA(tn~#D~0jx{U%5UoV}zDUDA8>_ohEeyQp{G;7fd) z@!6uC)3_t^0Rzuo%jTzme`{;&kT;(7BV$QnE=Dhls}5akCyw>SM9-%;vSVpqCEeOu zwB7Y_q#Y#BOAG9=j!E%nmm?7x}k->p$HW#iG?g!>6 zG#MKg@6bjdBm(*rSE8^Ss9@}bGWMvMTBZXYv zY6*V3rV6SLFp|EDnM%VIqn~%;TRt1dqqyDvc(f0f8#OK17p0YCYB<=e6{OJvi}KKg z9^=VLF_L~uwL&(egs6rhj3NOIZ3zqfJ)|n;qwE>-_t%w~JT+)&v z+8O=!8OpFLswwrgh3AjET6g_P8L^oBrb;Or?>}JYl}ooCl9BbVabeMl+8unBG1zU+ zL04#V#5l8b*~Rryp1iWl`*>VJ-?pAG_CK%w46C+=Z4zI@(vcMm9mQZs@n7_pg`^aj zK4zK`{{|1r-660hSFE4kSn08@SG2BMLQ9rbngMy#zD*oY(rC9HS+h`a#?PCYBl30X z>tgYuSFz|HUidg##`ttb{1tP?RjjtCliiIUMY)~!dNauAR&;8>??>6vzpZ?$N*~VT z{J35Uv=uD(eZV5?s{ZR1Gut`;-eo@6$-e#>k4DrJ4Zq|_9c`%2z{0vJ3`G>*8_!8h znDxvjExkq)44DRUw-g(;^0DtmTz6S7iW}&#t(0eMR+PADrmxT^dKSOkvr>9#xhX!&Hbt5>m^ICO|u91!c*A}L|}Vk zeR*4l>z-_yCpy4h^7%;m?P9u;%H!%5N~$onh4$&8tI~oxs-YVa>Ar#SSyhi%?U3dh zx1XNX#G*I7pklSVYs~ddVOmkv^lZ02@q5bj^74l*&^sJU-~lY9^2W7m7f!loFySOZ zsv0@M-+`oH(F;aIM086*wi-)N!0#Va)SxT>Ai*+S0CfH2`2YP^%}$9iu$n9DcAU%Q zh1s&sRU}SfN^V(IovRAoB$dD&#=83f0>^mU<*fw9^vO1yemjU!K36qwY4+4qa%M8_ zCdKm9NYh*$-4+~lNmo%azl5W~kwq{Z8Ithrx&7Rtd~fDpQW?pBTE6B8t1NpZJzG9m z_)K;tV`fs;=r`%S2dnGVV%ik?GI*-EecEUh-h9K-B`TGRbX!NyCQO=g6^Bb+ zDTJ&IzTdFUsI(G(nC>&_@7nPwgtC5tC87ohsGI}?HUJp z(K{(tzKgSUlpj@exmA=?F$%no!_+o$i#yP24yhx}YrKw$z1jZpmCEebmQ@R8iD42x zjVujJz6SSCyVw4<7AblWrV$uCZIO|~)|w=}u{8Qw+AwU4hDL>eAvgWOy_pJfpGf;W zX`)#BAByTEHV!hjQM5ZLo+XUABIbu%Ufk=0=Q6C&jD>_D%b9NbW#rE!FIASb9)_GL zd@WzD@8fMuEh4qis=;l2o3*IgMyuy?r{ZF!Zb;;ZFqPSr+W4%ehBRR>u{O=2x7PF; z(WY+E+AjFKfk|y`U9MsJ!xTXHI_xe-<<;2?P>|LRl?=7l4EZbVObN3Gt5AV98X;8B zBxPIQB=p~VN2yDokd~Q4xvzX!%3X3kFPJ~ns^?2sM$ko;2C@cM?hKNAku%~ z_vmLmy5{}++Z$8+(>{0VawzbB?y zi|n1Xnh!R|D54Z1zbN3hF>CzMjoOK+xN(s4tJmhR(MWQ<_d~L}KWy$i3sBXB=grZE zDyBQ#>4KQawi*??Z`?O8&~mdXsx@Sf1`uVs093Ejy7Z1GRarLoR;7O$W0p%csY_T! zab`nj{X*!OLf^AmE!Nat5*a(@1NDV532U^rxv>n9v|9y5t-u1pBBVaeSogLo=a=L; z(&Y<@RkLM+%Nmq#f3eP_F_1b;$fb+mN=r|yFUn4pEPwXczLQ(y+8f$!Z%(&YCfcil zHCxB_k^A{PKB+Sel+^i0<5oMiVB8`Z3~%oVw!rw+)cAbs0@U-;&?}_CRclt2Na9_Yf#0~ zaW6Hc=l3e0Gl$Vy?3$E8kWG9J=d(Y_B2**fed#+9y+NNWmESXSe34-iuOb88ul?A; zyy!X>JxbG7lRNidUV3q#^H`oggD3u}Xl#cb?{SP#^_Iux(&~Z?1i9=Y0WLc}+aug8nS^lq=cTDtqIyZZ|YoRvb^6BufRw`%2F?%gXYzn3#J# z36T`yjtsdmcb`GFwvr}!r9Z&fEeEf%^$elEO^MT*Kn%;1^bagT;^FD7<>kxj+I6i7iyJ_6mpMZ(7Gi(f@o=xsx zI4n#0DCKZi801b_H70+)QsW1hv)tMj_>c!-+%9Myz`f1?)$#nb1mm}ouBhO zKOg12(_^aB43`9npjp#v;Nx${h}f2Qgwg(j9WizHL>0ZJ!s%6qen)5ilKqecK6HjPAM6kvx7SA3SwCubf1WaTqN^CqL*Qb=-IS5oM>e z6sA;Kw63~mK^J4WUsOr%;O9}PAg#vGs`dyo3eJgLJYOzcf7C}tJP_|q+L|_GtA1T$ zGjXz{BD63&Nwy~ z>ZwvfngE?~Gn!lsM4rCMVv|bD;e5$SnYRhlvQ_|8mg78e(vsrupvR(!H zo9pkB45VVHdSxS67&GzvZqp*qwlnlxUQgrsHan&MIm#MV?(At8+(;h##~>7E3PTSR z2XPr2hCrKh44dY;Bn-2b3wp}=T9Ie@y;l&2qh!>RhSRBc+3ZtboFAS8n+~vo&p&kk z>zVuiiDiU6mDWJWb9@^!J1<+KKY{ViOyR39I&mmyqvb>H{r@+4py^El;zThuNmixP z;FB0E9HVI)J}0RA_ehztl+!rQxy^x*MrXX0-U6N?QhD*1Qi`3Yg6XfmBs4}0$=I`W~I5$zm%r(cUvNaSF4ozmutT2R|YyC z%i_-Zkj#42-y`_VNN=+X!`XZCq1p!}xm0sVS}8oY|HskccarNG>a%6V%LBerS*7zN zStb^6JZR+dNC)|>d3geU_ERpubK? z(&pmY;S^CYON7PcI-iRVlG^msgO_pbdy91e4Fjxw;2X^ajxQC%tM+*9m4*Jqj3CH0g>n=cvsl$f3dD)I!+&ahflPY!ar;S7q@U-@X*akP3)Ii!-nERM!4HWQWg>+O zog9`(wP3J*aOqZtch;DhAB~J3_#(xU#O3_Fz3j zZn*!_R6T~4-6bIF$8HiIdnrcjY*vQ^uyLMfv8 z4^r+Zbi`75n6Jm&67vLiB+)}dZ@=loT9YiY1bifWuvFHMFwsq!+Xqne_-N_{%$vRU zJ=eYJVqeP%gT6S@svhy_a*5vmuGwY_r9rhwIl5oH6^>#(=L1k*U-BjhrAXdpIkD3Ye$OJ z4-z=tV*7DR!#Tf4e;c=pC}%Iu+u<6vIy0DKniM$yx{*yzQwYj_o)hAcSM7OKV09$F zEc;f$3;CN9&66Vhp;I2T!9*Vk8Lw0`lh%Fne@f%9Ok~_RZd_#Ijlr)0&0fJ*jSyg# zTY)JbqbcVbGzw)s(3Qph`jckur!RcH@?1IENDaHBPz_s33GP}GPDXEc<@oue%D)`Y z=4Rd17IS0_<$kDZIrq-D>@nbXX0z3;OcR^~to^TvDRwrDnpuRs+MKa2;HdvgdmgBd z1f}>~Q#@?!zX5NW*wj&_WF31hTtXCWx-RbHaig<+eV+XsTEc1E-y~j*+~w|0P7iJe z3`ipZMW(&2t^Dy^0e!h0Z?IV)U+CDV0yjp*`q^c^4|C;#%0uh2%~U&HX(_KMv>zFR zd)A@6G({EQH*5WIgJKJa5?||8$*0nY?IjuFr7k}Ecz@f!B=c?j( zD*`pAz_=UcpSH?hW0(A8As9e^o&_q8hHtpI-`mt_XCJB#GYV5!8QFlp4Bbwzr#31N zNW`6Z`0MngAkohJa>K~G!D>QPR-;Y7^NzL*Ux&7jv6Xi(Zp>S;i#`fkN)5uD8{f*> zRE>!t>ic>LH{SoGa|l_ak#WQFOkZl^zBShR4H&e} zVKSlGZjBjK(+CQ?-p+-jY(Lj<7|<6$xQi~8Y&tfHizD@aWvCzpA8#n@a0>l*OzrMO zsLM!oOzo}6XU2-`Y;I;1?nKT}jGg(-o4To%uL>5~Aiu*Rg`JDLXMd!EO&V`1Ci|W; zewC@Y2hC$W{yBIAvWz9~Hsi)UlxB!`$cc9u-sYVJkOB_%yuGjw)_vq*Btv<~$nT3m zhP5k4m|-1%cj9}a1mO|bl5&V{Y&cNL6s{iWmySM)L_RMBwo z!0&fyginr-dtLGQ5KT3|RF|`Yf1-Nen7Fv3v;EM3Pu4RvrW=o z9%)DY-R1fH33lcI+O^UcxeC|ig-uxBo*UFSN(`mhFm653d7|F8Sr5PE<92OL{x>ds z1%-TDm~`Jw8Hqidl?wH)u}IPkEUfu>?HMRjzhFzfkO9qUpvQ-Os<0R7E^8bwNipBI zw6eYZ$5Jp4(7xOEi##b?S*vcuW^!8SVWIOxV6(NI+(PUFji%QW77JCe6sP4sYYs(7 zsrLg<^9rr>3*7I`Pz$SeQv|MFK;KSGsf5Qs9$H8BTmg4PN6U@$7SBKlaagx|p4o5) zc0Sd$s$er)0?K9Uk{oY?jFNklLk)n#XB1us?8A((5x*+e9@DWZ&!oWkqp%X?>wEnE zmEY89cjq(JB?@xCq?2>B@a?NmAQig!69+ zT!B9xrg8~lRqWCNw)*8}^T$N(;v%GyQW5+(4?3_!!mKbEKR2t+U>gb=vE?5JjZ!j3 zW31Ul?dxc5t-4wPD1o2`clY$^3aMx!&TRWBUwB5}jikCuCbhM0!?XVwO7uTz(75|W zT;=#~fuJLhd)L)Nu^v+EcYkNNvVWA2JP$->Z59(BBsP4JF3uGQdd0UTKlI(B&Bqt> zhaCpEx(x~^gLt!OpEB3cHFS}CtR%2~A*@{x3Quf0<+ndB`gQS5!2j&4Alv9u*+%cuFoHu4u?SKmnf zDcUH`NGg1Kf9rjg>hyb)?_w`@W-NmLsn!x>A8V|*y95(g|@ zq3`JymM}J6Lc)%}nVPD+N7f6IR9AcmZ^-m!{8|dY+~F;t$uPK*h}&289;9`zoEVOj zrd??V83SSP^7ZYPf8>X^Lq7cr%Ia_z;d#%gE^+>t2JXz5y#6v=h=u#$@_`>tKL)HYH@E!>^8BCEgo z&GyLK&;op?(uzBn1ze(_%I!0yr|(miH2KdAuqFET2b>bUI8Z&yK@n z`>@ZuK9shZo5;y3&Y3)%oy2&+4jti+CS>>&;sWkLQmfBdgFF8CQnSq4Zmtxd4t%l~ zpHG}B{}ky~q2~oAu5Vo}9}x~fdJs@y?i2+v+LW6nCSy4e?SR1KS{-?nm#rEfefgR1 zsXM}|(hFQa<0JVIY4Dspoq_DX17ONIovj9|5for}0Z{E50IFT$sbBfU-3xI01TbQ3 zvQR}WuZ|S?%^2}Rhd)XibtQTSUK`A1RXX=g{3Tz2m%Cafm;Vf}Eb|C} zoafq~#dsF(@8vZUn`}B*G+k9?OLxlbJMScQH2!!F-<|5^%sf;w(5)MuKwT6rcq3nU zal2`+VRwyVPZlr!_rx8V$l}(BkVO=;65S#Pa1t|`MQunWgzC{VwNX;50HE9~?bR3( zRifXWhIrOmDLLz$CMR$7wEWj68<^zNq^MSU(6>3&`$T?Qbu8NU7|<(oB!KOqZNjku zuJ}Uyo%)17)}})AGJWO~AkYNl{}_v1GNSB;v&ELVj~Y^TWP8rYQ`0N#9cgven{>^3 z>)1vILj_)t?2wqo`f&zO-&rBNxH43M1LC9E>Dg-yYhO-GuM!9Zp>haN9d7LP?(PM> zN=_y~EqRnF9ks_ za34UfK40mx|1c952Wef{WlZeB1Q`qZfTq?}x7?Wa&Dq!s~fpUThL+nEOn zQIo>~y=r|`@w;^Z9TRn>qK6jnoZg!dZV8ACr*AYeOJ2DFPX=(IWYyO`AA$dj=3-K^ z>j6&$uQjtWnMMAvZ~>a!Lg(!#fgyR&zYpBI`~+kadYL3Cu*Ogu6B84r`ZiPDYHP$t zU)61zN#2gJsnAiXQeUM8asY0r0&zXtCeu7Ed~Zy0nyg9H}W7Y zVJU@^Ezbil1HUvfFMrW2)${f1T5{_>=x$c{&4eA-G>u)e7{v1y(C4Q*2Yq@5OUhA_ z-Z6U}#Zq*fCkl*`a!o^wN2CG6);J!hJZ8b_;roIQ{>v|-w4_p)(03AajAK8lb8#5g zvjjkY#AzjhsH;y-`QiIl91?~jc8j{hwvl*=DA79>v}#e@j_f}S`U1GNz*En0 zMB5r^r-9EH?wS`6V~gH(J*r~~)O7Mori9msc}bv8Zr?Fb3ful@f4-cYl39Rza(jbD1NTdSp)#~JCS5flLv^WF;5D?%CVEB3dz4F2P1W58M5WWKj`hvn^>b&pK?n+Nm zmav|XBB-nqK=6Sp_d2V-li$vX6foj;ul((HC^GL~17cARr;h#s`w7zet-k7C0-oq^ zP#WH+2tB2pxj_pIgM2nF_7ybF2(N4QR&1i-yiXckow9mJZVN%8&fFZy1-`9wBvviA zn5S3Xqr%53sP_#`Hji&SW+iT~`uy>%w7>fHOpns`$mhU9N7u??tUWDFt%eu+v@=u# z&Fb)aHAdA}Y72DkU&`9?P44FVh7^Vs!OhcNdhBe$=2nQkCBklHHjt8`546Z0X+P@n z!+c0~16ifqHDQIyFC>^ONsVt$mULh6*`~kFpz2pDkTehidr3+5eeabLTpoLg2=~=75zdE~mmDnNt4+=FuIbQ>u`M<5PC+{1W>c197x5H#YI>Kiu!+EgU zT6~pyL1Qnun}1l5#~*A}TWsByMurm>ptl-tAuKJ*b=p(a5xJY$3i2_-mM&}vz#}Gd z$IwRHfyyAw6+4!;9K3qk1K|8zwU7&E)A4C^TQH0Y^Zz5!^eFwP9b|!GUk86c>_9ni z`!tH*UwFZ%+=0jqSdXXu<+S^6uWAQ1_y!G%bxEV?sq&nnSZQ$PziK7LnulK5ztZBaYIDPlY8|oG_bn|wR6SD; z7Z_S(xE%ieCisYj*b&g>gQ;3}3iEdukuqoKHStCj9hcZe>0eJrir0jU1$*x>!nRAd zZ4mF<_eSPLMM;=m#v(V~AZw7MyEf&;#zFV2eU#QVCB|>!HLdtS*o_KrQjt0oQ(!+8 z+~B-Z2iyAtVW9HV9WK5c#VLQRwv~=Ag{=-iSQ!HY;dQt8qbwrMgLHZRHFovZRClNT z2a{Gmpkdd|LkMo?R_l#;1J6SK#?2(F5+C32NF}pUH&CB3qQawZH3YO0>Fu}`-p>Yu zr9PI(Z!Q(+Z?}ucjbX+1yR*7xE=iT4F+QhXv~qYMoHxTvI@rl-{0uxr3pNzFb>5Czk-$;Otu^qe>9@n$yWIhkbjeK($%B5%| z0F1tBVPQ*@`rLdPK_&2?tUlCOzDf0AsLByN+47mx#p>U}eK~?%ajOSOMzLAo{5nufH`#JHuo9iiV<5@H-3A;lq&z z2L@30NkIjaT5uJbUI*h1RDN1&VdF`N3GiT24Pk6*OnE8el6Kxt@c81hgVCj5QP>x_ z^K4aFw;kZeTC|L8=^p~7S2MPJ(8T46A0ax{?e|nhC`8sMvQwEwLBa~59LkHdvoGaH zhAvNiwY7PQQ56HL&3;UO;U$1N^ck5lNmO@hFSJGavR7E+t z!5b|sZWnOGt9+g{CLmp6uwJh{m9{hIk~DzEbZdwon~{|Hkvht`#0l!;`1h?I2sB=7 zk#$*zTliEvn%y50=hqC?3LfG%1fLWe1yas*TxoXKu1lLiOyUNl>WtiiqzJDOJRRnR zF{Pr;Vl1AdAk(_*tAdz-w^)>8y15lzl5pw_mPS)EznRLOp6tMtLdS55|*W-0_X@ZtGI5L})Bx zH>vbh^(CBBjl_0sZ`HH~GU#`F3_L?(llu~#7}0qBrS1`}-{ZVwrileTM>HErH2PFX zgyhD2P;rjEFPT$Ua?0i1MoGjDh5tJ9f+VaT53eV_cVG}SQ{T>F}s8FYEGbYOH)Cpp2 zpv!>E2>jzR-KyP|33o}Cb*c=%IUT2HXv}x~aVvl~EHR~M8E`LS)7wuWW-f!Sxy!Hd z_WV1AMy#5(4tu|HDrn1NTXE48Qy9gY^vYV=!{etzQxD~e%)rOk>r0Oo@Zb^QV zPs_d2iLe|+Q&DjD?Ym1oma|G~)h4A+^qh;ps@6k#hZU3qVLLQOadmNrv95y6T7LR0 z5@|s(AA4d9Hs46h7`_{1Xozaq5@A3ca8s;x0ZZTJE1O2e#q~>{s5p1$$DIW|iv!{a zz#&CMB4;xff0K`1>$`WI-%VgqS45m%Xn*e3?p5KDnZky`t+o@s!(JL2ewHC0O>K@N z%ztg(ca1JNM6aG13|Kb=);uqO8aYmx=@_LQjp1pJ67n%BzawSd081l{Xb|{L1+-@2pd>e+j6fi z0&T?Z`)r@j%+~tb)~219>3&p5UJmA=H01Lt`MCd{D~nMq`I)9f-P@%)Zoc(nC)SCi zjpenqa2{f3+d|W6t9hFC7!FYp8k;hCahV_`MG|D`6dW@fBa{5^-?3#pp&Eh>FO0!z z#%zn3dk|N3OvstCf5L&OI86x87O{B9%0>*MAjZ>-3u%f-QhJ zylq2xgZY^_fJ?~n;J_#HjbMwG8hJtFG_Oe00FcFu?v+PNT{$)zNxGot$ANeS(Bmk+ z1XFdW+As-Er+(m;c=A|v4;%hN$k}J;28Y{`$ibWY2kSder+frMf*jf*iXGEz<$X?B zsSv?lR9#T9PBd+U)nZOA8gz>H;BX_U?2sWfo;{YtAMPh*mDZ<$$lW%gMTB9C#+7lS z=E9@|3FNyQd*UpSU;o{_h1rUBziH177w6jx4W(iW*u%lm%8S}6-&jD;fP6LPnnD5{ zIK+qusG3*8iGaT+FGqPOEoIUv)=M)#qd_2MejIQcaPmE)av&H1`roXw1N#f4ApU>n c*LCc1u!>C2e&sb3Wp*z3mZ@&(O{b^-1!u=z8vp