From 4c6341b03c25dfe5bb8e690bf78f301cab725395 Mon Sep 17 00:00:00 2001 From: Dmitri Soshnikov Date: Mon, 27 Sep 2021 11:39:54 +0300 Subject: [PATCH] Add perceptron section --- .../03-Perceptron/Perceptron.ipynb | 1085 +++++++ 3-NeuralNetworks/03-Perceptron/README.md | 65 + .../images/Mark_I_perceptron_wikipedia.jpg | Bin 0 -> 34143 bytes .../images/Rosenblatt-wikipedia.jpg | Bin 0 -> 36243 bytes .../04-OwnFramework/OwnFramework.ipynb | 2809 +++++++++++++++++ .../05-Frameworks/IntroKerasTF.ipynb | 1433 +++++++++ .../05-Frameworks/IntroPyTorch.ipynb | 1726 ++++++++++ 3-NeuralNetworks/README.md | 44 + 3-NeuralNetworks/images/artneuron.png | Bin 0 -> 8166 bytes 3-NeuralNetworks/images/synapse-wikipedia.jpg | Bin 0 -> 48364 bytes README.md | 19 +- 11 files changed, 7173 insertions(+), 8 deletions(-) create mode 100644 3-NeuralNetworks/03-Perceptron/Perceptron.ipynb create mode 100644 3-NeuralNetworks/03-Perceptron/README.md create mode 100644 3-NeuralNetworks/03-Perceptron/images/Mark_I_perceptron_wikipedia.jpg create mode 100644 3-NeuralNetworks/03-Perceptron/images/Rosenblatt-wikipedia.jpg create mode 100644 3-NeuralNetworks/04-OwnFramework/OwnFramework.ipynb create mode 100644 3-NeuralNetworks/05-Frameworks/IntroKerasTF.ipynb create mode 100644 3-NeuralNetworks/05-Frameworks/IntroPyTorch.ipynb create mode 100644 3-NeuralNetworks/README.md create mode 100644 3-NeuralNetworks/images/artneuron.png create mode 100644 3-NeuralNetworks/images/synapse-wikipedia.jpg diff --git a/3-NeuralNetworks/03-Perceptron/Perceptron.ipynb b/3-NeuralNetworks/03-Perceptron/Perceptron.ipynb new file mode 100644 index 00000000..8b4aad8e --- /dev/null +++ b/3-NeuralNetworks/03-Perceptron/Perceptron.ipynb @@ -0,0 +1,1085 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "## Perceptron\r\n", + "\r\n", + "> This notebook is a part of [AI for Beginners Curricula](http://github.com/microsoft/ai-for-beginners). Visit the repository for complete set of learning materials.\r\n", + "\r\n", + "As we have discussed, perceptron allows you to solve **binary classification problem**, i.e. to classify input examples into two classes - we can call them **positive** and **negative**.\r\n", + "\r\n", + "First, let's import some required libraries." + ], + "metadata": { + "collapsed": true, + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 2, + "source": [ + "import pylab\r\n", + "from matplotlib import gridspec\r\n", + "from sklearn.datasets import make_classification\r\n", + "import numpy as np\r\n", + "from ipywidgets import interact, interactive, fixed\r\n", + "import ipywidgets as widgets\r\n", + "import pickle\r\n", + "import os\r\n", + "import gzip\r\n", + "\r\n", + "# pick the seed for reproducability - change it to explore the effects of random variations\r\n", + "np.random.seed(1)\r\n", + "import random" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "## Toy Problem\r\n", + "\r\n", + "To begin with, let's start with a toy problem, where we have two input features. For example, in medicine we may want to classify tumours into benign and malignant, depending on its size and age.\r\n", + "\r\n", + "We will generate a random classification dataset using `make_classification` function from SciKit Learn library:" + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 3, + "source": [ + "n = 50\r\n", + "X, Y = make_classification(n_samples = n, n_features=2,\r\n", + " n_redundant=0, n_informative=2, flip_y=0)\r\n", + "Y = Y*2-1 # convert initial 0/1 values into -1/1\r\n", + "X = X.astype(np.float32); Y = Y.astype(np.int32) # features - float, label - int\r\n", + "\r\n", + "# Split the dataset into training and test\r\n", + "train_x, test_x = np.split(X, [ n*8//10])\r\n", + "train_labels, test_labels = np.split(Y, [n*8//10])\r\n", + "print(\"Features:\\n\",train_x[0:4])\r\n", + "print(\"Labels:\\n\",train_labels[0:4])" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Features:\n", + " [[-1.7441838 -1.3952037 ]\n", + " [ 2.5921783 -0.08124504]\n", + " [ 0.9218062 0.91789985]\n", + " [-0.8437018 -0.18738253]]\n", + "Labels:\n", + " [-1 -1 1 -1]\n" + ] + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Let's also plot the dataset:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 4, + "source": [ + "def plot_dataset(suptitle, features, labels):\r\n", + " # prepare the plot\r\n", + " fig, ax = pylab.subplots(1, 1)\r\n", + " #pylab.subplots_adjust(bottom=0.2, wspace=0.4)\r\n", + " fig.suptitle(suptitle, fontsize = 16)\r\n", + " ax.set_xlabel('$x_i[0]$ -- (feature 1)')\r\n", + " ax.set_ylabel('$x_i[1]$ -- (feature 2)')\r\n", + "\r\n", + " colors = ['r' if l>0 else 'b' for l in labels]\r\n", + " ax.scatter(features[:, 0], features[:, 1], marker='o', c=colors, s=100, alpha = 0.5)\r\n", + " fig.show()\r\n", + "\r\n", + "plot_dataset('Training data', train_x, train_labels)" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + ":11: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n", + " fig.show()\n" + ] + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEnCAYAAACpNTSTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA9CklEQVR4nO3dd5ycdbX48c/ZlrqbuqmbZEMICUmAAEuTFnox0hQviCAXNSpNUAQUf4IFG15FvIKCBe8VFASlJICh90ASSEICJATY9N7L9jm/P87M3clm6u70Oe/Xa1/JzDwzz3c3m+c833aOqCrOOedcSbYb4JxzLjd4QHDOOQd4QHDOORfkAcE55xzgAcE551yQBwTnnHOABwSXYSKiCXzVd/EclwY/p7YT7723q+dPJRF5QURe6MT7pojILSLi/8ddwsT3IbhMEpEjOzz1L2A+cEvYc02q+nYXzlENjAHeVtWmJN87BqjqyvlTKRQMVHVKku+7BbgZKFfV1pQ3zBWksmw3wBUXVZ0V/lhEmoCNHZ/vcEwpdvOS0IVNVTcAGzrZvg878z7nCoF3J13OCQ733CoiN4rIx0AzcICIdBeRX4nIQhHZKSJrReRxERnf4f17DRmJSL2I/FVELhCR90Rkl4jMEZFjOrx3jyEjEakNftZXROQHIrJGRLYGz1vT4b09ReQuEdkkIjtE5F8i8ong+y9N4Pu+QETeF5EmEVkkIudGOCbuzyCsdwDQEhqKC3v9+yLylohsE5GNIvJchJ6bK0LeQ3C56lLgI+A6YBewGugGVAI/AtYA/YHLgVkiMl5V18b5zGOBccD/AxqBHwLTRaRWVbfGee+3gdeAy4BBwH8B9wHHhx1zN3A+Nvw1BzgpeExcInIycD8wA/gmUA38GigHFocdmsjP4A9ADfBF4BigrcPphgO/AlYCvYDPAy+JSJ2qLkikva5Aqap/+VfWvoB64K8dnlMsAPSI895SoCewA7g27PlLg59R2+E8W4B+Yc/VBY/7XNhz9wL1YY9rg8e82OHc1wWfHxZ8PA4IANd3OO6O4HGXxvleXgXeBUrCnjsi+N4XOvEzuCX43rIEfoZlWND5dbZ/H/wru18+ZORy1VOq2tDxSRH5rIi8ISJbgVas99AbuyDH87qqbgl7/E7wz5EJvHdGh8cd33sEIMA/Ohz3ULwPDs6RHAY8pKqB0POq+gYWyDoe35WfASJysog8LyKbgu9vAfZL9P2ucHlAcLlqTccnRORTwAPAe8DnsIvwYdgEcvcEPnNz+ANtX4GU9HuBju8dGvxzfYfj1iXw2QOxoaFIx+7xXFd/BiJyCPAEsBMbUjoy+P75ibzfFTafQ3C5KtJ66AuApap6aegJESnHxtGzLRTABgEfhz0/OIH3bsTu0iMdOxhYFva4qz+DT2O9gvNUtSXsM/oBWxP8DFegvIfg8klP7GIW7mJsHDzb3sCC2Pkdnu/4eC+q2gbMBj4TvpFMRI7A5jDCJfozCPVgekR4fxthAVdETiSxYTNX4LyH4PLJU8A5IvIrYDpwKHA1OXBnq6qLReR+4IfBi/pc4ETgU8FDAlHfbG4GZgKPiMjvsVVG3wc6rpxK9GfwbvDPb4rIk0Cbqs4Jvv8a4F4R+TM2d/D/gFWJf7euUHkPweWTe4Bbgf8AHgc+iV1wt2WzUWGmAX8Crsd2YE8Ergi+FrONqvoMcBE2sftP4FvYhXtxh0MT/RlMB+7ElqS+jvVAUNV/YwHk6OAxlwGXAEuT+D5dgfLUFc6lkYh8C/gZtgR2ebbb41wsPmTkXIqIyFRgEjAPGyI6Ftuv8KAHA5cPPCA4lzo7gHOAG7EdwKuwjWk3x3iPcznDh4ycc84BPqnsnHMuyAOCc845wAOCc865IA8IzjnnAA8IzjnngjwgOOecAzwgOOecC/KA4JxzDvCA4JxzLsgDgnPOOcADgnPOuSAPCM455wAPCM4554LyOv31wIEDtba2NtvNcM65vDJ37tyNqlrd8fm8Dgi1tbXMmTMn281wzrm8IiLLIj3vQ0bOOecADwjO5afWVmhuBi9w5VIor4eMnCsqqvDuuzBzJixYYI/794czz4SjjoJevbLdQpfnPCA4lw9U4R//gOnToXdvGDECSkpg1y743/+FZ5+Fb33LAoRzneRDRs7lg9deg8cfh1GjYNAgCwZgvYLRo2HzZvjNb3wIyXWJBwTncl0gAI89ZoGgtDTyMUOGQH09LF2a0aa5wuJDRs5FogrLlsHzz8PChfbc+PFwwgkwZgyIZK4tq1fDhg0wcmT0Y0SgrAxmz4axYzPXNldQPCA411EgAA89BDNmQEWFjcuLwJw58MorFhQuvjj63XqqNTQkFoAqKmD79vS3xxUsDwjOdfT00zZeX1u750V/6FALFs89B1VVcN556W+Lqs0PrF4NLS123gEDIgeIpiYYODD9bXIFywOCc+Gam+HRR2H48Mg9gJISW+Hz5JNw2mnpXeq5fDn84Q82dLV6tc0RdOsGPXvCIYfsefFXtb0JRxyRvva4gueTys6Fe/99G6Lp3j36MeXldrcemltIhxUr4NZbrXdQWwuf+IQNCfXubb2UV16B9evtWFU7fvJkqKmJ/bmqsGWLvbehIX3td3nJewjOhdu5M7HjRNI3Xq9qewtKS9t7AYMH2wV/wQJ7XFZmcxp1dbYXYcIEmDYt+lxDIGATzjNmWPAoKbGvY4+1ns7gwen5Xlxe8YDgXLhYPYNwqjZ0kw6rV8OSJbbnINzo0RYgQkNIW7ZAjx7wla/AxIkWJCIJBCzAPPOMzT+MHGmBo7UVXn7Z9jjccIN9vitqOTNkJCIjROR5EXlPRBaJyNez3SZXhMaNswtrS0v0Y9ra7IK6//7pacPq1Xb3Huluv7ISJk2CU0+FI4+0Pw86KHowAJg1y4LB6NHQt2/755aV2VxJt27w61/bpLQrajkTEIBW4Juquj9wJHCFiEzIcptcsenVC046CVaujLzrV9VeO/ro/EgToWrpLqqr23c3d9SvH2zd2j4c5YpWzgQEVV2jqm8F/74DeA8Ynt1WuaJ03nm2iuejj+xCqWpf27fDxx9bL+Kii9J3/poaG+aJl4ZCNfZmNbANbWvWWM8ill694M03k2unKzg5OYcgIrXAwcAbWW6KK0YVFXDFFTZp++STNmYvYvsQpk2Dww+3Y9Jl6FAbjqqvjz7Zu3Wr3fWPGxf7s1paog8/hSsrg927O9NaV0ByLiCISG/gYeAaVd1rGYeITAOmAYyMd3fkXGeVldkY/RFHtNcd6NYtcykrLr4YfvQjWLfOchiFzhtaNrprF9x4Y/RhoJCqKntPW1vsndW7d1sgckUtpwKCiJRjweA+Vf1npGNU9W7gboC6ujpP7VjsWlrgnXfg3/+2O/myMhvuOeGE1KyaEbFAEMm2bfDGGzb2HgjAfvvZfoFBg7p+3mHD4LvfhXvvtRVH4QGhpgauvhr23Tf+51RW2tLU+fOjX/BV7ed4zDFdb7fLa6I5ki5XRAT4C7BZVa9J5D11dXXqNZWL2O7dlvL53XftTrhPH7swb9xoK2Y+8xmYOjU9d/Uvvwx/+YvdeVdV2Tl27LDzn3UWnHNOas6rCqtW2d4BVctqOnp0cp+9YgV8//v28+k4lxBK4nfooXDllZlN2ueyRkTmqmpdx+dzqYdwNHAx8I6IzAs+9x1VfSJ7TXI57Y9/hMWLbSdv+IVs2DBbY//gg7Zu/6ijUnvet96Ce+6x84TvW+jb18778MPWqzjzzK6fS8R6BPF2IMcyYgRcdx3ccYftfK6stJ7Uzp0WOA89FL78ZQ8GLncCgqq+AvhvpEvMqlUwd65t3op0ISsrs6GbRx6xeYB4Y+2JCgTg73+3Cd1Im9jKymzlzyOPwJQp6du8lqzx4+G22+xn9sYb0NgIBxwAxx0H++zjwcABORQQnEvK3Lk2SRrrQlZZacMhK1bsveu3sz76yJZyxvq8igqbiF6wwCamc0WvXhYAjjsu2y1xOSpn9iE4l5TNmxNb+hmqO5wqW7cmdjddWmqBw7k84j0E1zltbbBoETz1lJVtFLF8OqecYsMT6R6C6NPH7sLjUbV8P6lSXp5Y3eJAIPrqJOdylAcEl7zmZvj97y17ZlWVbZ5Shffes81cJ50En/98eiuKHXqo1S1QjR58du609BKpGi4CG28vLbXJ42j5g0I7m9OV68i5NPEhI5e8Bx6wC38o+2ZpqV0cBw+2FT/PPms9h3QaMcImRVeujPx6W5vl/D/rrNRNKIPNSxx3nE1qR+sprF1rvaSurAxyLgs8ILjkbNtmhedDKZQ7KimxDJrTp6c3e6aIpZEYOdLyC23b1r4jd906m0yeOtXy/afa+efDmDF23vAiM83Ndt7KSl/G6fKSDxm55CxcGD8NQrdutqxx8WI48MDox6m258/p2TP5C2hlpaVvCM85VFJihWROPdXy/KTjotyjh63rf+456wlt2GDnKS+3vQennmpzHM7lGQ8ILjnbtyc2NyASPVlaUxO8/jo88UT7SpzBg+GTn7Q9A8kkjuvWzVJRH320TeSKZObOvHv39ov/xo0W3Pr3z+xE8tq18Oqr1lMpL7eUHYcemjt7H1ze8YDgklNVZRfeeKJVFNu9G26/3WoXV1fbhK+qpX24+24r5nLVVYlXLguXyrmCRJWVWTqJTGprs81xzzxj33OozvK8eXD//fDVr1rRHOeS5HMILjkTJ9pFqK0t+jFNTXanvN9+e792//3wwQc2IR3KqyNigWaffSwv0YMPpqftheIf/7BkfjU1Nrner5+Vxhw1yoLD7bfbUmDnkuQBwSWnb184/vj2ZGsdBQK2AufMM/e+y9+82YY4amoiD+uI2IT0iy+mr4B9vtu82YLBqFGRh+569bKvf/wj821zec+HjFzyLrjALkxvv22Tp/362fObNtnQz/HH23xAR++/b0Ek1hxEWZkFlcWL4bDD0tP+fBaqahbrZzhggKXMXrcueoGdrmhrs57cK69YbYY+fSx19sSJsWs7u5zn/3oued262Tj/O+/Y6p4PP7S7+wkTbJJ1woTI4/mNjYnt8gUv+B7N6tXx51dELGBs2ZL6gLBpkw1JrVhhq626d7e9IHPmWL2Fa65JTT0IlxUeEFznlJXBwQfbV+giH291T6huQCLi1QAuVj162C7peAIBW3mUSg0N8ItfWO+wtnbv19ets4yqt9xiw1Yu7/gcguu6RJd6TpzYngk0msZGu+h52ofIDjrIqpvFEvoZjhiR2nPPnQtr1kSvvDZ4sC0jfsNLoecrDwguc3r0gE99yoYbIq1Sam21Cemzz05vEft8Nm6cLXPduDHy66p20T7ttNT/DP/97/b5omgGDkx/2hKXNh4QXGZ98pNwxhkWFFatsn0Ju3bZ31eutGBw6qnZbmXuKi21+Ruwn2H48NGuXVav4aCD4PTTU3/utWttWWssvXpZLyFHSvO65PgcgsuskhK48ELbWfzCC7byCGxl0nHH2TCH5wCKbfhwuPlm2+n98svtF9+qKrjkEqvUluj8wfr1MH++rQ7r08fSfgwYEPnYbt0sAMVa4dTaaj0T/zfMS6J5HMnr6up0zpw52W6Gc9mze7etJiottZ3fiaYc370b7r3XUpiDva+tzS7kxx4Ln/vc3mk4/vY3y2QbK4vrqlUW7C+9tDPfjcsQEZmrqnUdn/cegnP5rGfP5HMXNTe372YeMWLPJcKBgG0M3LEDrrhizwBz/PHw9NM2aR1p6Wtzs014n3hip74Vl30+h+BcsZk714bqOgYDsMejRtm+gnff3fO1YcPgssts0nr9+vacVqo2b7ByJXzhC5aS3OUl7yE4V2yeeMIys0Yb5xexfSAzZ1oRonDHHGMbz6ZPt42JJSUWECZMsAUDvlw4r3lAcK6YtLXZ6qR4ZUX79rUVS5Hstx984xuWb2rXLhuy8voPBcEDgnPFJLSJMFYtarDX46UTr6qyr0SoWtqN5cttqGnIEMtu66uRcooHBOeKSUmJDe/U19uqpGg2bkxdcsE1a2xF0+LFewaAoUNtNVKkNOkuK3xS2blic9pptoooWqGjtjZLLpiK1UJr18Ktt1rPYNSo9q+RI2246ac/bd+L4rLOA4JzxWbSJNu8Vl9vS0jD7d5tz0+dakM6XXX//bZZbfDgPXsHIpYGo08fq5QXq+CSy5icGjISkT8BU4H1qjop2+1xriCVlNjy0CFDYMYMy1IaulhXVtrS0ilTYo/v79gBixbZn7172zBUx4nl9ethwYLYy1D79LEA9P77lvzQZVVOBQTgXuC/gf/JcjucK2ylpVbV7uSTrZ5FQ4PlIRozJnaRm9ZWePhh26AWyqMUKno0ZYoVTwol1Vu50oJPvMnpkhL4+GMPCDkgpwKCqr4kIrXZbodzRaOiIvG9A4EA/OEP8NprdtcfHjja2uC556yAzlVXtb+WSGocX2mUM/JuDkFEponIHBGZs2HDhmw3x7ni8d578PrrMHr03r2I0lKbLH7rLUuWB7azWTV+UGhr893NOSLvAoKq3q2qdapaVx1r2ZzrurY2WwkSq6CNKx5PP23DSrF2OPfrZ2VVweYoJkywuYRoduywTXATJqS8uS55OTVk5HLE5s2WmvqZZ2xsGaxU5mmnWYEWV5wWL7aUF7H062dzEoGAzQ1cdBH86EeW62jgwD2Dyfbt9rt23XWx5y1cxvi/gtvTihXws5/Z8sPBgy1vTSBgq0DmzoXPftYmI33ct/gk828eOnb4cLjpJpt7qK/f8/UBA+Bb3/LJ5BySUwFBRP4GTAEGishK4GZV/WN2W1VEmprgV7+y/6zhY7olJRYYWlrggQdsrHiSrwouOhMnwsKFdqMQzaZNMH78nsGjpsYK+ixbtmfqiv32i78CyWVUTgUEVb0w220oavPnWxe+tjby6+Xltm58xgwPCMXo5JPhzTfbh4M6UrVhoEjlO0Xs9yra75bLCR6eXbtXXrGNSbH0729jydu2ZaZNLnfst5+ls6iv33uhwY4dMG+eXfCHDctC41wq5FQPwWXZzp3xa/GGsmU2NWWmTS53iMDnP283BTNm2O9AU5MFiE2bbL5g+XK44QY45BA4//zYw0su53hAcO2qqy0zZayUxqGcM716ZaZNLreUlsKnPgWnnAJvvAF33WXzS8cd115WMxCwlBXvvw/f+Y73GPKIDxm5dscd177MNJr16+Hwwz0gFLvu3W2IqKoKJk/es8ZySYkFgUAA7rknsd3KLid4QHDtxo+Hffe1HDSR7NplK43OOCOz7XK5Z8MGCwhDhkQ/prrachQtW5axZrmuSTogiEgvESlNR2NclpWWWh6aESPsP/LGjZYeeccO+0+9bRtcc42nGXDtewpiLRsNLT2NVorT5Zy4cwgiUgJcAFwEHAY0Ad1EZAPwBHC3qn6Q1la6zOnTx8Z9Fy2yVAXr1tlwwGc/C0ceaTtRnUu0foFIe1ZUl/MSmVR+HngG+DawUFUDACLSHzgB+KmI/EtV/5q+ZrqMKi+3ceHJk7PdEperqqvbE9fFq80ca1jJ5ZREAsLJqtrS8UlV3Qw8DDwsInHWKrq8sX69pTeur7f8MpMn2xLCnj2z3TKXS/bZxyaOt22z5HSR7Nplk86euC5vxA0IqtoiIuOB4cAbqroz9JqInK6qT0UKGC7PBALw0EPwxBM2Lty7tz03Z44NGV1+ORx4YLZb6XJFaE/Cz39uNw69e+/5ekODDTeG10ZwOS/upLKIXA08ClwFLBSRs8Ne/nG6GuYy7JFH4PHHbUJ55EjbfDRwoO08ray0HEcf+FSRCzNhAnzzm7bwoL4eVq+2fSz19dZzuPxyOOywbLfSJSGR0P1l4FBV3RmsZvaQiNSq6q8BT3lZCLZuhenTLWldaYQFZL162R3fgw/ahLNnOnUhBxwAv/ylJb1bssR6lfvss/feBJcXEgkIpaFhIlWtF5EpWFAYhQeEwjB3rk3+xeraDxgAS5fC2rUwdGjm2uZyX0WFzTMdcki2W+K6KJF9CGtFZHLoQTA4TAUGAgekqV0uk1auhG7dYh8jYnMLW7Zkpk3OuYxLJCBcAqwNf0JVW1X1EuC4tLTKZVaPHomtK4/Xi3DO5bW4AUFVV6rq2iivvZr6JrmMO+AAS0kRS1OTDQ2MGpWZNjnnMs5zGTmrkzxkiKWqiETVVpCcemr8oSXnXN7ygOBsbuCqq+zvK1fumWpg927LazRhAnzyk9lpn3MuIxIeEBYRwfIZ7aOqPxCRkcAQVX0zba1zmTN8OHzve1b45NVX29MSVFbCBRdY+cSKimy30jmXRqIJ5ioXkbuAAHCiqu4vIv2AmaqatZ0ndXV1OmfOnGydvnDt2mUVsEpLreKVTyS7OFRtT9ratdbhHDHCViq73CQic1W1ruPzyfxPP0JVDxGRtwFUdYuI+C1jIerVywvguIQtWwb33Wcb2UN7FlXh0EPhwgs9MOSTZAJCS7AOggKISDXWY3DOFamPPoKf/tQS5I4c2R4QQlU0P/oIbrrJg0K+SGZS+Q7gX8AgEbkVeAXPZeRc0QoE4He/s20s1dV7ZjQJVdHcsQPuvz97bXTJSaiHEJxQfgmYC5yEpaw4R1XfS2PbnHM5bMkSy5ZeWwuoUtbWBKq0lnX/v+gwdCi89ZZNSXkvIfclFBBUVUXkEVU9FHg/zW1yzuWBDz+EMloZtvItxi59gr5brXbyzl6D+WDsGayqORLKuiECy5d7QMgHyQwZzRIRz2XrnAMg0NTClPfv4vDZ/02Phi1s6zOSbX1HURpo4ZC3/8gnXvsF5c27AJtkdrkvmYBwAvC6iHwoIgtE5B0RWZDKxojI6SKyWESWisiNqfxs51xqTfzwUWo3zmFr39E09uj3f8NEzd0q2dp3NP22fMiB8/+HQMBWL7vUCASguTk9QTaZVUZnpP707YIrmH4LnAKsBGaLyGOq+m46z+uc64SGBkYtnsniqhpKW2TvPYsibK+qYeCHb3LQQZ9h2LDqrDSzkNTXw7PPwuuvWzKByko47TQ45pjoVUyTlXBAUNVlqTllVIcDS1X1IwAR+TtwNuABwblcs2QJpW0tHFRXzhtvWOegvENl9YbGEsrblAsOXISVUXGd9cILcO+9lixgyBDbK9rQAA8/DDNnwvXXQ01N18+TTOqK70V6XlV/0PVmAFazeUXY45XAESn6bOdcKjU2Ara09IgjYN482+AevjGtZ0844KASqip3Za+dBeCDD+DPf7afdXhuyR49LPnwpk1WtO7HP+56kbpkhozC/1W7Y0VyUrnsNFL1tb1GyURkGjANYOTIkSk8vct1gYAlXW1qsu7yoEHZblERq6z8v78OG2ZzBOvXw/btFhT69bOS3LIsYA9cpz3xhCUOiJZoeMAA2y0+f74F565IZsjov8Ifi8gvgMe6dvo9rARGhD2uAVZHaMfdwN1guYxSeH6XowIBePllePxx2LzZLjiBAIwdC+edB+PHZ7uFRWjsWOjd28YtevSgtNT2HOxRXbW11fJhHeCFFTtr92670McbDqqshJde6npA6Er6657APl07/R5mA2NFZHQwR9IFpDbguDykCn/9K/zhD/b3kSMtcdrIkZZM7Sc/gVmzst3KIlRebtF49eo906WHBAK2+eC00/boTbjkNDTYnyVxrtQVFdY766pk5hDeoX0IpxSoBn7Y9SYYVW0VkSuBfwc//0+quihVn+/y07x58PTTMHr0nv8pRKyr3LOnBYv99oP+/bPWzOI0ZYrV2H7sMQsQ/fvbP8yWLTbHcPzxFjSKkCosXWqrgt5+2yrU1tZafJw8ee8J+Gh69rQfaVubdbaiaWy0DPZdlcwcwtSwv7cC61Q1wq1B56nqE8ATqfxMl79UrTxDv37R75B69LCb0ddeg6lTIx/j0kQEzj0X6upsGcw777SnOT3xRBgzZs8ER0VCFR56CKZPb8/zVFJicyy/+Y3Vmrr6arvYx9Ojhw0DzZ3bYTiug507LT53VTIB4XJVvSH8CRH5WcfnnEuVhgZLjxBv7UC/fvDmmx4QskLE/oEuuSTbLckZL71knaba2j3v6vv3t9/VJUvgT3+CK69M7PNOP92GRXfvjhxE1qyx3sGkSV1vezJzCKdEeC6tm9VccWtttetNvJvM0lLbuelctrW1waOP2l6BSEM8IjYHNmeOXcgTMXKkVbjdssWmZXbvhpYW2LrVNqv17w/XXpv4MFQscQOCiHwtOH8wLpiyIvT1MfBO15vgXGS9elmXObjkPaodO+L3IpzLhPp6u1DHqi8Vusl5663EP/fgg20BxTnn2BDp1q3W25g2DW6+2Zb4pkIiQ0b3A08CPwHC8wvtUNXNqWmGc3srLYVTT4VHHrENOJGo2h3TiSdmtGnORbR7d2LHVVTYRT0ZAwfCWWfZV7rE7SGo6jZVrVfVC4HtwGBgFDBJRI5LX9Ocs4myfv1sQq4jVduQM3myrTJyLtsSmSgGG/JJVf6hVEpm2emXgK9jG8bmAUcCrwN+b+bSpk8fuOEGuOMO645XVNhXQ4ON19bVwZe+FH+dtnOZUFtrv7O7dkUfNlK1YZ9DDslo0xKSzCqjrwOHAbNU9QQRGQ98Pz3Ncq7d4MHwgx/Y6ozZs22J3aBBcOSRtrqiCFc2uhxVWgpnn22riEaN2ntiWRVWrLAbmVjLSLMlmYDQqKqNIoKIdFPV90VkXNpa5lyY0lLYf3/7ci6XHX+8DXHOmGGLIgYOtB7stm22Umj//eGyy7LdysiSCQgrRaQv8AjwtIhsIUKuIeecK2YicP75Nrf1zDO2276tzXoMn/98cjuVMy2Z5HbnBv96i4g8D/QBnkpLq5xzLo+J2EKH0GIH1fwY2kx4Kk7M50Xke6r6IjaxPDldDXPOuUKRD8EAktupfCdwFHBh8PEOrOSlc865ApDMHMIRqnqIiLwNoKpbgmmqnXPOFYBkeggtIlJKMAW2iFQDgbS0yjnnXMYl00O4A/gXMEhEbgU+A3w3La1Kt7Y2G9Tz3UxFZ9Mm2LjR/ulramxZoHPOxA0IIvK/qnoxMBC4HjgJq398jqqmsqZyerW2WqWKJ5+Ejz6ygLDvvnDGGXDQQbGrT7i8t3IlPPywLQEsKbFVH2VllgPprLMSTzmQa3butPXtZWXtefed6yxRjV2WWETexdJcPwZMwYLB/8lmgru6ujqdM2dO/AObmuDOOy0g9OvXnkRkyxb7OvJISxuYq4uDs0XV8u3Om2cpRQcOtOIn1dXZbllSPv4YfvpTuwcYNKj9otncbCmIR42C66/Pr6CwerVtfAqVD1W1f5ZPfhKOPtrvb1xsIjJXVes6Pp/IkNHvsP0G+wBzwz8Tm09IZV3l9HjgAatUPXr0nuu/QhUr3nzT/jd99rPZa2Ou2boV7roLFi+2K2hFhV1BH3gAjjnGdth065btVsbV1ga//S10727/1OEqKiwYLFtmOewvvDDyZ+SaDz+En//cgsCwYe0X/x074J57LMXHf/6nBwWXvESynd6hqvtjNY73Cfsaraq5Hwy2b4cXX7SqFJEWA4vYYPLTT1tGKmc/h9tus6G1UaOs2MCQIe0V7l96Ce6+2zJ05bj33rM5g47BINywYfD884mnLs6mpib49a9t7iM8GIDVsh892n7dX3kle210+SuRAjkCoKpfi3dMTnr33fgVqsvLbY7h/fcz165c9uqrNugeKXNcSYmldJwzx25Fc9yiRfFHAsvL7Vdk+fLMtKkr5s+3nkC01MklJTYsNmNGXsRrl2MSmYJ6XkSuEpE9alKJSIWInCgifwG+kJ7mpUAyt30NDelrR74IBGzifdCg6MeI2C3qs89mrl2d1NKS+ERrW1t625IKs2fHn+uorLReUaQaEs7FksgcwunAZcDfRGQ0sBXoDpQCM4Ffqeq8dDWwy3r3TvzYWHXvikVjo80fRCtRFtKnjxUoyHE1NfHrLataMMiHufLGRltRFI+IBUPnkpHIHEKjqt6pqkdjldJOAg5R1VGq+uWcDgYAEyfa/6BY/zuam22G0XMrt99Ox1l9RiCQF7OWdXXWzFhBYcMGmDAhdqeoM1pbbcw/3o8yGcOHx5/qamuzc+ZiRS6X25JatayqLaq6RlW3pqk9qderF5x+ulWliDSoGgjYePnUqbYUpdh16wZjxthy3Fg2b87Nkk8d9O4Nn/60/fNHuifYts2CRaoWmKnatNWvfgVf/jJ89avwzW/CzJm2Z6Crjj7aAk2sILNuHRx+uA0dOZeMZHYq569zzrFhkJdftgHY0JKTzZtt3uCkk+DMM7PZwtwhYpv1br/dbjEjDcA3N1sgPfbYTLeuU04/3f58+GG7e+7e3Zrf1NReorO2tuvnUYWHHoLHH7dANGKE/fh27YK//c2mXK6/HgYM6Pw5amrgqKPgtdeszR3n/Ldvt3ZMndqlb8UVqbgb03JZwhvTwK4AS5bY8tLFi+25CRPg5JNh7Nj8yU+bCYEA/PnPtn5x6ND2/A6qtsRlwwa45BI45ZTstjNJ27fb4qjly20UceJEmDQpdfsRX3/d9j/W1kYeTVu71sqB3nxz13YUNzdbicZZs6ztvXpZr2HnTusVXH21/Uo7F020jWmdCggiMkRV16akZV2QVEBwyWlrs3JPM2bYlUbEAsWQITYGU7fX71JRU4XvfMd6HVVV0Y+rr4dvfxvGdbH4bKg2b2iFcEUFHHGEVePykU8XT1d2KkfyBJCyAWQROR+4BdgfOFxV/SqfbaWlcNppNpxWX2/LWyorbXOa96b2snq1jd2PGBH7uIoK2xjf1YAgYv8UI0fGP9a5RHU2IKT6irAQOA/4fYo/13XR9t1lbCnfl9Lu1jko81gQUWOjXaTjxcqKChu6ci4XdTYg3JPKRoSypubyhudis3q1TY6+8Ub7Ra53b5tvPukkzwPYUe/eNqIWr3ZuY6PlCHQuF3U2IPwzpa1IgohMA6YBjPT+clqEsoOq2rr30ATp7t1w//2WH+jKKz0ohBs0CPbZx+bb+/ePfIyqTf4eeWRm2+Zcojq71uGJZN8gIs+IyMIIX2cn8zmqereq1qlqXXU+bC3NMy0tcMcdkZOn9expydPeftvW1afa1q22Uue55yxFQz4kmwsRgXPPte+hqWnv10OTwAce6OP+LndlbA5BVU/u5LlcBi1aZHvSoq3LF7GVqE89BaeemppeQmOjrdN/+eX2fEIi9tlnnmkFbPJgUzSTJsEXvwh/+YstK+3f39q9Y4d97b8/fOUrPifvcldOzCG43PHWW/GXLfboYUMjq1fHT3kUT0sL/OY3FohGjNjzwt/SAv/8J6xaZa/Nnm3PjRxplc7Gjct8hbDdu60NPXtGDobHH2/tevllm39parLgesop7VlUnMtVnfr1VNU7U9kIETkX+A1QDcwQkXmqeloqz+ES09SU2N24iI2Hd9Xs2fDOO3vXLgK74JaVWf7/yZMt+JSUwMKFtnRz8mT42tfSv+4+lI7iySctcInYaqEpU2yCvePI5ZAhcP759uVcPsmJCqyq+i9VrVHVbqo62INB9tTUxM8CHgjYV6yiM4lQhSeesFQOkYZR1q+3HkvPnjas1Lu3/X3wYLvrnj8f/vjHrrUhkTY++ij87Ge2HSO09r9/f9v0fvPNeZH01bmE5ERAcLnjyCPb00FHs2GD3Z1HW02TqOZmGw6KtLNX1eoVde/ent8/nIj1GGbPts9IlwULLAfSyJG2XDQUuCoqbBirvNzSPjU2pq8NzmVKIhXT+ifw1TcDbXUZUF1t493LlkUOCtu32xj6ueem5nzRMqfs2mW5B2MNB4nY8NYbb6SmLZHaNn265fiLNvbfr59lTJ03Lz1tcC6TEplDWB38irU2ohTwxXQ5KLTccdMmu3jW1sbOtQPtqaBnzrT39OxpwWH3bhu2uf761CydDBW537Zt79z9zc3tO38bGmwoK5Lu3dNXGWzbNli6NP732ru3ZR/1/QUu3yUSEN5T1YNjHSAib6eoPUWtrc02fb34ol3Aq6rgmGNs7XpFRfKft3ixbSRbtqx9NY6Ifeb550cvJldWBp/7nE2Yvv66ZQetqLDyBwcdZCUTUiGUafvOOy0Ndfg8QmhiOxCwn0u0ZbAtLckVxUtGc7P93OItEy0vj1+0Jhe0tdkw3Pz5FmSHD7e6CV0d+nOFI5GAcFSKjnExbNliY9HLltkdeejOd948G8a59lrbKJaoBQvgl7+0C+2oUe0XtdZWWxL50Udw442xq4YOHmylJNKprg4OPtguUjU17UMzVVV2oQ1VM4tW/aulJX2JV0OBpq0t9sqr3bu7nqwu3VautA2H69dbcC8rs0ypDz5oOQw/85n82Ovh0iuhEprRXhOR/4x3jIuvqQn+679gzRq7Ex40yC6IAwfa49274ec/tyGMRDQ2wu9+Z4GkX78973DLymwIZOVKGx/PtrIyuOIKm7dYu9YC4rJl1isZMsQ2wU2YEPkufd06CyL77ZeetvXsacNA69ZFP0bVft7HH5+eNqTC+vXwk59Yr6C21m4sBg2y34Phwy3D+QMPZLuVLhd0dZXR91PSiiI3b55doKP1AKqrLRi8+mpinzd/vgWRWHf/Q4daioh4S0wzoaICLrrIyk5ecYXt9v361+0idemltqxz06b2CeiGhvae1FVXpXdz2hlnWA8hUvlLVft322+/9AWlVJgxw246IlVqKy21HuTMmRaQXXGLO2QkIguivQQMTm1zitPTT9vQTiyDBlm6iDPOiD+mvXBhe5GzaCoqbPhozRpLypYLKiv3Hv65+GI44ADbr7B0qX3vvXpZjZ7jjos/Qd5VNTXwjW/YbupNm2zoqrzcAsSuXRYI0h2UumLXLruRGDIk+jGBgC3r/fnP4ROfgPHjLc2GDyEVn0TmEAYDpwEdq64L8FrKW1SE1q2LXxC9Rw/r+re0xJ9gbmtLPF9OIJDYcdkiYnMMkydbz6CtzXoGmbxYTZxoF8vZs+GVV2yIaNw4m3TP9Qvn5s3Wk4m0bFbVel8LF1qPcuNGW1Y8Y4b1Jr72NRgzJuNNdlmUSECYDvRW1XkdXxCRF1LdoGLUrZvdrce60Le12V1oIrlwxoyx1UGxtLXZBWHQoOTami0iFgiyparKAsBJJ2WvDZ0RK1jV11vm2qoqO65fv/blvVu32u7s737Xs7MWk0Qmlb+oqq9Eee1zqW9S8Tn6aBuOiGX9equZm8jQxGGH2XHNzdGPWbsWjjoq/UMuLrsGD7beZ8dU4i0t1jOoqrKbjOZmm1cK6dvXgsSDD2a0uS7LcnTks7gcc4zdAUfL/9/cbMMUid6dVlXZ5rLly/fOza9qSzm7dYOzk6pE4fJRaamlEF+7ds9d4WvWWC8xFAzKyvYMCGC9x4ULY6+ycoUlkUnlt1T1kK4e46KrroavfhV++1u7UA8aZHf4qjauu2OHTa4mM5576ql2MfjHP+w/fGlpe4nHmhobH86X4aJ0UrUKca++ahe+nj2t5zRpUuFUhDvhBLuwz59vF/3u3W3VWkmJTTqHqrh1/H5DqUE2bLCehit8otGSyYQOEGkAPoh1CNBHVTM+0lhXV6dz5szJ9GnT5uOPbTXN3Ln2nzEQsDX4U6fa5GVnNDTYhWDNGvsPv//+tqrIi7RYj+z3v7efT3m5BYOWFnt+wAC45hpLYFcImpvh3/+2lWqhZbsrVtj8wIQJ0XcrL18O111nx7jCISJzVXWvLZ2JBIRESqC0qerKzjauswotIITs3GkXpe7dfYw/XQIB28m9aJFdFDsGyNC+h+9/P/L6/XzV3GyBYNEiuO++2DcaoWXJt98efxWcyy/RAkIik8rLwr+AL2JF7g8HKoLPZzwYFLLevdt3K7v0WLLECvMMHWrBt7FxzzH2AQPs+WefzV4b06GiwoYezzzThg63bo1+7OrVcOyxHgyKSdKTyqr6PeAOYAfwaRHxcpouKYGAjV1ns4bAo4/acMhTT8Ezz9hwyosv2kUwFBiGDLHd3C0t2WtnupSVweWX27/D+vV7BsPW1vbUIZ/+dPba6DIv4RKaInI7cK2adcBTwS/nErJ9O7z0kqVJ2LHDnhs3znZfH3hg5uY1PvwQ/v53u/D169c+gd/QALNmwdixNqlcUWHBYOfOrleHy0Vjxtg+gwcesCy74RlxjzvOgoH3DopLMjWVdwKPicgFqrpLRE4FblbVo9PUNldANmyw3b4bN9pwWP/+dhFetQp+8Qs4/XRLuZ3uoNDYaGPi3bvb+cMvgj162CqvDz6w9g0bZr2ZQlltFMmoUVbfYt066ymUlNicigeC4pRwQFDV74rI54AXRKQJ2AXcmLaWuYIRCFjq5R077AIUImJj9X372tDNyJE2Zp1Ob71l7dh3X5tD6FiRraTEAsOSJfbavvvGThJYKAYP9qWlLok5BBE5CfgyFgiqgatV9eV0NcwVjiVLLCtotARrpaXWa3j88fTnVpo1y+5+a2rsvJF2c3fvbpOt69fb5Ksv0XXFIplJ5ZuA/6eqU4DPAA+IyIlpaZUrKLNnxx92qay04aTVq9PblsZGm1Dt1s1SfDQ02BxBaFJV1XZ379plG7omT05ve5zLJQkHBFU9MZTTSFXfAc4AfpSuhrnCsXNnYiVAS0rSv/Jo6ND2cpdDhtjkaXW1TXhv22Z/lpXZ+vyLL/begSsuyUwq70FV1wSHkZyLqbo6/oVe1XLrpHsy89hj4YUX7HwitnroiCOsfaEUH1u3WiqHYpg7cC5cl5LbqWoO1Ntyue6II2yJZ6xN8Vu22DLIdOdXGjMGDjrI1tmHtye0K7ytzR6ffnp62+FcLvJspy7tamrg0EP3vgiHNDbaXfl556V/iEbEEglOmmT5fNassWGizZvtcVMTfOtbVmvYuWLT6SGjVBKR24BPAc3Ah8B/qurWrDbKpYwIfOlLcOedsGCBDcVUVtrd+ObN9vpXvpK5BGo9e8K119oGtdDu5B49bJjo0EOzW4jHuWyKm9wuI42wTW7PqWqriPwMQFVviPe+Qk1uV6ja2mDxYssPtHy5rTw67DCrB1Fdne3WOVc8oiW3y4kegqrODHs4C1vW6gpMaan1AjyVsnO5KRfnEC4Dnsx2I5xzrthkrIcgIs8Akfaq3qSqjwaPuQloBe6L8TnTsPTbjPTq3845lzIZCwiqenKs10XkC8BU4CSNMbGhqncDd4PNIaS0kc45V8RyYg5BRE4HbgCOV9Uopeadc86lU67MIfw3UAk8LSLzROR32W6Qc84Vm5zoIajqvtlug3POFbucCAjOhGr79uhhX845l0keEHLA0qXw5JNWvEXEvg4/3PLphBeUSYeGBnj7baivt30C48fDxImW8dM5V1z8v32Wvfwy/OEPli5hxAhLAd3WZsFh1iy48kpLp5Bqqnbu++6z/D3dullxmieftApmX/ua1Tt2zhWPXJlULkorVsCf/mS1ewcPbq/vW1pqefurq+Guu6xyV6q9+ircc4+lf66ttfMNH25/Lymx+scffpj68zrncpcHhCx67rn26l2R9Oxpd/KvvJLa8zY1Wc9g2LC9awqDpYHu2RP+9rfYKaudc4XFA0KWhC708QqbDxyY+oCwYIHNHcSauO7f33oIq1al9tzOudzlASFLAgFoabHhoVjKy231USqtXBl/0jg0ub1hQ2rP7ZzLXR4QsqS01MbvG+LUnNu1K/WpocvLLSAlosR/Q5wrGv7fPYtOPTX+HfiWLXDaaak979ix8ecGQqUk073s1TmXOzwgZNHRR0OfPrBxY+TX1661OYZULzsdOxaGDIFNm6Ifs3q1VRDr2ze153bO5S4PCFlUVQXXX2+rjOrr7QK9c6cFiPp6uxhfd13qdy2XlNg+g5YWCzrhw0etrbYcduBA+I//SO15nXO5LSdKaHZWoZTQbGqC+fOtvu+2bbbCZ8oUOOAAG+9Pl1Wr4MEHbdVRSYkNI5WUwCc+AZ/+tPVenHOFJ1oJTQ8Ijg0bbPNbSYltTquqynaLnHPplNM1lV12VVd7kXvnnM8hOOecC/KA4JxzDvCA4JxzLsgDgnPOOcADgnPOuSAPCM455wAPCM4554I8IDjnnAM8IDjnnAvygOCccw7wgOCccy7IA4JzzjnAA4JzzrmgnAgIIvJDEVkgIvNEZKaIDMt2m5xzrtjkREAAblPVA1V1MjAd+F6W2+Occ0UnJwKCqm4Pe9gLyN+qPc45l6dypkCOiNwKXAJsA06Icdw0YBrAyJEjM9M455wrAhkroSkizwBDIrx0k6o+Gnbct4HuqnpzvM/0EprOOZe8rJfQVNWTEzz0fmAGEDcgOOecS52cmEMQkbFhD88C3s9WW5xzrljlyhzCT0VkHBAAlgFfzXJ7nHOu6OREQFDVT2e7Dc4lY9s2WLYMAgGoroZhw0Ak261yrmtyIiA4ly+2bYMHHoBZs9qfCwRg7Fi48ELYZ5/stc25rvKA4FyCtm2DH/8YNm6E4cOhtNSeV4U1a+DWW+GGG2C//bLbTuc6KycmlZ3LBw8/DBs2wIgR7cEAbKho4ECoqoK77oLW1uy10bmu8IDgXAJ27IBXX7W5gmj69IGtW+HddzPWLOdSygOCcwlYscL+LIszyFpWBu/7ommXpzwgOJeAQCCx40SgrS29bXEuXTwgOJeAwYMtKMQLDC0tMGpUZtrkXKp5QHAuAdXVcMABNqkcTWMjVFTAwQdnrl3OpZIHBOcS9NnPWg9h8+a9X2tshFWr4KKLoEePzLfNuVTwgOBcgmpq4Nvftgv+xx/D8uWwciXU19vqomnT4Nhjs91K5zrPN6Y5l4TaWvjJT2DJEvjgA5szqKmBAw+E7t2z3TrnusYDgnNJKimB8ePty7lC4kNGzjnnAA8IzjnngjwgOOecAzJYUzkdRGQDVlAn3EBgYxaakymF/v1B4X+P/v3lt0L4/kapanXHJ/M6IEQiInMiFY8uFIX+/UHhf4/+/eW3Qv7+fMjIOecc4AHBOedcUCEGhLuz3YA0K/TvDwr/e/TvL78V7PdXcHMIzjnnOqcQewjOOec6oSADgoj8UEQWiMg8EZkpIjEKH+YfEblNRN4Pfo//EpG+2W5TKonI+SKySEQCIlIwqzlE5HQRWSwiS0Xkxmy3J9VE5E8isl5EFma7LekgIiNE5HkReS/4+/n1bLcp1QoyIAC3qeqBqjoZmA58L8vtSbWngUmqeiCwBPh2ltuTaguB84CXst2QVBGRUuC3wBnABOBCEZmQ3Val3L3A6dluRBq1At9U1f2BI4ErCu3fsCADgqpuD3vYCyioiRJVnamqrcGHs4CabLYn1VT1PVVdnO12pNjhwFJV/UhVm4G/A2dnuU0ppaovARGqRRQGVV2jqm8F/74DeA8Ynt1WpVbBZjsVkVuBS4BtwAlZbk46XQY8kO1GuLiGAyvCHq8EjshSW1wXiUgtcDDwRpabklJ5GxBE5BlgSISXblLVR1X1JuAmEfk2cCVwc0Yb2EXxvr/gMTdh3dj7Mtm2VEjk+yswEuG5guq5FgsR6Q08DFzTYTQi7+VtQFDVkxM89H5gBnkWEOJ9fyLyBWAqcJLm4drhJP79CsVKYETY4xpgdZba4jpJRMqxYHCfqv4z2+1JtYKcQxCRsWEPzwLez1Zb0kFETgduAM5S1d3Zbo9LyGxgrIiMFpEK4ALgsSy3ySVBRAT4I/Ceqv4y2+1Jh4LcmCYiDwPjgACWDfWrqroqu61KHRFZCnQDNgWfmqWqX81ik1JKRM4FfgNUA1uBeap6WlYblQIiciZwO1AK/ElVb81ui1JLRP4GTMGyga4DblbVP2a1USkkIscALwPvYNcWgO+o6hPZa1VqFWRAcM45l7yCHDJyzjmXPA8IzjnnAA8IzjnngjwgOOecAzwgOOecC/KA4JxzDvCA4JxzLsgDgisIIlIrIg0iMi/sub3qD4hIj2CdjGYRGdjFc/YQkReDqa0RkauDufKTyi0lIn1F5PKutCWBc+xVq0BEKkTkJRHJ2xQ2LrU8ILhC8mGwBkbU+gOq2hA8JhV5hC4D/qmqbcHHlwNnqupFSX5O3+B7kyIm0f/D99KhVkEwDfezwH8ke25XmDwguLwRrFZ1SvDvPxKRO2Icnon6AxcBocyzvwP2AR4TkWtF5PMi8mawN/L7sF7EIyIyN1hxa1rwc34KjAkee1uwtxN+J3+diNwS/HttsBdyJ/AWMCLaucLFqFXwSPD7cM4DgssrN2MpzS/CctFfG+PYSPUHUlbMJJigbh9VrQcI5pJajdXeeAq76z462Btpo/2ie5mqHgrUAVeLyADgRoK9G1X9VgKnHwf8j6oeDPSMca5ELAQOS+J4V8B87NDlDVV9KZhx8hvAlNBQjYj8EMtCGS7d9QcGYon3IjkJOBSYbc2lB7A++NrVweR9YOmwxwJrkzz3MlWdlcC54lLVtuB8SmWwCpgrYh4QXN4QkQOAocDG0MVLRIYQ+fc46foDInIF8OXgwzOBc8Mfq2r4+xuA7tE+CviLqu5R61pEpgAnA0ep6m4ReSHKZ7SyZ++94zG74p0rSd2Axi683xUIHzJyeUFEhmKV4c4GdolIKB32wcC8CG9Juv6Aqv42OGwzWVVXd3zc4dgtQKmIRLqgPwt8RkQGBdveX0RGAX2ALcFgMB4r1A6wA6gMe/86YJCIDBCRblghpGiinSshwSGrDarakuh7XOHygOBynoj0BP4JfFNV3wN+CNwSfHkyEQKCqrZipVP/jRVDf1BVF6W4aTOBYyKc+13gu8BMEVkAPI31bJ4CyoLP/RCYFTx+E/CqiCwUkduCF+cfYPV6pxOjwFOMc+0hWKvgdWCciKwUkS8GXzoBKJh8/q5rvB6Cy2si8kdsWGckMF1VJyX4vnqgTlU3duHcBwPfUNWLO/sZ2SYi/wS+raqLs90Wl33eQ3B5TVW/qKoBbHVNn/CNaZGENqYB5bRXversud8Gno+0zDMfBIfSHvFg4EK8h+Cccw7wHoJzzrkgDwjOOecADwjOOeeCPCA455wDPCA455wL8oDgnHMO8IDgnHMuyAOCc845AP4/znfK4PpkDfYAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "slideshow": { + "slide_type": "skip" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Perceptron\r\n", + "\r\n", + "Since perceptron is a binary classifier, for each input vector $x$ the output of our perceptron would be either +1 or -1, depending on the class. The output will be computed using the formula\r\n", + "\r\n", + "$$y(\\mathbf{x}) = f(\\mathbf{w}^{\\mathrm{T}}\\mathbf{x})$$\r\n", + "\r\n", + "where $\\mathbf{w}$ is a weight vector, $f$ is a step activation function:\r\n", + "$$\r\n", + "f(x) = \\begin{cases}\r\n", + " +1 & x \\geq 0 \\\\\r\n", + " -1 & x < 0\r\n", + " \\end{cases} \\\\\r\n", + "$$\r\n", + "\r\n", + "However, a generic linear model should also have a bias, i.e. ideally we should compute $y$ as $y=f(\\mathbf{w}^{\\mathrm{T}}\\mathbf{x})+\\mathbf{b}$. To simplify our model, we can get rid of this bias term by adding one more dimension to our input features, which always equals to 1:" + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 5, + "source": [ + "pos_examples = np.array([ [t[0], t[1], 1] for i,t in enumerate(train_x) \r\n", + " if train_labels[i]>0])\r\n", + "neg_examples = np.array([ [t[0], t[1], 1] for i,t in enumerate(train_x) \r\n", + " if train_labels[i]<0])\r\n", + "print(pos_examples[0:3])" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[[ 0.92180622 0.91789985 1. ]\n", + " [-1.06435513 1.49764717 1. ]\n", + " [ 0.32839951 2.25677919 1. ]]\n" + ] + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Training Algorithm\r\n", + "\r\n", + "In order to train the perceptron, we need to find out weights $\\mathbf{w}$ that will minimize the error. The error is defined using **perceptron criteria**:\r\n", + "\r\n", + "$$E(\\mathbf{w}) = -\\sum_{n \\in \\mathcal{M}}\\mathbf{w}^{\\mathrm{T}}\\mathbf{x}_{n}t_{n}$$\r\n", + " \r\n", + " * $t_{n} \\in \\{-1, +1\\}$ for negative and positive training samples, respectively\r\n", + " * $\\mathcal{M}$ - a set of wrongly classified examples\r\n", + " \r\n", + "We will use the process of **graident descent**. Starting with some initial random weights $\\mathbf{w}^{(0)}$, we will adjust weights on each step of the training using the gradient of $E$:\r\n", + "\r\n", + "$$\\mathbf{w}^{\\tau + 1}=\\mathbf{w}^{\\tau} - \\eta \\nabla E(\\mathbf{w}) = \\mathbf{w}^{\\tau} + \\eta \\mathbf{x}_{n} t_{n}$$\r\n", + "\r\n", + "where $\\eta$ is a **learning rate**, and $\\tau\\in\\mathbb{N}$ - number of iteration.\r\n", + "\r\n", + "Let's define this algorithm in Python:" + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 6, + "source": [ + "def train(positive_examples, negative_examples, num_iterations = 100):\r\n", + " num_dims = positive_examples.shape[1]\r\n", + " \r\n", + " # Initialize weights. \r\n", + " # We initialize with 0 for simplicity, but random initialization is also a good idea\r\n", + " weights = np.zeros((num_dims,1)) \r\n", + " \r\n", + " pos_count = positive_examples.shape[0]\r\n", + " neg_count = negative_examples.shape[0]\r\n", + " \r\n", + " report_frequency = 10\r\n", + " \r\n", + " for i in range(num_iterations):\r\n", + " # Pick one positive and one negative example\r\n", + " pos = random.choice(positive_examples)\r\n", + " neg = random.choice(negative_examples)\r\n", + "\r\n", + " z = np.dot(pos, weights) \r\n", + " if z < 0: # positive example was classified as negative\r\n", + " weights = weights + pos.reshape(weights.shape)\r\n", + "\r\n", + " z = np.dot(neg, weights)\r\n", + " if z >= 0: # negative example was classified as positive\r\n", + " weights = weights - neg.reshape(weights.shape)\r\n", + " \r\n", + " # Periodically, print out the current accuracy on all examples \r\n", + " if i % report_frequency == 0: \r\n", + " pos_out = np.dot(positive_examples, weights)\r\n", + " neg_out = np.dot(negative_examples, weights) \r\n", + " pos_correct = (pos_out >= 0).sum() / float(pos_count)\r\n", + " neg_correct = (neg_out < 0).sum() / float(neg_count)\r\n", + " print(\"Iteration={}, pos correct={}, neg correct={}\".format(i,pos_correct,neg_correct))\r\n", + "\r\n", + " return weights" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "skip" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Now let's run the training on our dataset:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 7, + "source": [ + "wts = train(pos_examples,neg_examples)\r\n", + "print(wts.transpose())" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Iteration=0, pos correct=0.2631578947368421, neg correct=0.6190476190476191\n", + "Iteration=10, pos correct=0.8947368421052632, neg correct=0.8571428571428571\n", + "Iteration=20, pos correct=0.8421052631578947, neg correct=1.0\n", + "Iteration=30, pos correct=0.8947368421052632, neg correct=0.9523809523809523\n", + "Iteration=40, pos correct=0.8947368421052632, neg correct=0.9523809523809523\n", + "Iteration=50, pos correct=0.9473684210526315, neg correct=0.9047619047619048\n", + "Iteration=60, pos correct=0.8947368421052632, neg correct=0.9523809523809523\n", + "Iteration=70, pos correct=0.8947368421052632, neg correct=0.9047619047619048\n", + "Iteration=80, pos correct=0.8947368421052632, neg correct=0.6190476190476191\n", + "Iteration=90, pos correct=0.8421052631578947, neg correct=1.0\n", + "[[-0.66042328 4.90850882 -1. ]]\n" + ] + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "As you can see, initial accuracy is around 50%, but it quickly increases to higher values close to 90%.\r\n", + "\r\n", + "Let's visualize how classes are separated. Our classification function looks like $\\mathbf{w}^Tx$, and it is greater than 0 for one class, and is below 0 for another. Thus, class separation line is defined by $\\mathbf{w}^Tx = 0$. Since we have only two dimensions $x_0$ and $x_1$, the equation for the line would be $w_0x_0+w_1x_1+w_2 = 0$ (remember that we have explicitly defined an extra dimension $x_2=1$). Let's plot this line:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 8, + "source": [ + "def plot_boundary(positive_examples, negative_examples, weights):\r\n", + " if np.isclose(weights[1], 0):\r\n", + " if np.isclose(weights[0], 0):\r\n", + " x = y = np.array([-6, 6], dtype = 'float32')\r\n", + " else:\r\n", + " y = np.array([-6, 6], dtype='float32')\r\n", + " x = -(weights[1] * y + weights[2])/weights[0]\r\n", + " else:\r\n", + " x = np.array([-6, 6], dtype='float32')\r\n", + " y = -(weights[0] * x + weights[2])/weights[1]\r\n", + "\r\n", + " pylab.xlim(-6, 6)\r\n", + " pylab.ylim(-6, 6) \r\n", + " pylab.plot(positive_examples[:,0], positive_examples[:,1], 'bo')\r\n", + " pylab.plot(negative_examples[:,0], negative_examples[:,1], 'ro')\r\n", + " pylab.plot(x, y, 'g', linewidth=2.0)\r\n", + " pylab.show()" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "skip" + } + } + }, + { + "cell_type": "code", + "execution_count": 9, + "source": [ + "plot_boundary(pos_examples,neg_examples,wts)" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVa0lEQVR4nO3db4xcV3nH8d+zaxt711gJ9q4R2eyaSECgEApeUlDaqiUEpRAFCbVS2k0UUYkFSiNXAhESixd9sRUqCLAEVbUK4QVeQavwtyilhAKVaEnIGhJSMEUQ2Y4TyK4NUZLdxH92n764M/Z4PXfmztxz59458/1Iq92ZvXPn3LH3N2eee8655u4CAMRjqOwGAADCItgBIDIEOwBEhmAHgMgQ7AAQGYIdACITJNjN7BIzu8fMfm5mh83sTSH2CwDo3KZA+zkg6Zvu/udmtkXSSKD9AgA6ZHknKJnZDkkPS7rCme0EAKUL0WO/QtKypM+Z2WslHZK0z91XGjcys1lJs5I0Ojq698orrwzw1AAwOA4dOnTC3cfabReixz4t6X5J17j7A2Z2QNLT7v6RtMdMT0/74uJirucFgEFjZofcfbrddiFOnh6XdNzdH6jdvkfS6wPsFwDQhdzB7u6/kfSYmb2idte1kn6Wd78AgO6EGhVzm6SF2oiYRyW9K9B+AQAdChLs7v6QpLZ1HwBA8Zh5CgCRIdgBIDIEOwBEhmAHgMgQ7AAQGYIdACJDsANAZAh2AIgMwQ4AkSHYASAyBDsARIZgB4DIEOwAEBmCHQAiQ7ADQGQIdgCIDMEOAJEh2AEgMgQ7AESGYAeAyBDsABAZgh0AIkOwA0BkCHYAiAzBDgCRIdgBIDIEOwBEJliwm9mwmf3YzL4Rap8AgM6F7LHvk3Q44P4AAF0IEuxmNiHp7ZLuCrE/AED3QvXYPyXpQ5LWA+0PANCl3MFuZjdIWnL3Q222mzWzRTNbXF5ezvu0AIAUIXrs10i60cyOSPqipDeb2cGNG7n7vLtPu/v02NhYgKcFADSTO9jd/Q53n3D3PZJukvQdd785d8sAAF1hHDsARGZTyJ25+/ckfS/kPgEAnaHHDgCRIdgBIDIEOwBEhmAHgMgQ7AAQGYIdACJDsANAZAh2AIgMwQ4AkSHYASAyBDsARIZgB4DIEOwAEBmCHQAiQ7ADQGQIdgCIDMEOAJEh2AEgMgQ7AESGYAeAyBDsABAZgh0AIkOwA0BkCHagAwsL0p490tBQ8n1hoewWARfbVHYDgH6xsCDNzkqrq8nto0eT25I0M1Neu4CN6LEjeqF62fv3nw/1utXV5H6gSuixI2ohe9nHjnV2P1AWeuyIWohedr3H797895OTXTcPKETuYDezy83su2Z22Mx+amb7QjQMCCFvL7ve4z96tPnvR0akubnz23JiFVUQosd+VtIH3P2Vkt4o6f1m9qoA+wVyS+tNZ+1lN+vx101NSfPzSUmn8Q3A/XzJh3BHGXIHu7v/2t1/VPv5GUmHJV2Wd79ACHNzSa+6UWMvu520nr2ZdOTI+To9J1ZRJUFr7Ga2R9LrJD3Q5HezZrZoZovLy8shnxYDKGvZY2Ym6VVPTSVhPDUl3XprErhZSiZZe/ycWEWluHuQL0nbJR2S9M522+7du9eBbh086D4y4p4UPZKvkZHk/tCPzbr91NSF29S/pqbyHi1wnqRFz5DHQXrsZrZZ0pckLbj7l0PsE0iTp+zR6WOb9fjrdfVGeUs+QEghRsWYpM9KOuzun8jfJMQu7+iRPGWPbh47M5PU09fXL6yrb9wmyxsA0AshJihdI+kWSY+Y2UO1++5093sD7BuRCTFhaHKy+fDDLCNd8jy2nZkZghzVEGJUzPfd3dz9Knf//doXoY6mQoweyVP2oGSCQcDMU/RUllJIu1JNnrJHiJIJE5FQdeZp86QLND097YuLiz1/XpRvz57mpZCpqaR+vbFUIyU96qrUq6vePsTNzA65+3S77eixo6falUKqPtEnb/vo7aMXCHb0VLtSSNUn+uRpH8sOoFcoxaBS2pVqypanfVU/NlQfpRj0paqPWsnSvrRyS9U/jSAeXGgDldK4qNaxY8n48rm56pyYbNe+VuP0ixxDDzSiFAME1KrcMjfHiBrkQykGKEGrckurE8eMlkFIlGKAgNqVW5otOxDyuqyARI8dCKrdydVmPfOqj91Hedxdz5x6Rr/67a/0g8d+kPlx9NiBgFqdXE3rmaddeo/RMnFaW1/TidUTWlpZ0tLKkp5ceTL5/mzyfWm14eeVJT139rmOn4NgBwJLW+UxrWc+PCytrV28PaNl+sfqmdULwrge1s2C+8TqCbmyD1rZtmmbdm/frfHRcf1QP8z0GIIdhauXG6o4fLGX0nrga2tJuWbjaJmqjN0fROu+rt8+99vMYb1yZqWj/e/ctvNcWI+Pjmv36O4Lvo+Pjp/7/ejmUSWXvZDs3ZZp/wQ7CsWJwfPSTqzWh0Ly5les588+f2HJo0VYn1g9oTVv8jEqxZbhLdo9urtpWF8Q3Nt3a9fILm0aKjZ6CXYUqtWJwUELrmbj2CXp2WeT7ywr0Bl31++e/935YK4FdrOwXlpZ0tOnnu5o/5duvfSCnvP4yHjzXvb23Xrhlhee61VXAcGOQjGN/rz6G9m+fdLJk+fvP3lycD/FbHR67fS5IE7rXTeG9dn1s5n3vXlo80VljrSwHhsd05bhLQUeabGYeYpCsfDVxQbpNXF3PX3q6Qt70S1KIU89/1RH+9/xgh2pJY+N912y9ZJK9aq7kXXmKT12FCptGv0gnxjs908xZ9bO6MTqieZhvWGo3tLKkk6tncq872Eb1tjoWKawHhsd09ZNWws80v5FsKNw27adD/adO6UDBwa75FC1xcDcXc+efjbzuOqTz51sv9MG27dsTz2Z2BjYu0d369Jtl2rImDeZF8GOwjS7jNxznc+1iE4vPsWkTYI5F9g5JsEM2ZB2jezKFNbjo+Ma2TzSfqcIiho7CjNIteROdTO2v1eTYNqNq965baeGh4ZzvgLoRtYaO8GOwgwNJZeA28hMWl/vfXuqpj4JJstQvSeffTLYJJhmdevGSTCoLk6eonRVqyX3Qn0STJawXl5Z7utJMKgu/uVRmBhGxKRNgkkbV93tJJhzveeUcdXjo+Pa8YId9KqRCcGOwlT1MndZJsE0/tztJJjGsN44+mP8W9/X2N9/XFuOPCZN7pDm3iP9xQAPFUJQ1NjR9+qTYJoO1StwEsxFYZ11Ekyz4UJcIw8Z9PTkqZldL+mApGFJd7n7R1ttT7CjnfokmCxD9bqdBNNu9Ef9K/gkGIYLoUs9O3lqZsOSPiPpOknHJT1oZl9395/l3TfikTYJJi2su50EkyWsX7TtRaVOgvGjx9SsT592P9CpEDX2qyX90t0flSQz+6Kkd0gi2CO3tr6mk8+dbD76oxbWjWWRTibBmOxcr7rdUL1+mwTz+PCkJtYu7rE/PjypiRLakxkL6/eNEMF+maTHGm4fl/QHGzcys1lJs5I0GfN4tz63emY187jqUJNgmvWum06CWViQ3rsxWK4O/AoU7/a1Oc1rVqM6X2Nf0YhuX5vTQontaomF9ftKiGBv+qnyojvc5yXNS0mNPcDzIoNmk2DShuoVNQmmHta5JsFEFCz/PTWjdx+V/kH7NaljOqZJ3ak5/c9UhY+DhfX7SohgPy7p8obbE5KeCLBfpGg2CSYtrLudBNNsxMfGn8dGx3o3CSaiYEnG98/oC6vn2z0yIs2XNb4/S4ml35ekHDAh/ioflPQyM3uppMcl3STprwLsd2C4u556/qlMQ/VCTYJJC+7KToKJKFgqNb4/6yehQZxG3MdCDXd8m6RPKRnueLe7t+x7DMJwx9Nrp7W8spwprJdXlnVm/UzmfW8a2nTxScSRi0d/1Nes7ucrwZzDEMFiZH1dGXtfCT1dK8bd75V0b4h9VVWzSTCtTjJ2Mwkmy1C93aO7o7gSTMdiWJ+girJ+EqrUxwy0M9BLCrSbBLOxp91Xk2BiQ7AUo5MSy8wMr3efiCrY3V0rZ1Yyj6sOMQkmbVx12ZNgokSwhMcnoShVPtg3ToK5aD2QgifB1AN7bGRMo1tGCzxSoAR8EopSKcG+7us68tSRTOOqO50Es3XT1gvWrO54EgwwaPgkFJ1SVne0l5jrPdm337lt54WljpGUMdZ5J8H0uYGf8T3wLwBiV+krKJmZJnZMZArrXSO7tHl4cxnN7CtVnpjZk7zt5QvAGwgqjvXYI1HVYd49G/7cqxeA8dwoERezHjBVvXB0z95wevUCVPUdFAMha7AzHi8SaTO7y57x3bOVALp9ARYWkrAeGkq+L7RZXzGipQ0QL4I9EnNzSUWgURWGI/fsDaebF6BeVjl6NOnt1+vyrcK9qu+gQAOCPRIzM0mZd2oqqT5MTVWj7NuzN5xuXoBWK0amqeo7KNCAGjsK9/2/WdCe+f16ydoxPTE8qSOzc/rDf6rAicZu6/KMikFJOHmKaqjyKBJOhKLPcPIU1dBNuaOu0xObnaKsgkgR7ChWt6NImp3YvOWWpEzSbchvfKOQqnliAsip8ouAoc91e+WdZj39etmwm1mlaTNT5+cpuyA69NgRRlrZpNtyR7sefdZyTl2ekhDQZwh2tJalzt1qPHi34zCzjAvvZFJQLyYWFX1OAMjK3Xv+tXfvXkcfOHjQfWTEPYnr5GtkJLm/0dTUhdvUv6amwj53nv0X0cZ27W32WgE5SFr0DBlLjx3pspYviugNN/b0paS336jT0StFj4BJe61uvpneO3qOYEe6rIFd1DT7mZnkxKa79PnP5xu9UvTU3FZvYlmWKgACItiRLmtg92I8eD3k19eT7+0CuVm9u9N9dKLdmxgnatFDBDvSZQ3sqi1U083iXnk1e602YgVI9AhLCqC1flwXZdcu6eTJi+8veqmA+mvVbNx+L54f0WNJAYRRZPmimbxDBhcWmoe6VHyPuf5aHTzIUgUoFcGO6ghRQmlVx+7VmulVK01h4FCKQXWEWG0xbSleKelJE67oYz0pxZjZx8zs52b2EzP7ipldkmd/GHAhxsOn9cp37iTUMTDylmLuk/Rqd79K0i8k3ZG/SRhYIcbDp43kOXCg+3YBfSZXsLv7t9z9bO3m/ZIm8jcJAyvEeHjq20C4GruZ/Zukf3H3gym/n5U0K0mTk5N7j6YNCcNg68fhlUCPBLs0npl9W9KLm/xqv7t/rbbNfknTkt7pGd4pOHkKAJ3LGuxtL7Th7m9p80S3SrpB0rVZQh0AUKy8o2Kul3S7pBvdfbXd9ugzrC8O9KW8o2I+LemFku4zs4fM7J8DtAlVkHeyEG8KQGmYoITm8kwW2nh9USkZ3cLoFCAX1opBPnkmC3F9UaBUBDuayzNZqBfXFwWQimBHc3kmCxV1RSUAmRDsaC7PDM5eXFEJQKq249gxwGZmujvZWX8MM0iBUhDsKEa3bwoAcqMUAwCRIdgBIDIEOwBEhmAHgMgQ7AAQGYIdACJDsANAZAh2AIgMwQ4AkSHYMVi4AAgGAEsKYHBsvABI/apQEssfICr02DE4uAAIBgTBjsHBBUAwIAh2DA4uAIIBQbBjcHABEAwIgh2DI89VoYA+wqgYDBYuAIIBQI8dACJDsANAZAh2AIgMwQ4AkQkS7Gb2QTNzM9sVYn8AgO7lDnYzu1zSdZKYvgcAFRCix/5JSR+S5AH2BQDIKVewm9mNkh5394cDtQcAkFPbCUpm9m1JL27yq/2S7pT01ixPZGazkmYlaZK1OQCgMObeXQXFzF4j6T8l1ddBnZD0hKSr3f03rR47PT3ti4uLXT0vAAwqMzvk7tPttut6SQF3f0TSeMMTHpE07e4nut0nACA/xrEDQGSCLQLm7ntC7QsA0D167AAQGYIdACJDsANAZAh2AIgMwQ4AkSHYASAyBDsARIZgB4DIEOwAEBmCHQAiQ7ADQGQIdgCIDMEOAJEh2AEgMgQ7AESGYAeAyBDsABAZgh0AIkOwA0BkCHYAiAzBDgCRIdgBIDIEOwBEhmAHgMgQ7AAQGYIdACJDsANAZAh2AIhM7mA3s9vM7P/M7Kdm9o8hGgUA6N6mPA82sz+V9A5JV7n7KTMbD9MsAEC38vbY3yfpo+5+SpLcfSl/kwAAeeTqsUt6uaQ/MrM5Sc9L+qC7P9hsQzOblTRbu3nKzP4353NX2S5JJ8puRIFiPr6Yj03i+PrdK7Js1DbYzezbkl7c5Ff7a4+/VNIbJb1B0r+a2RXu7hs3dvd5SfO1fS66+3SWBvYjjq9/xXxsEsfX78xsMct2bYPd3d/S4kneJ+nLtSD/oZmtK3nHXM7aUABAWHlr7F+V9GZJMrOXS9qiuD8GAUDl5a2x3y3p7lq9/LSkW5uVYZqYz/m8Vcfx9a+Yj03i+PpdpuOzbDkMAOgXzDwFgMgQ7AAQmVKDPfblCMzsg2bmZrar7LaEZGYfM7Ofm9lPzOwrZnZJ2W0Kwcyur/1//KWZfbjs9oRkZpeb2XfN7HDt721f2W0KzcyGzezHZvaNstsSmpldYmb31P7uDpvZm1ptX1qwb1iO4PckfbysthTBzC6XdJ2kY2W3pQD3SXq1u18l6ReS7ii5PbmZ2bCkz0j6M0mvkvSXZvaqclsV1FlJH3D3VyqZd/L+yI5PkvZJOlx2IwpyQNI33f1KSa9Vm+Mss8ce+3IEn5T0IUnRnZ1292+5+9nazfslTZTZnkCulvRLd3/U3U9L+qKSjkcU3P3X7v6j2s/PKAmGy8ptVThmNiHp7ZLuKrstoZnZDkl/LOmzkuTup939qVaPKTPY68sRPGBm/2VmbyixLUGZ2Y2SHnf3h8tuSw/8taR/L7sRAVwm6bGG28cVUfA1MrM9kl4n6YGSmxLSp5R0pNZLbkcRrlAy6fNztVLTXWY22uoBecextxRqOYIqanNsd0p6a29bFFar43P3r9W22a/kI/5CL9tWEGtyX1/8X+yEmW2X9CVJf+fuT5fdnhDM7AZJS+5+yMz+pOTmFGGTpNdLus3dHzCzA5I+LOkjrR5QmJiXI0g7NjN7jaSXSnrYzKSkTPEjM7va3X/Twybm0urfTpLM7FZJN0i6tl/ejNs4LunyhtsTkp4oqS2FMLPNSkJ9wd2/XHZ7ArpG0o1m9jZJWyXtMLOD7n5zye0K5bik4+5e/4R1j5JgT1VmKearinA5And/xN3H3X2Pu+9R8o/y+n4K9XbM7HpJt0u60d1Xy25PIA9KepmZvdTMtki6SdLXS25TMJb0Mj4r6bC7f6Ls9oTk7ne4+0Tt7+0mSd+JKNRVy47HzKy+suO1kn7W6jGF9tjb6HY5ApTv05JeIOm+2qeS+939veU2KR93P2tmfyvpPyQNS7rb3X9acrNCukbSLZIeMbOHavfd6e73ltckdOA2SQu1Tsejkt7VamOWFACAyDDzFAAiQ7ADQGQIdgCIDMEOAJEh2AEgMgQ7AESGYAeAyPw/SugVn8RyP80AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Evaluate on Test Dataset\r\n", + "\r\n", + "In the beginning, we have put apart some data to the test dataset. Let's see how accurate our classifier is on this test dataset. In order to do this, we also expand the test dataset with an extra dimension, multiply by weights matrix, and make sure that the obtained value is of the same sign as the label (+1 or -1). We then add together all boolean values and divide by the length of test sample, to obtain the accuracy:" + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 10, + "source": [ + "def accuracy(weights, test_x, test_labels):\r\n", + " res = np.dot(np.c_[test_x,np.ones(len(test_x))],weights)\r\n", + " return (res.reshape(test_labels.shape)*test_labels>=0).sum()/float(len(test_labels))\r\n", + "\r\n", + "accuracy(wts, test_x, test_labels)" + ], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "1.0" + ] + }, + "metadata": {}, + "execution_count": 10 + } + ], + "metadata": { + "slideshow": { + "slide_type": "fragment" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Observing the training process\r\n", + "\r\n", + "We have seen before how the accuracy decreases during training. It would be nice to see how the separation line behaves during training. The code below will visualize everything on one graph, and you should be able to move the slider to \"time-travel\" through the training process. " + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 11, + "source": [ + "def train_graph(positive_examples, negative_examples, num_iterations = 100):\r\n", + " num_dims = positive_examples.shape[1]\r\n", + " weights = np.zeros((num_dims,1)) # инициализируем веса\r\n", + " \r\n", + " pos_count = positive_examples.shape[0]\r\n", + " neg_count = negative_examples.shape[0]\r\n", + " \r\n", + " report_frequency = 15;\r\n", + " snapshots = []\r\n", + " \r\n", + " for i in range(num_iterations):\r\n", + " pos = random.choice(positive_examples)\r\n", + " neg = random.choice(negative_examples)\r\n", + "\r\n", + " z = np.dot(pos, weights) \r\n", + " if z < 0:\r\n", + " weights = weights + pos.reshape(weights.shape)\r\n", + "\r\n", + " z = np.dot(neg, weights)\r\n", + " if z >= 0:\r\n", + " weights = weights - neg.reshape(weights.shape)\r\n", + " \r\n", + " if i % report_frequency == 0: \r\n", + " pos_out = np.dot(positive_examples, weights)\r\n", + " neg_out = np.dot(negative_examples, weights) \r\n", + " pos_correct = (pos_out >= 0).sum() / float(pos_count)\r\n", + " neg_correct = (neg_out < 0).sum() / float(neg_count)\r\n", + " snapshots.append((np.copy(weights),(pos_correct+neg_correct)/2.0))\r\n", + "\r\n", + " return np.array(snapshots)\r\n", + "\r\n", + "snapshots = train_graph(pos_examples,neg_examples)\r\n", + "\r\n", + "def plotit(pos_examples,neg_examples,snapshots,step):\r\n", + " fig = pylab.figure(figsize=(10,4))\r\n", + " fig.add_subplot(1, 2, 1)\r\n", + " plot_boundary(pos_examples, neg_examples, snapshots[step][0])\r\n", + " fig.add_subplot(1, 2, 2)\r\n", + " pylab.plot(np.arange(len(snapshots[:,1])), snapshots[:,1])\r\n", + " pylab.ylabel('Accuracy')\r\n", + " pylab.xlabel('Iteration')\r\n", + " pylab.plot(step, snapshots[step,1], \"bo\")\r\n", + " pylab.show()\r\n", + "def pl1(step): plotit(pos_examples,neg_examples,snapshots,step)" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "skip" + } + } + }, + { + "cell_type": "code", + "execution_count": 12, + "source": [ + "interact(pl1, step=widgets.IntSlider(value=0, min=0, max=len(snapshots)-1))" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "8561af1ae77c421f9ca068fe0bdac566" + }, + "text/plain": [ + "interactive(children=(IntSlider(value=0, description='step', max=6), Output()), _dom_classes=('widget-interact…" + ] + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 12 + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Limitations of the Perceptron\r\n", + "\r\n", + "As you have seen above, perceptron is a **linear classifier**. It can distinguish between two classes well if they are **linearly separable**, i.e. can be separated by a straight line. Otherwise, perceptron training process will not converge.\r\n", + "\r\n", + "A most obvious example of a problem that cannot be solved by a perceptron is so-called **XOR problem**. We want our perceptron to learn the XOR boolean function, which has the following truth table:\r\n", + "\r\n", + "| | 0 | 1 |\r\n", + "|---|---|---|\r\n", + "| 0 | 0 | 1 | \r\n", + "| 1 | 1 | 0 |\r\n", + "\r\n", + "Let's try and do that! We will manually populate all positive and negative training samples, and then call our train function defined above:" + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 13, + "source": [ + "pos_examples_xor = np.array([[1,0,1],[0,1,1]])\r\n", + "neg_examples_xor = np.array([[1,1,1],[0,0,1]])\r\n", + "\r\n", + "snapshots_xor = train_graph(pos_examples_xor,neg_examples_xor,1000)\r\n", + "def pl2(step): plotit(pos_examples_xor,neg_examples_xor,snapshots_xor,step)" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 14, + "source": [ + "interact(pl2, step=widgets.IntSlider(value=0, min=0, max=len(snapshots)-1))" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "8f45bf78e4c8471fbc6eea233dde2bf6" + }, + "text/plain": [ + "interactive(children=(IntSlider(value=0, description='step', max=6), Output()), _dom_classes=('widget-interact…" + ] + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 14 + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "As you can see from the graph above, the accuracy never goes above 75%, because it is impossible to draw a straight line in such a way as to get all possible examples right.\r\n", + "\r\n", + "The XOR problem is a classical example of perceptron limitations, and it was pointed out by Marvin Minsky and Seymour Papert in 1969 in their book [Perceptrons](https://en.wikipedia.org/wiki/Perceptrons_(book)). This observation limited research in the area of neural networks for almost 10 years, even though - and we will see this in the next section of our course - multi-layered perceptrons are perfectly capable of solving such problems.\r\n", + "\r\n", + "## Complex Example - MNIST\r\n", + "\r\n", + "Even though perceptron cannot solve XOR problem, it can solve many more complex problems, such as handwritten character recognition.\r\n", + "\r\n", + "A dataset that is often used when mastering machine learning is called [MNIST](https://en.wikipedia.org/wiki/MNIST_database). It has been created by Modified National Institute of Standards and Technology, and contains a training set of 60000 handwritten digits, collected from around 250 students and employees of the institute. There is also a test dataset of 10000 digits, collected from different individuals.\r\n", + "\r\n", + "All digits are represented by grayscale images of size 28x28 pixels.\r\n", + "\r\n", + "> MNIST Dataset is available as a training competition on [Kaggle](https://www.kaggle.com/c/digit-recognizer), a site that hosts machine learning competitions and contests. Once you learn how to classify MNIST digits, you can submit your solution to Kaggle to see how it is rated among other participants. \r\n", + "\r\n", + "We start by loading MNIST dataset:" + ], + "metadata": { + "collapsed": true, + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 15, + "source": [ + "# If you are not running this notebook from a cloned repository, you may need to grab the binary dataset file first\r\n", + "# !wget https://github.com/microsoft/AI-For-Beginners/blob/main/data/mnist.pkl.gz?raw=true\r\n", + "# In this case correct the link to the dataset below as well.\r\n", + "\r\n", + "with gzip.open('../../data/mnist.pkl.gz', 'rb') as mnist_pickle:\r\n", + " MNIST = pickle.load(mnist_pickle)" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Let's now plot the dataset:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 16, + "source": [ + "print(MNIST['Train']['Features'][0][130:180])\r\n", + "print(MNIST['Train']['Labels'][0])\r\n", + "features = MNIST['Train']['Features'].astype(np.float32) / 256.0\r\n", + "labels = MNIST['Train']['Labels']\r\n", + "fig = pylab.figure(figsize=(10,5))\r\n", + "for i in range(10):\r\n", + " ax = fig.add_subplot(1,10,i+1)\r\n", + " pylab.imshow(features[i].reshape(28,28))\r\n", + "pylab.show()" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[ 0 0 188 255 94 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 191 250 253 93 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n", + "1\n" + ] + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlIAAABRCAYAAAAZ1Ej0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABJF0lEQVR4nO2dd3Rc133nP/eV6R29EyABgmBv6t2qlhzJsuPY2Y1LsnaKnbWTbE682STr3XU2PtmUdbKbeB3HiRU7cZEtybZkqzdSIilS7A0s6L0MBtNn3nt3/xiwSSwAiDKE3uccHBIzr9wv3i2/d+/v/n5CSomNjY2NjY2Njc3MURa7ADY2NjY2NjY21yq2IWVjY2NjY2NjM0tsQ8rGxsbGxsbGZpbYhpSNjY2NjY2NzSyxDSkbGxsbGxsbm1liG1I2NjY2NjY2NrPkqgwpIcT9QojjQoiTQogvzlWhiglb47XPUtcHtsalwlLXuNT1ga3xPYmUclY/gAqcApoAB7AfaJvt9Yrxx9Z47f8sdX22xsUvm63R1mdrXFoaZ/pzNTNS1wEnpZSnpZQ54LvAw1dxvWLE1njts9T1ga1xqbDUNS51fWBrfE+iXcW5NUDPeb/3Ate/8yAhxGeAzwCoqJs9BK7ilguLCy8meQIiIjMkAT7Oe1DjUtE39VEceOydxy0Vje/legpLX+NS0Tf1kd0WsTUWOxmS5GRWXO6YqzGkLnbhd+WbkVJ+Hfg6QEBE5PXifVdxy4VlSPYyxiBtYgs75YvkycF7UONS0Qfwgnx8lPfgMwRbY7Fjt8UCS0Xje7mewrWt8Xx2yheveMzVLO31AnXn/V4L9F/F9YoOJ24ypM//yNZ4jXERfQ6WkD5Y+s8QbI1LAbstLg3eCxpnytUYUm8BzUKIRiGEA/go8OO5KVZxECBMmgRpmUQWDG5b4zXG+fosaQFEWEL6YOk/Q7A1LgXstrg0eC9onCmzXtqTUhpCiM8Bz1Lw4v+mlPLwnJWsCFCEwkq5gb28TpoUwPdtjdcW5+ubavTjS0kfLP1nCLbGpYDdFpcG7wWNM+VqfKSQUj4DPDNHZVkw1BWNWCEvht+Bmsyj9Y1hTcSwksl3HVsqqiilip3yRSbl+J8uSAEVFaGqCFVB+P0Q8hc+FwLL60LJGYixCaRpgbQwxyfAMmd9u0XR+A602hqkxwWASKYx+uZupviMPoAX5OODc3bhImIxnqHQHQiXE6WsBKmpoKkYYQ+mU73gOD2eQ0lkYGAYK5lG5nOzul8x1FOYqqteN5bfheVQMV0aasZASRsoPYPIVBorlZrVtYtF43xht8WlwUJoFLoDxedF+LxIl2PqQ1EYB/0u8gFn4SMpUZN5lEweJZZExiaxEkmkJUFaIN/lvjXnXJUhda3S/psVbLmhnf9e+xP+Zvgutn9rMxVvxmBPcRjVaiSEcDqRXjfjW8sYvqFQEaQuWdnSx4nBMoIvRdAyEi0tCbx6EnN0bJFLPUuEQKgqvb/YwGSLgbAEgeMqlX83jDTNBWkENrNACNTyUvINZZx82IMRMFEDef7zpp/xAd+pCw79g777efV4M42PBXCdGsHo7luwDm5OOb+urs5x46pTbAl28bHAAb4XX8PPh1Yz8f+aCR6egMPHrz19NjZFgtAdqJXlJNdUMbZGJ1k3NVGggHRabF11mq/WP4UqBHFL8tfD72N7fyPZXXVU7M7h2d+DTKWRuRxWJjPv5X1PGVLKhjZGtgapW9fPB0r3U6vqhPQUlgooi5gtRwiEw4FaXYlRFqDrfX5yQYnllKhVKW6oK0SZcKoGtwRPsMe/jJ9l1iAUiZSQCbXg7zNw7+lEJpOzfhteFISC0DQmmw1u3XAMSyq8Ya2kUlXBkiBnP9M23yQ/fD3xGhXHpMTfl0d/Ye+sDQShaSgeD9RVISbiczojN2dMGRJKYz25mhDdt7nIVBusXdVByJEi4khxg7uDctV7wWkPl+zF25bj6Y+txdlXS/X2clxdE5jHTy6SkFmgqChtzUQ3hEhfn+DWhk5UIenORvhRYhUeJccHKg/wlw9WEG2NsExtRRkaxxgcWuyS27xHUcNhRNBPYk0lplMgFTAdAtMhSFUKrKlJHi0NelziGzBxTBg4O0eR8QTm2PiCl1nxeFACfuI3NDBZrxHbkKOsYoTVocJEgYLEq+W4I3SMctUDgE8YPBDeT6UzxnOuVXQ2laDf0UTpfom3L4uy/cBVrdhMh/eUITWyNcjaXzvEr5e/wjqHiS5ULCkQkkV9exSajuLzkmopI7rSwR984vt82NePU1z88dzjPcl1/tO0OfuoVLN8pOKTdB8vpXmoHHVw7JoypIRSMCJXtvbxT/WvAHBvKoDQNDBNCj6pRYgQDD2a5fc2PM8zI2s58tYyml/TsXL5WRl/wuFAREKMbQwTOOVCFKUhpSDcbhKrSxlZp/EXH/8m97vfWdecmO94aB/wTPIBz5t8tfpNfp728DvOT1H+VgneEx3z3sHNCUIgdI3xjWHyvzjO37Q9ySbnOL/V+TAnJsp4Md3Cr7W8waeCR3n0jqN8e9N6ftB1L+EjGtiGVPEgLhMKaKnNHgoB5SWk64J0f0Ci+nNomknIl6bKm+Avlv2Q5ZobgO1ZhZ9MbOTx/ZtwdbmoeqMcV58bxqML/ndRAn7M2jJ63i9Zv/IUP1hxac+hM72MLlTudSe5132YPyg5DGsgYWXZUvtZPPvd1OzSkFnbkLpqtMoKhh5sInpbhk+Vv06TniEh4X8O3cRT+zbQ+moU0T/KgnfpQqD6/eQ3LufEI04qVw3zcNVx7vJ04hSeS55Wqji4y3Mav1BwCgd/uvIJjjTU8r1Vmxnb0UTDUyUFgyqRxEokll4nUQxISfW/Ovi/+x7mM594mlONJQifFxFPzKrRKgE/2cZSJh9JkHk1QMWb81Dmq0BxucjdvJqOT0jWNHTzSEk7W51jgHtG19nqHOPzv/BT/rr+fZS6t1Lyeh9GV8+VT1xEtOoq+h5dRvLGFH/b9iQ/jm7kj4fqcf5zGGfUoMKUfPWX7yF7o87Hgnt5yHeQ9t+s5LUX1rFioBZrZHRBlhdsLkTxelH8PkbvaSJVIUjWmyilWRrKz820mJbCUMyPcdqHIybQE+AesVDzhT7T25tGPdaFlc4g80ZxG/5CoHg85K9byeD1LoxNcdoqu/mv5fvxKxlUYeERWQJKhmr1nB9jm56kJPIGG27qpn9riD33NHB4uBJj/43UvphGeX3vgkmI3tFItFXhrnUHuCl4kpiVwSN0VCF4NhVk3PSRkxoJ08Vo3sdYvjD7/UD4IMu0MVY7CiaNU2g8uOoQOyLL6HZtpnxvHufTb81buZe8IaW4XFjlYcY3WGxa1sN6RxpQ6DXg+c6VeE454PRxrHT6iteac4SC8PuI1zrZvPUEn6jcPvWGf2kjCgqVpEY99+jucOW53nmcmzwn+NXMJ5g4GSSkCtQhDSuZKurlsYuhKyZKKIgVnUAaxmIX55L4dnfhiFfT8OkRakMxqChFkRIzm53ZhYQAXScX1HiwaS9PnbihUG9z+eLouIVAuJwkqh38zuZnuN17nNW6g5kaUQAlipvfCHZxYMVxXm7bQOBUBHV8omgNfsXjwSoLEVuf446mU9zomuBLI7XEDpWw4uXjZ5c/3DffxP7VtXwwsI8GTeO/Vz3L7cuXY1aEUBJJWERDSnG5QNcRLhdYJjKdAcs6kzcNLIk0TYQiQCigXHrmRnE6QdcQbvfUjLGFjCewMtnFratnlp09nkL5XC6skgDZiIexdaDUJrlneTu3Bdv5oHfg7GlZafBWNsjfld/F6WiEyQkPqQEHSq7g6pENeIlY9ajxDCKTQ07GkZnsRTcmLSqKiuL1QGMNY6tcWFsm+f3VL3CP9yQ6oAqBKSXq1Kxc3JLEOde3RhSTu9xd4O7ilwL7eb20gb913slEdxllR0uwJmIL0hfn/IJs2MKpGmSkzmnDgYokLxUeH93CaMaHaSkk8g4m0y7SKScISbpRZ62/D7+yn4ii4FF0bg8cJ6BleHKTg9h4kPJ5LPfSNqQUlfRdaxlZr/MP7/9/NOsxPMLNz1J+nhrbRPVf6zg6OjEWqRNXXE4SG2sZWy/4ccOP8Qmd2T4St3CwxmHy803/QM96J7/07GeJvO2n/N9iWPH43BZ8nrm+pJMnful2qrbFYdfBxS7OJTGGhtEDPiZMD/dVHOFvf+cu6p8M4vrprlldL+9W+I2S13mhbSWZO9bi2dddFD42QlWhNEK6XPDx4DE8wnHV1/xU6TYqH5zkO85bCbWtofyHRzBjk0VlTAlNI33naoY36Hzrrr8jZTl5LNaK9b0ymp89jTEePXusnoTTsRKOlFegOgap19xsqeth/72raHgqD9HoZe40v+RuWs1kg4OxjRaOCYWyvRZ63EDNFAwfNWui9Y8h3U4svwfLpSHVixhTqmBorZtUpaR0yxDjcS+ZUTf1T4Pv8BBGV++iGVNqeRlEggzdWkqiHgIbxriv9jAPBPZTpqbRkagCRkwHL2fOpSrRMQmpKb7R9CMyUmIBGSkwZUF/SmokpYOnopvYF61l4NVWQu0WgR/uLp7NMEKg1VUT31jFuv+yn08EnuUmdw9BRSVlwZ8M3Ec05yZj6iiiUF5LXtxY9mg5Hi17mxotyj+1PcbnfvmjtK9upuXro1inu2e943a6lH33EBVeDyeXt7Jr1Ub+z30JjNM+vL2CqtcnUKJxNMMkZKUIySRSSoSiMFxZx49WtfF399zBpzdv4/dLjnCfZ5itrn5WuIb40xMfsg2p2SCcThSfl/FVOumVWdr0GB5FJWZleHz0TrafWk5r3zjW2PiiNAa1JAIlYcZXaYj6BCXK5d/un0yGOJ6pImp4qHFGuct7jGrVJHzeeRoq5aqHchVWtvRx3KqhbE89Wv8YxsC1s9O4xhElscwkd9jJ1Q/Z84iUiGyO7ZPNhPQ01bXjZIPluISYXZ0S4FcELkce0yVAVa98zjwjnE6UQICRWyuJr8zjE853HWMh6TbSHM6V06yPElIK3gu6EPgVBxrv1lGtpbnZ286Lq1fS5y6l/LUSFNMqHqN/agPI2CqdzMoMrXqSb8ea+Gb7jVT05Aq7ZM97xoEuk+E9ZfxR5mHaygf5q/qnqPNEebM5ixFyI2ZbJ+aAbFgjVSmobx1iMuOk3x9GzTpQpsZEJSdwD/swXYK8FywHSPXdZZUKGA1pIuEkj9TupytdytFwBclgNV51ETfrAKnNDcQadSY25SitmOTh+gOscA6RkTovJesYyIfYP1FLfyLAeKywHKQISSSYZEtZD39ZvY2daT+7k010pyOkTR2AiCNFpTOGUzFYFRxiaKOfEV8AX+8a9M7hxd8UMlVPY1urGV2r8gvht2nWo1SpbvbmLN5Or+DF3WtQkwrC5Fxit0tURanC3to6SsNxbqs8VTC4KrNYHidC1+bdkLLicaxkCl0IwqKSTImfYK+FdyCH6OzDuNjLlqKiqSraMh8OTx6PWlgRcAodv7CIqAmkNr9tb8kaUkooiKwqJXDPIJ9r2EFEdRKzcvQYOm9sb6NyF1hDI4vmu2A21xJb7qHy/h4eqLhy2IUv7n4U/bAH96gk3gid95TyaHg3Nzsv7o39WPP3eau+hC92/Colhz04ryFDarWzl3XrO+nf01TchhRANsdzh9toqBnjgeojPB6pJORwIHO5qxo45eUcYxcQJRTEbKxkza8f4uGSd/tKWEiyMs9T8XX804kbeLRpP+vcBZ+nMm2SNj2JR+jv2jhRo3qocWd539rH2d6i8MfPfAavlFAkhpTQdITfR+DuQb7Y+CphxcW/dW6h9BseXEe6Md6xzOF5YifLnhSoKxrp2drMzi9Vs9nbQcXWSX74zD34HQ7kTJd854h0iUKq3uDrK79Do+bC2lzoMxQULCxiVo43MhVE1ATLtAQRRcMpdCwuPO7M/wEsLEYCWbpKPHwu/LlF0XUWIej6sOTz1z/Dp4JHzxr7fxVt5mtdt9NzpBL3gELNqwnKBqKEu04AU35/N63m+feVkfr4y/zR4UcwtkUoOZJHTxSeb0eNk8kGBccN49xee5LXt/4DP1i1gj/zP0TDT2pwLrIhJTQdxe8j+rEEH2/eze3uFBoeLCR/P3QXr7Y3s+rLXRhDI9ObLRQCNRjAbKnn6TtvIr0mzbKqMcxACbrTCQuxkckyMQYGUQYGqdl27uPLlt6hky5V+GjrHm5wn7rckfPCkjOkFL8fUVvJwJ1lTKw1+E/1r3Oz+xQKDr4R3cQ/HriJyp2S4IExrNz8WteXY2SDl+h6k09Xv8VmVxdQeAOykGzP6HTmS9kea8aSAguBftRDuN3E35EkeMrJq73X8WzJdeTCFh+7Yzt3+w9zsyuPMvXK4RM6y7QoydVZ1IyTikVTurSRhoES1clUaNzrP8i/Bu9C8fsKQVKn65smJRgGSl4yYipk81rRNMzkdcsYXaPxqfAh1jsGOd9/r89MsSNTwx/teQRx0kPpQclPIrfxpKtQB3NByDZmuX/1YX6t9DVW6Vx0J2qdmqDvl3NoJ6qoeS2C6+Twojugy40rGV3l497KbZSpk/z+4PVMHCmhrH0QK564xEkSkcqgZgsGdL02znJ9hO9E7iNUWoIxMLRoS19KWuFDb3+abFbDGHajlOQIBpJIKTAthXTagaJa6LqJplgoioWcWv4RQtIYGqfFN8xvlbxBleoBFLal6/jnvpvw9xowOs6ibbGVEv9hB1/lbv61YiuJtJNsrw9vj4Jn0KJ+1ECfzKN1DhV8Rs/DdCm4hgU37fh1nK/5qX4rgTY8Cbk8AOFhD/5OD5M9IZ6vu45bP9WOS+RYsXKAyR21OBV1UX3D8reuZXS9k4ebtnGf/yAaGtuzCk9FN/HGi2uoOCSxJuPTL6OUWMk0Wvcw1a8pZI47SfqqKTnVg1mku8EVh05mWQnJGsED/gM0aGlm4795NRRLfz0nCE1DiYSIt4SJXZ/hd7e8yKO+o0RUJ1mZ5+WRFsKvugjtGcA43bmoZU0sg1WtvXzAd+rssl5WGiRknm3J1eyL1bKnfRlYhc6s8pSF/3QCDpzAIS2qdnkQAT9WxM/TjavxN2W40XUEZWoZxSk0ytQ8rQ0DnBxoQGha8azpLyWkRI8r5AyVzQ4VwycRLhdCVZEz6GClYaLmJSOml7y5+Et6Z4g2a2TXprjJ3UONes6IspAMmk62TbYQfM5D5HACdh3Ed965Wl0tsetreM7RygZfN3XaiYsaUhWqgy9vfop/rryJ/lgDVfEQdPcual2NN3oZ2yC5038Ev5LhmRNtBE6Bebr7soOSzOVRcxJVWFSoaWo1N7mAQPo8CEUsiq0hVRCmwNoVIjAsKd0fJ9biI1l1brBxnXe8ybvf/vcsD9FRF+Gj4V1UqYUZqT3JZRw/XkNLfxpzIrYQUi5JuN1ATTtIBsvwjEoadkYR/SMXBCp+l6u0omDpAveYhfqSj/IdMeTew+eOO+PArqpETgUJNlZy4KN1LHcOcVPpaX7sr5txO59rxlud5G+e5EOh3axzqGSlwZvJ1fzkxBqq3zDw7OqcsQEk8zmMwSHE4BBuCiZJ8W75AVSVdLlOttRko9NCWWAjCpaQIaV4PMTfv5bhzQq/+/CP2eDqpkFLk5HwZsbJ1wbvoGtnLSu2jWKNLHIUcCGQFNboz/csePj4h+h6qxbXiMA5Llm1YxTMqZ53fKKQemJqjdpMJBHpDEoiQepgG/8qt/Ab4QP4xLlB2CkUbi89QWdzhNSDm/AdHsE82bFwOt8DyFyewGnJSGXgygdfBisaxTVczePjW0mMewjNTfFmz5TvRXxVnk+u3kVQudC4s7DYmVrB633LqX6hB2s8yjttBHNwmMDLKXLelXxl9AMk7nqeu7xHWevQLzjOKTTu8/RR2fAzvvWRm3nbWkvtUC1m/9C8+2S8C0VFcbsYWyN44Ja3adZjPJNYScX33fjao1iXGzSFQPg8ZAMK6x2DVGvv9idbcIRgYpVF8+o+ul+tR09JxNEOwp0uIs7pl+/0f1hGVVscHYuoleaFVC2P791M82NZ1FN9Cx865h14t53A53SAriNzuUJAyczll1KtdBrfaydA0xCqgjUZP+s6pHi9yFWNxJp9RFsVwtcNcVvlAT4d3slfj9zGC4/dQNWu+MLXz3cQX27x5bVP06QZnMwbfOHUR+jYWceyn6TQTnZiRRc+FtSCoqgIl5NsUCAdJifzBhWqRVBxXfncOWRJGFJC0xB+H9EWFb0lxqeDPYDAwMmraQ/bky3sPLScklPA8BjyCg1sXsuqO1C8boygyTLvOCrnfGE6h0qIHAb3WB7neBbz2MlLNwLLRFomVhL8HTDhDvLtlha2uk+z2VEY9HRUVroGqI9EGawP4OnzXvxai4yUkomMm24jRa3mBorDP2haWBZ6ykLkrs7ZVhoGSs5kNOcFRZIJKQTeYXAsJGooBGURAmUJtnhPo7/DYTwvTV6PrmBiIEDlSOdFfQ1lPoc5Nk7wdAbD7eJfGq7jdHUZv1f+IiFFIXBeZxdQXKzUJ3m4ZC+vNrcxuakafyK54NGVFa8HUVdFrsLglkA7PYaHtyYb8Z2OI0auXBapKkhV4FcEGirWpbx6FxDpMan1TtDhrkOqFLbuz2T7vqIirGWUuRI4hMW4Jfjp2HpcPQ60k11Yk5dY6lxAzNnsipTy3HlCoJZE0CrLMUv8ZMNOoiscJOolzpYYj9btY72rm8cmNvPM6dVUHc6iDk8s2kyN0B0oQT9myGC9sw+n0Og33Zw4VkPpcdCOdhbyzZ3x5ZuaXSv8Xzm7DFvM4WUuiaKiRkIQCWFGvKRDThJ14ApnSEoNiwuNW6+SxfRaaI0NWMOjhYDVc2xcLglDSgmHMRsqWPvgMR4pPecQm5cmf9V9L+0H61j1X49iJZKYi1xx1PJSco3l3LjuBH9ZvQ3tvEl1OeCiZPcosquvENdqGg9b5nOUfustKqoq+Wb7Q3zt/hj7rvs2MBWUzBMjU/MGf3RDOcleL57d8yZtVkhLIkyT4dMlfKX0Hr5S9SIXLjQUOVKi5iXCmLtdS5WVEwzeECFyNAiLNIGYW99I750ufm3Fs9zrTqKc11VkpcGIaXDwhZXU7TMK8a4ug7L9AOU7VJQXa9m3fgN/+QfwQHj/u6Kil6seHvTE8N//DZ67aS17f2s9yluTC9vZN9XS8cEwd607wAPefh45+lG6D1bRcuowZrI4fUSmg1vNY1RnSQ+7CM7gvDOpi3Ihiy2BTvyK4JV0NfufbKNqXw5zZGTeyrxgKCqKQ2fifc2MbBTccecBbg0e535vFzkpSUnBE5Mb+F9d96N+MURj9xDmyBjGIqZdUKsriN5YQ13tEHWagi5UtiVW0vrVURgefddSq9B0lFAQoSqF3cCWhcznr70crYqKGg4yeccK+u6W3LyuneuDHejCIKIlcLxjXtwpNOq0STavPs3BX2um8akAysETc77J7Jo2pISmIRwOxu9bznib4CORI7Q5BwAHWWkwbhm091bg7VEKWeeLwPqWHhfZEgchR/rstnALScLKIvICkc1jzdCXSRoGMh4n0Jmnc9TLpJXBpzhRECgIdGEihCzOiR5pIfMG3m6Vl0ubSVY+v9glmhmWhZKVKHmBMUcLHA2BKIEVWfKBShZrTmqywUndLT1c7zl1dgPDGToMk7cyTQROS7wnJ7GuNKBMzZ4yMob/pIvnXtzEnvW1+Ff+kDY9eUEIDwVBkxbjRt9J3nZsnA9pl0UqCqZT4lbzqAiGJ304owrkrxwcVagqE1sqGF8NulAwMIlbOZQ8iLxRyEa/gCgeD0o4hDuUock9AgkdLTmzMgi3GyrLMMMGq519vJSq5ftDW4gcNXD1Ti76kt47EZqGWlpCbkUVqOJdu1+VnImSNbBcOpZDIe/VyAYVklUKibYcKxqGcCoGO+Ir+EbXrYwnPaQmXWiDDlwjgrre0wUjZTGDjyoqRlWYoevhA6U96FPuHHmpIiYTmMlCcGnF40F4vWTX1ZOs1Ik1K5gOidRAWKAlBCVHTLwdCeT+Y8UR/PcKCEUgdJ14jcqt6w/xUMl+lusjmAhcwiSiGJgSolYGc2oMDSmCB0oPEb/RRadZT6BtI6U7RmBiEnNoeE7KdW0bUg4HSsDP6ANZfmfjC3zY33F262tC5uk33DhPuQidMguO1kWA9DhJl6j4tXMWcVbmGbEkigFYViFZ7wyx0hk87SPoo9WMWBJd5HGfFzhRiMVfYrgoUiLzOcLHDUZ1H7EbisfRejpIKdEyJkpeKzTcOfgzbwz0sKaih/8Z+uSiGVLxRsGuVT8BwHyHpoPZan4+tobwwUmsQ8emfU1zIoY4mqH561V0/WI1T5VvoqRkO+F3TObVax50MYipK6higeMTqQJLA00xMZGkxj2EoiDNK88+CE1j6AZYsbYHHZWUlWfIVAq7+K4wazcfKMEARl0pTaUjrHH1oEcVnLEZGlIeN9naEJGySTY6k3x64E72Hl3Gqh0dWJOT81Ty2SMcDszaMvpu9yC1Quyr89FS4JiUZEMCwyvJlRmEKif4QssrbHB1U6Hm+GLvQ+zqaqD0KTdVvRkcHQNYY+NYmUxROF0rDp14rZsP3LabR0Jvnw1JkZcqVjJV8NsSAiUYwKwsoeceB+7WCb69/lvUaXlKFDcJmeXNTIjPvvnLhF8LUHZUx8pR/MaUUMDlJFkr+cf6l6c+PDNmKIBOr5FmxHRgSQWXMFjlUPh4oI+PB/r4n6VreW6glclEGf5TDrANqULHhdtFdekE7/cexXVesMBbtv8moZ97WbY3ijIawyiSCiJyBnpSng34BtBlSL4TvRH3sEBOxGZl9MlcDmtoBF9XNV/qfYjfq36WDVN2VKtjiA+37uVn1bfg83rnZY141kyt3cdrNdKNOTzCZGKxyzQDZC6H41AP/hUr2JV14Vo1QcenGmj8tjbjSM/KZJqdB1eQWaVzd/1h8m6B4vUWXTqKbw/cwPFdy2iZ6J/xwCJzOazBYWpf8PL60PUMfSbAf6v5KfWa+4KZLxXIlOi4K8sxenrntPyXI1vqZvmmHm7ynyAvLfxHdUr3p6fXJhUFSrOsCfWjCsHbWT8/HN+Kd9DCGh1btEEqmnHTmS9DzGIlKru6jtH/mOI3l2/HkpLdpxvwndCRyWQhVloRIZxO8te3MrzRRdv97TR6x6hyXLjElbF0UpYDv5pBERbDuQCHY1V8+bUPoI9pOKOC8AmDhrE8js5eZDpdyFm6SDHA3oUQCJ+XdEThk5HtVGsGeanztYlW3hxuJPcLFcQbFHJrUoQCKSLuGL9VuZ1m5yBNmoFHKQwKLqGx3jHGn13/I77sfz9D2kYqXxjCPHF6kQVeHmmayFicwCn44ImHWBfso9xxoUH/f/bfgeOIBzULphPSjTnuWH2cr9e9wkeCu1nl7uMPHvxFgntDVOybm3Jdu4aUoiKCAXK1EZYFTlGvFbZmG5hkpIFy3Efpk4cxJxOX32mzkAiBdOrk/AK3eu4NNSNVRnJ+lLwsJMaczdq7lFipFO4xi7f7aukrD7HBUXACLVMt7vAf5Sn/LQiHA9KZ4sq/p6pkSqG0chJXMS4/Xg4pMUdGcI828XJiFfWhCU6u1TAjPpQBHSsz/b+zyOVxDmmMLPMSUnKYLhA+byEIXrEYvkBfLIinX8xulmWqnirt3ZSNhTn+4XLGqpzUYp0N3QGF3GDZkMAqCUDvAkUFV1TyXoU7Sk9To0WJWxLPkIU+MIE5nTYpBE5XnnJHHAWFvnyYvaM1eCaNRQn8Ky0LkTUZGg/wWqAFNS1QczPrWzIlGr/T+iJbXV2YSJjQcY3Lqw44Ox8IVSUX0Mj7oNodo9U9QLPz4oGIS5Q0urA4kS9hIBPEOaQROSIJnJhEHO3ASqWKYvbpoqgqli6o1oyptGLQmSkhbymMrxG4VkX5StvTLNdHCCp5/IrAAuJSkjQLBmFI0YioTj7oHWdPw1G+v/Y6Svf5EacWNy7WFZEWMp3GN2hy5HA9XdVhwp4L8+S693mofDONmsph+hwMpdwcKK+COlihOwkpfaxoGKJ7oHbOinVNGlJn1sH7PlhP2y8d5XOVL3Jmeu9k3uDfJq7DPVLYmr5oQeLeiaKiBgN03xfiP37ySe73tnMmuOEah+DPq5/nuoZ1VFWUIfsGZh0FOXAiTuaFIDuaV/CgZx9QSBR7uzuF4ZOFrb6LFM/mYghVRTgcRG4c5PG2xyhV3Zwu2h7s0gT2D/H039yG5yOD/POWf+bzbZ+jJFEL7aemP+DkDVxjkM7p1GtuJhshsLEB1yuTixaBf76wEglkOk08tYoJ0wNc6MjtFArRG3MYnhBVh/X532auqGgVZaTKFR4K7GPCcvN8qgXPiAHDY9N+hkJIlKmpn/3JeoaPldEcSyzK3j1zeAQxPkHLf6tn1FdDQ2IYkZiZgZD3KHzQ14WOSqJYOo1LYKVSeJ89gG+7l5M/WMaxcBuG7+KL43236hiNGT6zfhutvkEyd2oczbTg69bOJvYtSqTEmojhHTL58tAdPBrew82uPH9S8Qr5ckm8TeARkpCioQuVuCX4p9g6jiWqeHuoYDhoqsUfr/wpG5zDVKluPh5+k1V39fN/d3+I8s6y6UdBXwykxMpk8P78AK3bvIVVqXek0vIlD2OlM2BJ9GW1ZO92UfEOY2uuuSYNKSUYYPLGZUyuNPmF0n1Uq1kMnJzMG3w/toXv7LiR+i5jananON6ahCJAVTA8cKv7JBHl3J9eQyWgqEhNIq8yZ5VI53CPW0waFwYl01CRgstmdl8shBD4HVnKzwv4eK0hJ+NEjoXoT3qIqBkmVoKWKcF/qnPamxxkNot3wGRw0kNemoWt6nrxPa85QUqkaZIddfNsbC3Xu7ZfEAMNQHMamDOIdXTVqCpSLTitjhgBjqcqUVPG9JawhEDoGk7dwK8UjN6JvAc9LhA5Y3GCIEz5H8r+IZSpUBpX2mF5FkVFKy8lFxTnklTLPHpMwRmzFtxxfrpYmQzk8iiJJNqoH91z8R3ApaEaEmNuvpa9Hd2bI+RPk64y6b/NS7hiDe6hDMquw0WxQeldmCausTzPHF9DzboJbnQdIay4sZAEpclJw2JbuoqXY62cjpdysqMCZVLDNVoYWzI6HKivp16LUqVChWqxwdlTCBwb8iNGRovmRftSWJkMXOHlUi0twSgPYC1Psy7UN6/luSYNKVlVzsTH4/xK034+5BsF3MSsDI9Fb+b7O6+j9fcOI7PZomsEQtMwnZIWff6294tUBtdonqRZ9FnqlhTm6BhidIzUL1/PiOnmlrsP8nLtSgI/0aZdD62JGKFtXYxsbCRm5Wbl03JNISWhQxo/0jfzuXtfw1cMvdGUfXAoXcsbw42EJjPTmg0UDgfC7abMm6TOUdhSHs25cY4LRHZx+6HZJIJWHDqp9XUka+RZ3zVTSvyd4D8eu/JOzcXEMgtL6pd5br6uHnxCUO10ItesoOvBMlpu7+J3H3iOr/XfUXCoP+LFnEwU3eyMNAyc7YPUfLeWH4Y28NnwQTShYmExZOb4TvQWHj+8kaonHQT2j7CyY98FfZDi9/PcTatY5epntSNKWHETdEgyZZJsVQD9tApFNnbOGEXFaKllZKOHf7j+a6zQJ5nPtDHF0HXNCMXvxwi7aQgPUj/VYeWlyZCp8P1dWwnvVwtGVJHs0juDNE2seAJvv+APhzfxyfAb82pQ2SwuccOJnGGQTmmaWIkkzgnB86llOFZMMiADNO8OIUdGi+7FYC6wdBAOE/UiE2/5qBP/uFy45Xldw9IhqJjUOsap8k6SUsundaridCJ9HsKuCcrUOAoq0awH15hEZIvLKXta6Dqpch0jaGIheTHt4eex6wh05RCDI0Uz06+GglAagVgCmclgJRLTL5uUWLk8as8wtS/pDI028FuNv8Z1Nx7n+jWn2POHbUQOQunL3VijY0W1vG5NxPAedZJ8spL1x79QSAMkQc0I3IOCmk4D35FRGBl791hoWVjyXINLyCzjpokeF+ixzKx2jRcbQhFEWzxMLreo0yYJKfNr6lxbhpQQKKEg2RKd1f4RKvUJoBDq4ES+gtB+ncixTHEOOFNOtp5hi6c7VnOb/xgt+oV+IVID6XYU8jfN9j6aiulU0M6bzrCQZGW+MMOxBBpJUWNBRuqkDAeYl1mWEwKEUog2rExFHRYCoSooOTiYqmNjVR9HHXlkOICSTGEW4XbzWTO1W9Nygu4yeKfJaUmJHlNxTSxMfRWKQGoqlgZeoVCmxal0xzmtTjPdt9OJ9LqIOFKElBzgJplz4I6acA0aUkJVyEQEqr+wFLgt0cILXSupH0gUVRBHEQ6RbirB1aWixJSCITUTLBNzaBhlaJjq7jrytSXkrld5qOwAq+4b5FveWwmdKEHNZCCbLRoD0kom4WQHFeksZTsDSFVFWBYikz8bH+lSUwlCVVGmwuFYWMQtkx7Th54AJZbCLLJJiJlyJohsolag1yQoVVWcU075FhZ5IJ3XEcbcuU1cM4aUGgggIiGOfaGamlVD/HbpK3gERC3Bnbs+g3UgSNPP+5Gj40UXJO4sUuLtyzLxdpAjLTXc7z5xwdcfu2M7T9avo/bPlqMcPlUIUzBdppzZB++ppvSXeviVsu1nv3o57eJP2n+J8DGJFY0Wp6G5RPB1qPxR+yO0hYc4XZYq7JI8vwNWVLSqCmTIT6ohQKJGI10myK5J4fdm0DWT+6vf4LdLtpGXcLy8hN/++K9ScjBC8Ds7FlfcHKKuaCS+pozK+3r448afUKZe6AuVkRbluy2C2zsxFqC+SksiDBPFgKS0yMuZxTMzWmoYW+3ml/ynqVV10jJHPOUiMpRe1JRUs8bpZLLVYH1tPwqC5/paYWcQEeta7JIB5zYc9Xywhts+toc3v7GJ8u0aYnRs1v2b2T+EFpuk7+9X8ZXWZv79Iy/zizftpGdDmJP/0Erp9iGsju6i6j/NoWHE2DhCKbyKWKYJlzGEtJpqjJoS1kXaWescQMHFoVwJ3xu9Dn+PgdXTX3RLmdNCURF6IUB3+rZWxtp0nFvHubG684L0VjuzOs9NbsL653KWnYjNme/iNWNIiXCQfE2EspZRHq3dS63mZNzM0mk4yJ0KUHHIxBocnpnxsQhosTS+XhftyUrGggcoOS+q893+w6jLLZ5ZexthrRntRG8hUfEVNAlNKyx5rqpnsgk+X7ODJi0BeLCQHM9WM3q0lPpho6g6gaWIe1Qy2FXCTeUd1IRjyKZa1EQaYZhYPg+WRydW7yETVEhVCbIRCyuQpyKUwOsozFxYUnDa8NGmJ2nSxzF9FoZrgQNTXoZyX4KO8jDS50E4nTPbYSoEitOJUeZnsl7l1kgXWxw5tPO6osP5HDvSLbiHsgs/+yEK+3+VaTqoCd2BEvAx3uhiolVSp4+Rx2RbJkxm3IUSGy+6eEvTQSgKeihLrWcCgFjCTXBIIotkdk04HFgVEbIlki3+DrarmxGmeVUDo8znMGN5QkcmEVaAp7euZnNpLx8q28MfLm/FPVaKp3+osFRWJDNT0phZn27UlTK+ykuTe4TQVJfSmStl/3A1kUlj1rvFFxOhO1Cal2EGXKRLXQxv0sg1p7mrsod13p5CKBWZJ24ZPD5+Jy91t1B7PI7Se+lZu5lyzRhSybZKRjbq/NcVT/CwdxQFhf1TlnT16yaup/cUT7yoy3Gym7KRCV66u4Xv+Xr5VPD42QjkN7vybHHuQfm85One1ej/vIJA+yTsP3rZSyolEfItNXT8Brx/5W4+5h9CmQqtELMy/HRwLc3/MonoGyne2bolQvjgJErez8rbB7kneJjPfvJTqKkgWlog18VZXTnAn9Q8R5maxi8k34uv4aXRVjp+2oQclDgnTZ5rquXxti18+bYf0eoYINCuEuwong7uvzT+lJdL23j+7VsJmCbmqc5pDyyK04lorGN8tYfkdWk2eztxigu7od869suMb6uksadnQWajYGppz6FjOqBUdeNVpvf3VivKmNxaS+rDMZ7Y8E+s0BR259x87uVfoXy7VghwWCSD7ozQNFqqhtni78BCkou6CHRmkUXiJ6QEA/TfEsbSJH9+6F6qj2Qun+R9ukiJtf8owVM+OFrL8x/exLoP93DLPQfZvbYO3+GrC0+z2HTf7+Oeh97ig/4DZ1MzvRJdSfbNEhxDo9fc+CB0B2plOUc/G2Z5az/faf43XEKgCwUdFVUIFBRO5i1eT63m9X/aSv0rY1jtHZhzGFLlioaUEKIOeAyoBCzg61LKrwohIsD3gGVAJ/ARKeUsUnBfHjUcxlpew9BWHc/WUVr0YVKWyds5P/+7+2463qynqXviqoyojExxmLfIkkEgqKGRetFMXuY4yA7SpMiRQQgRvlqNMpfDSiTxHHDzVeVOmq8bZKU+Rr3mQUHgFDp3+o+Qr1H57g03k/eEKBufChwmz70RCk0tRFJ26AzcV028CR5qfYu7AkfO7rJJyxxPJZZz/LBO8vA/ks0XHDEvps+NB8nC78RRVi5nfEOYm4O7ruo603mG7oJxOa85aNRoHF+/g+OpSioDEyxb008qr5PNa7gdeUbTPv6s+0HShs5owku8O4CnX6V8fw5HNIuSzKLkg2DpnLyuglbHAMICIeWC1lM1A7uyeZq1PAHlwk0RdWqC672n+G7bHQizHM/p7ssGeD0/51eqQmdsrYDGJO9fcZRmxzBQeJHoNFJ8q72Fvb/1Q8TgJL2ZPDWyYd40XrK8CHRM3EoOqSmgXCRI4dQSbWJDDb0PmXys4TCVqsnenIMfRbcQ2qcT6Hh34vHp1tPFaItnUMvKyNeWcGN4N23OvrOBUsU0jZQFaYuKglQKDtbZjI6St+bOYJUSmU6jDI4RPBXifx++i3sbj3F77UkONa3DLSWJ00cXrC3OBWpZGbk1deRWpLkvdJCQojBqpvm3yXXsPN5Ew4E8InqhD+ZC9jdnyxkIIPw+UmuqUXMWjsE4jIxjjk8U2qCiogZ8CK8X6fMQW1dCvE5l07p27i45SoV64c48C4tRM83XR+/m5zvW03gsC8PjSGNuUzZNZ0bKAH5PSvm2EMIP7BFCPA98EnhRSvkVIcQXgS8CfzCnpQMoizB0XYDKW/v4Rst3qFIdDJkWT4xv4fSuepq/0Y81dHUZyAWCZtYREGEMmWcXLxKRFQzQSYRylolWXpNPY2JctcYzCYarX4kzMeDjp80byIUOUq8VnCQVBDc7LerCbzJxm4dn9A0Eugo7h4Qp0eJZpBBIp4qlKRhejbJf7OELNTumZqLOOdClpMn3+7eg9nlYkWst6OPi+jrlMbo5cdEyzyexthATDye5PXDsvLxqMx9EpvMMO+UxxhmunFsFF2KNjOEA9kVrWe/t5r83PXn2uz/teoj2gXIcRz24RiSlJ3LUnuzH6OoBCjvvTcAzVo5ruJzjH6ngocDMNM5VPdXj8LPJ9YRCbxG4SC68MjWGvinKiBqm8WeXieA+lfPLKg/Tc7cDrTnOP216jAYtRZXq4YwRBbAvW823T9xAc2OWio5+DCs3rxovhYXEpeQJaBksXUXVNWT2Qn1C18gvK2d4s8ZLd/85EVXFJRw8F1/Lcx2tLHtpBIbH3vWGP916uhht8QyyupT4MjcPBvax2TFzW2ch26IwwMqoYM3twCgNA3NomMj+MkxnAM9v5PhQaDefXrEFNRtBnF64tjgnemrK6L3Tyd0t+7jfnQJcHM9Kvn7kZsJv6bie24PxDuNiIfubs/eMhMjXROi+T0NLCUoOOQgeUVESSaxcISwHZSXkywMkq50MPZLjzhXt/Fn1cwSVd++Cz0iDHtPJzw6tZtXfjSN7BzFnEQ7kSlzRkJJSDgADU/+PCyGOAjXAw8AdU4d9C3iFuawwQqD4fMRXl1Lx4S5+pWYHFWohWuvBXDnPvbiJsj0W1tAI1lU6czqFG+dUjAlN6HiknyxpRuhnM7cDoOMgR/oR5kijeqKXklgJL/xsE6+ta+KGzd/AJ/SzyxwVqoPPlL3Kxnu62HZdMwB5S6U3EUIIiVfP4dOz+LUsv1K2nSYtcXY57wxjpuDUvlpK+wUBEb6svioaOMWRuZA2Iwy3Qk0kRkg95wc2mz2L03mGVTRwkkPhuSn5xbHSGRgeRX5lFV8PfghLO2fYOicMGpMG2sQ4IpOFWOLiMX7SGdTxBBnzwua5kPW06pUxfjZxG698spnPL3uRe93jFyzBOYXOl1c/xY6GFXy3cQuOTieh4xc+t3SZwmRbHk9JispgnN+o3MlKVz9tegbneQm1s9LgtYyfvz59N7XPleLrHsWUcsHaIhRCTzA8hre/jL+NNrPe3cVDgX08seEOyq1ViDcPnp2VUvx+lEiIga0ecs1pylSNnVkvO5Mr+MHjtxM+ZkJfd6EuvIPp1tPFaIvvxJIKFvKCl7PpsBBtUebyOCckiQbJ2uZeJirq8fr9Mwt/MA2U7gEqsnmefaiVZa5RoutNhHRR+vrCjxmzQegOWN9C/60BPvPoz7nbdwRVuBg1k7ya3EDoKS+hY5OFGZp3/N0Wsr9RPB6UcIjjn6umbv0AX6jeh4ng9P1lvNzTTGJgHUpOwXJY+KoSVAbG2BgY5rbgMZbrI3jEuQj2P0yU8kZ8BbcE2unPh/k/++8gvMuB7B3ESs6PD/WMfKSEEMuAjcBOoGLKyEJKOSCEuGjAFSHEZ4DPALiYQeRqoaCEgqTKFb5Q+zobnP04ReGhTpgefJ3g7c/OuXN5WiaJM0GQCDmyZ+85lWF7zjSa0ShKLkfJoVKGPEEOrg2wUo9RpRYeiVNorNZhtd7PJwP9QOFN+Wg+j4okopj4Fe2sfxXn3ddC0mukeTvbgK9LwTtwztC8lD6ncCMv0QHN+hlOA0uDsDOFS+SBi6dzmCmX04i8eJ2fM42WiZVKob+w55JqrjTfJg0Dkc1hXGbn2HzXU3mqi8jwOEdvW8brkRZud227wJBSENznibHJ+Qb1W8b414rrGHBeOMFgVaf55LqdbPGeplUfpVZzoqEC594cs9Jg1Mrx4+hG+jpKadvdhxWdWBCNFwqWWIkkrqjJK2MtNFUPs8kRJ1EvcU64KTnoLWQF0B3IihIyFV7iy00aKsZRUDiSqeXF4ZWU7TPwHR7GiMevOKAXW1ucD+atLVomzkkThMK6YB/PBRvxBwOFgXIO84ia0RhKMsXExCoGciF8lQnSZaFpaZyXejpdpiYilICf8RYfk80Gnw+fRBUuTGlxPO/mrYkGwgcnEANjmLOsq3OlUTh0pNdNuHWcLy9/go0OA12oEDrFE4F2ttW1EDdc+LUMdwcPs1wfo0U/M/Zd2E/uSCznxa4WRit9DKX9uA54CHbkZxWYdrpM25ASQviAHwJfkFJOimnmI5JSfh34OkBARKb9qqB4PYzcVcf4RpP7PMPo4tz2aIcwMbwC06XOqbe8IQ0O8CYr2YAmdKY7MTJbjVYySeCnB/D2tfDr4tM8fMcu/rxy9yWPVxCs0vULfn8nCZllyLS478UvENznoPqZARibwFwEfbPFnOEb8PlcKxqvhoXQeCbVRs1zTTzTfwP/4VPb3rXEp6FSobr594FTPNrWzkTrhd+7hCSoqOio6MJ90fr6WsbPj6MbOfIna1l1agyjfxAsc1Geo8zn8PTEOfFSE0+/P86Dta/xnx96gh9s3kzUaCNVrpBotNi6tZ17Ike4yX0agN05D1/dfyf+7R6q93Vj9A1c0Yi6VurpTGejzmc+NVqxSXxvdOBY18xtvmN8d80taNk6gk9Pzu2slGViZS2suE5vJsz9DUd5fGjr2a+L8TkKTUO43Yx/cA3RVvj4Qy9zo/cEFhJD5hk3s/z7l75A8ICDqpP7MNOXz0O3IBp1B9LjJJOHnnwJ6x0DnDGQ3u8Z4i53/9lDXUJFF5ce+Z/au4GGHwlGJhvQk3nqO48i0+l59Tqclh0ihNApGFHfkVL+aOrjISFE1dRsVBUwPNeFs3RAt9CFesbyBUAXJnkfmC5lzgwpS1oc4E0qqadc1ADgwElWpnEKN1bhMcy9xlQKfWCC0JEqXlu5grfCb7HGkT9vpulCLtexZaXBC6kKvjN4Pf5DDiJHsjA2gZVKXVFfVqYRV9FpzgXmVUawno5GKN6k7mexLGQuz0gqQGe+FMMFhktFZYHrqWXi60piOnz8j74Hub/kIP/Of+GlFQRu4cAtoGQaERoMTPqNLAdz5exIrOCnXatJ9ARoPTUKg4VkqYvVFgGUeBpfd4jORISszLPZ1UW8wsXf3FKLEkrTVDnKo2V7uMHVR5XqZldW8Bc996Mf9xA+kUPGJq8Yh+daaItnSMscI6aBGlfRYonLxig6n/lui9I0kfE47mHJN4duhZo0Q6oL93ALzr4Y5omOOYmHpFaUY1WXoUcylDnivB2tQ5tQp6VxPuupcDpRwyFwOkBRMHsHEKqCaKonV+EjWeVg5HqL0mXj3O07TIOWBtzszOpsS6zBf9RBuD1fcIu5jNG5YBrzOUQyQ7K7nG+Hb6Bt2RNUqxmCigun0M8G1IRCJpOTeYPDuUpaHUNElEI16jXcvJRow93pwNM1hogVEqOb0fn39Z/Orj0B/CNwVEr5V+d99WPgE8BXpv59aq4LJ0zAFJhSns21m5cmLiVHpsIgG1SZi5SmUkqOsBsvfhpEy9nPy6hmgC6W0UqeHMyDRgCzp5/yJyfpqGnlzwP3878anmCZNrNceQYm41aOr7Tfj/4vEere6Mbo6cVkevoG6EKbo6W1xWC6GoGJxSrjdLFyeZR4nMHBel4sbSMXkmRDKo5FqKdy9yHC7QHavat588bl/Lv7v35V14tZOZ5PtfDVI3fieClIzavjcPog5tQS/WK3RWtkjLIdOu33lDDebLDK4WSt4zS//cjfv+PIQoy2H0W30PedRpbtmsDad+SK28evtbbYb5o8n1yFZ0DAqR6sK8xewAK1RSmxMhlK9yY46mrlg594k4/euJMP+z5LaH85lT39BSPhKo2p7Oo6et/n4OGWHdzgO8n3X7yJkiOLX0/VcIj0ujqyIQ3DJSj5WQrhdNDzQCnJDWk+u/E5HvIfpFE7s4xeWIr7m767efvtFbQ+NYB5suOy91hIjeZkApHOUPtSKZ39jTz7K6u51dPO5ncM8BYWMSvHv03cyPeObObRVfu4zluYGf7ByBb2P9tKzbYMVnsHcg7DG1yJ6Uzo3Az8CnBQCLFv6rM/pGBAfV8I8WtAN/CLc1kwmclSuidKNhLhzfe5cYk8eanxpVMfoquzjIanJZ6OuYliHmOMQbrxEWSHfB6AFayhgZUcZAd9shOTPBQ0zznSyGMlktS8lqFrcAV3b/4dPKUpNlf3cF/kMB/1jZy3o+0cj02W8vOxtezY24I2qaDmBIFTktC+EayJ2Iz0uXDj5N27HuabYEeWg68288YHT7LZUWgQb6SaOfpaE/Wd049ZM12NTG2cKGosE2kIyKhM5NzkKvOkhx1Yi1RPrXSGsrcmcI8FWDX0WWhKsqZ6gA9VvE2zY/CSO7sO53N8a+wmhrIBJnJu+mJBYhMe/PtchPos/KdiiIFhzPM2iyx6W8xkEYOjlP2khLs7f598qYEjmOWepuM0uEdpcozQnw/TnY3wo8MbcLa7adg1geib3s7hYm6LZxhfF2JsHQSVLK+nlvMXrz1A/QmjEKF9GktmC9kWte5hqiQ8WXMDz7W0cvf6IxxvKOdkw3oiByF8LIFyug8rkZx27CetphqrLMTI5iDjGy3uuW4fJ+LlvNjbQtUbEt/pScYXuZ6i62SDKoM3QWXrEEdvawBVsrX5OFtDndzlPUqZIshLk0N5wVvpJn7Qu5nh16ppeCsHo+NXvMWCtkXLROZBS5q4h1X+4Sf38v/qb+PT67YxnPMzkvPx9kAdyZgLR68D97Cgss/kueqbeNp7EwCucUnt0RSOzpF37UCcb6aza28bXHKe+X1zW5zz7pvPIQ8cI7j8el5LtOIUBlmpMbijispjEtezuzHnKFhfSJRyNx++6HdndifslC8yKcevXPtmg5TIbBZt+yEq3nbh628lURPg9Y2tpFY7uN395Fl3ujOGowo8OXwPew810vJYCq17uBBaIZ3BTCZnrA8KGhcaR0+Uyp1l7LiziQ/7DxXKEW2kcqeJoyc67bn/6Wp8QT5+TcSck6aJyCmkDAeecJqc30HFItVTmc8h9x/Fd9xF4M0gI/c1sX/9cpStkq2hLiqDe9+VKw9gR7qFn7SvJR9zok2qePoElYMWwafexsrlkZb5rhehxW6LMp/DjOYI//w4kTf9pFvKSVR7eeau1ZSWxFldMkjHZAkD0QCVP3Hi74hj7Zv+DrtibotniDcIAivH8CsW+5N1lO1Q8XREsab5hr+QbdEYHILBIapqrmNsIswjv/oE3tIc26pa+KbndsBHSaoMdUzHmoghTQukhZzKOSqUMzkvFVAUhKZh1JUSb/AwfmuO21vb+R9VL3D/3l8lcShC1a5uzJHRRa+naCqGW6GqdYh/bXuMiVYNHYsW3YWFxEIhIy3GrRw7U6v58eA6Bt+spu71NMqre6c1AbHgGi0TLWXgHlcJPmsQXenih6ENTEx6yE86CO/TiAyY+HecxoonsJJJfF4vQi+YMTKXx0qlFsV3o+gjm/tfaWfHpzed/X35UC8ymZozI6qYkPkcpmni3X4Cn9NBxUteEr4KPhn8jxc9XotlWZUYRfYPYWazhc7hKv2MFhqrpx/v2AQTn6s+q1OLZfF2nji73POeYypvlPQbVLgnOXyiiUjP4vu/W9kscnSM0mcsyl73kghW8JKzmhectyAvsvlEyZqsmEiBEUcYJmSyyHwes4iSv14KMzaJSCRxjU/gcrsofTOAdGoM6PW4DYvl+QwM9SIz2TnL11UUCEGuLcWfrHwOv6JyNFZJ6c9OFXy/ihjfqyfwHwjypYFPkWgQ+LeO0NLWS2hjYSlyKOWn60gjvm6FyLE8Sr7w1NKlGpmIQqJeojQkaa4YYWt4Dx4lx7jhZdtQEzfs/l3qn7GoOtKHOTQy58EcZ0U6g7c/x2jGiQ40aBJ9ajhvz+fYnl7ONztuYqgvTOVLKt7BHMtP92CNTyxiqNcrox48jU/TkIZB5TEP1vYwpWYKzARiMoHM5jDj8UK4Ego+xpxZrVnEsa/oDSkzGoVd55zFlp759A4s85xz3GDhn0ttgj8TwPFaRmazhYE1Gj2rcynomgv0QZ3X/CvwdQs8Q0WQ40zKQqDCkREYObeUdTkf82v2OVpmYcYsn4PJSRg650+7pAyn8xEC4XAQ9KdY7+xDRyOd11GGeha7ZFfEjEYRiSSle1w4J4MMBEqYqPJQXRLjprLT1LmjxBpdRF0BTKeOMtWc0hUS02/gLU/SVj7Iav8AES3JuOHlzeFGBk6WEdmn4D02hNHZvbgiz0NmcziiGZIdQf5Hzd141SyKkKhYtCfKOTVeSvJomGCfIPz2MIxFMRY6b+UsOD9EgRWPw+DQ5U+Qck7DXcyWojekbGzec1gmMmvS9N/eRqgqMm+cfQOzsZkvFI8HJRKmLhBjlcODKS0Uce2YjTKfQx48jveQQvNPtUJqoqCfJz5xC5mGHJubO1nbcJBb7j7OmOkD4C53P8fzbr45fBs7+xt468Bygkc1fAMmgVdPszIzikynMYqs/ZnRKLw9QfMhBx2aBorv3JdWmkrZjTQ7wJKYFwm2aTO32IaUjU2RIrNLbNnIprhRzs0t5qXJ0Xye8aSHmkUs0oyZmqGQWRPLNBG5HBW7K0h16Rw90cJBXzPf8t+KMAQIsHwmIqPgGlZxjkFFzMLfm0aLpgs+VcVshEz51l6rCZSXErYhZWNjY2MDVsHHxJAKCSvLG6lmkrHF2z14tUjDQBoGzqffwgnMJB9NMfsR2RQftiFlY2NjY1PIDTgySubP1vBA2e+ipyTNg9MPQWJj817FNqRsbGxsbApR5TMmjmd3M7NwwDY2723EpZJjzsvNhBgBksDogt109pRyYTkbpJRlVzppqWu8xvTB0tdo19NLsNQ1XuP6YOlrtOvpFEtd44IaUgBCiN1Syi0LetNZcDXlXOoarxV9sPQ12vV0/s5dSOx6Oj/nLiS2xvk7dyGZTTmnkWbUxsbGxsbGxsbmYtiGlI2NjY2NjY3NLFkMQ+rq0scvHFdTzqWu8VrRB0tfo11P5+/chcSup/Nz7kJia5y/cxeSGZdzwX2kbGxsbGxsbGyWCvbSno2NjY2NjY3NLLENKRsbGxsbGxubWbJghpQQ4n4hxHEhxEkhxBcX6r5XQghRJ4R4WQhxVAhxWAjx+anPvySE6BNC7Jv6ef80rmVrXCTmSmOx6oOlr9Gup7bGd1ynKPXB0tdo19OZaURKOe8/gAqcApoAB7AfaFuIe0+jbFXApqn/+4F2oA34EvCfbI3vHY3FrO+9oNGup7bGa0Hfe0GjXU+nr1FKuWAzUtcBJ6WUp6WUOeC7wMMLdO/LIqUckFK+PfX/OHAUZpXw3Na4iMyRxqLVB0tfo11PZ8RS11i0+mDpa7Tr6cxYKEOqBug57/deZlng+UQIsQzYCOyc+uhzQogDQohvCiGulDzc1lgkXIXGa0IfLH2Ndj19z2u8JvTB0tdo19MralwwQ0pc5LOiirsghPABPwS+IKWcBP4eWA5sAAaAv7zSJS7yma1xgblKjUWvD5a+Rrue2hq5BvTB0tdo19NpaVwwQ6oXqDvv91qgf4HufUWEEDqFP+R3pJQ/ApBSDkkpTSmlBfwDhSnKy2FrXGTmQGNR64Olr9Gup7bGKYpaHyx9jXY9nbbGBTOk3gKahRCNQggH8FHgxwt078sihBDAPwJHpZR/dd7nVecd9kHg0BUuZWtcROZIY9Hqg6Wv0a6nZ7E1FrE+WPoa7Xp6luloXJhde7LgFf9+Cl7xp4D/slD3nUa5bqEw1XgA2Df1837gX4CDU5//GKiyNS59jcWq772g0a6ntsZrQd97QaNdT2em0U4RY2NjY2NjY2MzS+zI5jY2NjY2NjY2s8Q2pGxsbGxsbGxsZoltSNnY2NjY2NjYzBLbkLKxsbGxsbGxmSW2IWVjY2NjY2NjM0tsQ8rGxsbGxsbGZpbYhpSNjY2NjY2NzSz5/zlHSon0X6T7AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Because perceptron is a binary classifier, we will limit our problem to recognizing only two digits. The function below will populate positive and negative sample arrays with two given digits (and will also show samples of those digits for clarity)." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 17, + "source": [ + "def set_mnist_pos_neg(positive_label, negative_label):\r\n", + " positive_indices = [i for i, j in enumerate(MNIST['Train']['Labels']) \r\n", + " if j == positive_label]\r\n", + " negative_indices = [i for i, j in enumerate(MNIST['Train']['Labels']) \r\n", + " if j == negative_label]\r\n", + "\r\n", + " positive_images = MNIST['Train']['Features'][positive_indices]\r\n", + " negative_images = MNIST['Train']['Features'][negative_indices]\r\n", + "\r\n", + " fig = pylab.figure()\r\n", + " ax = fig.add_subplot(1, 2, 1)\r\n", + " pylab.imshow(positive_images[0].reshape(28,28), cmap='gray', interpolation='nearest')\r\n", + " ax.set_xticks([])\r\n", + " ax.set_yticks([])\r\n", + " ax = fig.add_subplot(1, 2, 2)\r\n", + " pylab.imshow(negative_images[0].reshape(28,28), cmap='gray', interpolation='nearest')\r\n", + " ax.set_xticks([])\r\n", + " ax.set_yticks([])\r\n", + " pylab.show()\r\n", + " \r\n", + " return positive_images, negative_images" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "We will start by trying to classify between 0 and 1:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 19, + "source": [ + "pos1,neg1 = set_mnist_pos_neg(1,0)" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAACqCAYAAACTZZUqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAI8UlEQVR4nO3dS6hV1R8H8H3kKuHzopZIiIgEooYRUYN0IGoZCkqiGSIiDq5Y6ERwEBENGog5UtTIBtee+IAok3yEICImSoMiqFEiKb3z+iqse/+TKP6steUc9z33dzz38xl+WXedlZ77ddPae69aX19fAcDAGxK9AIDBSgEDBFHAAEEUMEAQBQwQRAEDBOloZHCtVnPPGk3V19dXG+jP9L2m2cq+166AAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCNLQoZzce06cOJHN582bl2Rr1qzJjt23b1+/romBM3bs2CQbOXJkduwLL7xQ15xPPPFENt+1a1eS9fT0ZMcePXo0yfr6Bt/ZqK6AAYIoYIAgChggiAIGCGITro2cPHkyyZ588sns2N7e3iQbjJsg96JRo0Yl2TPPPJMd+8477yRZR0dzfu0nTpyYZJMmTcqO7e7uTrKtW7dmx3733XeV1tXKXAEDBFHAAEEUMEAQBQwQRAEDBKk1svNdq9Vsk7eAl156KZu//PLLSTZ06NDs2P379yfZunXrsmNv3rzZwOqq6evrqw3Yh/2jVb/XnZ2d2fztt99OskWLFjV5Nc33ww8/ZPMlS5Yk2TfffJMde/Xq1X5dU38p+167AgYIooABgihggCAKGCCITbgWt3Tp0iR7//33s2OHDRuWZF9++WV27Jw5c5Ls2rVrjS2uCWzC/WfhwoXZ/MiRIwO8ktazYcOGbL5nz54BXkl9bMIBtBgFDBBEAQMEUcAAQRQwQBAvZG8RZS+ufuWVV5Isd7dDURTFr7/+mmS5x5OLojXueOA/s2fPTrItW7YErOT/bdq0KckuX76cHbt58+YkKztBuapt27Zl819++SXJDhw40JQ19AdXwABBFDBAEAUMEEQBAwTxKHKAxx9/PMnefPPN7NiZM2fWPe+qVauS7IMPPqh/YS1gsD6KfPDgwSR79tlnK897/vz5JPv888/r/vk33ngjyb766qvs2BEjRiTZ2LFjs2NzG2O534tGHTp0KMmWL19eed6qPIoM0GIUMEAQBQwQRAEDBFHAAEE8itxEq1evzubd3d1JVnY3Su6U1xMnTmTHHj16tIHVEaFWy9/kMWRItWuh3B0wRVEUP/74Y5J99tlnlT6rzI0bN+rKiqIoPv300yR77LHHsmMb+bOZNm1aki1evDg79vDhw3XP2yyugAGCKGCAIAoYIIgCBgjiUeR+MmHChCQ7fvx4dmzu8eKyv4d9+/Yl2dq1axtc3b2j3R9FnjVrVjb/4osvKs07efLkbH7p0qVK8w6kZcuWZfOq7/Mte8y/q6ur0ryN8CgyQItRwABBFDBAEAUMEEQBAwTxKPJd6OzsTLJjx44l2YwZM+qes+yU4o8++qjuOWh9U6ZMqTxHT09Pkt2+fbvyvNHOnDmTzXP/vaNHj272cgaEK2CAIAoYIIgCBgiigAGC2IS7C7nTXxs5vThn0qRJ2bxsc4570++//155jnPnziXZb7/9VnneaFeuXMnmR44cSbKVK1fWPe/TTz+dzUeOHJlk169fr3ve/uAKGCCIAgYIooABgihggCDeB3wH48ePz+a5p94eeeSRuuc9e/Zsks2dOzc79s8//6x73nbQTu8Dzj2t9e2332bHPvDAA5U+qx3eB1xm0aJFSfbxxx9XnnfcuHFJ1qzNTO8DBmgxChggiAIGCKKAAYIoYIAgHkW+g507d2bz3Mm2ubtJyt5vOn/+/CQbbHc7DAYdHemvV9W7HQaj77//PnoJTeMKGCCIAgYIooABgihggCA24f6Re+x46tSpdf987lDErVu3ZsfacBsccu/+fffdd7NjV61a1eTV0IpcAQMEUcAAQRQwQBAFDBBEAQMEGXR3QZQ9Cvree+8l2aOPPpod+8cffyTZ+vXrk+zw4cMNro520tvbm2THjx/Pjq16F8SBAweyee6x94E++bdenZ2d2by7u7vSvHv27Mnm/XFCdVWugAGCKGCAIAoYIIgCBggy6E5F7urqyua7du2qe45Tp04lWdmpxjSmnU5FzhkzZkw2P3nyZJI1ctJ2mfPnzyfZli1b6l5Ds9x///1J9vrrr2fHrl69uu55b926lWTTp0/Pjr148WLd81blVGSAFqOAAYIoYIAgChggiAIGCNLWd0E8//zzSbZ79+7s2FGjRiVZ2anGK1asSLIrV640uDpy2v0uiDKzZ89OsrLv6owZMyp91unTp7P5xo0b6/r5np6ebD5s2LAku++++7Jjc48XP/zww3V9/p0cOnQoyZYvX1553qrcBQHQYhQwQBAFDBBEAQMEaYtNuLLHOy9cuJBkU6ZMqXveZcuWZfMPP/yw7jlozGDdhMvJbfYWRVG89dZbSTZixIhmL+dfP/30UzYfPnx4kg3kuoqiKFauXJlk+/fvH9A15NiEA2gxChggiAIGCKKAAYIoYIAgbXEq8pIlS7J5I3c85IwePbrSz0MVZbv3Dz74YJJt37692cv5V+5l6s109erVJCs7WOGTTz5p9nL6lStggCAKGCCIAgYIooABgrTFJtzt27ezeW9vb5INGZL/N+fvv/9OsoceeqjawqAJ9u7dm2QLFizIjl24cGGzl9Nvbty4kc2fe+65JDt27FizlzMgXAEDBFHAAEEUMEAQBQwQRAEDBGmLF7KX+frrr5OsoyN/48drr72WZLmTW2kuL2S/O2WnD8+fPz/JnnrqqezYF198MclqtfSvo6wzcmN37NiRHfvqq68m2V9//ZUdm3sU+V7jhewALUYBAwRRwABBFDBAkLbehOPeYxOOdmQTDqDFKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCdDQ4/ueiKC42YyFQFMXkoM/1vaaZSr/XDR1LD0D/8b8gAIIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYL8D+KUFeSspSmeAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 20, + "source": [ + "def plotit2(snapshots_mn,step):\r\n", + " fig = pylab.figure(figsize=(10,4))\r\n", + " ax = fig.add_subplot(1, 2, 1)\r\n", + " pylab.imshow(snapshots_mn[step][0].reshape(28, 28), interpolation='nearest')\r\n", + " ax.set_xticks([])\r\n", + " ax.set_yticks([])\r\n", + " pylab.colorbar()\r\n", + " ax = fig.add_subplot(1, 2, 2)\r\n", + " ax.set_ylim([0,1])\r\n", + " pylab.plot(np.arange(len(snapshots_mn[:,1])), snapshots_mn[:,1])\r\n", + " pylab.plot(step, snapshots_mn[step,1], \"bo\")\r\n", + " pylab.show()\r\n", + "def pl3(step): plotit2(snapshots_mn,step)\r\n", + "def pl4(step): plotit2(snapshots_mn2,step) " + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "skip" + } + } + }, + { + "cell_type": "code", + "execution_count": 21, + "source": [ + "snapshots_mn = train_graph(pos1,neg1,1000) \r\n", + "interact(pl3, step=widgets.IntSlider(value=0, min=0, max=len(snapshots_mn) - 1))" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "afbc754ef0d04c95a1af039574b46a6b" + }, + "text/plain": [ + "interactive(children=(IntSlider(value=0, description='step', max=66), Output()), _dom_classes=('widget-interac…" + ] + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 21 + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Please note how accuracy goes up to almost 100% very fast.\r\n", + "\r\n", + "Please, move the slider to some position towards the end of the training, and observe the weight matrix plotted on the left. This matrix will allow you to understand how perceptron actually works. You can see the high weight values in the middle of the field, which correspond to pixels that are typically present for digit 1, and low negative values by the sides, where parts of 0 digit are. So, if the digit presented to the perceptron is in fact 1, middle part of it will be mupliplied by high values, producing positive result. On the contrary, when perceptron observes 0, corresponding pixels will be multiplied by negative numbers.\r\n", + "\r\n", + "> You may notice that if we give our perceptron a digit 1 slightly shifted horizontally, so that its pixels occupy the place where there are vertical parts of 0, we may receive incorrect result. Since the nature of our MNIST dataset is such that all digits are centered and positioned properly, and perceptron relies on this to distinguish between digits.\r\n", + "\r\n", + "Now let's try different digits: " + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 22, + "source": [ + "pos2,neg2 = set_mnist_pos_neg(2,5)" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAACqCAYAAACTZZUqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAI90lEQVR4nO3dT4hVZR8H8HPfphJxLKzQippFZIOGTFEZFNEuxE3YIjAKFxYoKkqMItGqiCjEhSARWRkVgqCbyAKFslqJC/9ubKOgKP6JcuHoQPfdvBAvz+/Uvc2993e98/ksvzxzzsPM8cvB8zznNJrNZgVA7/0newIA05UCBkiigAGSKGCAJAoYIIkCBkgy1M7gRqNhzRpd1Ww2G70+p+uabqu7rt0BAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyQZyp5ANz3yyCNF9sQTT/Ts/CtXrgzz559/vsi2bt0ajj18+HCR7d+/Pxx74cKF1icHLRgdHS2ytWvXhmNvv/32Ips7d244dunSpS3P4dChQ0W2Z8+ecOy+ffuK7OjRoy2fq9fcAQMkUcAASRQwQBIFDJBEAQMkaTSbzdYHNxqtD+6huiet0WqBBQsWdHs6Xffdd9+FeTtPlvtVs9ls9Pqc/Xpdd8vw8HCYv/fee0X22muvFdmsWbNaPlejEf852+mddkxMTBTZ7t27w7ErVqzoyhwidde1O2CAJAoYIIkCBkiigAGSDMRW5E8++STMB+GBW2TRokXZU+AmMTIyUmQ//vhjOPaBBx5o6ZjffvttmE9OThZZrx/CPfbYY0X28ssvh2N///33IhsfHw/H3rhxY2oTq+EOGCCJAgZIooABkihggCQKGCDJQKyCgOkuehl6VVXV119/XWQPPvhgODZambBr164ie/XVV8Of//PPP/9uij0RbZNevnx5OHbZsmVFNnPmzHCsVRAAA0YBAyRRwABJFDBAkoF4H/Drr78e5h999FGPZ9Ib586dC/NWt5L2M+8D/nfqrvXo30bd9uAvv/yyyNavX19kV65caW9yeB8wQL9RwABJFDBAEgUMkEQBAyQZiFUQs2fPDvOjR48WWTsrBU6ePBnmn376aZH99ttvRbZjx46Wz9UOqyA6q1+v63ZcvHgxzO+6664i+/zzz8OxGzZsKLLopeW0zyoIgD6jgAGSKGCAJAoYIMlAvA/4jz/+CPMXX3yxyOq+oPzVV18V2c6dO8Ox0VbMuuNCpy1ZsqTI7rjjjnBs9JA9ethWVVN/4HbnnXcW2dBQXDHRvC5fvjyl89+M3AEDJFHAAEkUMEASBQyQRAEDJBmIrcjtqHtaPNUnwNH24Llz507pmHX2798f5i+88EJXztdLtiL/pe5Lxz/88EORPfXUUy0f95Zbbml57L333ltkq1atCsdGebQVuqqq6vr160X28ccfh2PHx8eLrFtfKe4WW5EB+owCBkiigAGSKGCAJAOxFbkd7TxsGxkZCfO33367yIaHh//1nP7OTz/9VGQrVqzoyrnoL3XXVDsP3L755psiW7lyZTh206ZNRXbPPfe0PK923HbbbUW2Zs2acOylS5eK7J133pnyHPqBO2CAJAoYIIkCBkiigAGSTLudcO04cuRImD/66KMdP9fExESYR+9YnZyc7Pj5+4WdcH+59dZbwzzaCfnss8+2fNxGI/4Vt9oFhw4dCvNjx461PIeXXnqpyOp2qZ4/f77IHn/88XDshQsXWp5DL9kJB9BnFDBAEgUMkEQBAyRRwABJpt1W5DrRE+dou2QnRCsetmzZEo4d5BUP/L26v/1bb71VZAcOHAjHRtf11atXw7HRl8Hff//9Ijtz5kz48+2IVm1EK36qKn4n8UMPPRSO7ddVEHXcAQMkUcAASRQwQBIFDJDEQ7j/id6FOn/+/K6ca86cOUUWfaQQIj///HORLVy4MBwbfYDz2rVr4dhOPFxrVbTtuW4r9OXLl4vs7NmzHZ9TBnfAAEkUMEASBQyQRAEDJFHAAEmm3SqI0dHRMO/Gl4brvsDczkvwoRW//vpr9hRCdf/e7r///paPcfjw4SI7ffr0v55TP3EHDJBEAQMkUcAASRQwQJKBfgg3NjZWZHv27AnHjoyMdPz8b775ZpjfuHGj4+eCfrRz584wnzVrVsvH2Lt3b6em03fcAQMkUcAASRQwQBIFDJBEAQMkGehVEM8880yRdWO1Q1VV1YkTJ4rs4MGDXTlX9KXbqqqqhx9+uCvnO3/+fJFduXKlK+fi5rVhw4Yie/LJJ8Ox0Xb8HTt2hGM/++yzqU2sj7kDBkiigAGSKGCAJAoYIMlAP4TrpeirtFu3bg3HHjlyZErnqtvGuW7duikdt873339fZMuWLQvHTkxMdGUO9I/nnnsuzLds2VJkjUYjHHv16tUie/fdd8Oxk5OTbczu5uIOGCCJAgZIooABkihggCQKGCDJQK+CiLbQRk9fq6qqhoeHO37+pUuXtpVna+d3MzQ00JfOtDRz5swiW7NmTZGNj4+HPx9tL65bwbBx48YiO3PmzD9NceC4AwZIooABkihggCQKGCBJI/qP89rBjUbrg/vUL7/8EuZPP/10j2fSG9GDyKqKt33Wje3lV2mbzWa8d7WLbrbrevHixUV23333hWOjv90bb7wRjl27dm2RLViwoM3Z/b8PPvggzDdv3jyl495s6q5rd8AASRQwQBIFDJBEAQMkUcAASabdftJXXnklzLdt21ZkdS+ernsheq9cv349zA8cOFBkdU+bjx8/3tE50Tvz5s0rsi+++CIce+3atSK7++67w7Gtrog6depUmEdfNf7www9bOuZ05Q4YIIkCBkiigAGSKGCAJNNuK3I76t7bO3v27CJbvXp1kW3fvr3jc6qq+i8P93LLcLfYivzPxsbGiqxui/2MGTOKrO5LxdHXuqNrKnrYVlVVdfbs2TDHVmSAvqOAAZIoYIAkChggiQIGSGIVBH3FKggGkVUQAH1GAQMkUcAASRQwQBIFDJBEAQMkUcAASRQwQBIFDJBEAQMkUcAASRQwQBIFDJBEAQMkUcAASYbaHH+pqqrT3ZgIVFU1knRe1zXdVHtdt/VCdgA6x39BACRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyT5L95QHHzClR5NAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 23, + "source": [ + "snapshots_mn2 = train_graph(pos2,neg2,1000)\r\n", + "interact(pl4, step=widgets.IntSlider(value=0, min=0, max=len(snapshots_mn2) - 1))" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "bf6f25d14c3b4d548ac026af97f70e2a" + }, + "text/plain": [ + "interactive(children=(IntSlider(value=0, description='step', max=66), Output()), _dom_classes=('widget-interac…" + ] + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 23 + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Discussion\r\n", + "\r\n", + "For some reason, 2 and 5 are not as easily separable. Even though we get relatively high accuracy (above 85%), we can clearly see how perceptron stops learning at some point.\r\n", + "\r\n", + "To understand why this happens, we can try to use [Principal Component Analysis](https://en.wikipedia.org/wiki/Principal_component_analysis) (PCA). It is a machine learning technique used to lower the dimensionality of the input dataset, in such a way as to obtain the best separability between classes. \r\n", + "\r\n", + "In our case, an input image has 784 pixels (input features), and we want to use PCA to reduce the number of parameter to just 2, so that we can plot them on the graph. Those two parameters would be a linear combination of original features, and we can view this procedure as \"rotating\" our original 784-dimensional space and observing it's projection to our 2D-space, until we get the best view that separates the classes." + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 24, + "source": [ + "from sklearn.decomposition import PCA\r\n", + "\r\n", + "def pca_analysis(positive_label, negative_label):\r\n", + " positive_images, negative_images = set_mnist_pos_neg(positive_label, negative_label)\r\n", + " M = np.append(positive_images, negative_images, 0)\r\n", + "\r\n", + " mypca = PCA(n_components=2)\r\n", + " mypca.fit(M)\r\n", + " \r\n", + " pos_points = mypca.transform(positive_images[:200])\r\n", + " neg_points = mypca.transform(negative_images[:200])\r\n", + "\r\n", + " pylab.plot(pos_points[:,0], pos_points[:,1], 'bo')\r\n", + " pylab.plot(neg_points[:,0], neg_points[:,1], 'ro')" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "fragment" + } + } + }, + { + "cell_type": "code", + "execution_count": 25, + "source": [ + "pca_analysis(1,0)" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAACqCAYAAACTZZUqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAI8UlEQVR4nO3dS6hV1R8H8H3kKuHzopZIiIgEooYRUYN0IGoZCkqiGSIiDq5Y6ERwEBENGog5UtTIBtee+IAok3yEICImSoMiqFEiKb3z+iqse/+TKP6steUc9z33dzz38xl+WXedlZ77ddPae69aX19fAcDAGxK9AIDBSgEDBFHAAEEUMEAQBQwQRAEDBOloZHCtVnPPGk3V19dXG+jP9L2m2cq+166AAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCNLQoZzce06cOJHN582bl2Rr1qzJjt23b1+/romBM3bs2CQbOXJkduwLL7xQ15xPPPFENt+1a1eS9fT0ZMcePXo0yfr6Bt/ZqK6AAYIoYIAgChggiAIGCGITro2cPHkyyZ588sns2N7e3iQbjJsg96JRo0Yl2TPPPJMd+8477yRZR0dzfu0nTpyYZJMmTcqO7e7uTrKtW7dmx3733XeV1tXKXAEDBFHAAEEUMEAQBQwQRAEDBKk1svNdq9Vsk7eAl156KZu//PLLSTZ06NDs2P379yfZunXrsmNv3rzZwOqq6evrqw3Yh/2jVb/XnZ2d2fztt99OskWLFjV5Nc33ww8/ZPMlS5Yk2TfffJMde/Xq1X5dU38p+167AgYIooABgihggCAKGCCITbgWt3Tp0iR7//33s2OHDRuWZF9++WV27Jw5c5Ls2rVrjS2uCWzC/WfhwoXZ/MiRIwO8ktazYcOGbL5nz54BXkl9bMIBtBgFDBBEAQMEUcAAQRQwQBAvZG8RZS+ufuWVV5Isd7dDURTFr7/+mmS5x5OLojXueOA/s2fPTrItW7YErOT/bdq0KckuX76cHbt58+YkKztBuapt27Zl819++SXJDhw40JQ19AdXwABBFDBAEAUMEEQBAwTxKHKAxx9/PMnefPPN7NiZM2fWPe+qVauS7IMPPqh/YS1gsD6KfPDgwSR79tlnK897/vz5JPv888/r/vk33ngjyb766qvs2BEjRiTZ2LFjs2NzG2O534tGHTp0KMmWL19eed6qPIoM0GIUMEAQBQwQRAEDBFHAAEE8itxEq1evzubd3d1JVnY3Su6U1xMnTmTHHj16tIHVEaFWy9/kMWRItWuh3B0wRVEUP/74Y5J99tlnlT6rzI0bN+rKiqIoPv300yR77LHHsmMb+bOZNm1aki1evDg79vDhw3XP2yyugAGCKGCAIAoYIIgCBgjiUeR+MmHChCQ7fvx4dmzu8eKyv4d9+/Yl2dq1axtc3b2j3R9FnjVrVjb/4osvKs07efLkbH7p0qVK8w6kZcuWZfOq7/Mte8y/q6ur0ryN8CgyQItRwABBFDBAEAUMEEQBAwTxKPJd6OzsTLJjx44l2YwZM+qes+yU4o8++qjuOWh9U6ZMqTxHT09Pkt2+fbvyvNHOnDmTzXP/vaNHj272cgaEK2CAIAoYIIgCBgiigAGC2IS7C7nTXxs5vThn0qRJ2bxsc4570++//155jnPnziXZb7/9VnneaFeuXMnmR44cSbKVK1fWPe/TTz+dzUeOHJlk169fr3ve/uAKGCCIAgYIooABgihggCDeB3wH48ePz+a5p94eeeSRuuc9e/Zsks2dOzc79s8//6x73nbQTu8Dzj2t9e2332bHPvDAA5U+qx3eB1xm0aJFSfbxxx9XnnfcuHFJ1qzNTO8DBmgxChggiAIGCKKAAYIoYIAgHkW+g507d2bz3Mm2ubtJyt5vOn/+/CQbbHc7DAYdHemvV9W7HQaj77//PnoJTeMKGCCIAgYIooABgihggCA24f6Re+x46tSpdf987lDErVu3ZsfacBsccu/+fffdd7NjV61a1eTV0IpcAQMEUcAAQRQwQBAFDBBEAQMEGXR3QZQ9Cvree+8l2aOPPpod+8cffyTZ+vXrk+zw4cMNro520tvbm2THjx/Pjq16F8SBAweyee6x94E++bdenZ2d2by7u7vSvHv27Mnm/XFCdVWugAGCKGCAIAoYIIgCBggy6E5F7urqyua7du2qe45Tp04lWdmpxjSmnU5FzhkzZkw2P3nyZJI1ctJ2mfPnzyfZli1b6l5Ds9x///1J9vrrr2fHrl69uu55b926lWTTp0/Pjr148WLd81blVGSAFqOAAYIoYIAgChggiAIGCNLWd0E8//zzSbZ79+7s2FGjRiVZ2anGK1asSLIrV640uDpy2v0uiDKzZ89OsrLv6owZMyp91unTp7P5xo0b6/r5np6ebD5s2LAku++++7Jjc48XP/zww3V9/p0cOnQoyZYvX1553qrcBQHQYhQwQBAFDBBEAQMEaYtNuLLHOy9cuJBkU6ZMqXveZcuWZfMPP/yw7jlozGDdhMvJbfYWRVG89dZbSTZixIhmL+dfP/30UzYfPnx4kg3kuoqiKFauXJlk+/fvH9A15NiEA2gxChggiAIGCKKAAYIoYIAgbXEq8pIlS7J5I3c85IwePbrSz0MVZbv3Dz74YJJt37692cv5V+5l6s109erVJCs7WOGTTz5p9nL6lStggCAKGCCIAgYIooABgrTFJtzt27ezeW9vb5INGZL/N+fvv/9OsoceeqjawqAJ9u7dm2QLFizIjl24cGGzl9Nvbty4kc2fe+65JDt27FizlzMgXAEDBFHAAEEUMEAQBQwQRAEDBGmLF7KX+frrr5OsoyN/48drr72WZLmTW2kuL2S/O2WnD8+fPz/JnnrqqezYF198MclqtfSvo6wzcmN37NiRHfvqq68m2V9//ZUdm3sU+V7jhewALUYBAwRRwABBFDBAkLbehOPeYxOOdmQTDqDFKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCdDQ4/ueiKC42YyFQFMXkoM/1vaaZSr/XDR1LD0D/8b8gAIIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYL8D+KUFeSspSmeAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAow0lEQVR4nO2dX4wkx33fv7XDXcZ7ywS8ISVcJO0sGQtBqJdE3AgiEutFQSTeixxYBmQM6QMt4KBlAjAB9MDDvujlHqwgAajQVHiJjrpwFpadf7AswKAlJkEAJzC9DPhPkM8k5dszLULk3dkJqZV11F3lobu1vb1V1VXdVd1V3d8P0NjZmpnu6p6qb/3qV7+qElJKEEIIGRdLfWeAEEJI91D8CSFkhFD8CSFkhFD8CSFkhFD8CSFkhNzSdwZsueOOO+TGxkbf2SCEkKR4/vnnr0gp76ymJyP+Gxsb2N3d7TsbhBCSFEKIPVU63T6EEDJCKP6EEDJCKP6EEDJCKP6EEDJCKP6EEDJCKP49srMDbGwAS0vZ352dvnNECBkLyYR6Do2dHeD0aWB/P/t/by/7HwDm8/7yRQgZB7T8e2J7+0D4C/b3s3RCCAkNxb8nLl92SyeEEJ9Q/Htifd0tnRBCfELxD0TdYO7Zs8Dq6uG01dUsnRBCQkPxD0AxmLu3B0h5MJhbbgDmc+DcOWA2A4TI/p47x8FeQkg3iFT28N3c3JSpLOy2sZEJfpXZDLh0qevcEELGjBDieSnlZjWdln8AOJhLCIkdin8AOJhLCIkdin8AOJhLCIkdin8AOJhLCIkdLu8QiPmcYk8IiRda/oQQMkIo/j1hmgTG1T4JIaGh26cHTCt6AlztkxASHk7y6gHTJDCAE8QIIf7gJK+eKbtyVOIOZJPAOEGMENIFdPt0QNXNo6OYBKZqHDhBjBDiE4p/B6g2bqlSngRWbSg4QYwQ4hu6fTrA5LIpTwIDgEceOSz80ykniBFC/GMt/kKI80KIt4QQr5TSjgshvi2EeDX/e3vpvTNCiNeEEBeFEJ8qpd8rhHg5f+8rQgjh73bi5Phxdfp0Cty8eTCQ+9BDwNWrhz/zzjtBs0YIGSkulv/XAXy6kvYogGellB8G8Gz+P4QQ9wD4HICP5N95Qggxyb/zVQCnAXw4P6rnTJK2sfnb28B77x1Nv36d+/qSnD4ngHDyyfCQUlofADYAvFL6/yKAE/nrEwAu5q/PADhT+twzAO7LP/PHpfRfAfCkzbXvvfdeGSuLhZSrq1JmW7dkx+pqli6llEIcfq84hDg4h+4z1c/1wWIh5WyW5WM2O7gv0iF1hSyWa7OwRAeAXanSc1Wi7lCI/19W3v+L/O/jAB4opX8NwGcBbAL4Tin9FwB8y+baMYv/dKoW7dkse382M79v+kz1c13Tp+aQEjaFqO9rs7BEiU78Qw34qvz40pCuPokQp4UQu0KI3bfffttb5nyys3PUT1+wt5e9b7PE89mzwPLy0XOsrPQb6aOKVNrfpyuqc/qcAGJ7bRaWpGgr/j8UQpwAgPzvW3n6GwA+VPrcBwH8IE//oCJdiZTynJRyU0q5eeedd7bMahjqynWxNEPdEs/zOfDUU9kgcMF0Cpw/32+kDyedRUKfOwTZXpuFJSnaiv83AZzKX58C8Dul9M8JIW4VQtyFbGD3OSnlmwDeEUJ8PI/y+dXSd5KkrlwXhs98nkX1FNE9KkGfz4ErVw76zFeu9B/iyV3JIqHPHYJsr83CkhYqX5DqAPCbAN4E8B4yC/7zAKbIonxezf8eL31+G8DryAaF7y+lbwJ4JX/vceTrC9Udsfr8Tb76WAZs20A3bkT0OZhqc22bwsIB4c6BjwHfPo9YxV9V3mMasPUB6yuxxlRYaEn0gk78uapnS3Z2slm5ukHf1VXO0CUEgHk5Wy5ZGwyu6tmQuk1XTp8+LPwrK9lALffuJaQCB4Sjggu7GTBtujKfqyPbrl8H1taywVpCSIn1dS5ZGxG0/A3UhS37NmQ4g54Mmj4jlsgRKP4G6sTdZ2Rb0cvY28tGwopeBhsAMhjm8/oJL6QzKP4G6sTdpyHDyZFkFNhMeCGdQPE3cPKkOv3nfz7769OQ0fUy9vboBiIdQ//jKKD4G/jt31anP/vsQX1wNWR09crkKircQL/2a6yHJDD0P44Gir8BXew+cNgdUwi6EMAtt2R/NzaAhx8+LPQPP6yvVyoXUpXr17M5BSQxUrKk6X8cD6qZXzEefczwrVu2oVjawbQWf3WZB9MM4PLkSNN56uCM3IhIbVarzeYTJCnQ8ZLOyfPww+b3hTgIWbadJK37XOHvL7uQmsJee2SkZkmPdXG2lHpnnqD4K3j4YeCrXzV/xueqGKp6VV7a2Sa9IDWtGTypzWodYyz+SC0mir+Cc+f079WJbx3V7ep19eqxx45u7rK8nKWbSE1rBk9qlvQYY/FHajFR/BXcuKF/77HHsvrQlC98wa5eFZu7lD/71FP1dTA1rRk8sVjSLm6NscXij9ViUg0ExHh0OeA7mZgHeW2WcVYd02n4vKc2vjgK+h6BZ6Ew0+f+yB0Arudvz9aWXsCLoIdyfZ5OpVxeNgt/l3Wtb60hkTFwcWvNwBtHnfhzPf8Kdevzq5Ye1y1TPplkPef19ayXP/TeM4mUpSV1hIIQ7ULLhsTOTubjv3x5cBWW6/lboFqfv8q77x51l+pcgzdvjsdtmgQjDOcDEH4gaAjPdWzjHKD4H0I16F/l6tWjUWAcZO0RW+EZaTgfgLCDzmN+rqmj8gXFeHTh87edqVsM3tb5/KfTA7ch/fABcPHVjt3vHaoAjv25JgDo869H57u3YWUli8P/0Y8Op6+uAqdOARcuHO5VcG9fD7jsCUu/dxj4XKOHPn8LbBZX03H9OvBXf3U0fX8/E/kRziEJj0t8dte+uVB+cF/n9XUe+jzTRdUdiPHoKtRzsZByacne/dPm4FpZLXFxOXQZzhfqWr7O6zN/Aw+THAJgnL8906lesGcz8/uqQzdpjG7RlrgKT1cDL6H84L7O6zt/HNCKGoq/A3Wr2rrM8F1dzSaN6TSK9aYlMT7AUMsi+zovl20eFTrxp88/p+wCXdI8lcKNWax9ZcO5c8ATT6jXygIYJdeaGOOzjx93S7fFl3+dfnoCDvgCOBqqrFrYrRoWPZ/XL/C2tGTWopEuJtg9Q5iEBPiL149lsTnSL6ruQIxHSLePzgU6mZi9CXXun2PH9J8zfY+9b4/0MSAZ0q3iy80Vo7uMBAGM89fTJlR5Zwd44AH1e8X3TWv/qHoZqjB10hCXuQApX5MQDYzzN6BzdUppt/S5zv1TnFcXjn7jBnvfweljrXa6VUgCUPxhntxlMwhbV9d1jUsx8DumTZM6p4/BzS52wxrKOAbpD5UvKMYjdKhn4QI1xffbfF/lQuU8mB6JdS5AG1igiANgnL8dpsXdmurBYnF4Ylh5wTfSAbaCnoqocjE14oBO/On2qWDyBkiHWPyiVy4E8OCDh/cI+PGP2+WRPX5HbOcCxBx7W/7RdasPDn3PWeIVin8Fm8Xd9vezlTptlo4HjkYStdETLp8ekFg38q7+6DpcxzHKFsott2R/aU2MB1V3IMaj6z18TZu41y3RYLP2T9OQb/b4AxLrwzUNRjV1T5kmqcTo6iKNAX3+9VR98zbHdGq/zo8PPeGyLAEoj/ZXH3AMQhhiIKquQem7wSPe0In/KN0+Kp+5zf69Kq5erd/6sUqbkG8uy+IZlY9OiOx1LLG3plhh1zWNisJft2tR364uEh5Vi+B6ALgE4GUALyBvZQAcB/BtAK/mf28vff4MgNcAXATwKZtr+LL8dQEdrhZ/00OIzK3kO/99G6fJEqurp0zIdfxp+Q8ehHT75OJ/RyXtywAezV8/CuDX89f3AHgRwK0A7gLwOoBJ3TV8ib+N+9THMZ2GW8c/hVD0ZEjFj+bjR7ct/LQmBoVO/EO6fT4D4EL++gKAXyylf0NK+RMp5Z/mPYCPBczHIbroza6sAI89pl8XqG0eYlzFOFlS8aP5+NFNBW8yyf725epi/HLn+BJ/CeD3hRDPCyFO52nvl1K+CQD53/fl6R8A8Gel776Rpx1BCHFaCLErhNh9++23vWRUV6cLN68Pbrstqzup6MqoGdM6PKaxg5/+NLP7+7AmVPHLDz6YRuhpyo2WqjvgegD4m/nf9yFz6XwCwF9WPvMX+d/fAPBAKf1rAH6p7hqhff6+/fqma7FHHYimrpGx+NFiLZB17qgY8qgi1udZAV2FegL4EoAvIhvMPZGnnQBwMX99BsCZ0uefAXBf3Xl9hnqq6rrPsYCyT38sutI7XVTEIfyYMd6DKZS1OCaTOPJaJoVgARlQ/AEcA3Bb6fX/AvBpAP8Shwd8v5y//ggOD/h+Hx0O+OrY2vIn/tOpvm7FWPcGQeiKmIiVlySpDkQnEiwQUvzvzsX8RQDfBbCdp08BPIss1PNZAMdL39lGFuVzEcD9NtcJLf4u5c8lLLRcXqkfAQldEbu08mK0EELmKdUQ1LFb/l0docTfxuVT6EpR5hcLKVdW7BuAoiwkUlbSJPTD7crKi9FC6NKlVq5wuiOGxlDKOH8rBRR/BW0MDtUyzabGQ8pkeolpEnrd/q5abt/XCTk/IJTVsliYF9eKSWBj7KVVoPgrcBnktRHoujpCyz8wIdft78rK82kh+Mqzrzy5CGWdZcZKY83oxV9V7myCDFRlrdxDLQyU2SwbNDbVtUR6icOnaSvsw8qrO4dPC6HJuVxC4VTdYZ/b2S0W+grJ7rI1oxZ/Xbk7dsxO+OsEvPy5rS1z3U6glzh8+vK/qQaLVlbai6QO1/vUXbvOqrHJd9NGjd3l1oxa/NvE8FfDi7kS7gDoQlBUrbxuYGg6rf9uE1zv0/T5tj2WpstSs7vcmlGLv4t7x1T2bc7F3mgChBaUJtPIfaNrbEz32aZHVPfdNhvSsLvcilGLv4/Zu0W57MryZ3kPTMgH3KTAtclr9X2Vm6bcy9Dda5seUd13bUPr2HX2zqjFX1XumvQGinoXevc79nQTQjX673pU3T7V87tGEdQVbpOF3bTg2Xy33Ejp8saus3dGLf5SuhlHpqOY5FXtUZsMKlc4xpUILhNFikKyvHw4bXnZXHCaxg83tbDb9IhcvstC3hmjF38VdXNJdEaTTfBDGzgZLBFchLcoIK7iWlcYmg5o9V2Y2L3tDIp/BRv/ve4ItUNXAY2iRLAR3tARO7r36/JmW5hCjI3oJspQ+INA8S/h2lvv2piiUZQIXYz+N/H5F91TXWhpdbEql2uvrJiXrW17P67nYlRELRT/Em2jf5aWwtX1ApbrBOhi9L+4juvMQVsLx5TPNuGZOnx1a2khWUPxL9E27v/YMZY7Io+O/Nta1F3gYuFMJuqGxbaiuAh30wEt26gq+kaPoBP/kBu4R8vx4+2+v7+f7XE9m2XbjPa15zXpkWLf2atXD9J+7ueAxaKffXCrmDZrr3LjRiade3vZPRX70NpuNu1yrSYbW5f3+C3y6yMvI2eU4m9iNgO2toC1Nf1n1tezun3pEnDzZhx1nXTM9nZmBZTZ38/SY8BWuKuU7+Hkycy68Xmts2eB1dXDaaurWboO1bP2kZeRM0rxv3ZNnS5EJuRPPAG8807WCFTLfl05JSNBZ2H2aXnu7AAbG8DSEvDuu8Dy8uH3l5eBlZX681y+nJ3rwoWsR1BmqSIZrhViPnfvNts+05UVVk4HRin+tj3PJ54Ann6a7p1RUxbUjY16l0hflmfZNSJl5o4SAphODwrvU08B588fFOjJRH2u9XW9tX377foKoXtWVVy7zbbPtNpQETOqgYAYj9ChnhywJUcwFZTYClHTtft19+BrOehQ651wwNcaMNrnMLroOIZXkp9hs1hZLAWmbRRN9R58LgftA9ton75nLkcIxb+G2Aw5EgEprbPRVnxtFr8KsRz01taBmE8m2f9d3O+IoPjXwLJEjpBSoQixImfdtnRlmjyrrS31d2waAFpr1lD8a0jJyCMdkZrA6Fw4bXfhsr2267PSuW8mk3b3Sw5B8a8hJSOPdEjqAmMjyr4sH9dnpfPbA653SQzoxH+UoZ4qmsw9IZ6xDRXskpRn8+3sAKdO1U9G8xW2avusit9Zhy4EtY4Yy0/MqFqEGI9Q6/lXx7hSNvKSJjUXS+zUhUeWrfoun73Ldo62FVK1uxLLz88A3T6HodZEho3fbQgumK7y77rcdFd5a7OkrqqCbm2ZF6CL3W/bwXOn+OfUbeISe1kZLHV+59Rb667zbxLEPp5bm92TTMZA3cqjMUdsdFQmKP7SrscZc1kZNHWWv25jklRa66YRBU13vdJdbzLpbp+B8ud87Z5UrqA2jUnM5aOjKBOKv0y/rAyauqUUUm+tm0TUtNksJrRV6XL+uoq3uqpv3E0VtM7qj71n2FF8OcVfNi8rTd1yqbuoO8d1qYEuW+umMfQFTay8tttEhiyApvupXtd0D9Op/c5j1QpqOndx3pih5R+H5a+rG00NqNRd1FFharW7eKCmGbC2P7LqHHX74cbsz64bU6jm06YBcw2/Uz1TIeyXiOgb+vzDiX/b5UqkbN44c9KYR3QPczrt9/q6Gaq6H7lcIKdTKZeXzYWxreUfEtdnEqoB66p7Heo6jPbxL/4+liuRsrlbjstFeCSEheRS6Vw3fbb5kW1DW219/l37GHW/ictz6rsBsyXxbvzoxN+X5b22pj7P2lo31yc5PsXNtTK7Wrk2PRJb68Am2qcvcVL9JqZeWqoC2ldl9lTmRyf+vizvOnelS3RbKmV98LhWZlM3suq6ATJfftNQzCaCEpOlURe1lWIERIhufN2z8CggoxN/X/WhrtdqqueplvWC1POvpWnYpephNJ1/4CKSdb7K2HyMQys4bcRE9SxshN1jgz468XdtOHXl1Wb8qqtxxy4J1nOJQRh8WspthNdWGKpHQKFwyutY8B3yZ2MweGzQoxN/AJ8GcBHAawAerfu8j2ifJi4a3X4T1WNoBNGTWHxhPvPh+0HVRfiozh/6ucbyu/VJk8bP9rdUCftQLX8AEwCvA7gbwAqAFwHcY/pOyPX8655zeae5sYh/EE9CbL5pH5aszlJvOsnINrJINzCsup+292oa8B5jT8AWF+EP2KDHJv73AXim9P8ZAGdM3wkp/rb1S9dbG6LbJ4hOx+ab9sVi4W9J4SaWf13e2oqITYM0tp5AHYuF/rnZRj4NMdoHwGcB/PvS/w8CeFzxudMAdgHsrq+vN7rxOky/UdWwWSyOBncsLw+zzAfp6cdk+fvG17018fmHzpfvBmkM6J6ZEJ1HPsUm/r+sEP9/Y/pOKMvftlynHq3WBO/3OmTfsc9ejWu0T+h82TRIQ+jB+cTUW+qY2MS/F7ePSsxcJm/SsPHAUFvPWHs1Pnskxe/muqyFy7mHUiYiKg+xif8tAL4P4K7SgO9HTN9pK/4q40UIKY8dsxd/GjZES4hejQ9RDJUvX+ccam8wovuKSvyz/OAkgD/Jo3626z7fVvxN7h3VJM1IGu2kSN6Aa3sDxfcBt41XdOeqisfysnkV0Lp8+fxhbM5p85mILGTvRFIhohN/16Ot+JvcO9Op3V4SxUqxkfymURGRodMMXzfg6zw2g1G2E436KKy2z8FmTMLnPYyw8o5e/E11SQj9Am7lozDmquU1KZELRJIGXAg/tq8HYTsYZTpvn+FppudQfu5LS+rPFfHTfbuYBtBYjF7868JubepZ0zo4BpIL4Q8VwWI6j4uQ2IahmfLX58QUl81eTHn0aVW4nquv7qznBmf04i9l5rZRWe0+xD9akeuI5Cz/ELHrPib2lM9lI5Km/Jm+FxrTrGCXCuXTqnB1MfmOarIhQIND8c9RNapthT9qkeuI5Hz+IWatmib2NFn9s1xYp9NsCVmX/HUp/qrBbpWl5VqhurT8Y5jPEMCKovhrsIn1rzNWoha5DvHaWw3ta62z/Jtc01SQfE22cnkmXbl9TKJZ3HfZ1+/S6Hbp849hJnMA/ynFX4Pp9y5H9+j2pE50DChuuuhGmFr9ppXbZLX14RdbLI72FlZW2s0YVmHTkJbz5LqRfchon/KzsBH+0JYeLf/uxN92FvYABv3ToSuhNFnpTTA1WrEMHm5t+QtptRVNlxVIu8TWzdPl6qX0+fdv+Y/dh98rXYUOhfjxTcIWg+j5uGdb0Yy9MvmaS+EbRvt0I/7JDVSOga5a5KH/+CoR8dGw2vrGY3+edWM0bYQ3hoY+h+JvIKLfKRhJ3WOXohyzpd7m+q5xzS4Nq22EROwFLZSR0WRcIyAUf0v6ru8hSNLA7fuH6PuhuVxf5dfXCfTS0tFZv75CWmN17+jQRXIUkR5NicydRPG3oO/6Hoqh1NVOCf3Q6ho32+ubQtF0R1sr1LaidNGAt72GrofUJq8+lubwCMXfgqGKZHJLL8RAyIdmI56213fxv6sKdVPxrPteV+G6ba8RotLb/iYdVUCKvwWm+mYKD47dPTTURi0oIR+azbltr++yG5GqUIcS6C4KnY9rhGjkbaOhaPnHI/5t1viJ2T00VHdWUEI+NNs1Zmyu38byDynQXXQ3Q0Yu+Rj0LazD6bT9OEsLKP4Kqr+P7ZpTPTfkjeh7/DRJQj00F39+3fVdY+7LwhNSoPu0/KdT+9+tK8uoxwpI8a/QpM7UHfShEyt8D5hWP2fqwhbnWSzCrloZSlSrFptq+QpXK3vglhHFv0LT3nKqlj+JjJADpovFUVEEDjZyMVk+PoXSt6iq8l3d2tLHPIaBQfGv0HScrK4nHSsDN26GR1u3iUkEdeeeTOJwkeiweSaurqwRVAyKf4U2lr8QUn7yk+mUmb7rLKkhxDIMpu/brmZYpe+wMZtn4pLHEDH+EULxr9DU5x+70Kvou84SA7qW2cZvb8L0o+veK8I/dfQ9YcSmILuMp/he0jtSKP4KVLH7pgYh1TLRd50lBkwRK6bCWLehhEkEmwpf3/50nwPlpq7/wCoGxd+SxUJdxlPuDdLyjxibmYV13VHTmj/lwjydHnzOVfgWi6NRNEA2sNxlxfDloze5vgZWMSj+GnRlaUjjQPT5R0ybQcw6wTL98K4WgamHkiJNXV8JQvFXMCZRHFJjNihsCqGN9a+y2E0C71r4m/gOY14TRXX/Plb0jBCKvwIfEwQJaU2TmH8bi71OsF26va49BZs8921pjcQiovgrsI3177uMkoHQRmzKAmwbnthksEfXI3Dd+9c2lnpg/vUYofgrcIn1ZxkltZjE3aeP0WXZB9dr1rmKbHsKtpbVwCJrYoTir8Al1p9llBipE9q+Qq5cextNZsi6zlOo+lib5JNYoxN/kb0XP5ubm3J3d9f7eXd2gAcesPvsbAacPQvM596zQVJnYwPY2zuaPpsBly4BS0uZ3FURArh5M3Tu7LnjDuDq1aPp0ylw5crRdN19T6fAj38M7O+br7eyAnz+88CFC4c/u7oKnDvHyuYBIcTzUsrNavpSH5lJlb094KGHsgaDkENcvmxOX19Xv69LTwXdfV+7lon3bJY1cLMZsLZ29HPXr2efqzYS+/vA9rb//JKfMXrxdy1f770HPPJImLyQhKkT97NnM2u2zOpqlh4T1665pZvuez7Pej03b2Z/f/Qj9Wdv3FCn6xoW4oXRi7+qx1qHqlccEzs7WW98aSn7y55KIMoP+t13MxdGmZWVLH1pKbMyTp06bAnH6NbQifnx4+p0l0ZNd+7JxC0vxA+qgYAYj1ADvrbRPtUjVsY0ca1X6taW73nrvsY0WcahbfSRaxgpcQKM9jlK02Wd19a8Z8UbXMfHgTYRJnUPOuUfQhepM5m0j8bpYz2VtudOPBKJ4q+g6YYupuVM+i4nXMHTkrZdpLoH7fOHCF2oque3qQTVHbRiFcS2v/MAutIUfwVNLX/XkOcuy0nKBmentH1QXVn+NoWq7cxh1Ro3rpVC5xbq2xoK/TsX9H2fBij+ChYL9VanTS3/GIQ3hgYoCdpa5nUP2tcPUVeo2l7HtLpl24oRsjDaim3IHdHKeYm40gURfwBfAvDnAF7Ij5Ol984AeA3ARQCfKqXfC+Dl/L2vANlEs7oj5JLOtpMR68Q/FpdLxEZIPPhoqW0WZGv7Q9QVqrb3UbeuvRCZr9+2cpQJZQ25iG0Xln8MVp+BkOL/RUX6PQBeBHArgLsAvA5gkr/3HID7AAgAvwfgfptrhd7MxcUFpBPzyMsAKRO5tfYz6gpVW4vDptC6rINSJpQ15FLRuvD5x2L1aeha/M8AOFP6/5lc8E8A+ONS+q8AeNLmWqHF36V8u6xiG6OehCDJ3kYKmVYVqpWVg8FWnVVua3HYFtrys9KJXbVLHMoaarL+UMhon8itvpDifwnASwDOA7g9T38cwAOlz30NwGcBbAL4Tin9FwB8y3D+0wB2Aeyur68Hf0hbW/WuzjoxT0FPfDPmRq8TyoVKNX/AtZCazm9TaFVzAZaX1Q2GbcFwyUNsYht5BWgs/gC+A+AVxfEZAO8HMEE2U/gsgPP5d35DIf6/BODvK8T/d+vyIGU3e/jauH6Wlg7KWSS/be/EVhcHgU4MdQ/bRwy+j/w1+ZyreMYothFbfcGjfQBsAHglf52c20dK9wCHvstbLETu8kwPk7i1edixClQT6yHWe4mQUG6fE6XX/wLAN/LXH6kM+H6/NOD7RwA+XhrwPWlzrS7E3zXqp1xGx1z2aPl7xvRAmz7sLqzlpoIc2noYeUMRSvyfzsM2XwLwzUpjsJ1H+VwsR/Tkfv9X8vceR8+hnmWaiv/YewEx9sKTxiSGpklZJmEL3UK3KQQhXVksnOHdPqGPUJO8XGe11/UAxsrIjSu/2EzsKj5ju59vaOu6TeNiE2rXVLDZLaX4V/E1qz1EPSIDokmrqCucW1uHP+cibKFF0MeM6bpJZU3yWteLGoHFQvGv4HNW+wiNCWJDG5eDKu7YZXJRVdhCL5vss3Hx2UvR5Ws6HY07iOJfwSTyLrPZ68rNSIwLoqKNILZZVkAnbFtb4Qpjk4bONZy1SUOiy5dugG+AFhzFv4KpfC0WdoJfNBKm8OXqwnGmPTHIwGhjwbZZUKwvYXOxdEyNhe9BWlW+RhSfTPGvUFf2islcpqN6vmr50tVB034AZECEtvylTFfYbAe1Q3WZRzQQTPFXoCpftmv8TCaHz6NqSGwbDjJQbCxYnciFCJ0szh+DH7LvBmpEIaAUf0tsQz6PHTuoP03mB5io9hqm00GWyXFgEts6AWoq1LrzxrRXbgyWdywNYWAo/pa0Dfe0Oeq2gVRtMKNaN4skTkgBVAlbFxO9fPj8iVco/pbUWf5tG4c6ETddf4DuyHHTtesj5PV8RvsQr+jEX2Tvxc/m5qbc3d0Nfp2dHeD0aWB/3+95hQDW14GzZ4H5XP+5paWs5ujOcfOm33yRHtnYAPb2jqbPZsClS2ldr+t7IdYIIZ6XUm5W05f6yEzMzOfAuXNZmRUCmE4zQW7DbJaJ9qVLZuEHsgaiyXskQc6eBVZXD6etrmbpqV3v8mW3dNI7FH8F83km1DdvAmtr7axt17p19iywsnI0fXk5nCaQnqhaGrNZ9r/JQtjZyazspaXs785Os+sBwGSSdXG3t7PztDm3zjKhxRIvKl9QjEdXPv8qbXz8k8nh5Vhc9r9YWzvskq0u60JGQKglGlT++ZWVo7tz+d7rlvQCOODbjLarfS4vH4Rt2i7AuFjY7ZJHBozLyoOukQAuhdrl3BzAjRKd+HPAt4adHeChh4D33gtz/vJ42M5O1gNXjZsB2fjDlSth8kEiQzeAqsI1EsAUVdD23CQ6OODbkPkceOqprA6EoBgPK6KMTPX96tUweSAR4jJQ6upXd/k8ffaDheJvwXwOPP00cMst+s/MZpll7kpRt7a3/YeXkoTRiW7VCmkSraOK+llZyaIK2p6bJAPF35L5HPj617PonwIhgK2trAd96RLw2GNu5yzXLRtDr0njQhJFF5b5hS+4RQepUEUZnT+fdXHbnpskA33+Hih89Zcv27tSJxPgwoWDulXn4l1Zyeon6+KIKBcsmxmChCjQ+fwp/i1pOiO4Oo5mGlieTrNeBes9IcQVDvi2RDf/5ZFHmvnqjx8/fN4HHwRu3FB/dm2Nwk8I8YthCJMUVK37vb3s/z/4A3MEzmxmduVUz6vrhHGGPCHEN7T8LVBF4uzvA08+qf+OEMDJk/r3r12zj/BhtB0hxDcUfwt0lrdp7ouUWbCEjvV1O4ue0XaEkBBQ/C1oannrfPhAJui6804mjLYjhISF4m+BLuT62DHz9yYTdfp0mgm67rwXLtgvAU0IIU2g+FugW3n3ySfVyy8DmYifPq0W92IyWJMVfQkhxAcUf0vKa/wXFvl8nk28Ki+PDhyI+BNP1Iu76ryEjJY2ewoQJzjJixASB6oZk6ur7A63hJO8CCFxo4up3t7uJz8Dh+JPCIkD7gPcKRR/QkgccB/gTqH4E0LiQBf7zFmOQaD4E0LigLHPncKF3Qgh8VDEUJPg0PInhJARQvEnhJARQvEnhJARQvEnhJARQvEnhJARkszaPkKItwEYNkUMxh0ArvRw3bakmm+Aee+DVPMNpJv3rvI9k1LeWU1MRvz7Qgixq1oUKXZSzTfAvPdBqvkG0s173/mm24cQQkYIxZ8QQkYIxb8ewzbsUZNqvgHmvQ9SzTeQbt57zTd9/oQQMkJo+RNCyAih+BNCyAgZtfgLIX5ZCPFdIcRNIcRm5b0zQojXhBAXhRCfKqXfK4R4OX/vK0IIkaffKoT4rTz9D4UQGx3ex5eEEH8uhHghP042vY8+EUJ8Os/na0KIR/vOjwohxKX8ub0ghNjN044LIb4thHg1/3t76fPK599RXs8LId4SQrxSSnPOa9dlRZPv6Mu4EOJDQoj/LoT4Xq4rj+TpcT5zKeVoDwB/B8DfBvA/AGyW0u8B8CKAWwHcBeB1AJP8vecA3AdAAPg9APfn6Q8D+Lf5688B+K0O7+NLAL6oSHe+jx5/i0mev7sBrOT5vqfvMqLI5yUAd1TSvgzg0fz1owB+ve75d5TXTwD4KIBX2uS167KiyXf0ZRzACQAfzV/fBuBP8vxF+cxHbflLKb8npbyoeOszAL4hpfyJlPJPAbwG4GNCiBMA/rqU8n/L7Bf6DwB+sfSdC/nr/wTgkxFY003uoy8+BuA1KeX3pZTXAXwDWf5ToPzbX8DhMnHk+XeVKSnl/wRwrZLslNc+yoom3zpiyvebUsr/k79+B8D3AHwAkT7zUYu/gQ8A+LPS/2/kaR/IX1fTD31HSvlTAP8XwDR4Tg/4Z0KIl/Iuc9GtbHIffaHLa2xIAL8vhHheCHE6T3u/lPJNIBMAAO/L02O8J9e8xlRWkinjudv37wH4Q0T6zAcv/kKI7wghXlEcJqtSZbFLQ7rpO16ouY+vAvhbAP4ugDcB/KuaPAXNa0NizJOKfyCl/CiA+wH8UyHEJwyfTeWegPjLSjJlXAixBuA/A/jnUsr/Z/qoIq2zvA9+G0cp5T9q8LU3AHyo9P8HAfwgT/+gIr38nTeEELcA+Buw77rWYnsfQoh/B+BblTwV2NxHX+jyGhVSyh/kf98SQvxXZG6cHwohTkgp38y77G/lH4/xnlzzGkVZkVL+sHgdcxkXQiwjE/4dKeV/yZOjfOaDt/wb8k0AnxNZBM9dAD4M4Lm8y/aOEOLjuT//VwH8Tuk7p/LXnwXw33J/XXDyAlXwTwAUURJN7qMv/gjAh4UQdwkhVpANmn+z5zwdQghxTAhxW/EawD9G9qzLv/0pHC4TR55/t7k+glNeYykrKZTx/DpfA/A9KeW/Lr0V5zMPOfod+4GsEL0B4CcAfgjgmdJ728hG3y+iNNIOYBNZwXsdwOM4mCX91wD8R2SDNs8BuLvD+3gawMsAXsoL1Imm99Hz73ESWYTE6wC2+86PIn93I4vOeBHAd4s8IhvbeRbAq/nf43XPv6P8/iYyF8l7eTn/fJO8dl1WNPmOvowD+IfI3DMvAXghP07G+sy5vAMhhIwQun0IIWSEUPwJIWSEUPwJIWSEUPwJIWSEUPwJIWSEUPwJIWSEUPwJIWSE/H9eZaOb4ecC7gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 26, + "source": [ + "pca_analysis(2,5)" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAACqCAYAAACTZZUqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAI90lEQVR4nO3dT4hVZR8H8HPfphJxLKzQippFZIOGTFEZFNEuxE3YIjAKFxYoKkqMItGqiCjEhSARWRkVgqCbyAKFslqJC/9ubKOgKP6JcuHoQPfdvBAvz+/Uvc2993e98/ksvzxzzsPM8cvB8zznNJrNZgVA7/0newIA05UCBkiigAGSKGCAJAoYIIkCBkgy1M7gRqNhzRpd1Ww2G70+p+uabqu7rt0BAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyQZyp5ANz3yyCNF9sQTT/Ts/CtXrgzz559/vsi2bt0ajj18+HCR7d+/Pxx74cKF1icHLRgdHS2ytWvXhmNvv/32Ips7d244dunSpS3P4dChQ0W2Z8+ecOy+ffuK7OjRoy2fq9fcAQMkUcAASRQwQBIFDJBEAQMkaTSbzdYHNxqtD+6huiet0WqBBQsWdHs6Xffdd9+FeTtPlvtVs9ls9Pqc/Xpdd8vw8HCYv/fee0X22muvFdmsWbNaPlejEf852+mddkxMTBTZ7t27w7ErVqzoyhwidde1O2CAJAoYIIkCBkiigAGSDMRW5E8++STMB+GBW2TRokXZU+AmMTIyUmQ//vhjOPaBBx5o6ZjffvttmE9OThZZrx/CPfbYY0X28ssvh2N///33IhsfHw/H3rhxY2oTq+EOGCCJAgZIooABkihggCQKGCDJQKyCgOkuehl6VVXV119/XWQPPvhgODZambBr164ie/XVV8Of//PPP/9uij0RbZNevnx5OHbZsmVFNnPmzHCsVRAAA0YBAyRRwABJFDBAkoF4H/Drr78e5h999FGPZ9Ib586dC/NWt5L2M+8D/nfqrvXo30bd9uAvv/yyyNavX19kV65caW9yeB8wQL9RwABJFDBAEgUMkEQBAyQZiFUQs2fPDvOjR48WWTsrBU6ePBnmn376aZH99ttvRbZjx46Wz9UOqyA6q1+v63ZcvHgxzO+6664i+/zzz8OxGzZsKLLopeW0zyoIgD6jgAGSKGCAJAoYIMlAvA/4jz/+CPMXX3yxyOq+oPzVV18V2c6dO8Ox0VbMuuNCpy1ZsqTI7rjjjnBs9JA9ethWVVN/4HbnnXcW2dBQXDHRvC5fvjyl89+M3AEDJFHAAEkUMEASBQyQRAEDJBmIrcjtqHtaPNUnwNH24Llz507pmHX2798f5i+88EJXztdLtiL/pe5Lxz/88EORPfXUUy0f95Zbbml57L333ltkq1atCsdGebQVuqqq6vr160X28ccfh2PHx8eLrFtfKe4WW5EB+owCBkiigAGSKGCAJAOxFbkd7TxsGxkZCfO33367yIaHh//1nP7OTz/9VGQrVqzoyrnoL3XXVDsP3L755psiW7lyZTh206ZNRXbPPfe0PK923HbbbUW2Zs2acOylS5eK7J133pnyHPqBO2CAJAoYIIkCBkiigAGSTLudcO04cuRImD/66KMdP9fExESYR+9YnZyc7Pj5+4WdcH+59dZbwzzaCfnss8+2fNxGI/4Vt9oFhw4dCvNjx461PIeXXnqpyOp2qZ4/f77IHn/88XDshQsXWp5DL9kJB9BnFDBAEgUMkEQBAyRRwABJpt1W5DrRE+dou2QnRCsetmzZEo4d5BUP/L26v/1bb71VZAcOHAjHRtf11atXw7HRl8Hff//9Ijtz5kz48+2IVm1EK36qKn4n8UMPPRSO7ddVEHXcAQMkUcAASRQwQBIFDJDEQ7j/id6FOn/+/K6ca86cOUUWfaQQIj///HORLVy4MBwbfYDz2rVr4dhOPFxrVbTtuW4r9OXLl4vs7NmzHZ9TBnfAAEkUMEASBQyQRAEDJFHAAEmm3SqI0dHRMO/Gl4brvsDczkvwoRW//vpr9hRCdf/e7r///paPcfjw4SI7ffr0v55TP3EHDJBEAQMkUcAASRQwQJKBfgg3NjZWZHv27AnHjoyMdPz8b775ZpjfuHGj4+eCfrRz584wnzVrVsvH2Lt3b6em03fcAQMkUcAASRQwQBIFDJBEAQMkGehVEM8880yRdWO1Q1VV1YkTJ4rs4MGDXTlX9KXbqqqqhx9+uCvnO3/+fJFduXKlK+fi5rVhw4Yie/LJJ8Ox0Xb8HTt2hGM/++yzqU2sj7kDBkiigAGSKGCAJAoYIMlAP4TrpeirtFu3bg3HHjlyZErnqtvGuW7duikdt873339fZMuWLQvHTkxMdGUO9I/nnnsuzLds2VJkjUYjHHv16tUie/fdd8Oxk5OTbczu5uIOGCCJAgZIooABkihggCQKGCDJQK+CiLbQRk9fq6qqhoeHO37+pUuXtpVna+d3MzQ00JfOtDRz5swiW7NmTZGNj4+HPx9tL65bwbBx48YiO3PmzD9NceC4AwZIooABkihggCQKGCBJI/qP89rBjUbrg/vUL7/8EuZPP/10j2fSG9GDyKqKt33Wje3lV2mbzWa8d7WLbrbrevHixUV23333hWOjv90bb7wRjl27dm2RLViwoM3Z/b8PPvggzDdv3jyl495s6q5rd8AASRQwQBIFDJBEAQMkUcAASabdftJXXnklzLdt21ZkdS+ernsheq9cv349zA8cOFBkdU+bjx8/3tE50Tvz5s0rsi+++CIce+3atSK7++67w7Gtrog6depUmEdfNf7www9bOuZ05Q4YIIkCBkiigAGSKGCAJNNuK3I76t7bO3v27CJbvXp1kW3fvr3jc6qq+i8P93LLcLfYivzPxsbGiqxui/2MGTOKrO5LxdHXuqNrKnrYVlVVdfbs2TDHVmSAvqOAAZIoYIAkChggiQIGSGIVBH3FKggGkVUQAH1GAQMkUcAASRQwQBIFDJBEAQMkUcAASRQwQBIFDJBEAQMkUcAASRQwQBIFDJBEAQMkUcAASYbaHH+pqqrT3ZgIVFU1knRe1zXdVHtdt/VCdgA6x39BACRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyT5L95QHHzClR5NAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAD4CAYAAADCb7BPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAArd0lEQVR4nO2dX4wlx3Xev7PDGcZDOoE4pIQN5Z1dJkIQ6iUQN4aIIHrZIPzzojimANJjiYEFLDBy8pBAD7vYF73wQQqCwAodR4xFQdFdWEoCBGIcK4wkw7CBBKaHgSguE625FLkkLUIkLcegTMCydysPty/Z09P1v6qrqu/3Axoz03Nvd1V39fmqzjlVLUopEEIIIb4cK10AQgghbUIBIYQQEgQFhBBCSBAUEEIIIUFQQAghhARxQ+kCxHLrrbeqkydPli4GIYQ0xdNPP/2mUuq2mGM0LyAnT57EwcFB6WIQQkhTiMjV2GPQhUUIISQICgghhJAgKCCEEEKCoIAQQggJggJCCCEkCAoIITouXgROngSOHVv+vHixdIkIqQoKCGmLqYz6xYvA2bPA1auAUsufZ89SRAjpQQEh7TClUb9wAXj77cP73n57uZ8QAoACQlpiSqP+8st++wlZQyggpB2mNOonTvjtJ2QNoYCQdpjSqD/yCLC9fXjf9vZyPyEEAAWEtMSURn1vD3jsMWB3FxBZ/nzsseV+QgiAGSymSNaIlfG+cGHptjpxYikeuYz63h4FgxADFBDSFjTqhFQDXViEEEKCoIAQQqqECwHUD11YhJDqWM0ZXU37Wc0ZBejBrAmOQAgh1cGFANqAAkIIqQ4uBNAGFBBCSHVwIYA2oIC0CiOMZMZwIYA2oIC0CJcaJzOHCwG0AQWkRUpHGDn6IROwtwe89BJw/fryJ8WjPpjG2yIlI4zMrySEdHAE0iIlI4ylRz+EkGqggLRIyQgj8ysJIR0UkBYpGWFkfiUhpIMCUjOmYHWpCKPL6IdBdkLWAgbRa6XWYLXtnRy1lpsQkhxRSpUuQxSnT59WBwcHpYuRnpMnl8Z3yO7ucsRRK62Wm5A1Q0SeVkqdjjkGXVi10mqwutVyE0K8oYDUSqvB6lbLTYgDDO8dJomAiMjjIvK6iFzq7btFRL4pIs93P9/T+995EbkiIpdF5J7e/rtE5Nnuf58XEUlRvibxSdWtqVVzESMyU7iC0AhKqegNwEcAfAjApd6+zwE41/1+DsBnu9/vBPAMgBsBnALwAoCN7n9PAbgbgAD4BoD7bOe+66671GxZLJTa3VVKZPlzsRj/zPa2Uss2vdy2t8c/OxUu5SakMXZ3Dz9mq213t3TJwgBwoGJtf+wB3jkQcHIgIJcBHO9+Pw7gcvf7eQDne597shON4wC+19v/EIAv2M47awFxofVWXaHYlChShZeBDBAZf9RESpcsjBQCkjMG8j6l1GsA0P18b7f/dgCv9D73arfv9u734f4jiMhZETkQkYM33ngjecGbolTQOoXbrEKfQIkiVXgZyAgM7x2lRBB9LK6hDPuP7lTqMaXUaaXU6dtuuy1p4ZqjRKtOZfEqXFerRJEqvAyTUVP4zgbDe0fJKSA/FJHjAND9fL3b/yqAn+l97v0AftDtf//IfmKiRKtOZfEqTPktUaQKL8MktDby4jtKjpJTQJ4A8HD3+8MAvt7b/6CI3CgipwB8AMBTnZvrLRH5cJd99Yned4gOn1adyu00NlEQ8Ld4FfoEShSpwsswCS2OvPiOkgGxQZRlLAa/AeA1AH+B5UjikwB2AHwbwPPdz1t6n7+AZfbVZfQyrQCcBnCp+9+j6GbKm7a1D6K7kiJba+wYMYH7CjPIShSpwsswSupA/9yC0q2BmrKwSm0UEEdis7UWC6U2NvTiEWrxKkw/YhbWUXKIXOsJhK2TQkC4Fta6cOzY8vkcIrIcj5sYLpA4xmLB8fyMybHE2Viz2t5mXGEquBYWcSfG0T7mrO6zu8snfubkCPQzKN0+FJB1ISZby2QlSuQxtpT7ORNyBfoZlG4bCsi6ENPd01mJjY3pu4yt5X5moIR+cg4EGSU2iFJ6YxB9AqZKE3KJJK955LVkxlbtgX7iBxhEZxB9Mi5e1L+FMNXxXSKqMckAM4Dv6yKpYBCdTMfKWf2Vryz//vjH0/pPXGeVreusu451nbVug2GxMlBA1oFUT1fO+IPr7PY1dMb3b98xzRO7Jvo5Ss1hsdkLW6wPrPTGGIiGlcN6NbU3hdM8Zfyh71Df2Rk/ru7Ya+SMt03+r3XWuo2Ut1DXLDc2yr8Wp+YVBsCZ6GskID5PnIvVCTH6qdaecCnf6ri1PG2FMBnHVvUztWHVNcvSBrv2fI8UAsIgegv4TtnVRVr7hASdU0VwXcq3ovH2GcsccwZSJwLYmlOpBIPa7x2D6OuC77KlLhHVEKd5qviDa8R3d9fvuJHU6K+eImdg6nqnTgQYa5Zjx526nmuR7xE7hCm9rYULy9d1pBs7pxjXp3Be28pXwPdQq786d7lK1DuHa8e01ueqmXKV5cOAMZA1ERDfJ26x0ItO6cjiqnzDJ2traxlML+TYr9lfnTNnoES9cxlW03FNfZacza3mfA8KyLoISMgTZwpMu5wvtNW7freyJyv3uykqq+47lHonR67roTuuKdBe28hgKigg6yIgSvk/caFdy9Du4WIxno7byJOZsydesyuj5pFXSly8pnOrsw0KyDoJiC+hVivEoqR+U2H/uBN123Ma+ZqNdKoXVfan9KTwRKa+9S6Z4+v2JkQKCAXETMhTGOLTsHXvQp7MAt32XHpV+6tbfes9FIzNTf2tD7llOWMktljIOkEBoYCkJ6S7bHMwhzyZNXfbPZlRVZzngMbUM/f1qtmlOCUpBITzQFyocYJALkLmepgS20PXqapk1cAUt35Oy3fZXk45hu8ty33r+SbEhMQqUOkt+whkHbsrIT6NsW7pzk74dco1WcCjXilvfW1ZWKHlsQ02WxiBkCWgC2sCASnZmmuzOiamiHrGToD0PN5cDZnuUuzvh7/Pq1QMxKXZpWiaLT2KrlBAphCQkony6zbyGZLyqQ1Qg9qD36HoLoXLos22OaBTZmEtFvoA/uo7qbLM5vgoUkCmEJBc3VDbE6I7785O3HnXlQA1mOsIxMcNNVbXWnrjprcArIy87jM+93Cu7YACMoWA5Oh+uBzTJbOp9S7QlARYgbn2PH3cUDWPtlzrEFuvuY5EKSBTCIhS6btcLsbM5SkXWTquY6ilO5mbQDWY4+UZuxQ6I1nzCCRGQDgCoYBMJyCpcenSLBZuT0LMS5fm2sXWEWv5MlrOqY3y8Hz7+25NoaYmY3NhAcvPMAYyDgWkVQFx7dK4PCGmrlBonKX1rlUOMlqRWgyUi4jV1GQWi2UAX/dYrK4hs7DGoYC0KiCuFmOxcIt4jjljY+IsrTt3c5DRctZklG3kbDIhRnq4rErBNwI0RxMCAuAlAM8C+M6qwABuAfBNAM93P9/T+/x5AFcAXAZwj+34TQqIUu5Py/6+XUR2do4eKybO4mq5QrtlKbtzCY9lPFRGy9mSjrs2mRRzUefgJqqZlgTk1sG+zwE41/1+DsBnu9/vBPAMgBsBnALwAoAN0/GbFRAfTIKwtXU0Gd60WNEwzhL61IZ+N/X07kTHsh6KIxCllNslD7ktLV2DudCygFwGcLz7/TiAy93v5wGc733uSQB3m47fnICkDuTq4iSm93umKE/oE5/SUiQ8lvVQaxADcSVHaK2lUdhcaEVAXgTwvwE8DeBst+//DT7zJ93PRwH8Ym//FwE8MHLMswAOABycOHEi7VXNSQ5LYXJvuUwtTn1e2xOf0lIkPJbpUCuD+QtYqFc2dtV1tJ+F5UJomVK+EaD2EUiN982VVgTkr3c/39u5pz5iEJBfHRGQnzcdv6kRSI6nxHVdihRzRlzOO7MRSIo00KlIHVryrbfJ09q/LWPlbG0UplSbZe7ThIAcOhnwGQCfXlsXVo5xeuyssJTnnWEMJMVSGFOQ2pj5avPY+cfKYipna735VkdNK6oXEAA3Afjp3u//E8C9AP7lIIj+ue73Dw6C6N+fVRA9V4sbPnm6p9hFqHye4jXIwnLV/JhipKhC6qbl29exjTxWdarJ6MZe99bjNi0IyB2dIDwD4DkAF7r9OwC+3aXxfhvALb3vXOiyry4DuM92jqYEJMYv4NPKQ5/S1sfkGXC5lCWS2YakNma+Tcj1/LbPTTUKSXHdU4vh1COw6gVkiq0pAVHKrZX0nckhgfDQpyPkicjV6ivxZ7hcyhhDksoI5TBmPk3I9fymz03Zf/G5XrqmWKk31hkKSIsCYsPmTHa1CiEG2Lcbm6vVx3bpEwuP7ZAxvX9TEp1PmVzXsvKpn683M3YtrSndWz6uSVO9TNfI5/qVcO1RQOYoICZnso9lSnluXSvO1eobc8HlGIGY1sjUVdPljYKuxwrtRbucX/e5KWMKKUZMJnyva4l4CgVEzVBAXNa+ytUtqaXVhx63UIQ2xgjv7/vf5kozoaNJURYfEXPJBjM9hiF10b0PjiOQQtvsBMQ2Asndo65h3K057nVAvbKxq35vP17QUnu6Qo5n81bq9LLSuZjRjF2PVflcrqmvkI/dMxcPMrBc6MGESXx0YU/GQCgg8cQ+RVOS0pdiO25v+zG2x0XEUdBqSTaz9RXmPgLRGXCX/JGx78bUxfR93WbCdCxdeZiFVWCbnYAoVUf32PUYKaO5I+e7rnkKX9nYHf+Ow7lrcduYeqm2nnPL2T8u5w3J1tJdS9toynXU4dNWFovw8kwFBSRUQCpJEZ2kHCEWIkYUEltnnYBcgyEzzHJNndw2E9wb3aXa2LCfbn//3fUyNzbiVqkp8TiYYgSm3rtprqzr+qGuZQkR9z61r2JAAVEBAlKL/2KqcvgadJMLzeUYKZ3qi8VSKEaONzoCccR6SSa6N6GnyV28KQTFJVdEd49sIzff62Iry9ZW2IuqajE1OiggKkBAavFf+KZphJJyTQqXY0zgoL8G0QfSHbA+2D51iLS2IV/P2YRL92tMm8t8kZTXc9XEfUZ3Y4P3GpwdY1BAVICA1JJ24pumEYqvtfHpGo4c4/f2F+rPJJEF0pTlOuB/rAFGQ5NqllmmMqZuwv3z6NxAGxtpjaBv3GEYdkt52W1lcRXm2kccQyggaoYjkNRl8W3VunI5LKmyOtVDWKgXsauuQdRVMaTd2ih1r1zPm7F8ptuW8rQhAeRURnGx0AuWrV458kx053cV5lpMiysUENV4DCS2xfqcy3VcHZGam/wBqjVFaEXG0WxIFtKweC5GNsSVlMoomh6BEo9mbPutxbnhCgVENZ6FVSJNY3/fPpoIvD5ZHqBS9yrG+ia4f7Gr1sZqoG2LNYouIx9TtniIWIaUyUfAOAJpcGtiHoiudS8WSm1uHm5tm5v50mnGxCNhKy/1AK2qmfOVs9oTZxohhVxLl1jG8Pu6PozNtRR7T00jn7H+jG1BQ5dlSVyaRIwQ1eLccIUC0oKA2Fr31tbh/21tpXMwD4+ds0tpqWou+nGXH6OQuyvDCMn3WrrGMvq3eaz/0m+CJhFxHQH4JgKMHdsmpqaExhLtsQbnhgsUkBYExNT6c3bZdV3LXF3KjqkfoNUlfBG7WevlQo7AruvxXGMZ/cthMrxK2Rd6tI0ATPNPfZq+zZ3n64ar1aXkQ4q2RgFRDQiIqfXnjLr5PFEi4U7jwt2t1SXUTTicKoJZ2n3hYkSHt9ml+fVnvA83mxiY3Gg+1yt0BGK6Di2Tqq1RQFQDAlJqBOLzNIWsg1HaYnbUMgIpHUB1NaKhZTZ91ncEMJYIsLOjn+0dGgOpfSmRUFK1NQqIakBAbDGQXEZY9/TcdFOaUUNpi9lRPAbSUTqF0zWryfYdXUKeSQx0/z92LH05fLKwKunjJCdVW6OAtCAgSplbfy43UO4Mr9IWs0fOLCzX21ODnvaNvcO8z0Pfce35j9VPl6+xsXG0CY6VI+baTZHeWxscgdQoIHNqaX1LsnJEp65TDRYzMz492Np6uymas80tthKp3d3lwHbsMzffbC9H6Ko+tV3zqWAMpDYBmVNLzFGXdfIP9PDVyDn1QZQyG3afuIftOvjMCXH5Xsk+zFRtgFlYNQlIjS0xBFPyfmhdbDGcOVnMAbm8dK1cNt/sKt1ma3o2V5nu+773x/UdKqH3x3fEWroNUEBSCUhF/vxgbE9haF1MVqR2CxiJa7/CxxjUPHAb1uPMGT+hMG22a7NY+Dddn36fbk7LUERi7o9Pe6mhDVBAFEcg72BzWIfWxcVXkWr2fGW4ZgelWOzY5/a4CJZvcHls0p/u1vuOQMauzVj5QlyGsVlkGxvp7o9rP7QWc0MBUYyBvIPJ0MfUxXWCQeoXYVWCzRD7GoPYwW4KUXPJrrJtoYswrpqK78z1kPvjWs9U98fUFvrlNF3TKaGAqBlkYaU6r8nVFBtAd7U2qajBQeyIr8GJ7X26fN/2Gdc+gW5bjVhiRMTF0NomF9qaiEs9U45AdMJ95ozbteIIpMDWxDwQHSlHPjlHUS7dp1QCMuFoMHeqq851FFM9F8FKuXaUaT6JzT3kuw0Xeoydf+tSzzNn0t6fMdegSzkYAym0VSEgoZZI9/SFuoOm6LnrZrincmFN5CBOpVO2AdrYMWNuU8wIZJX34BrDWLmVQnv6OleVyxIjpnq6NhEXcRtrVikfI1sZmIWVeANwL4DLAK4AOGf7fHEBibFELjOoanPnLBazmOGeUqdsPfGU2ucaA/FZyX9MZFKlr441X5c6pFiD1MX7mjPuYMos820XOczA7AQEwAaAFwDcAWALwDMA7jR9p7iAxFgiF/+HLtpYUlRyiprn9QwtSg6dmiob3CXDauw9H7YtJplusTg8stjZCQ9+K5VmBNI/zxTiPjyvLas+hUDHMEcBuRvAk72/zwM4b/pOcQGJsRq25Hddy3dd6KhFPJ6WmAcrh6eslvTM0LiEbSDpm/EU0yxTxEBcjxeD7pqY7oHvAti52tUcBeQBAL/e+/vjAB4d+dxZAAcADk6cOBF3FWOJvbsmh7BvmksNLi4fdE+f47AiR8aMy+UzFbuGbPCY7CjdtbPVLYeRM41qQkaeqQfOpmsSur7XGLlGtnMUkI+NCMi/MX2n+AgkRdqG7vshXcmYLK4xZ3UuV1UCa5tiTkWIEbLNsXA5ZgkvoMvmm368Egnf4/XxEeT+Ao4x1yzF9U/lZos5TwxzFJD2XFhKxbfGvlj0V8/1mR4c07LGntTNzaOR2JTd6QRPRQmXUYpz5h6pjB1/a+vwnArfly3Zmp3u/7brEtN/Kp3xburApLzHjIG4Fga4AcD3AZzqBdE/aPpOFQKSAteA+ZiohHT7Vuf07a6mss4JxuUlXEYp3Ak5MsB8vYC+186lmYSE5mI9uCV79LbjpBxlMgvLtUDA/QD+sMvGumD7/GwEZOrUEpccx1hLmaq+lmpMmZBmLbZDgVL5tFN4T12vnSnfY2yzZWHZjmnKIYltjrVc/9LMUkB8tyYFZOzJDWnVY9NcY9OQcnT5dNegwadvrNj/ZHOh3trZffd+Weo0VQ84NbrRwtjmcittfSCXPk7pmMLUHZiUUEBUgwKiM5w+TulhekpfbFzzA0NSdVIb+Eafvv4AcE9G3sVuuYdT+OBD6mNzg/muf2UzyD7zaMc+EzpvpdG+S3IoIKpBAdE9Dbp1H1yc1yHdKNcRSKq0l8rx1bLVbXgRu27XMMH5hp/PGdDXrYzrujCgi5CZHoWxcqZcAMHl+udOQizdd6KAtCggttQNW6uyGX6fALprhlfOtKYKCOmRrm7DNWSK9DqUz2f5c13T0jUn04sth8fyzeiy1cun2ZvOEWOkc45SahkBUUBUgwIS2220df98u58uju2cCwZVQMgtWd0G6wgkgWUwlc+1Jx0y4c21KegGxa6BdBcj7+uuCzXSKfJTbJRIPx+DAqIaFJDY7oepdfsaK9dMrJm+LGpFSCxhdRsewkgMJLHbLzbWYTJYISOQMXR9kVQ9a1+jGzpimWLxxanWTLNBAVENCohS6cfWrt29Ia5xkJSr7VZIrLF5CAv1InbVNcgyGyvxtXItn65ZhUx4C3k7YM6etW+/K6ZTYOtLxcYuTLGfKeMiFJBWBSSWsfSYkJY397W2HIl1d+gWFwy5JWPfcymf6TM2w+6ahWWrQ+6etU95YtySpn5UisUZxu7V1tbRJIHccREKyLoKSJ8Yl1jIXJCZ5jumzIqJESTTCrSm8pkM5v7++P98VoR1wSerKjcxiRG66xiaLKArX4pEhBgoIBSQOL/BWGJ/rvUj1ojQWxJzK029/9xBW1vgWecBzZ3KGpqarROdnCOsEnERCggFJLzl6dJ4z5xxiyTWkMheKaG3JMaImEQii3Hq7v91iLoqu+ohLLz6HLWksg4xNeucQlwiM4sCQgHJ091dLMwCMrbV8PRXQokRSEwMxHbcIwZ15GQ/xrZRRIZiVUsqqw9zmxtCAaGAhLc8W7c0JD4y1dNf+egnRwzE9by6YHjK8ryz7tdgexG7zk2jllRWX3I2vambNQWEArIkpOW5pOb4rtY7xdNfq+9jQMosrBTfS9lEdLPvr0GcB6em+SeV3crZQgGhgITjmhvatzqpRyA5hG8NCdFUl9Rd3W3Wzb5fjUCGL6/SBdB1/RPXzLPaaK28FBAKiB+x80dMIrIKrPuUJYfrbQ3x1VSfyYNj2z/bOXqAP5Nt9QtYeI+adDPeXdcWrYVGBsaHoIBQQNxJ0cIXi6MzqVab76SCEpHmGTDWyzWNFsb6Byb3kU083mkyibrbIXNZa6TFZkkBoYC4k6KFLxZK3Xzz4e/fdFOY8UiZflxhVy+HO0NXdd0kNN27rXwMdv9YOdwyvrkaoQPN3O6lFgfGFBAKiDuxLVw3+ghdJyt5bmk95NI400xvn5X5XUYaU/WifUUxdNZ37j4HRyCNbhSQEcYMrE8L739/Z8e+5HutT3Uhci1L4fMqGVMP3nQrc85z0AXt+9drtSZoyuYxhXFvsTlTQCggR/GJkIa88TBmFDNW1opHEiGYXESx7gwfQ2j6rG0OaewtGcvVCGmSqZrHVO6l1pozBaQ1AZmihcVah5onEDaA6fLFXiafXq7pszl7y2PH1hlw33eOuJzbJ2lg3ZstBaQlARl7sjY37QnzvoR0t1yS/3WbaeZXbsGssMtnuoQ5XUG+n8116UL6HylGBqUEs2UoIC0JiMuTFdOqTXEOU3crxGXV33Rrded+aqc4foCFNQW61wGfPkjKEYhtlBErmBX2VaKhgLQkIK5PVqqAtKthje0y6rqLuf0GOY8fIU62nnDLRijGAzqWUhzy1kMdOeMccx3BUEBUQwLiaqhDWrxt5KFzkqfwN+gMdm5fTk6LESlOY4Z2MiPkYOVDhMy1/KYcDl0WVgpRzdmfmGsMhQKiGhIQV1dRSKv0NaYuZem/oHlnR6kbbjj6GdMcEJM45ZwUkeKpziBOkxghBysfKmShWeC5R1r9fpBu4mQsLU4SdIECohoSEKWOzq9I8YJlpfytk23kMSxHiGPfJlJTpiT5HDNTGlVsboOTIXZoB6GxghqNqCnja4rEAI5AKCBlSdVV8zWmtsWTht+LWXZEd55UzulUXd2YOJIDvkYoSB8d7pNtMmKOl1LlYqoyMQZCAZk/PsbUtxsaM7U69imfyh8SEkfywNcIBV22yBGI6X81GtEpR0WtJ0CMUbWAAPgMgD8C8J1uu7/3v/MArgC4DOCe3v67ADzb/e/zAMR2HgqI8m/dvknzm5vh7rYYyzOl1dJYo2uQZAbD5zYFGcfIGIjtnLUZ0RpHRS3RgoB8emT/nQCeAXAjgFMAXgCw0f3vKQB3AxAA3wBwn+08ay8goUZWZw1M8Y5Q6xEicJliEVo051u9JGnq3nawcYzIwmrNINc4KmqJVgXkPIDzvb+f7ETjOIDv9fY/BOALtvOsvYCkfupLR0tdMsRiy+KYZ/tjbKuHsEhmSH10tIRxnPqcKUY0tY2KWqIFAXkJwHcBPA7gPd3+RwH8Yu9zXwTwAIDTAL7V2//3Afym7TxrLyCpDb6p579ambfEWl7DcoTiMNPvGkS9iN1D4jG8pCm8hrZXv5YwjlOdk6OH8hQXEADfAnBpZPsogPcB2ABwDMAjAB7vvvOrIwLy8wD+7oiA/FfNec8COABwcOLEiWwXuAlSj0Bc56vonvhYC+QyY39rK+uMM5ccA1/j56KLIS+IbLX33Zq7bI4UFxDnkwAnAVzqfqcLKyUl5kPonvgUZQk5rw8OIzZbNUKMX+qVbFrvwZf2lJLKBQTA8d7v/xzAV7vfPzgIon+/F0T/AwAf7gXR77edZ+0FRKl8XVEXq9d/4lN0K11HQKlddIMymi6p6bL4nja0Wq334Fsv/xyoXUC+0qXkfhfAEwNBudBlX13uZ1p1cZBL3f8eRYtpvC37FYa4WL3+E5+qW9m/hjleGhHZddddltVkPNfTxlSr9R586yOoOVC1gEy1VSUgc3sqfGdn5+hW5nTRBYq8ac6Eqar90+7sLKfXhFZrDj34OfW1WoQCUpuAzOGpHrK/P24tVy+v7pNLQCu0NLZRhOs8yZipNXPqq5DpoYDUJiCt+xXG8BXFCo19DmzevSPGPMN1qfFS11gmMg4FpDYBmeMIJGYhxZYtiaX8LjGNd277mgwX1qSas4ECUpuAzPEJChFF23WoXVzGyj+yTrgt2/kdjZ1jx2KENanmbKCA1CYgSrVhHGOnUIfOmludz/XVdqWuo6d/ymo45+jaHGFNqjkbKCA1CkjN6Iy37n2j/e/5GHNTl9ylm1p6JOcy/6VXXmtx16RrvibVnA0UEAqIH6YJDKmMtS3H1aWbWtoSucx/GXSrjRobIYg27a5pwFta94kfFBAKiB+u62nEGGvbLDvdy6n6CySW9oV4Rcg9julp6V1CSbUZ7JoEjZihgFBA/HBdT6Nv8H27v6ZjKuUmIKVHIP266UZoNrdfAmyXYarLRFGYJxQQCsg4uifedT2NlUH37f4Op1aPWTWX0UXurnVIIkH/8/v7k3T9bZdqioFajaMckgYKCAXkKD4ptLp1plZrbYR0f02jGdP3ht3m/f13y7exsfx7iuvjwkRd/xpGIDUMBkkeKCAUkKP4PPGmmEho91e3rRgz4MM3K+Xq4S8WaRZnTN3114yIaoiBlA5HkXxQQCggR/F54k1iE9r9dTHOtlUFQ1YqtGFz3/lYxJhuuac7rHQWFkcg84UCQgE5is8Tb+rChnR/NzeXowmfLrGvKyz1dQmxiKFdf9MM90otNGMg84UCQgE5iu8Tb+rChnR/fbvEU6QW287jahGHIyff98NPJZaJYRbWPKGAUEDGmfKJjz3XFJMbTedZBehtdUjRFc8oljTyxBcKCAVkWnKks4YurxJSdt15XOqQIhiQSSzpZiIhUEAoINOR03/fn7S3ypIazl9JISZjx3EVhhTpSJnEkoFuEgIFhAJymNR+DJc5I6n892NvPvQZIYTiKgyprHQGXxNTbUkIFBAKyLuk9mP4zFpPYVR1VjDFvA0TrsJQsZ+IIxASQgoBOQYyDy5cAN5++/C+t99e7k91vDFEDv+9vQ088oj/uZQa/9+1a+P7X37Z7xw6HnlkWeY+Y3XY2wMeewzY3V3WeXd3+ffeXppyROBaBUKSE6tApbe1GIG4uD1S+zFcMoZSBbtN58o9AlFqFilM1irMoI4kLaALaw0ExNV1ktqPYZuzIJJufSpTdtJECxe+wxwNbcXuN1IOCsg6CEgpH71LDGRjI40R0mV4rQQqxqj7fHeuhpZBEjICBWQdBMTHNZUrC8vmxkolIql7/r6CEGJoY2enTwHTtMgIFJB1EJCpe49jhly3tHsNPVmT8PheO1MsxnV2eo2jF45AyAgUkHUQkCndKrpz3XSTWUBK9WRt18a3520abW1tHU0YsAlrLUa6YtfcHENOrUABWQcBUWq6p8xnsb/SRnKx0GdorWIzvj3vmLkvtYnrkAotdcW6thZQQNZFQKbC9yVRqZ54X+PmYuh1dbGVd7FIKyA1jEAqhZ61sqQQkKiJhCLyMRF5TkSui8jpwf/Oi8gVEbksIvf09t8lIs92//u8yHImmojcKCJf6/b/voicjCkbCeDEifH9OztHZ6qt9sdOprt4ETh7Frh6dWk/rl5d/n3xov47LpMclTq6TwR4+GFzeVNODORsPiO6uaCp5oiS/MTORL8E4B8D+N3+ThG5E8CDAD4I4F4A/1ZENrp//xqAswA+0G33dvs/CeBPlFJ/E8C/BvDZyLIRX3RTmn/lV47Owl4sgDffjDe4ITPoQy2MUsBv/Zb9czs77sfc2Xn3uuzsLLfKZqrXiq6/ottPKiR2CLMcCeF3AJzu/X0ewPne308CuBvAcQDf6+1/CMAX+p/pfr8BwJsAxHZuurASM7WvPCTFNDRW4xqTWCzcX7PrOpmywhhEaRgDKQtKu7AM3A7gld7fr3b7bu9+H+4/9B2l1F8C+FMAHl1BkoS9PeCll4Dr15c/c/egQ7qhYyOl2PP12dsDvvSlwyOuW24Z/6zLiCbETbcGVLy8GHHEKiAi8i0RuTSyfdT0tZF9yrDf9J2xMp0VkQMROXjjjTfMFSB1c//9fvuBccuzv7/8CRxd4HHF5ubyuCdPAseOLX/qjPhQSH/0o/HPDd1pFy8ePv6nPrWMu6Rc6HJGTN1fIYmJHcIourBIDDlScXSTHzc2lvM5QnwmLuX0TQOuJcWXrCWo2IX1BIAHu8yqU1gGy59SSr0G4C0R+XCXffUJAF/vfefh7vcHAPx2V0nSAsOet6t7Jkcqzt4ecPPNR/dfuwb85CeH97mOBFzWTHddAn8Fo8WkcWLTeH9ORF7FcnTx30TkSQBQSj0H4D8C+D8A/juAX1ZKrV7ssA/g1wFcAfACgG90+78IYEdErgD4FwDOxZSNTEiMjz9XKo6PAF296iZ8P/VT7/4+lsLsc06m+JI5EDuEKb3RhVUBMW6oXKk4oZlaY+eOXVJ/zJXGVCNSGFTswiLrRIwbKlcqzpjLaXMT2Noyf2/MpeU6V8UlO2x7G/jylxktJrOAAkLiiXVD5UjFGROmL30JePzxd/fpGAqfq0CassOG4hgaMyKkIkQ1Hqc+ffq0Ojg4KF2M9WYVA+n30re360/qP3lyGf8Ysru7FDLfz5m4eHE5Ynn55eWckrfeOhzQb+F6kVkhIk8rpU7bP6mHIxAST6szwlwyq3w+p2OYZPDHfxyeDUZIRVBASBpKzAiLdQO5Cl+sQLqm93IVQdIYdGGRNmnJbXbs2PjqwENsLrG+G+zEieUIqLa6kmagC4usLyGr+JbCJZlgzCXWH2HdeivwS7/E9bRIVVBASJu09DIJXUqxael3xk1IA1BASJu09DIJXUrxm2/qY0aMm5AGoICQNonNjJoa3yQDV2GoUTDJ2kABIW3SaupwH1MWWWjchJAJoYCQdmn5ZRK2BShD4iaETAwFhJAS2LLIQuImhEwM54EQUgLd3BCRpUAQkhnOAyGkVVrKIiNEAwWEkBK0lkVGyAgUEEJKMIcsMrL23FC6AISsLXt7FAzSNByBEEIICYICQgghJAgKCCGEkCAoIIQQQoKggBBCCAmi+ZnoIvIGgKuly5GBWwG8WboQGZlz/eZcN4D1a5l+3XaVUrfFHKx5AZkrInIQu8xAzcy5fnOuG8D6tUzqutGFRQghJAgKCCGEkCAoIPXyWOkCZGbO9Ztz3QDWr2WS1o0xEEIIIUFwBEIIISQICgghhJAgKCAFEJGPichzInJdRE4P/ndeRK6IyGURuae3/y4Rebb73+dFRLr9N4rI17r9vy8iJyeujhER+YyI/JGIfKfb7u/9z6uuLSAi93b1uSIi50qXJwQReam7/t8RkYNu3y0i8k0Reb77+Z7e50fvYy2IyOMi8rqIXOrt865Pre1SU79pnjulFLeJNwB/G8DfAvA7AE739t8J4BkANwI4BeAFABvd/54CcDcAAfANAPd1+z8F4N91vz8I4Gul6zeo62cAfHpkv3dda98AbHT1uAPAVle/O0uXK6AeLwG4dbDvcwDOdb+fA/BZ232sZQPwEQAfAnAppj61tktN/SZ57jgCKYBS6v8qpS6P/OujAL6qlPpzpdSLAK4A+FkROQ7gryql/pda3un/AOAf9b7z5e73/wzgTC09Iwshda2dnwVwRSn1faXUTwB8Fct6zoF+O/syDre/I/dx+uLpUUr9LoAfDXZ71afmdqmpn46k9aOA1MXtAF7p/f1qt+/27vfh/kPfUUr9JYA/BbCTvaR+/FMR+W431F65CkLqWju6OrWGAvA/RORpETnb7XufUuo1AOh+vrfb32qdfevTYrvM/txRQDIhIt8SkUsjm6lHOjZyUIb9pu9MhqWuvwbgbwD4OwBeA/CvVl8bOZStrrXTctn7/D2l1IcA3Afgl0XkI4bPzqXOK+bSLid57vhK20wopf5BwNdeBfAzvb/fD+AH3f73j+zvf+dVEbkBwF+D+3A2Ca51FZF/D+A3uz9D6lo7ujo1hVLqB93P10Xkv2DpkvqhiBxXSr3WuTte7z7eap1969NUu1RK/XD1e87njiOQungCwINdZtUpAB8A8FQ3xH5LRD7cxTc+AeDrve883P3+AIDf7nyYVdA9nCt+DsAqUySkrrXzBwA+ICKnRGQLy6SGJwqXyQsRuUlEfnr1O4B/iOU967ezh3G4/R25j9OWOgiv+rTWLid77kpnEKzj1t3QVwH8OYAfAniy978LWGZGXEYvCwLA6a4RvADgUby7isBfAfCfsAyGPQXgjtL1G9T1KwCeBfDdrvEeD61rCxuA+wH8YVf2C6XLE1D+O7DM0nkGwHOrOmAZV/s2gOe7n7fY7mMtG4DfwNKN8xfdc/fJkPrU2i419ZvkueNSJoQQQoKgC4sQQkgQFBBCCCFBUEAIIYQEQQEhhBASBAWEEEJIEBQQQgghQVBACCGEBPH/Ac/WVNwz6pe6AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "scrolled": false, + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "As you can see, 0 and 1 can be clearly separated by a straight line. This indicates that in the original 784-dimensional space dots corresponding to digits are also linearly separable. In the case of 2 and 5, we cannot find the good projection that will separate the digits clearly, and thus there are some cases of wrong classification.\r\n", + "\r\n", + "> Later on this course we will learn how to create non-linear classifiers using Neural Networks, and how to deal with a problem of digits not being properly aligned. Very soon we will reach above 99% accuracy in MNIST digit classification, while classifying them into 10 different classes.\r\n", + "\r\n", + "## Takeaway\r\n", + "\r\n", + " * We have leart about the simplest neural network architecture - one-layer perceptron.\r\n", + " * We have implemented the perceptron \"by hand\", using simple training procedure based on gradient descent\r\n", + " * Despite simplicity, one-layered perceptron can solve rather complex problems of handwritten digit recognition\r\n", + " * One-layered perceptron is a liner classifier, and thus it provides the same classification power as logistic regression.\r\n", + " * In the sample space, perceptron can separate two classes of input data using hyperplane." + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Credits\r\n", + "\r\n", + "This notebook is a part of [AI for Beginners Curricula](http://github.com/microsoft/ai-for-beginners), and has been prepared by [Dmitry Soshnikov](http://soshnikov.com). It is inspired by Neural Network Workshop at Microsoft Research Cambridge. Some code and illustrative materials are taken from presentations by [Katja Hoffmann](https://www.microsoft.com/en-us/research/people/kahofman/), [Matthew Johnson](https://www.microsoft.com/en-us/research/people/matjoh/) and [Ryoto Tomioka](https://www.microsoft.com/en-us/research/people/ryoto/), and from [NeuroWorkshop](http://github.com/shwars/NeuroWorkshop) repository." + ], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [], + "metadata": {} + } + ], + "metadata": { + "celltoolbar": "Slideshow", + "kernelspec": { + "name": "python3", + "display_name": "Python 3.8.0 64-bit (conda)" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + }, + "livereveal": { + "start_slideshow_at": "selected" + }, + "interpreter": { + "hash": "16aeaa504b544176258e5caf576fc030dfd6fff62d0c15825e7863ff13e121ff" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/3-NeuralNetworks/03-Perceptron/README.md b/3-NeuralNetworks/03-Perceptron/README.md new file mode 100644 index 00000000..6ba59682 --- /dev/null +++ b/3-NeuralNetworks/03-Perceptron/README.md @@ -0,0 +1,65 @@ +# Introduction to Neural Networks. Perceptron. + +One of the first attempts to implement something similar to a modern neural network was done by Frank Rosenblatt from Cornell Aeronautical Laboratory in 1957. It was hardware implementation called "Mark-1", designed to recognize primitive geometric figures, such as triangles, squares and circles. + +Frank Rosenblatt | The Mark 1 Perceptron +-----|----- + +An input image was represented by 20x20 photocell array, so the neural network had 400 inputs and one binary output. Simple network contained one neuron, also called **threshold logic unit**. Neural network weights were potentiometers that required manual adjustment during the training phase. + +> New York Times wrote about perceptron at that time: +> *the embryo of an electronic computer that [the Navy] expects will be able to walk, talk, see, write, reproduce itself and be conscious of its existence.* + +## Perceptron Model + +Suppose we have N features in our model, in which case the input vector would be a vector of size N. Perceptron is a **binary classification** model, i.e. it can distinguish between two classes of input data. We will assume that for each input vector x the output of our perceptron would be either +1 or -1, depending on the class. The output will be computed using the formula + +y(x) = f(wTx) + +where f is a step activation function + +f(x) = \begin{cases} +1 & x \geq 0 \\ -1 & x < 0 \end{cases} \\ + +## Training the Perceptron + +To train a perceptron we need to find weights vector w that classifies most of the values correctly, i.e. results in the smallest **error**. This error is defined by **perceptron criterion** on the following manner: + +E(w) = -∑wTxiti + +where +* the sum is taken on those training data points i that result in the wrong classification +* xi is the input data, and ti is either -1 or +1 for negative and positive examples accordingly. + +This criteria is considered as a function of weights w, and we need to minimize it. Often, a method called **gradient descent** is used, in which we start with some initial weights w(0), and then at each step update the weights according to the formula + +w(t+1) = w(t) - η∇E(w) + +Here η is so-called **learning rate**, and ∇E(w) denotes the **gradient** of E. After we calculate the gradient, we end up with + +w(t+1) = w(t) + ∑ηxiti + +The algorithm in Python looks like this: + +```python +def train(positive_examples, negative_examples, num_iterations = 100, eta = 1): + + weights = [0,0,0] # Initialize weights (almost randomly :) + + for i in range(num_iterations): + pos = random.choice(positive_examples) + neg = random.choice(negative_examples) + + z = np.dot(pos, weights) # compute perceptron output + if z < 0: # positive example classified as negative + weights = weights + eta*weights.shape + + z = np.dot(neg, weights) + if z >= 0: # negative example classified as positive + weights = weights - eta*weights.shape + + return weights +``` + +## Proceed in Notebook + +To see how we can use perceptron to solve some toy as well as real-life problems, and to continue learning - go to [Perceptron](Perceptron.ipynb) notebook. diff --git a/3-NeuralNetworks/03-Perceptron/images/Mark_I_perceptron_wikipedia.jpg b/3-NeuralNetworks/03-Perceptron/images/Mark_I_perceptron_wikipedia.jpg new file mode 100644 index 0000000000000000000000000000000000000000..48600994eee4f30c038992841c22dca63e2d2a7e GIT binary patch literal 34143 zcmbSyby!r<|K=r!k`kmFq>*kI5G4dbP#Otokd|)gnxVU-r6i>rW@r$Q?jb}ZhR$L6 z{(k%HKF|KOyXVFqb3gZ<^UgW{I-zcgm0%&LefcCfn4@-aofQ5mHiHU*r zc*4TM!UhrGfF2tW9zHGs2@xqN2@whL(`U34PsymsiAgA#D5+`b7#J9yQZTbJ)3egj zGtmF%A!v`cg0Mk^I5>p#WF%zt|Ih8A8z8|!3qljeKw}2bNzgDz&>s2$hR1PYq5W3_ z{%1o&$H06XB@Qkg{-Z!6F@TPSfr0)w{zm|h(*BRv0ZbAsQYHa;?5CP$AZ8ab!Qg~% zI4mz~d&sqZ9kU93a{Y>n_l$y)ikj^?I|nD1u!yLbxP+v_tJjK3$|`T(f6&&^)zddH zx3ILbwz0Ky`{M55>E-Pc5*ijB5g8Son3SB7nwFlCSx{J1TvA&0y}Yizq48%^b4zP) zU;h9MJ~%WyH9hltc5Z%Qabpv)wY{_ZXYViaI@|0OH7>Dd-!nfKUToxg%WAaa~zwn;1 z3U9C>{{!v6kp15Q`}+Td?0*6K-?$b50t~dr$-^K4z`#v(jgz1ChAQ{a=fgXNQhArB zWQk%q3Y>KEH~la4SfLqSpDicmIOaOZvY))T6=+a)zuqO8r5L_066`6 z0OA;q)2?*ECxU<0uILyZ0CE#XUh!y&ukbneKqm#ApGqOL>R;fUHY3tHYzWbxDe(X_ z%|8HF<=1E}>{FSFARoSGvVHXI#|u>Sj8l;hz>mQP0G5U#H6gs1Yp8$h#O1LQKhKbc zXlLx0;0s(NdqmeCsmH%y-6#<8Mp|j(M0`$o;tu~mh+k$JQS%A?6PY^`y=@)O4oKrf z2_viiY`K)r=iPceN2Fr)qS6Mt5 zo6lOQZD#IUjlT9l(`RZVej7oOvEX*jscQ`Tn~&WNaCj-0k3nU#!kFTECf+zx9;kNG zt?irt0Z01I1i=F1zyfjEK9SrSS9W0;2gogpp{I9$s;t{AWDa6n z3ZpDqdY=$Hl?AauM}GqNMyCg>$qC32u-xq=q?OKAa*+~FssPzUW|{w_ZJlTXh>we6 zsCg0%+N8MFYkTPfC4tQ#&{Qs%E1uopUHdx>02CYT0hBsO``d2Vpz~vKp|vd*4{(qH zzk=Lyx$MP0bOb*DZOnPIS4q zR`_i8tj;TY7uAw?T!i(_+9@_&2U@pOyF)HK@cV51H1Bq!t2gb|P<1@uH}uwOZ9|T} zEYnXn0BzbK$2sov2>kP0+8+Z~1FsSp<{LJv)45^x<_VS?l>6V>znKou;4c!6jykNl zq@O;O?qJZbZZ-!}ye+k(&}mu}+3YOF3{KEgvFaIw)0L4NQMf43>Yq#|O-GMIW~o@I zq6qzUYRr+Spo`fDU`aC6z+o>+?dNm_3ybGEB%K~CI2hn{!`TIu=i1T82_4(9?8=4}9*&-7PFBK^L^G5|J3P$EzhW%>j z>zO`)-q)EN7dhaoIJP=`djNiNqsV5EN?}pgkIF9Z)*pIjXNe>VU!(DpFKU`P0V7K= zO^%^9rPV3c4?uwK9Vd8$a#GK4Q3iT{ZE{?}UHyk3;G>foSheBtjmr@6r|UedG7U(> zMG2!Rphz@`@~%S`phe8_R|doMU;Nc z2O5O6V{BA+EnGc<`GFEyIC#Gt@Fk8?Hd&954n==u^8k#*5EV_JMDXtZ70zS+%qB_FLN-ZLi46l?CfMl zh(3q^HES;3aj0_Td>T`z1T83Z_NxO{yTB>rF6|$~7zTe*b`XsLM$5SDG_O0%T@B7!k z--?clO`biuC|Qq_AbYa$a{I$B==-%I=RBciC&M2+3dzPP?2qd;cmn&wx70Bx;*X&0 zziH8!0cI=o{OuP zn{4B_d@^wK4FYv-a8{u=*c2=5K!t}r%b0L5pi-9;bV_uko!ZQD`zAo?3$50R-U|Qj zI2}juUi#y@J%18G$DxOO)9ud1ALOg~rf{3#4Ofuup2-tGBR5Lqx z28ld76Gl4(55V{89ay`zHj9@5z`D7@5^_+G`(ateMG91e}OH~>PfgU^J~LV3(c-x4<@d)uaxT9C8k z%@E!BE@pp8e{)EvO=B1F&Le!CzH@0@CZftw{)_Aahx1=5Me=ZM?GGqAklmEMy=siw z!mEnZ(S-bK)!5t!BurrMNRHOd)@iE5|<4_iyLzOI&j|V$-az)zUx3>G;+8g_b&WvBrv0k;AVnPc$c=QSzq3C-1%U39UGI zr~a`NrVyf0?S}83y}$mwq`xSG=g+g$ZJz6Hm6Ig+T`L67mnePfQ{y48wi6=()m98P z3o%xWUP8h@e}CVz3KCn+pI?3^Yp|?Krhqw@iKs9<6^3(HP6t8c{O(PxQk=12TPJQw z_fj8_c9+Cgb_nHLOw#uHsLqA2R(uXIms$Mo)QKYFpKgW+E09d z{eESth~L}sWAcIS>)o#a!wZc;JOJ_RRO?i<`kNQF7qj^^6iLq6HAm+7*5KX?6LlXh z4w~D)*41`1bHF5a5J*#E3WB&7LLhFLyV`fp6UGSJ`5A=vilwvB)oon+G5}`jUuz&j35E(17(uC%5{S zvNRmOJrQl0B0C8>Iww@6vXTAqK2yr;_cNU+OqJ$!CX8$3G%}w^@&O2q8Ay1^D$DtM z=3r=G37MO7KXZ?JR>p?>5y!%$6y2D8YdnM!S#FB@>@YUaYAZ@wqeAA`Abj#P2ttirdufC!-Dx;zU>pVh zdEOy(0r}WTe=H8a>(f95lLM|5NQD6qU@zG%v41fDzGd@#07%Zkq*w1%8#^Ng47R6+ zmkhrkf0%k*bN3Y7_RWbLJ^&3|oX4H1?+&+W;4WeLgiHZkUPb2Au*XOx*!D4UspvYc zuHDSB?-7$_qUW3~;Ez%cm3C+|omf4&z31*uM#Sp2-tj%Et1!;;@C5mRsUUUmcKT+J z_~q4)&6H1ty0hZq4*;|bikhkd6ZpMEI@SlWmCc>oc=TJW?7X_ip_8uNH`eF3DRnm; zmX}fFuDa4{yHn+WV=tV5N_2uGpV)6&#|V#HHafUzr#T_LYY9;Fnq@qlJYKE7&8F1j zC7tHwTkY&rkGFzrL5-#Tvm?sS42!?)Su&ar1Z%S+fA zKgcv4pv=;)qOk!Vv?F!)ObG9ccQ>6P!*%0cBCfzoGPiTfuaJ+w>X@#Pwkyh|F)WI) z93fB!qJem+AjZCJX-G&T;I?!W=uw);?EN7LjVwEFZcoj^(vNr}Bzx;L4_PEg>@tIn zmoQiypaBqqX9 z)uu~IH%BSG}?64eRXCOrY@rvTeG$HE1B!JX$O6o%zop?KYpdyc2#fzik5 zcN%`uE9p@*qj+e>?=DL32$5K65l4+NOG9kWRUM)S+tO=YCZ6XZLm94A+y#PM9J8&H zF1aTDE!aAFZ&nOen?>88CWe)RBUUlpZ{7*+M!WNhnIJ_DF0$GU@jt}kVo!q(BP@iH=|^nk3zDPO@6*|9#*VvjWv-m1!W`|ofEf_I9%pUh}&$mk;|>6@!HG{eU8 zK0Bh_Z$WECgA`}f5bWO>jFq;4`3(W8!fV%KDWbGPf!IVr`G3k6B}-ikR7I4i4yT_kYio=vAzEj?O zk$#8OOx%2m3p@ChGGh^hyB2He*ZDP`OT)3uKrITRA)-WaBQ`hG^)YB29`j^$*cxiG zch)0yCd9cd1!zgmDqIal2+(L%8%*<(m6b>6*Mi2y^Ras!W{M3zA|T+$ZSuJ4q);CR`7H|0Ob9^U@rpI3!1#}k>@ zocriN(!MT7kf#Ugm6g9F%gmT)IzOrGV!A4Ut87(S-FoG75~gDhuPx|APhrX|$KZJE z&x@a3S=GMXHDW&Lpmdy2h;?V#Q}2LqsOZy>&FwSH%I9L94p2T zyAgEdMZaJKFa-f?1}VWHp^$Eg^<(clEISR>PxHLLGef!b((_!8h1zU)Ss;0Fnj=L$ zsjrf=5sU!N!8BCeMUJ|YwGQ+q`eu8io@rh!sCMw56k!hZq?)t(qh;FtGl}eQsVo*M z%?acA>N!Kbe&w{qq_>VWc|2I~QEc(uQjDZn#IkjI^JkVULon~h1s(HtckOYBvx_6S zr+*fW>N^IOn^_C*=N9Ghz!Ng|?-2xs(n8qS;HjBFMAh52FfG+^Z&PczZk*r~m?r3@ z%)MLY+Eqjmovu>TPa(F}sQx$>LUJ9kPk~Hw^l#VCEawg=Y*&(k1oB$_?PmgQ(<3ME zo+;TsGr9BNI$a}p^%N$GEUXN+kAYVd)D9ry(0L6n@fRycQ@B{ zr#p2hLMNuu@UuJB((>(5qpQc|5A5Iom-8P5D3R+-tKj05J|4(cina_IlaE8s(Z1XS z5z;zD;Ed5RvNW%zhZW0Q8t#&AUn!Z zuwaDodQbu(PV1}t@wik~^VnWz=2e{&*ml&9{Mvl|Y{G3dNP1~N7KG$^c1b`Z~b?h%4Vvm;Ioxr+0X>&_D8Ud#5Dck=VtucrcSgA*Mn`< z{ZqerqCBbr?eacyAQ@vy1oh$lL7D|g*&i~Zspwh(!(u-$Qsq}Ci&tcNA%O_@xAA{@ ztD;16Q=RLMDMUTk>jikNd8t7(T&|&u`1Od`E|q!oW4OYu75*r2$3ClQ?9Es zz6^H6f!q1;DYIosI0`JrlmB+b!BQH@4V*CWjx?*?MmF3ZX`*rWTpJJE8jKSXn=ZxSl&_dzGv*-A{T0N|K2Yp;^{EMEx z#5Pnv5Gz)5`pB@0LUUoaSMlF;6fsTF;Gh{d{eL^JaQPL9>&ZXn11-1fxe<(KXL|!G zKWsfDt-8J&?9KM#n9GrC>Ic&)Q~mYtu#<3LHS|KHki`CCXzi{`8VCKZh57&=vX!S) zo(&ZyD4xG<3vtGl-*RVlr}de3ED`EI0wr5$^XPs{DkA`OVL-0NUKg@2EczA=o7){O zQ{e4pTbLh0w{;3rXV-RG;$B4@4&%c3B|Sh*^m%W9)LER>-%Vx9{Gw8H-@3cS*}IHf zhBLpo8jac2!rhpOQfgRk`<}!moMW(5gFerOef4hQS1$5a-!>FyM5L^r_MDxSeTqIE zKIvWRm(?4N5>1r2Zk7NA!)Jq-?771;2%2JliEpXJG&*}rC0m0hAVt%iQ;FemW~QPk zv2a&HL_+sF#p${12jIObx-qv0YC;iE|y zaymWP3F6A4QEu7YJZ++pr;@25Qhxw)UOSbl+DdXeMN`ncD-#DLgsI| zKTb;c7m#OA$Xt`0!Rk%GavA(y*4oJ8meyH@18uPaD;@t!TuCfHQEZxdY>JjL2E+}; zT)9U$9Sh~$%KBqGFr3%+bZe=W#Ga#9Uj8M-(y zTE0>ISvxwhX@At(|9#~&E+|*bJR$f>XE(wub?t2;=V4Ks+YW=E+99|#P|Kf#1mp%Y)A(;%k`}t7AtuSFQ%el-WL&q_lxon zI$d_#u&j+(l71(ce_A8pCDL^>a5FcRM{&5g3RlSb^f~-cLX23QU$+{Z;5{xeLw#@bIDhM-gCDz}*Ph^es@68r7ui?9v89ecmt<0B8a+szV(-QaG zODIBer(mBtN1BMmW1$B?Z9o`{&gBRb7!`q+>y}Hd; zG44Fo_$uZsvVdmJ&?F%z+HB9p{COA2Rg9|l3$t8SK^UW@4oskJhXvw)GnalN{+D3iH&!vJ zh^Fhy?{7%wX;0_P8^M@mADgdL28*!#3p=N$ySlD7v%jApwr^KEVbwV5fzT$6Dhabs z*rb|6yMMqF2swJV`^0~|{L~>ShC}3@egY%C#80iBXxQ>>$#0;b(G7?#l}7;Wv99@=`HVZp`=u1`+V2cZVECE37KGx%{#7gkg;rQ zh!*J=sQwsp2Ac$dbA3F^9M0wpJw__4D@Rv9l zah88xCc%&=mlJMN8NxsKF20;>Tj_8;I^m(T063=U_=4PLbyeSC)?$G%X)B&s{Bzv$ z<)M!S$x>$S>aVbYfqs`lY=A2}_E=F}$)w}T;&Kc930OnF`t`Ec;=7+J%ajrmGA*oU zi0s?x{0!6-Q4-5mQ6f(_U1>=C!nn!=_#Y%i7+udi^BL9tDJKe zzD)2*!Dj;gT`I7;zt@o*smXG8D#sC-rJTFF$Hw6MzpfH8AUC5T&-mAjUZ2oSV_6BE zi~eE672RG|*Z12@vmefVQB%twT;sz_f`RgBsdqm$B}z>-S>`fR7WU^qSv8jHp_6Hy z#7If~kW*^;QlS{8Sn6aouZ4bo zJdiq9jOYS}=WMK=cQ`hNPUng?(~UsMwgq;v8Qxct1zKIX&4B3cDUKfi@S6&$r)4FY zX=46J_yn8LeHAv3pR54HB&T%JbMK1iu|8@Ysz)#$a{tXl&Ch1q@th8%(omzeJQ!Oh z2og6jDfXK#SvM*PZ0FwVf>Kl!Uiqpv$HfR%l*fa#$S@|zCK;r!e)QzB9QQj9RqZ_h zR?$}6G*Oy;^j*Cg`^GKDTG$$)i_2{R`qKZbAqjH~$kZnF0Yd^S$TxDmOt9TzP6!ZcE1bC_Q>O%hmUx;}$YgJNsqma&9yHvSWYJVwN}2BWTeW`)QHH2_ZiJ zAiSxA@pqN}hpLYCRkp>qZGj`QrFSwmBq0I8e%$LtpkSV8@yklXlJ|%|ADmyk(kYE$ z>WJuGC5?t@(hYi&v{mz~tjGK4au=0UQ9#Vz8e60~ab*8cdx!l32EC>{@AT$}uhAh6 z-Uo~({wL=nB|oJqz$BYx$Z#uSefL&_TS?JDHnzLX#*C=H;u^6XrLLwQ+)yv5#dAwE*Upn5cf}sbG46 z>)m!M7i1oTQu_9N<5G^FNyMCi*o@BWsBA2lGvQhq5R3HyFvs|Mgs<$H-?kvMLhlcC zd1QmoT@HOsCox3}{EA|CNTXL(u@(tEf&6o6K4oe1klCU{eS;ekce;A1n#K2>hT`2Y z4QW(va;+-+(!nq6?fBELibU1^N-z1n#%$-6Nh!yFYZbay0NtiJZLS&$kQ^Bs^!j?Z zmoXY9F%FFxQ%W!++DyD0CbfTkX=2QFQcLZqDf5=Ki*b(6y3cNNX_45oi_?#F%o+S7 z6x&-b*vqy|&F6i!ZL8KvozQnS%}>A95s>u2aMnrY)`Exf@qb$Bwne`7Dm66=Zl>r) zNT6PP@4@uM;8V$CurqX_RaO;c#u;9~kapEOf23KQ@84%ifCC+#Tf8zllWHTt2+5Ua zCA>Nq&uML2z#fH3(O>f@G|V_?JZ9H)d{I$gAiqk=n2 zE#~>I%xgfyj@>>~wNayi>*pt3lT}l9fB=#XlM&xf_Llu+H@Wf_lsPI->a2q20uyKO z#R%D(O7%uf@Wzc^nz4GmL&Dn35sZgaELh${bhVnR2aT;cpW+sYqYZ4;otwFrfd z^7;6SC$J0t3EIKJAwBY!w8YWAyL2~m^ibB5eC6DCLch})T9$8hUzhJgZHo=ybonWC z%V-Uv%Z^mLkE|SMZ;Ohb28;3%W2SWNaDBo}%@# zJUv1>7^>m-sZltLdFr3z`_n;rI)OUljNJ~G1g|=7LkXQV1#i-q3s&)m5^qbvFU0q+ zX8-P;TxGq19I5?8!7~-#j$O?8eH{;b69h=p2ND?oYpuXvDj01Qux#xy}G6SmNGPQ2nRu6tv?p$sD8n)*C&!1dDbGlAo*mwsn-F?cr>oByIi`r>$SCGV>OJ?4RvY*(U3Yg1uWG#->~8RpM@``UQS ze4|6s>tnDYZx5Zkn*082CPwBJv{27>QDJFcx3&Ga?lrUY9LS5K5-;3BnywFmEmsX) z-v|r9w0leG`K~FfDc1TjAmpcjG0kQm^}aE4qq0bb&L2e}v@T2l(G0^@Mr6-5+)-t8 zZ7BThC#A$64?IZE$Gh;*Iq@a@R3ljIP>?MP!@!|hlh|RWU0mqr$;e@C)Tzv~9 z%$y^_(;p-+a(51}oM2M)^`P5_HeY3IzM5uuY0JAk{cDg!1ur{zz6L-sh8X767muhQ z>Hm6aKM~2U?C~L}%LINlk-% ze#FUBXna*`v_J&5jppgWA`Zf&zq_c;+a!uRpTPCH)p^UTXMU?@#%dMyF?#D57jv_{ zmpjR=x@zNNiArC}gsUyu4Up$dWXGY=Dx0hwsWwMmu2hkAn?!r@RmfEo`+WiflIXIc z!S3|H)Mjll&7LO-_w9)j&?IQS&7k(7HlFhQ;E%CAsn|$6u;z)<4?f!0)&Y}V8;zAT z{p6PcQL?$F=jSynmu)p;`?odXH`X`5 zIRiK7DR}5_wgfuD@SX_G3Iqe0m!!FC_uy6k~smK2yEB?D>_E;I!M=p`yTpw<2bqCnuEu?IcK?kKXkLZ% zzgtp&{<uvMODZR^L9g6E?%nM4pOZpFPpCQcV3EC-LzmMf_e!ut{!1eR^qPEIT zXh9umq)5if7NC&octr=*7!JJkiAlOrU>iV zrA6M!&Y)4hjH7?oEKoW!ou?ic(*nmx$ZW1+(SK61s1x@de3K2X>G~CFx9Q2wZy^19=<6XR_vSzLfzihfNuortQ$;02y-^}Rv zZkUiC4UU;8>5F>j4c-2i$fmC-TF0cl?lx~d9eh5()(U*b486{8s1>Xw#QtXvEt9^{ zp1*5hcH-62!4FBMqI1i3%WGTf0N5(WM9L>(m)eC>$QcNwngxyi zRmGx}B7wA{vcHs_T}|y(*Edx6CyGbHa3)G*kJ+U|RG-Y_vW~t~b?tojPW?m3DM@pX zGa9Y$MRz{FnBejXhhaLcX=>*a4{maK1w^FBhl7#A6Gqii2z%jr1wW=g?dsFNuhEW_1oQ0hjI;I_p86}BNOR+mTH(Nlq|+pC|+L< z`)LnWt0()9@8yfdeQdciwq_1>it{ve=#H2PaLwlexs8sC+i~d`?O#12;VFqmXWa}h zH5BG%OH-KJ8*(foXp)9aU3(NBoz^*wCF~%&P4dR3>B{t1S(!8O@|fW)r5GV?cSxiD zG-muLglQ9%?qbkavSec=x-c8BB~B+NeAa67h+Q}iF6;UEM3w6-RJeMFU7KL6W#yzB zZEr0a`14|`wL$T;dLoS@Rf61zBr$s7C^#N;a=9-p6(L4Yi6a&llDlS`+0y2Zw-MK$ z8P{(!LQq{kC~vh`8sq}$GU;t)z~_katC450`y?whO~RelR{k<9&b}71 z6$lxwPn1<(_wifHqtE#}>g{{aLwi5sv~&LcqCUxeHI2m8qduoC4TH&-PziD6nNLL6 z2yUm`pP!pzplHZAk7IumJ@w0~wK-~5n2Sh4VZyOv%x}dZXWb|BMiqh8pD7wuG$QPVZUPgj6zcbSBQAF2RP4-*_`yWR#jeBSwPKSi(vQ+Mb&OK7<9~Md2M=YqtxX{J zo%AD>^5v=*z9UWbqB{#}W&r9L?B6a#q8wb`QMl$ctGwYgZCThtl$V7S2n!MpVD@?} zl8QE5NvoHyQ16E6~}WqX7Kh#>K@5Qsge+ z3r=%#e1E{m#5<98r{;h9G0cTJzSco;UA9WHYbnhY)cpj)vNs8_(;E)jf5Pg-)=|LW z?OCG278nAmmH7(>8{JZkr`b*2XXr6zn+|J?=E+H2mUU0RNHl-O2hLeAf$i*J(Z~JBbeU<(5p}daW;Qc z&v*3~1BUl$ z{=LC@L6ffwHIygSid`Bl7aMGV?O`kvZtG_fkO0FUq7CGeL}UlJCrQ6{I4FwTFcW%F zy-c?5lb0M2n9}w1s=jY=H+r{os(Q3->IDx;O=3EG$ixFs_8b50+r@20FEg8;8M-?w zeKl4$>*)M40%t%XzoX?7>@9t?l?IC~SEWw3xtUC;)(4aN4dVNp_d8pmCNG>$>11ra zJ~xqFi08~e-%!S)N#T)gx!g?UDDDjR+Tx{&5a3|vK$8G9m2N!NIWi5?tKa{tP2)if zGrBaKgFH=FkZV&Vf}rbF>%Ie5a&OkHa70%Vxaz%F-CNw&uV6k^Yn*aHWs|fSe@P$r zostvMJxgWH*;lSr*vsp*+dj@x2wKZB^t*b>V(&p)<;A=<@zHOs)}@Rg(15d{jqu~~ zxwkd2UWFa>Jb}LeyaiFMG@Iw_xAZo#J~b{j3EMnH6XI*$OsBN1y~*IDA2u3V`>ytz zNh-Kh10Hg90W_7XZY|XtDlv0pdFw8(5UrZWE2)E5>CeJB9Jf5tn!9cV{@4q&rJlNb7EjrJ!ekXV|jk+wmvr7}Hoeco0mX1S?2}4Y;GM4bl$>?#6 z5Oj}8gu$u3x)HwOGH+!V#;L2zhbZ#qb;6vUy-9Q7 zTYc->kE1+GIzr4KaV!N#BWLdo_%#{eHSGLJK=nzVWT*D8Ag%NX#{0LHG}m{fjX?#9 z#X+$S6E`)28j}HyO_foOjK1r{DUsg^6e&#~$ z!Ar%!mrd>H*&S?w#)&ks%t3roQFgYhZ2#9zUY0aw(%2MyIP8>Qo6MHmtF!$``td2R znzk+9v}^nqsiii{3(}DDwt~L~eqvg^mlx%*gXj=`%n%pOaUs;?-2ghTA^KuTDh^|4 zNcL{qZ=%IvwN=}M#5Hb#9zTe-4}r_{42s<8?Tqkkj<4$1)tg0qGPz>Xcm2dBEK~>D zfp+^N;iZW-P+lid~ z9QoAq$hMXZstVFCDKFZ-FzGdoCHmLuGXR^@pJh3X2TU)Y8pf!Q6MU1u9m!QP`9VIY zf~3%!KC6HdF3NvSRrD0ZCQ#iv!JW`3Ri7Vo3>cq#!rm|SJ-d3Zip>hY8?!EN|8@7# z(U+wvCDII>t zp+$13%d25+TW_VM7PagoV>0TWg0>rSFiUCvzAb<0KGA?_I4klQMA16^7bLx;WSeT3eyOy62DcZbno>s-+lm6a~^=I zH|A~j5lfZ*K(iwLHoCTGEPU>}^zK=y;p^`}9#Se?j^O&f`?@YbV-sPmuz#RNXunA& zFjjcwC>4WaYSKvTP(#InID`lUPZxW)Dm>}x4_9V_u#(VnacE9@kniP!^^;VO|1>un zMy39XNKu^JZ9#9Jbp`c*4ctpWy-a$_k78TO`a$#8xnO&Ng-S0pZc2JXh3e*gp!FEH z`i!4jx&B|BXLB`uR!dr&3(l0vg;L(YIr$@<-5hi9WJEYqRd@&~d49+6{$npkVM+2ST z`7Kg)=?%7|pqk{3?E4Eic^Vbn?m$W?PU~2 znE7AH8Zsn@#G1NF22Z%@ur?&?d3!KeoGy&CFlv*wU;eB~m(RJm45HecB&vVY9#lv8|>uN{K0Wftjd*n5OfY#O}-}_|xQ( z_bG*MN7qW?9eN_2+;tGu`YF`l&z)_){5KmF`Xqrc+$h>N7?wJ(7sY|CXX9=k8pIvV zd1>ZXSm3l-nOYx+GBjQzuFoHHLN1zxz2jmpvV)+vHpyr^CImx)kv`rYSTQZ1umxRz z&!c`b=BW24`{!qqn%G`blmwWJAPv{(l)#s&}MPf zYOq=9@&u1dQxdDq$i!Hy&@>qu!kG8iB+`$QmegKH72SxG#WKZ+R5lcWtbPDC4n`;M zcz=KMrZW{7YE&?_490u_n2bvMXMXd(E)F}aX-!s+k!)Ic#ikZBJ%eEQ(c!*|F3oG6Y5b7f10Ex$cH72T~%nJj#Q70#&fw& z&AeGOF?eDlK$s!1x0|hC;*v&}^#fI{7~`MMz&7@xO(&pnoFRw)+&p6$8Sk~PI8#^E zue6HrtyrJNo^<@daD(R(&E9EOiX6sR@NH?|?_d8drIJw~F}5{#680%KkhV~-59MCw z7iT&BVQtT=b=ohqcVks`LG|x|tu1MLwW_vrZxE*LbwiD8SaqKq<7$WUyYBy047JrL$5_Pv6pMa^xoGSC)F z9y9L+Z3t@Cj&d|CwVRbNllAsvDJ3y&>BoNPMJ>GQL;BtedzQ`+_c~RMkr(lO0^!&1 z{}PGs8)%laboh-_@HnP9180v$#?)oTvn={5;$M+}q53ImynW+ml-ntNo+q8u6grRX zr%j_hF}yX0*hkj>Zkp-h10W|p@rLICs2vP~7aPH$4dq1j&G-{XLVL|;6UM)7;8oJ_ zd%h`F{ohG;&v@yvw?Ch;p0PB_n9c@&oV)lZz~@55&VK3tNM-1%;+N2f%=FJK-T)IV z^(PrDUO!e~$QS=McVjX*8JtQz5#@aWk8vV^{H<-`jXk^g@ew)L{EPno!WH``& zesdD3OknQlHlzGBu1J<<2}5JGrW?~rA9FMF=0PqkS6a(jttD?kjXxkKp++u-r~B`V z?<*bbrU!Vu^)cwy56ya5eu4sr%LZJ?!gn@TcH+Wc)jB=JMf*%&kuWa4yEKo1r=7AD zMQ)7#>i{PM^ECGRp-?asS@r5^uVbPI7ZW9hy*O@LI9i0}5}i^$?Midz2#K&y=wy3B z2Buy3E6V-mlnGkogvOhCcNSI(DDP=%SLPgeKyASd$hjCfs?bgh-1l;lu&eQ z*GUFWp5seu@K+3(VX(UuPmjywApq_bm^_my!!R+9ru^p@70g%&7-36w$RDiDU4JkT z94VD|PIXxlO1BbJGxXSEA`+K2Z`4PoLMEAqw#RT;wjrX|8W=$A7A<{1%J|yLAu`BC zfa=eG-gQDGy}2NDkn|+T<;f1L^?v{$LEpZ(X?)4%j^Gb5cHm_&a5@9pm*Xwf%pMbh z_R>d&M3gZj836wPKBBQSO*2<Fqa903q&d@S*&vEI=t)Gq&xbS2ULI5&c zBzQmP+JHD;g>EGHho#6HcK36;7|u)AQTtQxO>)_UVO7}t+dwUZUis&*p!{o-_(vN> z)}kd4eE$G@3zPCMQVxGV!j6D=#gv6^5cR_jE9ozeGQ+Av zq}$)9NE%`Sl6eJ_JvpzNZ2th`;y?F=eG&1$SCdS!(={ldhDl+VxIzxl0YQ(U%?J)t z;s%Ko)vH{1gIQ&`NFotTwxjN<{mr66I-b3F>06%+8s0yGTAH)FMpo=OQONz@IQ%Qk zJaO?3>%~?k&g)ILf;HM<^5kRXk=L(L)MKE}(;kEHwoShV??^z{YpjFN{_Ic$+IEg6 z@I+|(q$#CMd-jW%0&XTu;g~k;jsV~e)z5q=)2y_=h_|-3+KOHxTR2t`G9SLZLFjSr zezl8b@fv+M!>yoN=@2S;jAWQf*^m>?RPp{5Z{R$L{yDgJUSzhBfFzuJ%6gu(0pG)6 zb1#au`;9kIg39acoDW@;vTIforD()aTk`#>_8vwV0YrY%J)+mw~5nHF!_7sRw!7J zob!zI1m}tX``LczqJhv+R!AD+wK1ZdO6SdzaVWiyehXvF$I3pKuGEU=ZzR=)?7=Ny zXui-D349+fA2T0()Uu9wPlt@if=iIX;^CJp<1Pp9KLJ&V@Xy${3%itz*v5Bxt`C#D z5xMzcF`NTJ0|`@7{70eb2=As_k>G{hYDWEsUbs7%!c{l%JVCfOn(?DP+-2 z`26iv3u_j6Er19z6v}B)G^N9Nm zw$D9OA3LmMe9iR-j8<2OZ*@rYhHH&J+(zpnNDSMyoZ>&Oc=}TVn)ugnEW8V%9Y0W) z%NHyXbaG38j6uod@&`YSc~^t)u5Nrxmlw0kI|$n>#IZbAseEyQ_V>X$Fp}5{ueGsw zQBg7$cdak`y4S;|di?_x>8; zw2g09(dKz{y+-cFR~uR2l%o;X1diU-+k8_OQ~0{*M>8@607kEj| zK_ofEW-+z$M^An_eqNQH{vFmKpX~DK^2IL0Cf6HI(VVsqZVx`joP0m4h=iikbYz7L z7+{h!fPVu(8!-5;@_)3Jm`6PEvPr*j1QCKk2ZB91Qr!5DTUeoEZDk~JKJ}3!$GV9GB#~gc`@g4rZ&YSi-xEQ-zUZa0_Tb0k`)WFxY@kXF;9%$o3E2a!h zybge8gU?R?0O3|(*6vylE^9dg?%dKwSwZ0E?+kwt^sEAP2l+%Ago*vtjG+90Ir`H< zmwKeu@~J=b)W7(O0PR!78*h-wmUUGKv6W!zIsgw(&V|fHtSq^yx2crG^WM+9e6JLIXF< z&BjP?;>X?XQLd^ATv=M%+*_GXC{)DXcsJevocy()q}y9Aqil*~P-Ri_XB~abcXDPF zI9VmYzzhx#;AjJ!Z9?Jx&MtKS0NleF8D!nKazWg2^LPCDH56LNnjJw-k1ViU5rc=HO8v}fcTepsg}TE^|R7$F4kGsr%?{{Tt==cI>JnJmS< zg3lGRx6kKnX51qjsTl-p?Vi5W=~GiWgzIUd&eKd;UEDEHO6|z`c>_7*e~9TMJL3x+ zrIWEZBk&bT=KC`^1>|=aE=D-?_n-*%zY*&93oBjNL}w_`BCuzak5V#1k8l3~U2}5l zULulKX*A1eE+hmQ9x_`50qe&~+L4Lu0PVC>)GCeU;vt@(xXoZvfV2hi8I_}VMq7e@w`WfM&CJdy4p0=DJ! zQO5vtn(-JEwCba~SF`x83%wFpUhP)eL2DwGVzL!&%6bO(!%VSrx+(jQ!Ga*S&W>1iHGn_d)GtYD=UdU9$s9k20+qtxNraj(gO>`{{RjAO{Upg>wjWK z(^Pq-(qKxd{{Uq3-_o7%iROdEEprR(3kgSH25`iBf;)Ov&xn&+o5ELGoxZgJ)1*BeRhUiWlyRD?t(vD~-PvCs3TWtvWy3 zS8@m7KHd|jLOJ{^uqL?+$tE}QFZL?3{jdQi%gD$2&%>v!Lm8Bdrg`YeuH0lj!(hxl zWBh%(FK!q!`gW^k(izt-I=7K_Q`DE$To-0EBWNe}B)*lpmMz)mNTD zZvO!BOvFdoW5jt+nC|}oRgd<1`ceW(MT1?F{k?I!?PyAQqXD)IN7JDk)-Q=x*k}Mn zuS;^^zSU_|`GClv{{Y_^;Kcqd0o zlJ=1a&K6eO%6AspNc85suTHvx@5j>3Zzz&C+(QA@ffPO@z?WV)ytZ;e$>pIt_TwtN z)Y7I2Zt$26`XHX*|D(R<3ii`yi`~k&s$Z&O2un%E+QU867SGiaV^F ztMCH1P`SrkcIjIm67`!O5O~FIbUT4|iKUod2+Bb${g?yBkA^!6s*dFa+WMWLC+(P^XXZpDPZvXppDcEgq0I`AYofPjvKf5 zbsP zo<;rQs};%Iat7@30LZ}^J^fPN-rX(ajt?_yvdE`y4+In6r^*I-{Tp~`$ij9jhaRT7EBlvzwqPnA5Jlv(fbQrMF5uS93)r( zou@fr#yG|s{{XE{o(M!2C``_m2L@!2D<)VDr01~r<2>}B49P9TkwYvN?Gsx_e9gou zY_J0(t_LHwGuIr6UQ3_&n3ce9-jodF2#F(ZJ-Hxt=xWkii)(0^8h3RVXj(-_BOQ;w zQ|>T;X!=z0C5t?fN55^FGchp}mMzCzb;mW!C9a*Aqng`K5sihIS|;*L zMC0k6PagHBs9fqd_SSLP$1HH9YN+amac!L7o}Kx?&N2Y3I~_VDk|`ZxZN+(W>W4Ww z{{RoA0BC7xW|P8;qu7YZQ@!m$cKy==LgSu!{41mJc8{a4fUV>jZIp5`wFW^5R^`(? zfu3rfhPRN*WMhR8w$U79kXta3yN-jN^{eCgtg>3R|!QWMne-B7i*8;TR3^ z2S~{*Jkx?lTn4W~_{Ss1u&&pEGo_?u@H^J%vCGC32= zw}!zPc=AE@#&KQ?Za;p%QC{Eidg&~!H0y_HfsNUNMB7+kr+N?+z9M*T`%toiO-ky@ z&hiUb);~3hMq^d!mOw*uj>C_7(D-bSFT#7!WmYb?JGTD-3E7N$*CFvY;w6uZtfGR` zN|M3iK)Zy1@8o2h0iK_&biWPlrGJGtGX%_Q_M&icoJI*Bl>lC{@cqr-!}u)al_Jum zxNDMjmTx(f0z+*pv#%LFJ?ohGU8RjH;tkc8+pVsQFt`s1BrwS5k;l;0bohf~ZQ(KD z&+NOV+i+dxGpXZuMH>MfJ%6P|@JYjX(s$c}_Rhd(IQf*&Grh61xPKRTg66^E^P1dS zv5{Gamf)7)0(SzdgU3@|Z{m#t-%Qo4u3^-z>& z9FdBa1j;9GD6fdsfHUqpSLXEZ{2#4XcrMHg4~f(_*`pi$wc)pOCNun9FjJ|Ra2?!-PDoMHfJGoQid-9}$B0mm zwISG1P82UOb?SQO^r&Nk?`&D|8pr^faq`#Z&)x6F&)3?mzLQ{OG8;Iz8J2Q({{R|{ z2K)sPX&Q840@*}1d7F6YzwiEZ0h8j}C~mwlq+8!QY5`|rWGjIo=jOotYss|xXl?!= zOBK9J8Yo~DMg@BBjGEU?&@`ySMQs9HvM3}ffrjIn@_RX=o8k=545=cJ*eclv@}NgG z@obOyN;Pct*q)#B%|Fi-7r-4l!&LG5T52|s-N0?6V;rg(Ndtp}Rz4|}e-`SoyX;b_ zmc~H++WJ#I0bc3)U&I*hZ}l5Xcs|W4nPrp9P)HaVYzhW;9}_gaGfDBf+Spn}XK30> zJESYML*`Sv1Ha4H*j7UMvDgPK8@!=ZhF}9W(x(HU$pC}yG0;}e#7$b$RPnP*VLTUc zM`PuOf?(M>em4yC^!N6xgb8z_{h8o&WsoThxhu&4V;DHw>Br`10gVg^s>X&lRgM*4 zOk-~wzjk|cZaDsRJ?Kk~GB{uYMUji_C!B_MC2`JvY-8!|odpsZWWH@}R;6g?PyzoD~ENXFpm1rX+$ph@g{kxKW+b=W8GC=Kyp7 zjQ1n)Sdv*FXr1uWxGbeg=NQgE87_BrFfPi`?pGnqC) zaRgToNL3xgff#)={_*1jsr8@|O+ICloq-FC)?YTwd z43Ke>SC7XzpbRVfsUR@kq(Eix#n;RroE~$KFn0_A-;YY#j(FrZms27rRwg+?VA82B z0mo6sPs{I7TiXSQ$vk^ew>Or@Xk{!x>W7@zSYYNeowPh^q@Q+afES&%hkFBy;Wn$4>Px zkuIfh_V*LUi-IDFRI>qr)Z>nFeR4b1J6K%YO*D*N5R9kh1Oc&>g*hZ}dY{Lf-~v~d z&4~}$*^$X+g_t7a2X06l@JD~jqnczz!R12o(4K(9@*lwEo~Reiuzd< zk~UJ2C@6zC$0w7K*!Rb3%r`*X!~}CNl5Zpj7*q1{cK$;=@jx4T*0nq7dVZf3p|@ce z<+m1WCJTCV{cD-DxtPlyz>E-Z(*xc!jDTxzP`--eN3+GLlgWux9ogB0BN;gF_25@E zJ><=85)mUvWKswi<8C(N7@!W1NWHjA>tptWmPZToX9of%7{|9CkF9KcQ5g75Al>{H z<0t*(tWO7Nvq>V~OXe3X<)moD5WY)eEzo*a_r;jX{3n@uVPGHl^3VsH{3(&7_=lpi zFk)r^wmR@@Z}yf~J|qrgV!mMkO0Wy^FG6@fhvQg32*%$Y=wm}bJ~y`M~gfou6R~>(^s_-thw@}M+~ozPUGuDt`nXRTjFc4EYcRyIaeag_uRIQdWJYg@p{ zC;TEeQ*3eu@HixAf@lLf#UBqmL#EkFt;Y)8X{U6JG;A`9p5y#8N${fK?R-b}h|FO5 zMl-x*VTk9b^!2WX#L;+-yf=G&2B{pHY_gk4EnYIM<{ogpbHN~4=RQ<@=7mm$dO`uvsYvVa0w^AZZs^_hH zC&Y|@X6c~h=JmgmSCeUqWAU8oNGEaW_)-zid`FMX@t&m^F^$Cr=?U0BcD;S7ufpvg zNz^=QY%kR=t;B7QBo*BINf^&O5mr7UT+40a4NB?)7Fdchk^#XHuVelOruYkSEyu)r zX{W@JymQLTK4n|F4Uy9uk55cdY?jTYt18P1#53}+a)cHecXZBAUjG1{0xpY~t|4%3%sYNm z1@If5;Ep=yr*5EeQ8k5x610D1Xmj((cLKq%PI@1Qg8t{$S41*!`m;AMlYpgClgH9v7S^sOR(iDCh^0zyXLwh%n7i0n|6kRbq- z?nW!ctpNK<90P|H?cW@sk594F@8fcj%Q1M>2qge3{zo*BjE@}nc3o~JnZCiNUc(Gq zV>fDIbW{U|z{4DzgZ+8EVT%Yrh4XDdSYNp7h*(q3#sP z<`4$bxci@(tPh7;MaGBYo10si?c`|f+YJPNBk(NV8@aZ+H(qHg zbdexs89a68>&0MvDUqMWQpqD0cY)4NB%F!{dM1?x&-^9&#P;wd&ii8vB$EY1RvfbN z_s368YtQ^a;jiqCcEai#m?X5GNbtUBGRAn})O7m$*ITFf&ra}%jc?}EA-K4P;sB-; zMLBLjIVXYbUU{hMX{>n7FXBlff$mVp8yxHz9C7cO0QeId_0oU5f1N`TD@oUrO*rYASr0W8A&<=-^vvRdeJO#({9lgZeGg8K)^iMo^|1c{ z@D{Ho@XE}-EM%T@EQ(t$e~Xb`xA8sXT|Y=Fxd!?7f8Q;S>0VQ!ocP}3aKQxs05M63 z=l&%E{wURu5tb?dC$Gr0L*d1p-S3NHO*Z~Z1LU^Jqb#gIUYrnl$3I%q_=I-&r&cOA zF~C^Ha>SyK(=?w17=OZjZnzkV8A{}J!sqqJ>p;zq6k1>CzAug&-BL|XSS&a0Ir&tE z`FSUf0sjDj8mE03wzjoc)mqIIMqTg&ADa!2Yyf`0Lt9=xBzWn6caSVp5=L@7!bX1% zKLb?sTLzZa^2SM12j+#GW9MU$jPf&|^NIkmi-B<*j~KavSQVb#<;FPM@`K+aIRiZY zawK;D0A*HLCoF#YN>y;Y=NKbA5sZ5NSCZb|R4WS?Yi2POAdFxFGwas}f;s1)sID#H z`!SAFEOB5<8!V-WEJ!Q)k;n9)MQiJOm~GNHZOn1*#^_WxJqKSv2>fcLtVm=Lx0YfI zS&t);oDI0-@=qD9iDHr{S7U70;Ty|B2F`E~9e5`rp&s={-I69QSguHRX&CN*nQ}EDE>z+yUWQ{LRc>D8zA%<G9Wn0?b7QOjI#+9Dj9c5qx68_u+N@IpJ6jxf2kTYm zzErpnPi%pZ7s|4nWBb?~jFIX2MQLmg;%vIQ**~XSvLhByAXxm^*oc1Z*5*Kc~F_cQ&%ZziMBx zt1_&h1t_Qol$_vwX?{Av_OA(9L>W}I@{mtV`BEHKO{{ammp*0Oa;aBc(Ek7%Y+f_d z9>8|%T3;U6!Qgv091(nj{{SJ#4FGVz4GRAN!pYG{jK}_xK2kA`yn23Bq5DPu0LaM2 zbIhkdcl)E$pU?2De}xj4@!o{!yW8z58=iSQ{zA3?0BDC__(qwwq-tq8ul*`MHh*wIHxSQhESC)vt-A%0 zvP^$Br(OqYNDf!W9uC#~C3>*m#kvVe@+SieCOYxdb?;qof^Ors@Ey!78O!;HA(Ufq z+6D;(^ydPwJ|aot1NK;TtFd<)k2V*{-L=5mkWT{xkH)R|NXh>I2(7}27)Q94PuMQT!^IE&M`fe2}pf3Ngt(=L7RKz2a~9 zNZ&xbxV68IZ6%o7NfUgeKOZOqslgpaE0gdJ{{Zd85wTK77&tk|pbo>t9t_lcTP?=1 zqGnlF3N6?LEHl&+amO{mX!i2K;yY_wged8BsUnfkhU6SqqgzL=+$46|73#rys?t1f zft6C(In8hBeTVG*Dd?!{{VRZ06OSJ zbC(DA&;IeO*+dL}YG|RexQb~sfDc+urvi|g6rARm96!caypO~BYgjJpkGD3#w|6*d z@{b9%+@39Wyd%ya@8cZgcCTajmZh}qDTh_RBz{d^O`w(k0EEV7Z@%S-9@!L_nSN5;pw93^CI^YkS6!i=P~7 zHv5W+NR5vN0IC9U?Na*Yf#hJLaiDZXxc1J~Gl&@x>`yg@dupWEoK zc(8;y3ma`#$saEV03TY~wPXS|ia^_uPDsYlkaLhRkjhWK4P5w_YPOpEQV-t9$i>?{ zmMnSw20tH4rK2^x+H3+j00MSaWl#!|0Uo@LdU|xA4O{5oj{D3=M%$t0Ob32AInDvk z@;-owqgA$mK^*F+aH<#PTzd5x9G-h|??7dfX+^`lOTigCcCUYI{{UWjst!UC44kEPH{Gj=e`dl~%!~Yj%=W2nv#! zA@WLrfHA=wa5(0!SzR{CjAmyX0;)#k#zDgE&pk=~2XI5;-qI_`#wf-gY3Gs2To0ko zvHH~n65QVHT={q;v5aAP`Ac!rgU6>zy}TC;(3NRhW=U)wc+aQa9XaE@SH6(Yk@0ft()r!O!98O~i?D`-x12jv0b%z&YhX z&Q5ds40XsdJ6jJ`R)5>0g$hXQT|V?iOp?m(v>g7x~;UM}%{ z_lq=mblpo%g2FVojzo20a(P^H$3`dcuCm7Z?&;)Ihsjn~CFWHp{YV`L2d4u*^?&14 z!&>+z#^yZkj_x)Y{{XIU{xkvRp9$f3ekf^(4&+;np@6__1B3Ziuk8-IyiDXUIF#fE z$sZ`~+oe_bUcyg_x>SmC9^km|)G@2yvanpG?`8_%h!j6D>N&~!Ct@zVT zNxsDl5&2(ekjBjtE=JXZ>=}LR?&aL19Da+c~70o0hMjNDxbvv02k=GrKP&~ zZfRhAFPrkRk}!A&J&j%XZ-?*=`IX28=G@&#`36tD0yH7gq`dHBSxmP}=iT|@W{pV6 z00bj=&u^OoxnBn8cX~&RXRy@ajb)A&B}opfr=tVhR^_LTG$?!#{h2%N)V#(S5jup) zI5`A^kEL^e4}Pyf4qO4X38Ng^HW)=u6d?0qyved9%-34qdzq;IX{ZGFcNX(JGRDYNk+4SzP<$`(8tFbTgHqJwd1FMCw{0%L zijl|HKd(x6#NYWC^?#~*`Xw*H7-P8c?X*)svq$B~STcqf+IhjCXHnvsNjy+v$<`y;9cR5f|7> zOA;=@lf8k!=RGQ=pYL=DmPpT?BY=m2jvM72v$sF@(Bgn6n(8;ao;6HPgc9NMIc8iB z!$0BLlHOA&E%s+;DipGkyTHzOFG6|x^UXlktt?F8pLz(UDxelCgU;M@f<1dusCZ;p z*}hW8X$(gk4j1tKIO{+Y-T@>p0klJhSwR_G9uEg3X9Kt8Rs>OytZiB4oOf7$bRQBV#C1-0_cGoSwB}GLge9(Rm@>K(6G0 zv~!XOBa@!E&t5ZHP!^IiE6&UTXG65A=O7%Pu1`*YeP|g;r;)Bc$kwh{xCOUu+BWb` zLEw@HwraeR%<(}4@XQ^Q?m~g~`=JmdS zBwRav;0oZ3_Q4n$T2M5SX0+nX#0uOow;a0NR@)YccbyN&|#F(mo1s^Ix-K2lFl!{vXSuW+-D?Q#54Z@0BVBISb5@#7>?$4++3yy z`ix)>r|aK}vGJiqzYMK1N`o{xKA$0=4g=v;vs(N|(jaW*CC4W}!fNlw_EF*~@(Ct? zQ^(8bc>JoD!bfSoA!zK<7d~O<7|+VXrF1_Q^f`6U6kbIX=@E)u$9lIs55w}KpdL1y zzvEufeQ%}slK#U_n7q-IcV$?dmK+W-&tBuwyq{arE_8iBTTWH710#-%E4CLBTU}d7 z9TcQ&H{$?&`0ezd4pYZ|7P|1hvet45Ufss%CTNUd$9!kgrFI?zxwg0P4csp`?_~2% zHlLR&00iS5s~_Sm=D(>$X8NP~_VI4}JnBhTQ;$-84>|8m@ViO>00{NWd(H28bG~)M`1$WFudkjBZoStfr!&xDNT{BN6&>||l6OwCx#Es%5(XRf- zFqcOwCd^q=n^=UrGFS0i+P(j@-? zc>e%8&5VJMG{Sq(=aWq)kjG||Jkn;GTjmr2%zRLqICPyQ<&k4SsB&1~0fw(B(Ap>Q zwDLtLkyx=(4ngF1uTJ>8Zynx&ra>uXN50xh^}!4)%DguWcD^lmBgs@Opq{-&3`Qr! zk0|(`RT2^xE0U{|(Legs{{Vw}j+dkO+6&A3)wY3QlXQ}epc20*;F?by$L9FQQH%F~ zF&Xm@$_CZz-jm_pg{SL2Fl{qX7=&B2g`qhlfC&4*;($7j7HUvx-X*%!Y-C%d^7g_* zU~c3OeliUf=36~JGOrky%?&fRI2k3nbUZI3``G6dQ&6(gv`t7_K(Dq-bArwZ0Fp^0 z40Xp*&>FW8PqN{dvIB_$A+iqA3Fm?td!C-7ng&>vguYB6BPhy41J`!%uS}9hxbK>U ziSop$``Myhm>FVo^5Aom&#yTAX+^w}EA0}plCB&s3mgzfr%r!5u?5$bU|oRZA$L{A z2|ay3N&seZ^%1;CrbJ`g9HNJjh{{Srx)!m-J4t+m5 zwIfCL1VHN-npnx5?f#ImHR6GU+%$a?Z{GJ9Z*b~zN5BTtCjAP^Y$Ge|Lp z-q>8=@z(>6KcvDjEwe{7R`agJSsqSAsh;Dp0loVVdITa{$1SqP@sOu2wMNauDs#p` z#yx#Ib!?6o&$jaJ8)gVsLc7QTzEgln@4!6|(v-vG%@ig`)@D}Oo!yyBcl)@58%~nQlunnWT@EVC0;Pa6ujWo_O`mIhr{C0C^Ol8Fx5A zm(DPNgTcTU_vhU7>`+g29I`VBqLXr}s{$2@<#K(%<2?xi zfFswTK?Q{4cJ)>BGtr6uT-7~1Zt{GX&~L{I03UI(I^*#*E~Nqc4XfmwM}}M+vbHzi z4F3T2g=T5K7Hj!oLwIeG+W~}S&mf)vKkVXwI?oK--0D-r7hsGy1)T^V>Q5a<93Ohv z_{J1`Fo`_G%Dh~k_f2v3pAH`9VV75o2|}|mwk#KN79Ypg<;QA^#$OHN(flBFv8pY{ z5s?tK+4qS4Eyvf=fHQs#;Cw*QfMsu-@;;4QzZo@4-7CcRGugM7=1NuD$F~hlk>QEX;y@r0yl{HdJ_xc$ z@Ll_rk*3`sP_9OBqXPi*&1(EbH=0p_bVgZ)!l>Ql1_S_jJAoW?{uNKbQA-zuH7Fm6 zeA#(Xj1wC4{AdPGXQ#t?;WtTURhHvW6M4cJcwM;KNCX4&_0B$(%Xo7|pG@&&mRf{T zEU}>=BV*{mkAF(-G_6c0gId~afffG%n0&bMLv89ZNc#81VB0P0c&-_znt8Jugl8qV z13VKz&5aNU_03ThCfF^C02W7+EScSv&PNA18R?F=u1NTP{{T$!W{-C$GFne}0E%!9 z);otB@(%}tU5<%=9-rckKwjPbmfhkQk=4j8v2lz7JDSSi@1r% zaEg4vjt{md1MH>gMGMl3&_jnxztC;P|w)UZWoH6J*qzBf}z zT9Jv9=|Ly1IL}&XVTw)J;($4iiYNXgx>u+6fY0k*QK0kVDZk!t=U%7rU`*Z~(}lPO zU$*>!^v)X<ZTi^5#oF73x*U*T=Ov2o%ii$!>nTE4U!6!l zbfOX0-?u*W8@rbXw=x+FX9TIlha`}ru6hB^dQ;)BmepCF-Pa(FI9XTTdup`sfqI-X|tD9@bh^X0x zgaD*JHYCJ}~FmO@bHa`Jkfha`F) zJAqVf0o-Pq3+W;X0mZSydLNW$jCG)uYo|t%8A(tIp>-hjJT`gHO!L#yfGo?aS|!_C zP7ZR+%n1O1a&X`d-8t|0)U#b_*AN&({hXhenAnz4{qCoell@(V_xVyL9Be+=C0Ad0Z6OQEa-Fjz?ckThu#T<;FMRsI?03}H%cP~A& z*N%E~(wOmyqg0vW62`zqCJ16c>I#lI$6DujmQ!ZrtvMs49+IN9hAPEOyT&!tj^ z4Q2^eXahZ{L5QTz5bz2PoDMn8cqhF8PYva{k;TL&H;L7RsCOQ?$-(MK>^`EWXd*Xo zO9i}3E09`M9I(!NW7EIkPIlB~QrA{4@fI65?U9$M%A6iPxIHPaJ=MFyiygd3pcmdk zoNPp~Xfs@}o2d6{RzAG*ZmWEj40J1LK{XkwD)BWy; z@U2}+G`Y1w^C2ycT)57CM?HoIKU%A6q}$t=%(os|9hv4%kH{{Yslzlr|<@<`HE zZhKVoT0tagvPKzK?t|LA;A_4l+2((1>A=D0oz`<#XHVQj26nWQ{_razO9KKc#uakBNLkaD27aED`h7;^QBJnwsy#dg4mR zHJcz;(*z%;W6c3$xO*#NK4=jAMMQ0_)Pk`fBc~lJ&*SknzbvYUSGNI=IavPy_>o!{ z`nIEFkj*x>@_C2|iYs(%^Ng0p0O0Y~fI8VBiLy*;#PO0T+B6WZvt|3F{{WAK9Y5L? z<>$q|ATPPE5kX`7zb!w*HMQYQZvOuO#2RjOaRC=m?v3Ei@0@TL1N!2DpJ$GgQVx_> zgpaO1;~sRM?;qz~c&kL`OJ#GV;8|01DwzXKg5v7#M+1el1g#`AHub*@Clk<95OJ;Ye_&LbKAXU zc=;z!6yJzdO8}tkPTLneb`?Xx{vgzREv{;t^CF{$@<9l;g34%EN+nkhlzad>;~3-l@+bpcMZ@YQIq#5pXyJ=PpjAZueaohp# zS~H7hw~9%AY=sWOHpkuMjBp20_r}Pe2Iz`e&Z?Y6)#c)J<;e2GE46tB^-I>yw;!9fd`wTt#i=MQ?Jq67L2x zBmAS-a&esCjQdaoHnZwzV1#+Hi5GV8=OE3 z{@3uaxlI+*;hOMpkwk zI(OQmKT3Q)B(Y1V)pY1XZOX{5>KK94DJRpfGyuar)y0e4>B(sFZcr7r=bZcdb;+ke zHmn*sX4B@BoCh(eTwtHOf%A0!W8SLEt6SV#NfcT@f<lYpIb=yBgYg#$Had%x_6e9LEratRV)b{zEE$y3venyqgg!$i?9 z`nK|eGbtGx$OG5#_U5R;;%zF&!|`ddSnbW*EokTE7v1KL+lXu(gBqbgnKhiAfE% z*z`A3h``Or7}({w8+hxAui{ULgH`Zk`U>dp3rP~7klY3w5OPM*=|Ie{f=50((#Je* z{zj+x*mroMal0XICCK@B`@Wq1bq~QJU&q>UAwe6u4{F=^krWqRD85;f%n5M#1cQ#0 z0p+W6GOGn|l=jAHi7T!^BL}Y)*rt_YPyBnQnP!>0sOWx1AF ze8xXFt#;!`)AP-$W9dNAtTT~rgX%ia2N@1aZaqy{mg;7VM$MHS1$D2d*&xT+qsaEC zO_iD8+r~c%0OD^WFY^wfulOycJ|@#P?gU;$2_Nr{E2K7dn8CM?KT3wjN)6&YE^AMb zmhuwF4g-ASpRE9Wsyb0hy(p~&EqTq)iL^DEKyTDCxdd!)U!{64Bv;HDjlzvW<0OyX zx38(FV?8!c6zEyz{eSO<{{RY!{8OR!zEl4I0B`WGDS5$fyniZbkRRd{fb~E4O0-<# z?R`!@<9>ZARq;lLpZNVBgl+y6OuWAFUDf0k z@=I>pcCsNMh*cls`BV#_>aKXxu7LG0xA<3dSuL)-lyFf`4ZEk~TM=rOa3D7q2ms&` z*aMFA2=m(?3~F#lWPLht9d@uWo<&-HB52{-miKX=K$*|-uS<_mvbcy8y@8aE?*my^ z=HHT+F*nhu3!Fk}cNidjr#R^7*sEH7t<(x)b(N#eer0U_99F!u#tA#u^Qg_tfAi6Q zItFJoj>T}kYa@@|MCe%$>67^5{A$UiS|qD3FH0!|EMdqWP6zX?R=JD&>G{(Q#3TM% zr{_Q!>1Y|6MuO!_WbJ5+D9z{Z#6|>(`OpR|G3tw^hzi`S?1UBti9cTXsVyz;jj)dKWpWP$MB9hsjCb^{K_dO6 zoy)|2zVsPXe;WRD0SiMEu?X))%!~)h?Ga!-uydcsr)sM9Dx{=A3V#qbyHuZCZ8`kE z8oFjv{A>BseAv(NujfD!GZP-yvw3iMK?5oeex#Gff6P=@D=g8JHu9ipn80( zpUnMg&~*O*o{Ra?G?8QdV*Ydi%-(4Cm?GNNWsLS%is$N2HJwobOBr3bjzWLkH22e|3Udsf_sOk$X`(O8i}}@)vAN!-@t_9$ zCAYeR;x&x3q@p&;hH!tofH|(4;vL)G>a$xem{INSrBO1c`o%cGIPaX*Ujcx8TVQfN zS##Ie)4W$Ir--!z?xQy{bwBMI0L573MFhhxd;aae!j$gE`ndpo4C9G0JbOt>|xT12TChI6pkzA%|_C3bvZWg%drPx z^{=IX;*hARupcW)Z76J*q#y7TP)if8epVk*UsvoDY#xG@fcY!;s@=aDktBV?BEGTM zJt#4+prXRL^S_I+C&K+7I6vu3KlEjCR#xIWGTTW|%bX8>E9*Cf7+|WOsHj@f1J)SN z)kPK=;*vg_ zqZu);2mJF-%D#?16!7Q$0JPu2 z*?+>2cu&J;riWqw0Dc$zD6k(lO$>fnA|(8UV#*jf!hkF4(cupbFvDne2maWKY4Asc zk6#Yyr2hcd0Y!lMJyo}J#a*^S!x^ur4}p9lf974&zX(92UjukeTPvVh52SQh4-W7} zppRa$c2HZ)&T-E}Pc1Y?t+ zC;kLVKlW-lSRX7qCp<6ZSNtonyiuh97bf-}{R~&q$KgMKh{4bh{{Y*}-~3%V&*4{r mEiWOvx6rmt1dAkX_Nk862O!|SI*K`19f%xJQf;EBNB`NoyrTdB literal 0 HcmV?d00001 diff --git a/3-NeuralNetworks/03-Perceptron/images/Rosenblatt-wikipedia.jpg b/3-NeuralNetworks/03-Perceptron/images/Rosenblatt-wikipedia.jpg new file mode 100644 index 0000000000000000000000000000000000000000..30dbec1fa780a5e277a2c195380ccd361f46f1b5 GIT binary patch literal 36243 zcmXtfbyQRT|Nq2543Lr(q#L9`N~K2$NJ&X|cPpZFmr|oUq?@TYQW!8o86gZ9us1ni zY;5DV&-a|)y?@?&?mhRsp0DSlp7;9C^*_KPpoX>v;Kq#`fIQL(xP}8%0r$ws?~>oS zclYky`}gkMr(}FY`S2m-6Z*&0j9lzI+??#39K1rZV!Q%U0vw#rUW!Y}D<~@~^N6Zx zt14>ADk&>me+S&Zd!y;b#~U}O0oT6(tN;MnP0}lF0RI1RgA8!<*6lmwcS%=YJh}lO zyK(a-*)7s>hl~tBeuIpf18`IP7R^gTyW5<8AMVg*9!r$9{vfBD-Js`sWn_QyxQ$oR zpMjevHp}698F2T1PgVh(N*rpuce*j@^c6X+7)9vFsx}hFmTKLEjiM^BxlFA|Z4?xn z1M!NNJC_wE({8H?JKUdT-p9NqS}3k%*RAGk^!jH2O7gD=Yu_+vS56*fX{Nm5sf2>2 zFG68XAqRC#Rsk<7EAJU-*IL8HcWzip7-~_Dej+6C2o;U8LxOiC)P0 zTgW()EyOGjqs>X}8PEJI(sQxko2P!`mfvV~C9if$z?&jN++NpqRL_>iii3tiY-}=r z&0YmlaHf|xMgC(~qSLQiT~e0FVm7Q?Qp!e>a693wy*QQfoy#5+J%b+yPQFY48^1C> z%*tEQgr(~5WDQVwryzy0R~Yn6`qYs*G^(iDLo@#{&w2c~TJ~xtZLw^Gu{PtEx%KC9 zV~w6Ylf4l1WVNi(=|%Q8+rL+%lvzgDqIk#8co*hS?`NjhfLI5+jgiW39u^(_E0N<} z4ePd-cO-|z3|Aap$TD?Ej}{RLJ%LQ+_Vv*#iu)thwGztO#x_2w!cK2CC>9p%2!w%C zH|>jw)>rW$X7;amI#Z9m>XgdX^2OhM-WTPHt(vkR=Bw_@u-=&1=l7cV5Z3@%+s8g` zaoX&H5T8+LAGY2;_al>#%6B@M8MC&=O9z{K{rr)$2aeZ(M?bTUW-x>T>cSq46`)eR ze+|{%FwVre7;E$Etzq&5L~6h_Aj!nUucf#8(5U`Us-#cnnRu17y=Hr#hy#!B9=+!H z+l6nX@>R<@1&DJDT;sH#G>6CjNclOiFNqB3=QUXADFZcG)iIhp(OwpiQ% zA;wx;FXGaQH~O$4`E-0BkxSjb>?d!Y(ZM&*=mh=+f$+b_QkMllny-{}Q2 z{OOvo5IS#8F65GqfOVA-{b7N>;n>_WYDMKB9=LF#mbQ`yXuCR_#=z|r4U_zwHjjYl zeQIf*&ua8)KAN`|BHL+q_s7JOg3FBBnXg(dgRcRX2C4>xYXD(2@~tXEU^06yuUXQF zPg37h?4tLC>DO(nktyr?5+>zxJ}ae3`0G>kpAKx%B|7!>j*eb$>oo)V7vqbH2$7fd z-X-;JZNq7+qf|UZB2g{~dYFM6E?ZeuOiLXTmh(8MWe)Ztd&SflD>IwE zmnw*_3HbVw)ig3n9EU!rxI0)j^%K+;NGG&i7T*F|Po=o^kJtP0jWE`34VB9 zIEz;WJB&FoxeyB zKZ-xGiH-F)le*3o>iO*p=Q}#K5U)&ICc_#-2MU~@+6Hl%o|M`zZ$Pl~z6*9f#af-r zt83+Ws#tbLKT#5}Jh9q7C zFhM8JhUUJ%OJEpL4A4+&x-rO|rxvCS>5r1{KA2XWcAbeR{Aw#RXFT=wN+fE3O-Nfn z(tNb9`>EEeCY36cgH+>%v5bx<-*JAahLlC?X%%?+!6g5g#jdkO`)p`Oxm zn={k;Qd5t=R0g%!o4|VBW^%Dz18PvIGMo507Hx}2E+~r7jxf}EyO7Pi4PDY!s7k1G zvKxK=q>-T`+9GHDCJ~qT$7Ghi#SV$x^-^kObJrt0d8~E23fQ{4`{ehi(yuE+rC#uM zRd-K+tA1eczC@~p&LgVra;C)W2T@P7MvU+cR2%l|#Zvb3ra-R8>3r;a_1*frkjhq; z$b#QHipQoOzM3TsK6;&DUG&gnJ?8T|)n4CFVdLt4c+}u-$O|kY^5N{&+mRq}#Ue7j7o)sTx8+zV!M`?BPRV@7!`{WfmO{!zEKi4kVbx_ui=Y=o} zMt)|cPQ$C|yV^*0Yup#WXFOOmpb6%efC zbb?;(bD^x;Y1-p4n2W~DoGN_GoddP36)INO_Cu6;*s8^jz3RDrAkL(wGm4+`dI?{O7Q776r0jNRgfFj#Ft`IA`b860{B-oYqXjM>Lm{}qFV_Ib1-moPC^lp1B`hu9SgoQtOXmm%&jbEyR%s68;$_DEtB98#}}# zB{$f%CSTE_j>!IO?;3F5pgvnS#7VXpdPEJ}uY!4?ocf4LZ(6o_AX+E6fjX*v5jnHZ8Ys@eZek z4LKmTZlr8bNF^%#uwsM#JubSvD!Y4-vgBDx-mDR9cDnGG0rLDPC6c{%1%ny=KA6nm7^M3~g-{4Sy9QhXv^_Smyp79EzdNWK`NqqI%AY~{!h-uFip$)X znCc(AyYL+GdBZ?_Lkb5XctfmSeaM=T{L5CQ)``!+z;$} zrSJOXYyCo$7fYpRzazXU(t2LNTgl)4z8jKuX8-s)A&uJ+%yve7Y-SEeQhT#t?17>9EA>S(v$ zCvee$VOUq|5&oHk_`NbXvqBn>{mo2{5s&kPhlb1o>ko)C^kOy>ZNb_))s64)3bV}g zIbX~iN6dAQs*DOu9jo&Y$gZNbuxp6iHY5$oq&3C*tB0N#&oG7t0V7@mY zwZ$UE#9_LSY75ukVD^Jnry9N-jy@lK&T7g(sG#uTyL2f%4}W!is%ozgqg#uf+iv*k zQGC$r{qR89;H`@~=de+j8QvBFesl>nq^xONu^vm&5U*5fIL9H21Fp7J-B-ox_8J@p z`x&DxdO2^ZX(TNaT89|Jyn?;XSu#F>g&jXce@HdbuQwbvKY!x!_j2@CR|PgA%E$1o ziDAzyGWon_pE5h7VgGq)NH<27W-lj~7(PU*rZfyNuz%3zzYFGTK;F2H+(_fDviynI z=Y#$yd}h7vUCHpr$>&Pa7tmw0Z~njL&&J&2M<&_Oh?eC1)f@C0GHaSxBIrYpbu{@>D)B3-K{W=Sc(` zLL-^Q;->3Rj1y`pRu95!u28~8RW2Wt`}ip>+n~)!;@Qxfsh;l(Jy`x_*raW~$*G|@ zhmAL%dDoHi4dr?BE_&ugVdq-)_at6ut}9*xym+_ndQCLl`GEL@8DH%7amsL>+RK;>` zVO1(CfmF$Z8)f)sI))9t5z!>ZsAo7IGbjCOB-PpConQ)IF`1m7d6SkV8Pn-L)l>!N zin3(gtJ|M)WJmfQT{76Jd;-&^8;G6$d;;aU3Xix#mE4?Q$iU1`NKYcwPE8DBqNv1J zDL+}aForD-C2tVStOv1AM1OaO@M&oXrP~@TvFn8^=i`+xhmJ#_F5XcKeK$YP(pSTgH}JA2t{h_VM+Pi*S}XF~D8nTt;%s zyvQV3^Hou($g5AAbzI8}WB;vm9J872p>V!?j2GMdakGvv8_l^E731P}`*!di)m{N# z`U@yKeC;GgQ)Au*7{U<`WyG8j!lVo9?@ovh3A6bkFR&MlFuD;R>1u~_$cNHNcJW{n zRPf?@;L}XEWY#G0)VmP#R3koFF9#I9Ww~QkBw2t$j#tTGHNvkXh_~_9XQx|UrA;Py z12?W`l?VLmmJ3hZISxp)nv3Ys9{Oxwpoypt>d_aRq^@ zlbfJkabn0aiL@=mAaTeyL0s5z#u?-3+Gsn2hkXK!qpV<GsVGj!xP>7LjYfZ+U1z%blUb-r4`I0l6!CU{T)L&MyG^z1W_scdEsa zEhixxF8>H-$g^+RbKXV**8r4_CR7D(^Nv}Q=>-v&YJ$Bt&sqD2LeG6Q^{da3sxzPs z`%+A~4~O|qByTlS!>UNt+LOH}d6op|k3^S$fq{Ze@j1 zUkI0a^t+rtfAd-+|G7`fO%M4IpAy-Ycb~&r_MWQNtQ)=jA=NervI;kq-YoncCZwy@ z!X)&37f(jLa=EU4yP0sACO$p13W#&KAxLx3PU%Nl$S~d41)<+tzUo5+{yngfQ;d)a zD3)1qU>to?spegPhRzWe0~$f}plv*DeX((OyM&MXOtLz(lC_W zOmY9qn*QR*l$ts>rz|v;OsRqPL;s-92TN}EFTWe6*->XO4i<5Zf4D!_083)fW=Gen znVylL&)DzlAQzme2@kW*U7g&2QTV7x*Pr4yMkl3!jbTDK(BkpnIe?{#g=jnZ`pfH2 z*2;m}Z#c7J5>ul(Y6zTL?XM&bIr-HYm*@}bx?MaxcsJwe{Av#DH6{;}OC)({7zPTOUb|pwUl}{E4dqed&LuC zNf?M&wi`5@KbzUVK(zL@V>4D5IawLoPLi+u3fs40NZRz_Q6zZ-e!`|Oc(s>oK~ZS* zzb(~ICeA`s&8cQ`l|V(0&P!tlk9xe$oY$cq7dDAQ?-K}W`|i&{zNoVufT zFD*U8FdhtBXHAh`UkYqAJyTCLqm*92OLjpK1XfkvZM2C6F1qJ@WN1nL8~p2?4F21b z_~n1{jnb*iht3Z-B~n8n=GOqt8HlOvX^cq0t(QCwntS+@!tsKj_!FsyllBe zvh9SP5YW@z*MNA5#7mY9#FW7`puOdj*Vz?|WvlgE#q&sdE0R_yaXy0vu& z**4(Q3RDGi$qM}bEh-;)h1^nrmw@oUh5pPmxI|s5SSM&849V)oqspJAma!Pu_%^AW z*RNR3Uh#n?lnI8>ka6S^5`yfur3aZEk2Tt=EWJo%sosBO7^t#zg-jZAPbjA@3|@%3 z27vd58~eAt%aXE;udV@$c}9^{tI}1kKDg&#W)pyZ$uI44ZXUH%5h*Wazeh8*vo+LR zrt9=Cj-?Evsj~Nuj>gAZJVd)UPP4Jr!QbgQyU7Z>`PdqY@>1RRyaUJiybTX{d9(z= zP8a<_Ez;F8^~O0b^?tB!sRT8~|O3QVr8?A2H#TBn5 z*DvqO@W6XJvXakyFo-4QE1&PtLa<27jeaFF?f%Erw5dCSzC{CNqf!CkfA3B_V)Dwc z>lv9>JGbRgf#(FtBzXRmaAF+w*5=h0kOT!xChaUQ{k{gU6FK3RpUF{h0#Ds&rLfcwE447FP)tR8(YXF(2lICV}fnrTS>@=H??K`f!2bqVR>F!?<$zY>v zfG+%P=r4ymz{omo?+Gx{Y@dY5UOhijEKI(g{^1ikQ&MV(x3V2E4pzB$F7kn&+Tj|2 zB{O40ET14PNg8^p{TdK=#x5KRx(0;U((IKnLH&Vku^curOIR?}l6&M9cn_Fk41@2X z5b06=LmX23+sl9Lr6K5saJj{`t^TCAcwXw0`aWoWeO6T>R}NHYdy!Z|jZZL2adM7| zA{f|K`J%{8Gxn<<35wA9hP&sXIcn0RyG!F7SR7We>UCJ;T2%C8&22_jJSlOb6@)0z zp2g_Ac221+nMt@?pNWf05ko^(0w4cK=Np{A8cPFp#dtM5-g{nva)YM5cH-(;mvOa* znx7e+$=v(TmiDP(dN1CZ{#C^mh`JhVe)Q=r|MYfslp{Itul?Ge4td|0<}%0pp*$I_ zQ_}XWj?1$iVK0w)u-0MP25~QR}EMQu7a?|XcZr+(4YMQ25!_lnd z#Zn=dr*!2qT-tkYU4s;6#e1JKlWCbru&{q9&1Lz#xz^TmW{uFH|t3%q9C z9g9H-xH+Jq?6MnCsHeIaF&Wv^uwf<>8X3A^9kOYVgg3C;!>!ezOH2PqTbWfwgby|M z+WsW3u=Qdb=jM}*hrJ~-okW8-SVoZQ^9mBmrfqyuAnWj^mJ``cHIInOKPyE=9jTw$ zn|osz@O~%hNe2qXt1`ikJ1lxahv=j8>4E*_@Hy9TTj{ra@e`)~ptvEC*5jB|ub^UW zvB{@@DSgtU&*WkfV&eb2NFI3Gb+E<2w+@|&bSHpEW8y#GU(mILIvFA{|G7`4kHo|~ zLgIgMQ5C@B-YZ3~38j1aT?3e+pI&O(UIqlXl%qKM`zAO54G9}9^#35vNoF=+yrK%8 zuM|6AFiP|u66u_spm|u=61JRRoUR|RhdXsY9Iv#$SgFu+vLRn*4Zo~kv|dYCOcu$V)BS^k=H0QW zgydb6B7;j%N~_lFzd@Vv3BGY5q1jo}Co}QlUrN3rUobMgMb0?ad|4)A0>{AiYF;-<`Qz^;gyogT}m)yV*);pcLpkI>x@6+nB z9Zi0G8S;3hbS4R-AtG&OZVJx7K!?P`DX7hJt-B%E4Ja;bYeCF_3CL|GBR_u)5M%pO zyvQkADf48UZZ+3iZf;OG&Vt?~-HQV@RobCW3~xW>?e-(GDFyqcshAi_c_UYfabG()*{+SwmicfmsH0r4<3*2pocR0^$9fhq~-j^V&HRS<8A zyR=Pi3pgfT;UftoFN*$eg;n_<`!;7-R>4FTkKla!H~T1BS>&fS{Y zpC2N06^&Xu&dB7w8Ac89D_8n{4Oo9Hg)6wRYWnF1og(O(j~P(Utsk_UXn7zQM0; z`QEf`4TvqEPU(yBDQwyQ_;U0DJuNqnPwfaB<6#(m?sj&D}zE0(N9s zFD%YfkjxCGM_D4XT_&X5cQtw$1Jh86#nFP(NRWr^!-{S&g=lrWlNl0Sz+um~r={Pl zA^nj$_rIx~4oxvcFaB%NgIkYqJ_Qbxx>^VCnIXTwHd`R#_Bez8nuxC0 zXtD-eAcsxgYIgVC>&X&RJia3Iiq_X<0dKcS>c6Z%M;v932hM%KzT`;xl%PzdiAj%u z_?fqMx;dn8R~7|yI(0z6271b?*MP|lCx;Xj*Un_o@FFl21fK-#FLqr(c{=>=34AL@RAS&6`Bx}pZVBO^k2(V*notC9F8yr>xle93NQmE6OZ`(-mkwjE5!I>w{71u(ZN&mYL|4b^-p+_MRF_2& z3$*XOaE!SpJ1sYp{gJb^hOIzpHY=;|ooQ%H&C!xxciL70KhFSPaTE%#tN^}!M&pI02z^#8u>PQv_a5xc1$P_d?YIyTv75&xv_KN}4c6}|VP1sBi$cYITH zipdDdsD}HHIrkp*oJ8#;;9b~+{n4~|jQdEBW!uqePNsbON?@xdjBgz{=FNqG8K{eF z-Gsbz;@K-3D($>f!mA~Nrf$8g{daL zKF=00@}>VZ0GgDFytKbIk!=6l{>KUt{+oHbH7fsepk8W~1wF?sWO|E~EhC_cQ8D07 zCkMfUL%O6?`4-)1f?S|O)VES<^|!bvsT~FGk!s=BawH9di&&%6Y^8iGFJ45lp+`~U zRk7Un=oxGO8XSR05eCIg`Txa<|Hf&*p5;K_fD&LN*=~k27`DpnoqS zF+-(d0B?|K7<&3a3xOf7VbTX;;^Pc{DBWeickCd%Imw=Y2W%N4OeKjQYOij!O+H8 z(QKzlCVHt!b4=*}ma1;I>s|3VAj_#5+W%oDF$s6)kokp~u9fte}g*`Do5_2{Ns$Zj}y;_0&*#enGCqRz$Gf?AG9OTdPR9ZA&=xi2cfEsH-}-O|3@$*Cc3+x^Nb z1kfpAAwT4YyxOyP<_P&APU(vV$)a`!Ozk*%5PgzwLz|veKh82UdlQB;JH-s>*cruI z@xCvj;zOAcj&3=u(m&&$jl2d7#pwt?;1>mk5=gFIi<{v@{u@?T4KBE|4h!s7yq6bs znxgB7kVq@NCn-C>o6oO-O=cKNKRP;^L4G>-UGUkspT=|$m)(tM(+6)= zx(euN1pmr$B_T8bRmH0_4p1=+E^NL-y=I6Noy~Bhq11z!Ev^94@pPILfen-uktV;B zS09Po?moIwL?6_)i3GTw#Kf#ufB7h&@~j-V*r3Ol=}0F&R^)L2e!qH2e6%~38hL&V zxaz}7q)&WYy}|Dk6pODLQF0X-WX49Aj%zGG3&;H}s+yS-{k&x_>1K2fqZsB>5*F6i zEw{iw+4Ieb#w9L#NXaeo-iNo=e3k5tobxB!jO};Rds&&jIZ{Gy%K|eQk-F|U3=BBg2V4;EM=FCSfoqQ}$qS(8u(Gd=rNGb@c-@L{Az9qn=n;00yVdzK z(BnsS4z{a~$3-4#WQ5Jzj>};7{(CE9TwLU&`sk?)kBnLB3yn8m;l{TABF#Y6vA=q4 z9K`zvl!?S44~QctyKq*GbL4Y-UGE>&2cb_iHmlufPd*Qntdkg)qz+5Ixc zHU0QM(!>JDTcTu3z$82Q+ZKyFI(zLCgX-x|!#)Lz#~mLI{fg?|!KS4RJHOabJ3u0q z+GF-R-e*wdc$+>04tw*54&9uo8OeH$MX<1)2#ZNAE&iYaGdHC;-A%^7)^$lokw{w3 z-S3PX{KhL^4Yo&1ZWis;>jr%Gjc!2f{v+J}@iw*lwdre`g7J?igY<)N{aLzeKr@+Q zy?l!mK{7qc?3wnP{Y@>hOJdpoFih_Ug`D@THUga34G9R>9#?4etYez$1-F30!SrPz zO!Rs*{FpP98t_noAshCXo{I!Mk;EdGVl#o8y+Yo6Sct+^_1m?rlj4GUHbd;po=5G0X zl9+u<7N`E8^@mn?^d)8hST8!OC$J)(=BmUPB&t&pcq zJxhDV7HA-MIxeY2Ci-$#+|({Ug{14dnULx^W}kSv!#1I$A)wa7O!fkq*-2EbPLJPj zd7SHKX8-SD=y0llRo37MvL!}{ZB4pIi)=O9?s&gzb|l4QjDZO55~d-d^~IUH2LXk% zqYlc88uGvq+&8LEL zut`q(-bWi((FN^D(++qOp57jKIYqZWi1-G?ItbtLU<3L_$;m;V*90Yz>|Mv-rqo+C zMK}W9y=&ZP9U3+5nGZ5OIh_6JreJ;;e3ATcf+b)t8&sDMF%s$~qfYn5w>KD*FU4Yw z8EN+B4-V`F#k1+|m7N?Mg`W5S+7zeG&`xk}7zL}uJhZUq^SIe{3XyLWKfgG?KwD{O zENe?7xhS(o`k^)(1mJADS`BkIv z^ovSfey7_{PLBPUt~fgZsh*tB)9SBx3MhAFQO0l?I#7%;hc! z;N}8x#@<#6hPs`=`=6Agn&v;|Wjs_>y447eX#^LeWg~ac1JjCT{e|*;`KI>KABL= znALN;+95rP@y7>^8}{%jK#De9o|J!7Al&&KLs;_BflYKt;Oqg`V!OK!9j$yx2r#k8 zaE0F&>JtfwR%#{ft0z*T)NtTfl03=U_tFCX>B0{)i@)`DU}4zSiRxe)ZT)WLjvX&s zUE`_x^;S+#2_x}Zf3}<5cDp8)CWGD!j)J0zkyjpvD(_Rl#3!>c#ORpQpwqE0bx(TK z{NNVI(RFf>FEEy<8A^D*iJFg0@DWn=o2xw(k_#{V#S z!AW;=g+QfATa9z)xc4{jvJhr*rYT#knd$=kv$3sEwrXFQh<0DDRRao+<8(QattW{v z`JJgWA~mOFg)UKCuPGQX5qkOwe~`@`)-bgWYAG@?wvQ?k{dwgKoCgqy1!iBhUZza4 z8}NpbSX|Oo#2b}e5brI2l7h3*(}jqe2$2vPsY90_!|mi!q@qt^z|Nu?4PDw$_Lw7_ z?=HRCTNTJGt{lFlzxP#ge}%h z;Iv=DDgR|$6`?rzKf2zP!|gdeVuLs9Nc9Ietk{Cr&9brGB8{nwi4wqiv)`u44zhII zpLUAL{f@Ff^Kf8b-naC|$?wL$pLbkI-e3(V3AjQSVlDjN{5r91$L%IA5B@Tr>_4b+ zVna&9IcqD=38X&9{Hly^JHO};BK3v6+F|<1>#s->T32 zPlkG{zXgdvRi|W<)qgqJ+Z-AIEuLJgf2T>m20Yk}s3*oykIX6*fje*_Lc@G!_`6uKUrn)3IW{vCrBFoeVUdbd0?wJEQ~xuEV>nJj>6`WfdNM zQiXM#YLlZmTmK0ZeM$Y0LaXk5PJ*JVH&B&ZmkR}b-o(SWu%>xX-SN1Ng?1$6J)5|& zS@O;q(*>k4_;*W1y%Tqa3EAv9>|jFTAny2n4no87ojt5q2y+$byX*M_KLZdQJg;d; z+gahDk>J;`@D@h2jcOp zdSEeamR{ZTwGM2#-1-8R7Ie4AKZSbm#W&;2(&T78Da%h5XpNpzmlxUsJl@J*+I3sE z8s0IC1nT5=|3jq2bQ1fDAzJ@UP+H!++SkH51!e8`4h5={=MEG5sLdyP?j<{Qp5vl| z97EWUbXni&HK|TRPojO;z415OCXYkZMrMy1|CjOaoWB2kP|dHUnKQaB*HZSjPrMcr zLgRjF9sW0~2)DrH^|sZi_i4H}%P46;XkZ=m`gZ#pJub&8hz={P5Hn&RCWm+v^iaL^ z{-G*e5&2O*olU#9YG=Nel_Rg)m9pUeHDIr=`mFsfO(``$C6(Pb5d_EZn0sCM#l`(~ zuRPhf8c+>LS+IRO!s4;F1+s)aW`8MZ{&iji(E_HId%l6gp1fH}F@N?N>%82nr2GYq z&=9}-io&`r$I8TZDgBKfD&+kqh8y`2_i%`K<#{CkLO~^Pf?kGo{oB=*%3ND(BBRDD zdtEChH>)%5ZTGWp`r{!j+r>L%^MS%#!9W(}$JW8CjI19AQQ}-lX`{ zq1gR0{ndFt&E6LE;U}{ue+N*vvS1G(lj+!2k_BD8vR4v*fbVWehrN2rZ5SuWbs*gg--XDs(410&$<|+z`&acF<)NsK_5@68YRIiPFSjb8 zbv8-dZhyKvB-QYlt^>5kh27h7A)*k(O;jfV5Z}?{S-{&C-(2r&0EIT!#5#D;cF;yJ zH%o(7;-Vsc30M{woT<%Je+>W<-@18yd#&2#5Zk(U0A0qf!9i ziaA0qx1|^*6PXaZyIjl`_JSsh?Ca0pAFt6N{?GbVaOd&mUrW(R@$S{!E9)wT+n4e$ zRp&~w3jPJz5xx}8q%f_a60t|1vpY%$<#U<_@B(ge-0W<#v3R3AE#{YbnahN zV14twEI8F4cZEmf-C^LBj{c+Jdh@TiWRkwm*kjvyDzk$enOX8~zg+p@ylX(yrQm_o zaZ}I8e|9|y=42KwM@uR0#cM~unSVQe#0!PBxWKL$m$LB=voVF2Z%Rk{9V=KFqK5+B zONP>l4rziZyry6K=2ZE9xe7N+Zjmix4Z+QTteIx`{Ji9$OBy3aUfxw{K1-OrGFSaZ z{){N8Ir2W!wMWg;JkJhwUedC2Ckxe|d2}<@iOl>W~LHp;(&sNO83D`&35a+P* zl$U5(rXMCY;I(gy(t7J|_76El5;a*7x1H_@*1gqV1ny+0eaE8DQXK4d$G->CvAp3G_7&58I*klfI`D@{rYZU`FM&jue` zAQ{CtT(XTHaANL3B(tE>JA-Y$F3Gd0@3Cm z#Du1ScK9%6ztd$n=AjSC)p@vv!pO$(dAPQGnYfC^BjT+Ry@*z0(vh~jac&O!)XYLg zO5dc=Y|-cLOYZ5rlI zApNePJ06@@uLYENu+EUK2{84p4a_3L_Ns-}Yrm4`?TBzoFhR7Fx(bhh6Oxxg1f|wYODk?f3o)`tMxS9Xf=0T6J`M}`HQjhAX z4_59sw4#qR;c#*xJVd`-Zl=>AzBDx^?IvLY@$~PEmQmSkKWr$kOEJG z#a;wMNL-u+wf9^!c7Tx>udG;*$(T7N1Mxm4_5F2_`)_W1ZI5|}1FH5Hw|S1ZSG|&w za?AQ~J*p$o#t=fH_T7X3#3pZNHRX;nKh*s!!1>GiMHhGWvzmQ3K^g7z6Lo0sK^-tB z)4X&6`C2_pZNwuTGm|Tw+}3+G-3N-dy$19>7W{c)8zAS0fHLD;eqzXc@l$_Zk&T#c zf*zkZ2ZZ)_EJZlf?`<$%oIlEB5|5}BQ3l%%&RjNGQr7r{D@0nCuGa0FDSkbjGaT^- zBL=B{hKC-WN6e<)Rg?JmDvC9<8bvwKZAs+yyy%Zo(e8aL{>ms!uHwA$Byja_20rFQ zAmJ?K;`@Mk;*>n5#q)BpmQ<3f)N??|ziW9XUBlhoxu-4hu)F~i#b(fk3e9CcL?&9~ z#H;L2!Su7-O^#2eml%Xie9e9|(?TF@w@cQJuk}e!jz%v-N_dfV(Q|AFq>K%;snE|$ zN_Xq)kT7`C$w=5DucxmR#wJRt`?b2bkm5dVd4Dato@@YJc_!pJcMmOhZTm?1Yx*}e zf9ILYPF!&A_XT-WN1OJo$8O#cQ*YjkyCJ4a#?&ul>xsWHNPFz^N}NQXp+xTiI-b*I z{Lg%v);&-R6_rr11!)>else_}V#@j-@v}QQt&P^g-`|JNPDbY*n76g2sk_En@vL(R zOaa118Re;N`&Aw0?~K%u8Cpocf$!5#Y|;56OhQD|UfU1fV414vUTvmtCIf0U2mqz+ zUR`=neY4y7w$@-jp;mw`HIW?=TY2E{k-RyVX*92@D{;fXJCta?)6vE>V&K|eXxfmh zc*WL@+3d86@1;}tfPh}uMu+*De%=%>G09P@IEcm`I};xAZt1qxW>Gv|XM}{4vct1B z_5ySJxLW?v143ZF7&eimFEa*;LB2AT^giV>MT#B4jUPRpt5>&g4=Gcy;Et{2@TOYd zt17bD8~L8Z!%UN^VMni>sJ<~@ulrutK+a%&X}OK6Q1%Oy@p#WDqnSH?b|w|V+9{~i z!ki1I4tvZ~BEL{9>lo6tItGbzaYZt(_bP$ngQ8SS=4kDAUdMbT^#=F)#lfS5^Wxww zjLL5%<#qo@ISp)ry)AQa$?zTTpr|)IRM>V;`vBuN690*}-?JC&M0Ze;Dt;`XMCN6ZrfT+lXhr>2AC8{nVRe=os?gG)bI z;KeHTbqUk9k`BomV8tHS^N38inTtQQ9X4U7_ZeIM1vFzY%)8!V2794#JfO_k`l!wx7x1LL^KNT?!JoKi{s;s2pPARcr zAqmC8@fWK`j25FF&K}qzGj>Z|2Mr^x2Ty-GLnSt}Q*K&y6V&OFs=$Vd@3c;bDkG*p zFx}#cJ(O`K$RXjAn;n1%dyKPi*wC|*^zy4*RQ+GW8#E&Vk!H-)<9Yq%MgAa|aYvDZ z`(UEe%_{;z=U;B@NRg#;Earvx;zPtZaXE|0QV$4dNLKQ|po)c}n#x<-C!_mJWwaZg z<}4jOzZGF&QNS`t;m5mwYFhm-GmB9`WX9>QbP~O`gw_fJ|zEZH-N*-?GX0grZ-ic(A*dab8W<|T|kEwMZyP6%tP3-rr=Y6+DM&3tO z)nuhg(Wfv_0$aQu%K}f0Yqed6?9-$oqTi|`u(HUky8bFi$)=Qg>Cke^pPMzqU^RdQ-v~>fn4a+>YmGMXL8zlS*?-pp5pflv`J zZx#?Vf@#ii7{UbX=(;oU}zm%8$UMNJ82S~>#Nlq`tg}zy}$d>_0;V$0W-8#GrqdJL>{#;YK9ZI$$cZiw8 z9{5u%>_Sr1ml;iR{MP$3 zqWgD(g7}Z!#;(H)53Q?AAcJ{M<&<_elp`=(PxYC;z?C^kXEwHz2%Yjhlhr1$UDG@$-r?^j1+O=8 z9hvzuqys}GR2*w}cC1Bi9)y`_t28L4R2dSnaZ!DObz3J8m*(j*ZunS?L5y#QD5vgb zaS^TVR-LSIweX908>v$X{(W~JNG1UTpM+X#43}Kjos-8Z37@L3%Q6EJ1QvP4c6DQ* zex`HpuAGUwTI(%eMf<_T$U_1MRH3Z3$@3?-4|KNpT&!4nSc zLz=z?!IR&g23MMf`IQinQkcr-GYzq$LN!v=N#;#8#BejfL#UZx?keeIiSWXuUVGzb zQg5(RJ+ve|l&P{1V&@?1Jp>r|b*U17b9cA^0ICtx-q{VyX>D_~< z>le4q|A;1mSgglSdkIgqO8g3bF(`O_<_FH?_bt<55Z2>VF9XeydYOLblv(Bq=lXkG z0@H(5n{kiL3JMGTL8a{WDB;!ZPBYVBXdzmy1$`N2BL*@mlR>QBmUWR&W0d`aWlX2C zu1M^5wvw{`5iz3;Nw+zf^JXrlM51G=wBHseUi>4{(p47LD_b)i)zW?+q~54rOwLU! z8K%XPMID!Zaa4V;t>`n^#w}j9w^ob*8P-km#joY_orr%g9{o}JHSXs)KfMKyoRu^M zfAJ=F(u|GaH491i;g~u(-{#3bqEslKm`E=71L1o5CKqTIpu@IPm^>|O;vnEKhE!kD;$p z%ycAym@aK$Wi$+!vq46sDYii^T%^|aRRq|l9gfCTjd;!4DVo2mP2q(O+0VRs6hB;= zu@LZsshU3227Gi7B(t#0;B%=OBky@f@{}2n#3+-a;5kceAUM87N>;ns>0@v76BIC~ z3UZ1c$sR+bE{PhYfgN()*DSkpbTDUO2QzAOsQD%zJ`<;B;Se3rAB9$SB9L2imC%TF zhOp^oZv0qg_iaI*OwQeal;<1Q%(=*0Cv{Fl>cl3VY3Wu{u-aP{tNuLx zlUuxuvDXLR+?lZ}gdo6JIwlH$&>=*gcjHVBN0Zcb9%0Vbr&#!({Jd;5>+U}oYj$do z6L?}+@gl((?4Tq_fcUzNo&!vBkn$DF9&ZkcRHBCI-Q5xQICS-w-H_wAZIS2_(Gd4d0_S`_go1(5zp* zXEYq{y+xL&fwU|n|6<}sXE%NVY@x!lI@Lrv3WK{lE-AFQU^z8H0LBTI7;J;x;{jmP z5bi`_OVGld{_ibsMJo2*hWGZ#cLd{V0~kB=ws zdBw??1cQ{f1i!6OmGsgfJCOigfF2$cn@rheL;dLVW~Ot+{!!DzDm7*5Ynucjj9nQ`vn1Yr3AumlDk8S3=Kh}saOdniqSTqo zr*NI3^KGI3zOkSFYTvkeqZQ=ywY3%XPBYImddQkbA{@D+WMdb8vCb|n> z9D~e3P~&-;z1`VBB;Hf3YVutFU=p+eXeu8nkv+lv~wZXM)Qlh!65< zln$-h5;teDSUub2H6$vL3V28SVO?^VuXf)-6u{jGUdvijB2m zRy2Ukc}Ca4_mg+UhiZ54;&piS8S>XEAh9bMXHc0YM|wJ5C*2XTzV}t4A1JKG^yh+R zFwbW$XH&^5+u7>@fn6Pk!BJGeHjUM4V~UdE+d^_8sKn`+glnP|0IjM-P@F9CwRFl* zMZ8Clz9s)wfi@AYaVFOJ8W)~muj1xu@aDw)Yi35>Wf&Qcb6I42UTpML_X-Der$j6-nx&Bv=WW$ ze#!Rd9;IUY*^Sp0aysAS9}!hv6i1Qei$`RT%?Q?MUueE1%*a-ilJ)o1#TkLdm!WP* zsTZ7gUjrcUGaAyldO1m(9#4$E+0~Dgx2Rh-t5WQ=Pv9@@^aL^^2l6w=dElRo>5$8kqwETb@^*4147q!3jHdUgT@cl+(b{uh3&j<+fSDw z*QWhZ@CC7+qxwgnW*>stqYscJ<5nn_fH`g0N6&F`cQ!8v*0NqSMmA))rQY(RG4MgCg8x6F^AYykcRdlAWDwUexpuz0HD!XLHonIh z`og|dwr;d@$%$GnJLek<6g6dUX#!U}ZbVB^eX}zols=M2*HD0Ouj5?~Hg&ClavDfW z1$y>zx_nH5`pwO?*KPwJ1>dz2AEs1<5yc#8y8Wpj3(@r>g=YTHm1yfuWT!+iuciZ3 z3Lm{92=TOu!kdYBVE+;A?+p)e)=at-orE6k1&3MUv{V}$wGi3l@&4h9b2s`pf2dPQ zs=yD;V0sr>M&^WXBrSwVX}d+Za}v`-t~7XlDsf_|pHX-u_f3h||0jlRDVL9Cr7@H9 z<$5jgEgQ}lUBu9UR-ODJZa;F>iV&cVApx!?MWr}-9F#h`Y%N{ot@Jv7>zrb8?#r+jrAC&b5mwrpp9e(G$A#xn0dXNx7{7YvBv-3H>+LWk-hSt+ zBCVzoZH-SBVKXd!=DsqeNUY2*AE+{p2RU_R!8?Qwl?6of;!#^SLN~zt$H~c$>ltX* zyTTA2X2BCv11DZO7Byje@kjlH4v3<_e=m|=RXfg`Klzh;oHI~^YEpRMdRG_ec$-5; zLiGV+rD@*v9?kJjpRmOsP$w%)U3bTclMPbF`c^bdKit#wMTy1lZ8?eL^;`pM8?e5}_sV zL4~gH9exM6A)}uW;-aR?ZOKzcA45R;bEYzcA0D#CjdnqB0K)7~x8uomV@+UuHUoJ% zyn366e#r7HpL+$vyFC=zy{elv+~4#`UWM3JKe7~CqwOpoqDHP(LjYfQzz^Jd4h91q zNd|c>E#z#$VZv3z`lZJDOXUY=1TU4E{M$_)>wbDmC=htVjhilrt%Ex$#0#9dPvTnSa_#+v z+Z>h+vDv|YM8Yg99ERo{mq)!}Hd>Msrwikti_(t+U}2MAU12z+Ldk5JQ!Z%YV`-ZF z^-U)LKRo$vZV8Qgna=m19Z=RTN0wI*d*_mQ`E(4>J}!Y0dl@kBz2ONVS%`o{PU-*1 zR&jTb;D>6go%Q)uox_6C7il2;%gl zgiS@ndUh`{Ox$BI4kvt`)WUP4eob=@MaycZ^wzw;)LD$)q* zdU|?xC3O|OQUEK%K~}Jk>eeJC z3vMem6u88L-jEw?V;J$llY;Eur~inY^X(hlT>1GPQcl#HNSK-yltZkD?+5(l|6o5Q z(T@*lE_>1Z)b5w&r6l1c_hPD4Gab@Bj>Hna6SvgpX+1@2NGA%27hmHpc8eRv)!s@%%uY zm}vbc@9nt-HkhYV(J_Z~Zq;~})Yoj3Q9mt>7)pFZd+xuW_>>mQD^M0%#sZt{Ye-f` z@ic0&tTuatncF!jRBID~BRhDPaO+csD)eUS+Z%88)fd?I>pslc*uh?s)XZTX!5S1< z0uEkaqdQrVz4l!*@%%L}_M_ivlk$Ss#~?2Z_iN?%LRugRykNpXtFxwsnb~)_j&VXi zc>%488S~Ec5{cw}+ay%GzRW1CSrb#SjDc@_(JrOeR8<8@iwER+AJlYOW(zd23blQ`IH>vWBUVQln$Nt?st~xBpr#(l z-Mj5;yTatMgZCIMubiKq6Y}-3rP#&PMa&S^r9Jqr++~*NbG$$WLoO=!)udRFd55|0 zH?NcKL1-4%eXMK`6nH<)K|ow#I6RyHNjszd5k)OAN|D6^Lxn09(V)ICSWkegpRBz@w$WR58CXK($TXnH;};1CXJzvy@y$d7XP!T+w;I-<_c_C3Q8 zKf}lxa;$vYyO3H$Y))LYYz9lvE@LSsI+J~CJk?ctd z{h<4N^xlt4+t&KJ3UR7=2$Iu^Hcmo@-!{X{;L}cUGfXAdw4H#LN}Gy{9}kH*P4m-` zRYPoq+BII0NNX%mR&3V1*&BNpVZ33R5R{aNVXz^%g``cs-q~m7!(hOBkXSo)vrpr% zxg{SG_zX&emkXCjW&H#nPEI=fo=(pO-I?!q)8zD!RYXLDfkIJ!?z9J)N8+YDgvH7; zub7i$Hl59gDc7xSYr0nfNzzlZ3R-0_yWi^kEZ^)D> zPSW7%R!U_%oRv7xQ<^osO-PLIEI9N>mIdp>jipx?LhGn|JdQzkmRGw+U!Qd@Z8D18 z<)*E5#RW!eXy5T?XI!`n#=BXTvKp4XVc1H6E_;R3?O*%Hy|2Mb>MB^0D=cyY00*zO5HeHxM@UNuuHN z)xeqs)E?7K?VP-ws@;TydmV%1G$JlkBXBj}dP@%u*eo{rZ5^E?y5reknPOG(fYr1w z!&TX5!&OOwiK8S?C$d2~=xiJ%&qIyE3iaiK9L1^c$AvCQSq)1Uze|@svW9qQ#Au89 zrO1kMwvlK3c%1b&tyCD(EAEEhp+p(?nS^{2)}F^?9&2&x68*S5DQf<6Vi@ul6P=5T6h}7>VW;wI`X0%@UIhanL@zW~z!slMeMjDrZ-5IAKLY_Y` zLkE>l@4wr{$dpy7cbE0U2N}Ik8^j&051}@zbmX|{i;qSsqPw8YPBNCVw(l6mCM%nk zDEVHGcVTbY$qP4ko5ox8*aOlb7E?ER&*!RyTyu^IEk>QWlI;?CzyA_`QziM^|Jo~W zlpP#{d0I@c$%5qt$$c(QLgzxaG!U>hO-DMYt6IVK;d=4Zt>cdN*bpm7m=oskTMxtC z7&X+NcT^}lFy^1>_bHK+q@DYtPB~G?xboYS1I*(FPHwcBYE_^2&g~-541d$6;9I|* zz{A&cf{)-m-d&1f!~W6lCWo(x{zaR;0VTUCW!vmuUQ{2MSIWistN<3HVdb?`q>&rX z(RP#uQ4meQVwXg@kt!xH#l)}=Y=ZYa9hiuU9Oyg`?TK}4?+S-iUEHM+T*iD)n#up* z5bjVigfgHv#MU?f6Ucd^PVgTy`tuS#223gi`IGMPlz8Dmd(-ZrVRP5Qw1bXq`4wLc zk2%8+WVWm=ljox5?Yhu2d!;ezrl+_5)h*~>9xhBTQtHMTyh~$`Ctmh4E>?q$s_yVW zrQ8RJT%_fz!P|*IS$dT?%^Z1{%rD5Di)S~JvP9JA4p5Z&_Cd*z8@=lhC=LG_%e zDOJqQ#`C*GOJ&tNmAgeH4dnxysTLRPJm=AycA4$AiRgmt+J_bn)2~KR#1dGCVF}rx z8x1&8A;5blCa{E4rXL6tPi2|t6t03Z&kpIP#Z;txHv@+qVD0ay#|WqYsg*XK|3~E7 zUz(n8Fg4a083}7uO?z^SLteiUx&%`li>(oF5(GECI`lh+Rczv#yp=Sf)^BpITI`J? z4IRm`Nv;#lpXw}@C664_V8riev%g{iPmhe^uk=PMou3k`&nc&znQVHr1d{w_5ySywU+)^L6~1 zR}5vvjk07OeQGI%Y4geHkU=aLwyeD#M#pJUTys(7(@i;SvSLC^Gk(nXKcXL>q?y<+ zQ&hyeNHLF{fHap!WqW%vJu>rf_`O1s)?rtLOOQ`;tf!V6u7f!gaJHtbX}@sKhx}gG zGck-g@4=Tdak{a<23T&^kcjHnV7c(y`BH3Ljo4MOzM3{VI}B|zi_{6UGpIZKJhp*Z zrg8&#+yHWhr{3F*VC^)O4TCnDZM5Km5Z@EA4gGjK>H>TVlZsLM0LrQQ3mYZw_jPli z*56PQD3>6ld{>+V)^2M@Y@DdKQWthACM8%MmU-ASZ@uMXgbFgZgwi71jxv3u^&ZQ+ zaSx~U1_3WFMvSDmxh2*TfZO4$&EtMy#*;0@a6r*RG%)5K!RXuzd;OV`T-wr?(2hXq zi^Qxy{410|iX*p26+Bv+QA1Y<-fg0McVtviOmU**`e2H#mhPU`vG-23*qXNjuUUE8 znwr0tOz_*{NE)uX%=W3OMtz^=%o?RDSAgkyk;zWQvq{PKqLl6ejrnFUVpRhc?wl?L zTjIDF3MM*oHn2-!H3EA4H`BdX&`27(SH;=#USI2R5=zR5`%%g_JFRXV6gA4WJyq1zF*P1yr=Wbr*+e6 z+CozkL-w>9C*lqa{th3#rb8h!y~f7+$!#<~yk%A+!|lXYEYR^fzn99zt*<%qE8e^1 z43C5q40cx%z7zKaP@$HgQsYcgUE>)?LhA*)_fyUQ{8l`)1=v?PpP@(nc;F^t5bzpT}dS67*Z} zAAb^ozR5B@EjjPYd28UJ71q>~i;-yW)Ly%5&J*%(^_r)QEg(biBUFnVr1^K2kW2CK zC>t{}!OCajc@$GOm|Rgs(jPL=R*(7SVXA#iHx022&tj4i?r_jOi1P-2B*!M4&u5QV zR7u;r{WdZ;H~Y{~U!-JIMRBrMc0c1qcajxhx$rgpursHNg&BYPoc-0^NHd>ih0I0V z$8!sMfq0fxu=e)QzCz|y?`4PcAdNDBTA53YoPPE5vc#iT3Uo7R*Y1W=szQfvB*WCn ziX*f|;x6w!-OdQ}E6T`A+kt0j?By$CG$s7&qF3})aDjF=!ZncY)*j#{dG!{`iZ97? z!}E#icV#iQ6A0EMkxOYc1V@ziQl)E=>&w0BXCVB8S&?99Cn{8~D& z{Tv(G{66v7k;yqi{97IoA(RUgn2k=tBx)>F$vl$IE{V7z(&GN* zcJGSt*>u=oK#h0w-q`<&$kpJy3|9)(FO^0Y$Uk*rMyqH0;g7lg1OS|)kuz2kJnmcB zDvy_iq7ID~y{tVKY)}gh_9$kM0_`6IhQmxHJf;mQ>1=1Jutnhi^ql@<{J2SlygI9K@MfM z!IC@oEMza#mZ#^1zw<-Yhvu+7f7iu3o=whL+AqT$B6jmzYVMS;fL4_{ZyNqc7C zSx#As453E-&a*C%_DR!!ovN*?ca`Vr>5j!DAE5Gpw=1{v+cp^?OUkIY?bmNO}UJmrQzb zT7OCtn)IptXwHjt)0G6hU6+JKrA={&)khR0q#^lfOyNjDQnWF7yo1AxW9*kvzCp{d zql7oKgVp3H`m%#x5#{)8GRKWFjyNC}BmzUQ4y4UJd#6YrhCrEJ)S^@R1tcS^UM94@ zr~Ipz^5C9Ro@sGF!Vy&IaAah_?|Cf8)7$nlI8*0%QG8kbE^UxtH{H#xFI98f zsG|ct)n24Ms;=b(i>*0leuYb2iAN60^^>+)wO3N^^y;=*FM|-eI z)b>S5c}&MiMEUHNxf~AGm&+23?S1|*E|~vl4?=zDrYJ{ztz++LabuzcXX94!lo_%X z9s(U7(skM|g$uN(UMGxvk+g?VzqIo4u@;i-&NO44_SbcghDQgoryhjv4Ep6O^;e(* z$|7c(Y!;Frjfwt-?P3h>EiiWYofh3gP#z@AnY*NY9ib&^7#mD3n=iI`y45BySCb3B z&3UoQ)keXeMqX=J=qcGaqr1^Xa%T)i?^qIQ^%1R_**jh6eWzSBWE@ui>V93~?wYZJ zldwRNFFK~DMkl~@AVz6a_Phu_jjYP6tDrM^X zV8x?Ak1h{i6`G^)o-JHpn7vJUJ6|tqL=sS7PCZf&OwyMFfQ;0HtmSGe^a zoZOK4%D=qq^1MB!r!{#({qd)S z_C&>w+{0@IPdC`%(tlcAYhC)ur%-WxTUE_z3!XHojEgA|;!H5YKL=|*65k{66mI3_ zB+&mj=jfDo4fRwZwy&+ctQ9quu7x275 z&Q(*oj_4r{ZxRMPIkk;FpVvJ(O42yh->&oWkBIJ~oAkgksVFp}h3Ta2%5$Dw0bmJU zx16^uq3m^$TN4dC&UZiG(ywvloC*8kRE9zo4u$%|Ur2iT{gsMQ$(C=0vL9RlEd>!P zS16F@?SYPL?5wB95I@Lq4MHZf!eU@~@qi?jP((WZs7@jCGQ$Pr;UQvHd97et=STDa zZye9Iq^$s z&<5#yjipj*+Da-1`~0n?sQ=Lm zff;WKs7Jq#6A}V%#+|J;22ChbgR?NX?l9G#0evV^;7*R`Q1A;a7|B9+m!2)i#ek|y zA3-dc6RUyQ-u`?ixKBtVYzSIo zWmV-vbxK_OEd-`rQgG)(taV}#8^_n#y}3hG(-N+@-B9SKy$GVsK#841^C zMk2aWn;$T)OeO0z8TTWt1aoZb>8$_+qofq3vmb8nt=l!xI7(=Ew5^#rHg@m`R1%X5 zTLNVH{t=l@F?>aFr^RS6^Yxz*bSGa?Z5f}^PtK;a0tR^FG`BPstgEB-ws5*|(Wk(< zMx913H?Q=k4^PShvt|Dfb296Ow_9JD`Fb_$BydDGi6G4`CV!O)e;sY|yCE~0;s_w{ zDyZuHxV1gg8H?g;nxn3gevM1-LTs))U^INmIX?s#9T9c)9T$Ezd3NuWa0=!KHDqz_ zJ!02^7fd}A`gPv!wG`PN?aHR%4_ivqu@2J~-4P41O3d`MU?3NRQ#icT)pdmG{O!(X zI=BR-ws`Y@`20f7^69~ucY@ZXJc9-`g}rJ|*Lao2AcUokNl#$8O__W_jVbwyQk~o` zzAR`EyPjmyHZd0Pe0Bll_{t&YJK#B03Z8K_UbOgnpR*a&W&w(XgTQPVH$o6L@o-U{ zYyB|lIY6<$ZL^HJlA49fX%}!z@W~X&hQ{%+$9~z;=+xoU)s8UN6GJg0K+$V7VdF0> zrYKHf_<^_8wNow`yhn3wQI=`!?st-i-8UXp41#SRdx>h~e?%=@??zTxL}pddF``0E zI#;2bt|#pjV#jfqwrYyoTb5E8du!*_rLXviSpP=`JXraU@pgmsh=>Py72inPP&Qwr zB=S+6#}DAi%))(c5+lgogxmnc<7+{yWYnCXe2Pgb1*;SbuO;g3*^ z&G5VI$HgXepJ?5XC;~XtsqOP5W$io5*7?>Su(UNe&2L;y>0szz^B3JW0M5L#KU_Inw~`#kg6mt?G4uKyMcZ^G03Nr6tRCO zvc(&AkChYQi;b-a zm>2Pm)ED;_wmCYzr{^vk=jV;I(@Y0{5Kk7w@0EWG2z*3)YLF}hZmZJrl^Q#|DvXyK zT1aE3O`~Xz-#7{vio0O)A;6N|6@LuGLJD0|^xQAbp@4#mx(K6N=4%~bD=K4p3pg|h zx90Eey}EH1u8}5;n*iavDwk%OpDbW}LKAIQKDa*ru|sGZFD9+Kw`YG&9L3mO*0h|e3KKObXDaTg`o>DmhIEEIjcJ7h#%m7Pv z!~$lOrKNRIpZfHPI{p!{bv5UX(2tX3VKq^1o{xWXAj7GG`jcd?J}P{iokxKkFYE38 zjHiY!Ep`|Y#I*($ws5w0^3qfeCri?X*XPQCe)&G(eCs{44nZ=Y3*+j`(7Do%HS#&^)txq7q zI+-r>Ag@ExU448qoNZT%CHh%A*}!gl!ce3w<6ob=_+_6gw})+!Qlsts_?s!qfJ1*t_!=*om7| zv6%7dk!aGZDu|8@K{)lz#-fV_Br8ELK&cHajy6JeitW(JTaADJ8vUfhH-2sS`^-?6 z(BI)@!eI_65iJh{?_6LqVQMr-P*oMSd#3Z!skAEcw({+jXY*K4Og_i^64XTuKR`cJ zpMCrP(|?&Q8uAyM#VaR3l7B=8qLr-R@9*!xn}JgKJ=skyPZ$Y2i_4297n(fR26u5z z>dKNnxrn`L>evTJReac)!SRq5{?%8ulqvXI=pk%q+7-z2ppF3uH1I3My-Lq%5%g=9Yaed-7g>cLc_?9tf> zJ(1QU4bzgpb=yoMfgZ*41u|h|)(?IUSCze}GX|M}on)9N2@mTtRtfB_C%R>g<~XvL zG-!-PhfLg2lGb{o9_-x0sgiaZyml$GZ9RI6#iCUk+3AkuyiRWZqfFSuHIDnq5@1z5 zIMQPf!%0zVkjfaHcI6$i9!|d-h@bOkxI%Cp$+kR5bYCv2J&Kl`F{xcxj4T(-(9N5R z{y3iVF%%GrUrI2U!um7#Le?Knva|W1;l2Y;b5E~ia@5YGui%k^;xOx;w5#{v{<~JT z{O2Wq%mHsEnS}npLcV%O7QWKEY;+k4t}*vE9g#Y@p;;q^h0dWCaZYQHX?5wS}l z@jY!9(OvfG{h#Q0f-Qfc(mbX-my85?KE}s zjk_M#Ol(t0N8W6SD-+U_eoOa(BfSb&&HVz`5s>O+=_B!@Zh3mtL^v~qizVH40g}? z!2w3SqNdm# zN0}$ztoD)+Zh~L*^W+)H3RbUNSwJ$Gu#GL!jt^1mL(uflylHNR zvCe4^KVkN9W#jBzKChF|m1EjaF{3cDo^bYCSGiQuOWI+h{Sc#@M+I)yW1iBv$%&g7 z%atBYz&pfWz*vGAk4i;a;_=p zYkOBC!C|#?^{ChfDZ&^L1W-=PY;IBs#R(I<_)Q=52jJ~@<614M5#W`94rJ@SZ?C3M zNb)x=T)#meXib$b_b`i3Te!kuP$Q!3o29*ryPo_>q5i(egaLufR~*qZDSeqT#n|L22jXfq` z_|Vn$B*J_J4o&ZT4z?%i8fg_e^jguqrG=EZJlm@54`WgOrxZ@t}eMj5;Z1gOPGM&WKgpcg1PUItxd zjv|NS#V4m;5+3SfLNO{eSX89ikhi^_@BK5*8^+Rw50mqYUzut%L7;HmZ=~zl4B$p} zCrgv6J8`1%b=Mlrlj;mio`cB!QXP;N*6_F58tMCny=YT2_XDh_-;Ivhlt#=S28v6Q z?>bapX&EzvV9P1d-Ap5i8CIGwSk!%(-Fh8Ul&ZOLY*C}{Rg|38H>KT1UgRxG5!|0odh}7L8Q8&gv!9? zUI80-d-vf%>=}n6Nt@@WKboQcA5jfsLnu!!e{UN+yck3fosXp2E-a<#OT(3VSXN~p zo)j>`Je%Axek`QB=PdyN*;o%ubi_SK28Xp$SjQ78iwgk#8K^OE?Or)b`U(hdKTO}w zdiFNpNE^D{MTZRK%zn(2tA9}GR2LTWB2n1B@|PHpylxvhlumNF@-hheCvx#tHtc=!4CyJyz(bWI-=hQ zP&;Y$a+pIP9d>DB+}*8JlV-gSFiFr&5uRi?*Ia$Fv@(?dO>t{U*@n9t*if1yo=KpB zb7khh!@JXjNSCmA8KK8A**D5If?Q?eZtc{wl5VtcHOSS>!I~vn+l2a_W6NQV_PS`<@)*g)uXiKIw4A_m1_ggD$|>}wt9i0p}f_a zR2G7XZ^XHXu!D*2ZVwK3^pEImBZ+Z{zvlDsl9TV7oD%A`oxo^I|6H6$D>MckUuB_; zqQ%JCEk2c#9uQ8Vftge`F#DcZbsHJSmP*!Rxl+hZZLUq^t@PBnPN36Ujfd+y3g6Fw z%)3VS>L6;jjQ+Bk0T+%_Dh}v225;$!ytuz%pVG7TLbwL=H;Bf$id3j9pWizrmYx z1gQ;Pt84b)z!NAE&sL(y0y`Vr@k2sZ4*u9?dmN~inJgKt;7vMbay#9UaQ=ruefe!E zwss?AzVn^vU<)3^tySogYzFC!2P$nRBi7-gzBvZQax4A6G}xQlITRy=)o*c#I@fAR z{J|k|<&hFXFaz`#4Hoin5>Mhn;7#&_0NGJ}W9$FXc?`ynMccf?MXfY9?VpN7E%rN@ zov&7pAOBrPTx&{g0R3#9?NE3g3@PyOr5vMj9`Rb)ZjQL^B%R2<#akt8zM20t?D>wl zdwW7i60UbwK7wK=Ffic!9KlzX>`yYGM`n5C_<6wAwc14Q2+&}~0#t#4QZZ>x%7&B^ zWECGaN+e)Ip$2O=5t7dhFvd~g+;=6C@+2n7PtX@{d`xkstXE-s+@cx4m;=vFO8h59s~W zA@9BAAug5GJ18FhmXTJG#iZ;zVJ#unKsqY@rln_lj)b#!?x#!X+#VAbBAEA^C5O`< zN13#Qf8PSfg;dFG9!#sV;XO}488C@vU?o_%=0;p;`m9s?t#mg?r%RRMW*3Bc+?;qf`N@sl!`H1sWv<8Ak1#E9;@r8EZU0yda=q&o_?S zr*hUcUT)~zWf-V&0aFx8tYFOoP zwq7*iCiVFFb#dkH2Lm3@7ed~Wj9%VB0zX7{{qlhd&9r`D{G4=s+__%#z@asob@?3z zq<_A0&z9VYMdCnceIuMz_434U!Ad5o$juTES+d-`R60%I^ZT&UG^1<7Nc?h??{KZWkH6OXyzIV@gCQs zTPBvFz?LHIJE(c9!Bgh_kf$DrZ~hUbiHmcMYZZ)Q|Ely}-f)=p7d}K%SHY}{^ALCTCP{}W7ULoP1&C3M)iT>F;LCjUIGYpa zqtXK`-DzRY5MAQg)DNOuo2_$A6tZAF^yn%uYHLIew^*T|8_B9&vToDYQPlY^i`#e! zPYnctjX7+1Ag&gk{c!1JhjE77GeQR+fnpg-+Z4tsCB9C2G`^Kb?G~Xk0oMhGBre|6 z{sqZ5eq{*J?r2j^VeVLo>j2(zIarR-}UHkxDN-Is>LvZGOub1 zwlfFkh+>q!r+6X=A;Fe;S(j;V(mkpZ%xVALSR7K_2aSoPlm?l=0UcK}i%=>}=}l$* zvgH_iE*R~t*_r<`fUt``3)ATlhvCiY=g_&_kj&7N>oaJDSU|wqG!ULzFq@zL2gmnh ze0$E`Y=+IsL}${o+c^lmqiUEi+gEFSx89P`X#AUBAU?lyU`Upg4%^3+j{A3p=@Zf-B5c57yER`a4#0DCF6isER6X$7246Yvk{CJFCJ>$&-8F%r{Lu#6 z=0_Ri!C~q!F;WH6rd!rL0Ll32!XuSQEp(w?X>Z;kbQ?*qldr&SsP@FlhJBA$TI6JI zjY99@H*P#e_FUSg{BB5Rw3??HURk!R$`AigKUR~mh8fgYdM{n}Nqc@P9Voql&-8b9||83z^%DSCmX&<#y`gp%eiH-PR zhi%N@+pr2>?Or%T61%o4wlS&9Cd9be+Slg{Hh1+DqS}}b@ zip4DUdwy#&do*U3BFV@hVMtiNKZ@-8s6&U{$$-^-PCwG)O!kop$s*w%QcQnXz=pt_ zWzvVR2GsYhJJshAFGA(qYIfP-a5$n(F2zE5VW6Sju!`~c+iJCyV07hndn{a*4mL3f z%C9hV6ul)sMZhDv<7e#cJ*^ZU>>#+=4B3@th)n_IhpmE>4bm3%DU&hUOl4@-`d`ds z?#!)z6SLrQNp*_MO^;>vaK14@cUy3en)5QhF%cb+cvoF-jk0>{5t@&x!tc9O*g#+T z5`{i6ZS%GOQBL3W8WTw{`hXt@aq1R)3qDF7O?gi;b!n>35y#!Q*NYE235-`A^+jh; zcyX{sXa=84R?FlB|6y%o%X-ZLCGnY&p~6xcIW*9YhEjay3+|VR09I~k9S@rE8@{3# zF$#<8y+Y={EmVwefj7pFMx5=9rhMqNqp12GfQ;xOYXNV0r_$TTnr*DOZ!znr^FocV z>Q`&}>nR%}GY6O6m9604 zCo51y5|Ta$tK*J|Bvv$!;DTcUB4G5FffiGQlegkF!@GoGO2m=rXGM7hK*ssEnpz#O zCj{sJLj3;%EDO{0KQniGcX~aLnTW~q*EaAE)5)RIZqKIaVfj$}sYL%bKltBe~n`lDZeAm8n-4#k<}e+dYSB-+mIOcW@j*ei3j=TpW=C~X6jhSYBPi- zQ`#iR^*AfX7hX|JI;LCl_=CF%KQy9CZ@R5~g;q}B*EnTLse3tk~a<^meV?LA|+V zHI+O77*07n>nu9?PCN76+BRL6@?rU9X0W&Em(W{o?_AD+= zROG9q$_;U^OVNB6YvDa6*5>zB6Mt&w22C;+!13VY773AcZ2H~Km!{v2*52ml<;Gi& zQ7VIDa-jFGV+(rop9c8%MIN$8Q!U-y+3vW8&Rp`mUd- ziM_Gh;6RgDr!;sR(?%+j;k_qavAOx5nHcPJv+p6=GM(eP&IIC%Mu5QX>UwB=S9UYp znsltY<@t0HiJi94N=?d_q0af5>&%Dr*Unlm>$1Ww_LRY-7pfA7Z{t96W{ z*(C-HX}12apL^%dfxG_zB#n1>M%(o`_u%@WWd@!dqy{HzC{-Cm^!k76NXk;$MQJ$# zE;G*sdhlwyv?~E>VyVs^K93%GG}2<-3eVwcm|t{{SzyAeBYGEOqdnuXAr| z_VyX>c1Vy*AV0U8aZT29?YmQyrMJ_ew=bEyLR{J8eLS9(OLbjl+-tGjVF_@`&gTn` zey2Vvo6hw+^J8(B%R6Zgu1o>XrM4X2-JgkXuy!OyKQm7h^7F=W_!l>N#l6(A+uC-H z;8@imSCX8W@#3kwD!=m`F7pHYzi#GFl}mrW)6W)jt9H7LtJ_IVr4~7RgX*06Up$)G zCvts6xA(U@zdp~JRZ@JuEKI(wnKh$ted~61=e~~a%vkOn;4r+eqCCD9+r4j$j-%o6 z>9!b0Y$MrnV<^5qN5`&KlBZG8cI_MeN^foLK%CgW#`vShsAHGZOj~_acRwbhI2+eR z@g4vuwBUV^s<&3pUp=}10GZg^l&wijT8b3soSZqX5lhwyb(>og-Rid{OK))bcJCj{ z_5ByqS>W|Q8YeBbr5KiQ8M?0*{{Aace+=sSSWk5&8KEJM7j=ip;=H_2h$pzq;jos< zo3Y*Q#r(7Y{S5xK=JnO-x*6>3{{Wrc-R(C55{wL(%wy{K@`c*9uCMvHbbSF1t8!70 zrry*uDv^PwoGo5Kaa7yWd`oMjmfSbUZ_N@=-9_8(Fk{aR;>t<{*Eix_cl<}K>EW`w z!zz^3iwy93R~S6w7BhRP-P+sS+jpuRM(kj1#x+nHTQFzKVBXVZrcy*)UUljO+O*v_;*W@R9U=KUP>$Rl*%%3tF5gN#`sDL{e~;~Nd_&-l&c^Ixx%q#a zxsdJw^Ndz3#u2|?)1Q$a%bcoY2}^Q$u*WAg!u#NL zpz38G%^AjIWJ{1rtl7n4+}1hHtlsM(wHa@U{jtuoy+ZYS`&c8ilV#|>KHh%VT6KJ;LR*UF zG9-H*Zn&&pgARDruH;C`>>k;P6so>T@#|lA(@MrscPRohB^o)p{nwueY2DuFNh{@; z$YOPu>(_SUb34*vjK zE|&o?7dgrP-=>$UWbSWlENVTdk*7cUHB@GiLzg4z`~F%}dVQO!jYzCy&yVN&>aM1J zvt#O#=gzY)YN5h??@zX}M*e8-cMIXyr1-RU_Bsui$87%qlXA1m+za*3(5^GAJxFbD z_H7Ie&Ci;jzCU=?G3vKJ%aCb=H+x9Yfa1QFqqp&1tEJs~5`FtjyI9L2b4mC6>$M)O z_>&p1Hr=u@##C^BPwB;LH^wC|hxD-6gqaiElPmLI`}=E;)XVbs_V(frY~)*;ul?WG zSNt>LJO2O-_=eu*#HuooN*)u_UZ>;V6?(0>>_cKZQQg}ojMAPy*#2B&Tn?XokHf#q z#5-g%S2tEu_4ce|T~CB=k@##kk)Gqc?8TF011%ovexGxh&`w=XUB4~a%1`kY-Ad?w z-`1UU{xvO*m*T88_6vnay7Jq9+Wps7Y;Q^(S4lmxFgGFcnR^jpH_!g-Ptxw+LA3~# zy}h+LetVe@e{b%pjB1~W{3G)EfAZg-C$>nh}Ac z-m+CCmH?Iy>-y`5_-bcS@g2>L!p+z+TjQO5AK>2y>AG#GPp63obG9v)nMTL$dG*H{ zoi|^u@L*Z%;+$M46DNOT=DxlYt^x6E;p zs=k+vH%rj;rszjz1KBES>WBAVU2Bf(A^A_AYBzrO{-1mH)pMG{a+0gHZTaW@HJ_l9 zJ(+IIB`w&plbiW*{Xa_9t<9!A)gV7B!NoYQ*B+X$Z+Vpe03-gEJ-p-e{@S~5ep~UJ zS|u%r%};yB^8Fn=r~d#ic5Z*<#z?%W>#XzM`F9kyTQS_b&GD%{$F_d^cKxcWRlBUC zxbac&`5*Ppj5j57j9O$xJ+Jxc*)W5-ej=Kz>K1a9b1UbJUO#^NGu>kDLfee!O7rdM PtM>L|*@;h|U4Q@C>3al_ literal 0 HcmV?d00001 diff --git a/3-NeuralNetworks/04-OwnFramework/OwnFramework.ipynb b/3-NeuralNetworks/04-OwnFramework/OwnFramework.ipynb new file mode 100644 index 00000000..48a8ac16 --- /dev/null +++ b/3-NeuralNetworks/04-OwnFramework/OwnFramework.ipynb @@ -0,0 +1,2809 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# Введение в нейронные сети\n", + "\n", + "## Эпизод 2: Многослойный персептрон\n", + "\n", + "Дмитрий Сошников | dmitri@soshnikov.com" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "notes" + } + }, + "source": [ + "Данная презентация представляет собой введение в современные нейронные сети на основе Microsoft Cognitive Toolkit (CNTK). Идея однодневного мастер-класса основана на Neural Network Workshop в Microsoft Research Cambridge. Материал и фрагменты кода частично взяты из презентаций [Katja Hoffmann](https://www.microsoft.com/en-us/research/people/kahofman/), [Matthew Johnson](https://www.microsoft.com/en-us/research/people/matjoh/) и [Ryoto Tomioka](https://www.microsoft.com/en-us/research/people/ryoto/) из Microsoft Research Cambridge. [NeuroWorkshop](http://github.com/shwars/NeuroWorkshop) подготовлен [Дмитрием Сошниковым](http://blog.soshnikov.com), Microsoft Russia." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "notes" + } + }, + "source": [ + "## Обучение с учителем\n", + "\n", + "**Дано:**\n", + " * Обучающая выборка $\\mathbf{X} \\in \\mathbb{R}^{n \\times k}$\n", + " * $n$ - размер выборки\n", + " * $x_i$ представлено вектором свойств размерности $k$\n", + " * Известные значения целевой функции $\\mathbf{Y}$ ($y_i$ соответствует вектору свойств $x_i$)\n", + " * $\\mathbf{Y} \\in \\mathbb{R}^{n \\times 1}$ (задачи регрессии)\n", + " * $\\mathbf{Y} \\in C^{n \\times 1}$, где $y_i \\in C$ (задачи классификации на $|C|$ классов)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "notes" + } + }, + "source": [ + "## Задача\n", + "\n", + "**Дано:**\n", + " * Обучающая выборка $\\mathbf{X} \\in \\mathbb{R}^{n \\times k}$\n", + " * Входные значение целевой функции $\\mathbf{Y}$\n", + "\n", + "**Необходимо построить:**\n", + " * Функцию $f : \\mathbf{X} \\rightarrow \\mathbf{Y}$ который _точно предсказывает_ значение целевой функции на новом наборе входных данных $\\mathbf{X}_{new}$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt \n", + "from matplotlib import gridspec\n", + "from sklearn.datasets import make_classification\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "# pick the seed for reproducability - change it to explore the effects of random variations\n", + "np.random.seed(0)\n", + "import random" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Пример\n", + "Рассмотрим пример двухмерной задачи классификации на 2 класса. Примером такой задачи может быть классификация опухоли на 2 типа - доброкачественная и злокачественная, в зависимости от её размера и возраста.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "scrolled": false, + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "n = 100\n", + "X, Y = make_classification(n_samples = n, n_features=2,\n", + " n_redundant=0, n_informative=2, flip_y=0.2)\n", + "X = X.astype(np.float32)\n", + "Y = Y.astype(np.int32)\n", + "\n", + "# Разбиваем на обучающую и тестовые выборки\n", + "train_x, test_x = np.split(X, [n*8//10])\n", + "train_labels, test_labels = np.split(Y, [n*8//10])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": false, + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "def plot_dataset(suptitle, features, labels):\n", + " # prepare the plot\n", + " fig, ax = plt.subplots(1, 1)\n", + " #pylab.subplots_adjust(bottom=0.2, wspace=0.4)\n", + " fig.suptitle(suptitle, fontsize = 16)\n", + " ax.set_xlabel('$x_i[0]$ -- (feature 1)')\n", + " ax.set_ylabel('$x_i[1]$ -- (feature 2)')\n", + "\n", + " colors = ['r' if l else 'b' for l in labels]\n", + " ax.scatter(features[:, 0], features[:, 1], marker='o', c=colors, s=100, alpha = 0.5)\n", + " fig.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "scrolled": false, + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\winapp\\Miniconda3\\lib\\site-packages\\ipykernel_launcher.py:11: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n", + " # This is added back by InteractiveShellApp.init_path()\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEnCAYAAACpNTSTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOydd3hb1fnHv6/kve14JHbiOAtIAkmALAghgx2gYRYoFChQoKUtlA7oD2gptJRO2jJaoIPSQVkFQoAQCBlAAsQhg+yB7cTxTLynLOn8/vjqIlnWtLVsn8/z6EksXd177tW973vOO0UpBY1Go9FoTNEegEaj0WhiA60QNBqNRgNAKwSNRqPRONAKQaPRaDQAtELQaDQajQOtEDQajUYDQCuEsCEiF4nIOhGpE5FOEakQkVdF5NwwHe8OEbnEyzjuDMcxA0VErhcRJSIlQX6vRETuF5HxIR7PSBFZJiINjnHd4WW7GY7j53j4TInIz0I5Lg/H8Pibhmjf5SLyTD++V+I49+tDP6rgEZGFjvEs7Md37xeRxWEY1qBFK4QwICLfAfAKgH0AbgRwPgBDeITrBrwDgCfhcRGAqCqEAVAC4CcAQqoQAPwYwALwtzkFwH+9bDfDcfw+CiFCePtNQ8HFAB7sx/eqwWv2RmiHExV+gvA9j4OSuGgPYIjyfQCvKqVudHnvPQBPi8igV8IikqiU6o72OAbAZABblVKvRHsgoSLY30Qptbk/x3Ec46P+fFczCFBK6VeIXwDaAPwpwG3HAfgngBoA3QA+B/AHl89nAXgJQCWATgB7ADwEINllm3IAyu31jOPl/n65y/dyAfwJwGHHsXcDuNltfNc7vnc6gBcBNAHY4vjsGce4TgWwEUCXYyzf9rKPEpf34sFVUzkAi+PfnwGId3y+0MPYFYCFPq6lAPiu4xpZwNnsYwAyHJ+XeNlniYd9Xe9rW8f/fwbgOwDKALQCWAtgqod9XQIK0Q7H9XsRQLGf+8Ljb+r47H7H38cDeBu8315zfHY2gDcd594BYDuA7wEwe9j/Mx7Ody6AfwNoAVAF4I8Akly2M67h9S7vGffBiQDedxx3H4BbPZzXmQA2O+6V/QBucny/3Nf1cHw3D8B/HGNrAvAsuALudV8Ecg28/Lb3B/rMDdWXXiGEh08AXCcin4MP6l5PG4nIOMe2HeDydR+AMeANbVAMYAv40LQCmAqaPMYDuNKxzcXgA7AVFBYAUO/4Nw+8wb/k+LvbcewMAB8CSHZ8pwzAOQD+5JhtPuo23H8DeA7AZei9sswA8DyAX4IP+JUA/igirUqpZzxeHfIPAF8GH7QPQDPEvY7z+gqATwHcBuBxUOhudHxvp499/hzAjxzfeR3AFNAsMl1EFsBp7ngSgA3ANx3fq/awrzdAgX8vgMtB4eC+7TWgsLgdQAKAXwN4TUSOU0pZAUBEbgWV7t8BPAAgHbzea0VkmlKq1cu5+PpNDV4D8Ffw2tsd740HsArAo6DQnen4fh6Au70cy5V/gr/zJeC1uh9AI3h/+iIDFNa/B8/za+C9tEcptRoARGQKeF0/Ae+TBAD3Ach0Gb8v/gdgOoD/A5+VKxzn6U4g1+AUABvA5+pJx3vGbxzIMzc0ibZGGoovAMcA2AbnzOMI+JCd7bbds+DsrjDA/QoojK8BH6ARLp+VA/iXh+88A6DSw/v3gQ/LJLf3n3aMN87x9/WOc3jEy74VgCvd3n8HQAUAcdtHiePv4+EyI3P53r2O96c5/l7o+PvMAK5NjuN8nnF7/xrHPr7k8t4HANYEsE9j3BM9fKZAoRTv8t5ljvdPdfydBqAZwN/cvlsCrmDu8HN8b7/p/Y7j3B7g/XIPKNRNbvt+xsO5/tRtH8sB7HUbu6cVggKwyOW9RMd99JTLe/8BlVqKy3ujHL9buZ9zOcvLvfYWfKwc/VwDBeBn/X3mhuJr0NuzYxHFFcGJoOPy5+Bs42IAb4vIvS6bng1guVKqytu+RCRDRH4pIgfA2X0POIsTAJMGMMxzAXwMoExE4owXaIIYAc6uXfFmb7cBeNntvf+Cs6wiL9853fHvv9zeN/5e4GfsnpgLCiH3ff4XgLWf+/THO0qpHpe/P3P8W+z49xRw5vxvt2tcCZrnTsfA6PObiMgoEXlSRCpApdMDrnSyAOQHsE93Z/FncJ6PLzqUYyUAfOFr2Of23bkA3lRKdbhsVw1gfQD7PwXe77VeDPQahPGZi3m0yShMKKVsANY5XhCRQgArAPxERB5XSjWCgrfS+14A0NRwJrhk3QKgHcBs0CySNIAh5gOYCN7snhjh9rcnswoANLoJRQCodfxbBM/nZ0TtuO+zxu3zYPC4T6WUVUSO9nOf/mhw+9tw6hq/iyF83vXy/cYBHr/XuToCFpYBKARXEbtBG/hF4Aw5kPvF0zklBvA9T+fS7XbMUQDqPGxXC/+RZKPg+14DELJrEK5nLubRCiFCKKWqROQvAP4AzjI+AZfU3mbREJEkAEtB08ofXN4/IQRDOgo+nLd7+XyP29/Ky3bZIhLv9qAWOP497OU7htAZCeCAy/sjXcYWLK773GG86ZiRj+jnPgeKcczrXcfkgjf/QaC4/yYTQHv5V5VSX6yUROTCAR4nVFTD8wy9wMN7nr7r614zGNA1CPMzF/Nok1EYEJExXj46zvGvMRNeCeACERnlZftEAGb0ncVf72HbbtBBHOj7KxzjOaiUKvXwClRYmQFc6vbelQAOwrtCWOuynStXO/5d5zJ2eBm/Ox85tnff5xXgxGdtn2/4J5jje2I9KPQnernG7krX0/GDOXaK498v7hcRiYfzukabjwAsERFjnHDc+/MC+O4GeL/XXAnmGljQ9/oG88wNOfQKITxsF5HVoI23DLQjLwFwK4AXlFIHHdv9BExaWy8iD4FROkUAzlVKXaOUahaRjwB8T0SqwRXFDfC8qtgJYL6IXAAqnCNKqXLH+zki8g0ApQC6lFKfAXgEFJbvi8gj4IogFVQS85VSSwM811YAvxKRXNBmfBW43L5eObxy7iildojIcwDud8zg14M24vsAPKeU2ubYdC9o/79BRBpAAbnHk7JSSjWIyO8A/EhE2sEIncmg7fgD9C+Ryohouk1E/gEKiW1KKUsgX1ZKtYjIDwA8LiJ5oAO0Gfz9FoCO7f/4Ob6n39Qbu0Bn/s9FxOYY73cDGWuE+BnoeH9bRH4DCt/7QLOPzygjpdQ7IvIBgCdd7rUrwAAFV4K5BjsBnC8iK0CTV5VjJR/oMzf0iLZXeyi+QMG/DLwxu0Ab5GYAPwSQ4LbtBDAC6QiceQiPuHxeAgqSVtDE8xioRNxjr4+DMwbcNWY91bH/RvTNQ8gGFUMZOFuqc+zjDpdtrof3SJtn0DcPoQLAd9y2M/ZR4vKekYdQAT60FXDJQ3DZ7hbHNbG6n7OH8XjKQ3gcjjwEl+0CijJybPsTcKVjcz0HeIhQgYcIHMf7SwCsBuPnO0HF/zcAU/wc29tver/j7zgP35nhOL8Ox2/zABjr7379y+E5ymii2/7uB6B8nSO8R7Ktcb/OYLTQFjjv9VvAidPmAH6LPPBeboUzD2Gp+30RxDWYB2ATeN+65iGUIIBnbii+jLBAjSZoHLVwzlRKjY72WDSDExFJAxXkG6p3Zr8mCmiTkUajiRgi8ihoIqwCI4FuB1eqf/D1PU1k0ApBo9FEkiQws7oANOt9Aq4yt/n8liYiaJORRqPRaADosFONRqPRONAKQaPRaDQAtELQaDQajQOtEDQajUYDQCsEjUaj0TjQCkGj0Wg0ALRC0Gg0Go0DrRA0Go1GA0ArBI1Go9E40ApBo9FoNAC0QtBoNBqNA60QNBqNRgNAKwSNRqPROBjU5a9zc3NVSUlJtIeh0Wg0g4pNmzYdUUrlub8/qBVCSUkJSktLoz0MjUajGVSISIWn97XJSKPRaDQAtELQaDQajYNBbTLSaDQhoLsbaGsDEhKA9PRoj2bgKAXU1QEdHUBqKpCXB4hEe1SDAq0QNJrhSk0NsHIl8P77gM1GQTppEnD++cC0aYNPiCoFbNkCvPoqcPAgYDbzvMaNA5YuBaZPj/YIY55B3VN55syZSjuVYxuLBaio4L+ZmUBR0eCTM0OSAweAX/0KsNuB/HwgPp4CtaEBaG4GLroIuPjiwfVjvfkm8N//Ajk5vNlEeE5NTTyva68Fzjor2qOMCURkk1Jqpvv7eoWgCQtWK/D223xGOzv5bNrtQHExcNllwAknRHuEw5iODuD3vwdSUig4DUSAESP43iuvABMmDJ5Z9eefA88/D4wZQ+VmIAJkZ9N09O9/A8cey5tQ45GYcSqLyBgRWS0iu0Rkh4jcHu0xafqH1Qo8+SSfz/R0Pn9jxvDf5mbg178GPvww2qMcxnz6KX0GrsrAlbg4ICsLWL6cM+zBwKpVQFJSb2XgSkICP1uzJqLDGmzEjEIAYAXwPaXUZABzAdwmIlOiPCZNP/joI77GjeMzaiBCOVNYCPz971zFa6LA+vX+ncc5OcD+/UBLS2TGNBCUAj75hM5jX+TlcTuNV2LGZKSUqgZQ7fh/q4jsAlAEYGdUB6YJCqVoJvIV2JGURPPRRx8BS5ZEdnwa0IbnbSZtIAKYTHT+RIKWFuDjj4F167h6yc0FzjgDOPFEIDHR//etVjqRfWE2R+58BimxtEL4AhEpAXAigI89fHaziJSKSGl9fX2kh6bxQ3MzUF3tfwKamQls2kQ5sH8/fZydnZEZ47Bn5Eigvd33NlYrFUJaWvjHs3cvcNddwH/+Q/9GYiLDRp98EvjJTwB/z7kIMGoU0Nrqe7vWVi5PNV6JmRWCgYikAXgZwB1KqT7rVaXUUwCeAhhlFOHhafxgt/P59Bec0tXF1fsddziDQeLigEWLgAsvjIwcGrYsWABs2MCL7u2Hqq0FTj0VSE4O71hqa4Hf/IY/eEGB8/2kJDqDa2qA3/4W+OlPfa8UzjkH+NvfgIwM79s0NgJXXhm6sQ9BYkohiEg8qAz+rZT6X7THowme9HQ+y11dvf0HrrS20jIwYgQwY4Zzpd/Tw7D4zZtpLfjgA8qD5GTKptNP1xO8kHDMMcBxx3FmPmZMX6XQ0sL3zj03/GN57z3OIrwJ8pEjgbIyYOtWYPZsJtFVVHAFk53Nz0X42cqVQFVV35tEKeDwYaCkhCYojVdiJg9BRATAPwA0KKXuCOQ7Og8hNnnlFWDZMmDs2L6fKQWsXs2J4Xnn0cnsSkcHw1VTU4G5c/mv1Uqrgd0O3HgjMG9eZM5jSNPWBjz2GLBrF8NPU1OpkZub+fcdd1BxhBOrFfjmN+kvSEjwvl1DA51SU6cC77xDpWDEMU+YAHz5yxxrYyPw6KMMQU1I4Iqiq4vndcwxwG23eY+sGmYMhjyEeQC+CuAzEdnieO//lFJvRnFMmn6weLFzdl9Q0HsC2tDAydrUqX2fTcPRbDbTn5CQwP+bzcDo0Xy2n3qKeVSTJkX2nIYcaWnAD37AVcKaNc6l2CWXADNnUkGEG0NY+1IGAAX7O+/Q2VRY6DQdKcWZxUMPUYHNmAHcey+3++gjJqRlZwOnnELFMZiS7KJEzCgEpdQHAPQvNgTIzATuvht44gmu9uPi+LJYgEOHmK08fXrf57O+ntaKrCxOVJuaaBEwSEqinHrjDT7/mgFiNgOTJ/MVDRIT6bi22XxHCB04wBuipKT3TWMk0iUlAX/+M30NqalcDYR7dTNEiRmFoBla5OUBP/4xV+9btjCopaAAOHKEZmNPz//Bg72jIe32vtvk5tKc3No6NOqwDWvi42n7//RTRgl5wm4Hdu9maru3GX5qKm+sTZvoaNL0G60QNGFDhCv1CROc75WWcvXvic5OriSU4islpe82JhNf7e1aIQwJzjqLEU/d3Z6jiGpqeDOMH+97P6mpnHlohTAgYjIPQTN0Of54ZxSSO4mJ9DN2dzPoxJP/z27ny5Oy0AxCxo8HbriBySvV1TQfAU77YkcHbxp/yWlmM28ezYDQCkETUZKSGApeVdU3aXTsWD7/XV3eLQRHj9Ih7SvcXDPIOP10JqCdfDJvjIoKRh+ccw7w859zZuBP2Le10cegGRDaZKSJOAsWcCL43HP8NzmZVoGODgacTJrkuSxNdzd9BxdcEPkxD2tqamjnb2ykx//EE0OfEDJuHHDzzVwtWCxcERiOpvnz2bNh9GjP37XZ+NLxyANGKwRNxBFh4tns2fQpHDjA96ZOpZx59FGgvJwOZCMPoa6O/95wA3OqNBGgs5NVCD/5hI6b+HiGib70EkM8b7op9OGpRkiaK0uW8Eapr+87U7BauaJYsqR3prOmX8RMYlp/0IlpQ5O2NsqgFSsoAxISmKm8aJEuZR8xrFbgd79j4lpxcW/7nVJAZSXt/z/4gf88glBQVcVEuqoqp9IwEtSWLGEzH3/F7TRfMBgS0zQaAMyZWryYL1/ldjRhZNs2YPt2mnLcfwARmm/27KEpae7c8I+nsBD42c+YSLdtGx1NhYVMonNPd/dFdTWwcSNnGmlp9FvopLUv0ApBE3Gqq2kCEuHk09fzrJ/TKPH22842lJ4QYc+Et96KjEIAaLY67rj+2Qy7uoBnnmEGs8nE6AaLheMfP55lLUaMCPmQBxtaIWgiRnk5Kxzv3ctn0mDuXJajCWaipwkz5eX+G85kZjKbMNaXcXY7S2lv3sxQNnfz1+HD7C99333DvsyuDjvVRIQDBxhBePgwn8niYr6KitgX5ec/Z6kKTYxgNvtvn2m3Dw67vWHaclcGgLOXQm0tC3ANc7RC0IQdmw34058YkOLeSc1sZgXmhgbgxRejN8aA6exk+KWnzLqhxIknshyEL44cAaZNi+3VAcB+yykpvseZn88oBk/1UoYR2mSkCTu7d1N2+MobKixkBYMvfzlGKxTv30+B8emnzg5Ap5wCnH02NdpQwyhZa7X2DQMFqOU7Onj+sU5Fhf9MxpQU3qSdnZGp9Bqj6BWCJuzs3etZprhiWB4OHQr/eIJm3TrgwQeBHTsYXTNmDM0MH3/MDNutW6M9wtAzfjywdCmFqXu7zY4O+hjOPRc49tioDC8o4uL8z/yNAlqDwQQWRvQKQRNy2tpYZ6y2lgmnVVWBfzfmVuwVFWzN6FqHH6DgKCyksHzsMeAXv2Am3VBBhLH9ubnAq6/yOhgml4wMZgguXBj75iIAmDWLNdN9zfwbGpgi763N3zBBKwRNyLDbGcX3yiu0NMTH07JQW0tncna292fSZmPJ+4YG9lAYO7Z3JFLUWLXK2X3LE6mpLLC0fj3wpS9FdmzhRoR1hubN44qgvZ2mlXHjBtdM+rTTgOXLvVdUtdsZ0XDjjZEfW4yhFYImZLz2GvC//zF6yLWvwahRFPKrV7NkhWvfdqUoazZvptz95z/5fOblARddRDN91CahSlHQu3bp8URuLs1KQ00hGJjNvWuYDzby84HrrgP+8hfmGrjmV7S3s1bTmWeya9MwRysETUior3f2UXb3F8THM9dg7Vqa4Wc6EuaVYjLsjh2cuM2f73Qot7YyMunoUeDCCyN7Ll9gtXLp4s8BEh+vY2ZjnQULmOjyv/8xd8Js5swjMxP42tcGj/krzGiFoAkJGzbwefImO4uLaXnYuJHPZXIyhf3WrZzAzZ7dO7ooPZ3m3JdfZmTj2LGROY9exMUxUamzs/eyxp32dv9JXJroM306b6aqKs44kpIYIDCYzF9hRisETUg4cMB/kuf48Xz2LrmERTOXL6e/75hjPPsL4uP5Wr0auP76sAzbNyKsyW/YwbzR1ARcfnlQu1aK7UVra3nuxcWhryit8YAIsyE1HtEKQRMSTCb/ia3GdiefTLP86697VwYGubkM/Q9aIbS3s2TqypW0ZyUm0rm4YEFwkve001jX5+hRz7VuampYdvmkkwLe5b59LKtz+LDTSqEUe91fd51/l4VGEy5iIY5DMwSYNo3hpr7o7GSQSl6eM+zbn9nWZHJ2VQyYujrg/vuBf/yDBcwKCxkquWoVcO+9dBQHSlYW8MMf0nxUXs4wqPZ2KojychZ4+/73fZuUXNi9mxGqbW00g7m+yspYwqO2Nsjz1WhChFYImpAwaxbNO52dnj9XilVOzzmH2xlh/K2tvvfb3Byk/6Cnh3X8jZaK6enUKomJTCrLzweeeoo2rkAZMwZ46CHglls46IQE7vs736HiCdB/YLPx0FlZDMF1VYYiXBlYLOwkp9FEA20y0oSEtDTg619njlZ2du/Ivp4e+vGOPZbRfQZLlgBPP+29qoBSVBjnnBPEQLZvpxnHW52MpCTO5t94gwI9UJKTGQN7yilBDKY3u3ZxgeGrhEdBAR3tnpqDaTThRq8QNCFj1iw20EpLY2Sf8aqroyK4887eiaCzZtHRXFnZ1/+gFL97wglsrRkw69b5r0WTl8dUaveSDGGmrMx/QIvJREVaWRmZMWk0rugVgiakHH88y/4cPMiioHFxTGz1JKMTE6kknnoK+OwzCsOEBJpN7HZgzhw6k/2lAfSisdF/+QFD6nZ0RLSQWaDdanU4vCZaxJRCEJG/AbgAQJ1S6vhoj0fTP0ScjlJ/pKdTKRw+zGii5mb6aSdMYFjmI4/Q9j5pEhPXCgv9CMzsbFatTE/3vo3dTukcoCM4VJSU+HeQK8VtRo2KyJA0ml7ElEIA8AyAxwA8G+VxaCKI0aJ39Gj+/f77wG9+40wkFQHefZfVp886C7jySh+ml9NPZx0MX4Xm6uuBGTMi3h1ryhSeT1ub90PX1XGVpUNPNdEgpnwISql1ABqiPQ5N9Ni8mY7m3Fwma2Vm0ulsVJ1esYLFN71iSNO6Os+fd3XRVLRkSVjG74u4ONZPO3IEaGnp/ZlS1FMAcNVVER+aRgMgxhRCIIjIzSJSKiKl9cYTpBkSKAW88ALzvzy5AcxmmqHefLOvQP2C+Hjgu9+lOai8nNNxpeiYqKxkkP/Xvw5MnBjOU/HKtGlMWxBhRenycr4qKujrvucenUiriR6iAvV0RQgRKQGwPBAfwsyZM1VpaWnYx6SJDGVlwAMPcGXgy09QUcGM3oULfeysvZ0NbN5+m1PvpCQWU1q4MCYkrs3GxkE1Nc7SFSUl2qGsiQwiskkpNdP9/VjzIWiGMc3NzgAgX8TFOc0rXklNZRvIxYsDS4mOMGYzS1VMnhztkWg0TgadyUgzdElICCw002oNMkAoxpSBRhOrxNQKQUSeA7AQQK6IVAL4iVLqr9Edlaa/2GwsZWE202LjTy6PG0cXgMVC5WAgyo4RR/cio/kQFIDutjE4Yeox0PMZjSa0xJRCUErp+IohQEsLE4bffpumfKWYR3DeecCM47ogFeWsZ5GT0yuxIDmZYaWvv+60p484uhcnbXoaKR20EXV2ADNSgeLH84Cbv85yqRqNJiTElELQDH7q64GHH2bNnvx8RgwpBdRXduOD25chxfwuJpX0wGQSJhqUlABXXAEcdxwAdqE8eJDhp1MT9mHep79Ed2I6jqaVoK0NSMkFZpwGiKUJ+OUvgbvvprbRaDQDRq+5NSHDbmdxu/Z2hocadv44WzeW7P09Tmtejt31I3BQFTOpoLiYZaQffpi1hUBT0be/DdzwNYVZ2/+Oo91pqOvOQnc3i+MtWMAS2sjKYnbX3/8eeE0IjUbjE71C0ISMffsYEupezXNc2Srk1e9EU1YJknsEe/dSYYgIzUaJiWyg/LvfAampiI8HFhUfgP24anTlF0OBPog+2clZWTzggQNRyyvQaIYSeoWgCRkbN/Z2BgOAyW7FpH1voS1tJCCChAQ6mnsllqWm0pPsmlNSXQ2TKKSkClJTvZSqMLzU1dWhPhWNZliiVwiakNHezighV1Lb6xDf0wFrXBJS29gKrNOWAavVLW40NZWNABYs4N/BhIoOJKzUamWjgg8/dFbWmzeP9qlYbr7e1UUz23vvscJrRgaT7k4+2WFT02iCRysETcjIy6OcciWttQp59dsRZ+0GIFAA8i0KqbuKgJOmOoWXe6/MMWP4r6+kMsN3YGwbLPX1wB/+wJIWyck0XZWVUTkYHdFycvq373BSXQ389rcsipSZSXva0aPAX/8KvPwy8L3v9f+aaIY12mSkCZq2NvYG3rmTMslgzhzKdENOZ7RUYmbpn5HU1QxLQjq6kzLRbma1usSGasamdnRw4/Z2JiIYFBfzb9cDuHPkCLcpLg7+JNrbgV//mvsoKWGrsqwsFsYrKWGLt9/+tq+Gizbt7SwF29HBcWZnU5llZfFvux341a+42tFogkQrBE3AtLayb/0dd1Dm/Pa37JD2yCOcZBcVAbNn08+r7AonbXoaNnMijuQeh3hLO+x2Wmhy80yQjAzmImzbxjftdppqDESAG26ghqmr6x1JpBTfs9m4TX9MRh9/zH14qjNtt9OEtXs3sGZN8PsOJxs3cjWQn+/585wcKo0PP4zsuDRDAm0yGiJYrZzMJib2teOHgpYWRofW1DCXzOhiZrcDe/YAP/sZ8KMfUT53dwOH3y9DQk0FmkeMRXt6KuLrq4GedowcnepsUpaWxpn4nj3Al7/ct4nwmDHAvfcCzz7LSnCG4FeKCWnXXutsohAsK1b07Zlgt9NktG8fT6Knh3kOe/YAS5f6boYcLqqrqWFtNiqBt99mcocv8vKAlSujUuJbM7jRCmGQU1MDrFoFrF1LpSBC083ZZ4dWfr30EitHu1tnTCZOshsagCeeAH7xC+D224FDWeXoOAwc6RI0NaWgPuU0nNC1ES2HmmA9KkhKMaG70wZ7Wxc2ZMxGzZFLsXgfo0d7TfhHj6amOXyYygOgRioq6r8z2W7n6sC1pZvdziinykoqqsxMvmfYx7ZtY93qSFWjq6ujIty+vfd5bt4MzJ3r7AinFBWXCLW0CH0KtbUcv0kbATSBoxXCIGbXLobuK0UTeHw8J5KbNgEbNgA33dTbCtNfWlpogfDV1jEnh3X99+yhzBxb2IPqbIGtmhPalJR01GIREjsa0FVRh+4mKxKy05BZaMPhCadj62bB3nfKsGB6E86/KB7miS6NmN1bqg0UEefFMpY6Bw9SGWRnOwWw3c7tCgpoL3v0UdrJwt1688gR4KGHGJ/LhA3nZ1u3Ah98AJxyCm+f9KYAACAASURBVFcx+/ZxO4CRRpMmcSWRmKiL+vUHpbhK3LqVfpqCAkZuZWdHe2QRQSuEQUpTEwNkMjJ6tw82mym4u7qAv/yFMjSQ3sa+qKjgc+Kv2b3ZTMvO5MlAlRqFA/sV0ke6fk9Q1TUCdTIC8VmA1aJwslQgo+corvn8HqS2VKN1m+DQWoWSCXEsXX3xxZ675QwEEc6yP/6Yqw2lOPC0tN5CtKPD6ehOT6ft/tNPQ6NlffHCC/QDeOrbMGkSVytvvEGTV1oaHcpKUUGUljJy69ZbtUIIliNHmCB54ABv5rg45sf8+99ccl9+uf+HYJCj15ODlA0b+Px76yWflMTJ7apVAz+W3R6YbDGZaL0AgLcPTYElKRPJ9vZe+zl6lJPXuDgg3daEurYUHL/9OcT1dKIlqxjWUcXY1jQWliyHvfyRR3iioeaMM/iwWywUvp2dvbPqrFYKWVe7W3p67+S5cNDQwGN4W46NHcvVSkcHx2uM2TAVpaYynNZiCe84hxqGk+zwYV7jMWP4G4wdy1nVW28B//rXkC+TErRCEJFUEYnhjJ3hwbp1/n2L+fnA+vUUxAMhL4/78Pcs9PTw2VEK+PDjOOyZc+0XiWkAZZhh1k62tiJLmvBey0y0ZRShOykTEIHZzG3qm+L5MO7aBaxePbAT8ERJCXDNNTQTuYa2KkUF0doKzJjRd/kVbkFbVUXh7s32b0RAmUxUHsaPa7c7FcW8eVQqhilJ459VqzhbGTmy7+zHbOb9smYNTYtDGL8KQURMIvIVEXlDROoA7AZQLSI7ROTXIqJLTUaB9va+ZSLciYujmdyYtfeXUaPo7D161Ps2RoTT9On4Iry0dvTJ+GT2txBvaUNmUwXSWw8jv+cwCrrKYYINb2RdjYNSAmtcX5NQdzf4YI4cydmZ1Tqwk/DEmWcCd91Fs1B7OzN+m5tpL54/v6+tra2tfzkPweBP69bW0odRVMQZQVsbx9zaSvPXwoU8n54eYP/+8I51qNDTA7zzjucQZAOTiQ/U2rWRG1cUCMQgthrAuwB+BGC7UsoOACKSA2ARgIdF5BWl1L/CN0yNO7m5nCD6Ugrd3ZQd/hSHP0SAr3yFoaXNzQzAcT9OVRVDTg1/a3Y2J6hVRbNQO3I68mu2QZWVY2+3oDl3IspTpyKjfAtykuo8HvOL0NmUFGqio0fp4As1U6fyNWIEl1PFxZ59FoaWO+200I/BlcJCHstbhFBPj/P9E0/kNbFaKazcS22Ew9Q2FGlp4bVKTPS9XWYmHc5DmEBMRmcqpR5USm0zlAEAKKUalFIvK6UuBfB8+Iao8cQZZ9Cx7IvaWk6CQ+FbHDeOk2mlGE1UVeUMkT96FPja15xliAD64Iy+xzZzAqqLZuLg7MuwJudS7EuZDpvEoc2SiDmZe3sdx/BXuKckDNju5Y/LL+cD78nMYrfzpE8/nQI7nIwYQUFfW+v58+RkCi8j+slspiBzVwZKeXcwaXpjMgV2f9ntsV3fKgT4XSEopXpE5DgARQA+Vkq1GZ+JyLlKqRVKqQEaJTTBctJJ9BHU1nqeODc1UU64CumBcswxrJqwYwdD861WWlVOOqlvPbV585gbVV/vFO4JCcD48YyUjIsDRmfF44T4XegGHahKcbI2dqzLJN1i4caOsL/mZgYH7djhzE879dQQlBwqKGAS2h//SOGfnMyHv7OTdrfFi4Grr45M5M4VVwAPPshchLy83sdMTaVCOOUU78KpvZ2RR7okeGBkZfH3b2lh2J43mpvZ0m8II8qPzVJEvgPgNgC7AMwAcLtS6jXHZ58qpU4K+yi9MHPmTFUa7qiPGKaujmHxdXW8j43JY1MToxG/+10K4GhRXc08ifp6jicxkbLq008pZ8+aUYevfnYXmtOL0NUTh85OKrk5c1yi+yoqgHPPBa64AmvXMlfLbndOftsc05PLLmOLzgHL654eapvSUl7MwkIKX1/25XBQXc244QMHnAX+RCi8Ro1iwlpJSV+zUk8PHZ8330w/iCYw1q0Dnn6aD4ynm6iri8EHv/71kMhJEJFNSqmZfd4PQCF8BuAUpVSbiJQAeAnAP5VSfxCRzUqpE8Mx4EAY7goBoMzasoVBEg0NFLwLFwIzZ/L/0cZiYdj8++9zApadTTN8Vxf9eDmrX8ZJh15Fz8hiTJwcj1GjHBNfpbj8SU0F7rsPH+/JwmOPMYrJ3dRryMDrr6cpbcigFHDoEFcsNhtnsccey/effZYOzqQkmrqUolPcaqV2PP98nYcQDFYrU+1LS+mwN24ypbgyaGgAbrkl/DkoEWIgCmGnUmqKy99poFLYCWCxUmpGqAcbKFohDG6UAnq67ZA3lyPuzWUQwzlqlEydOBG45RbYcvLw/e9TUXhTcl1dzmQ9f77BIYGRUfvee7TBmUwM8YqEn2Oo0tPDiLYVK3hDiaPvd1ERa21NmxbtEYYMbwohkCijGhGZoZTaAgCOlcIFAP4G4IQQj1MzjBABEpJMwCVfAs5ezKVOXR1nvVOmfFG2Yd9uCntfGddJSXyGd+ygT2PII0LzRjRtgkON+HjgS1+iibKsjMvbzEwmqQ2T1VYgCuFaAL2CwJVSVgDXisiTYRmVZviRluY1pLOxMbBdiPjOlfCJUb7i3Xd7e6zPOovKSReJGz4kJNA0NwwJJMqo0sdnuui6JuwEWs5bqX6ai2w2liV47z165o0U8H37WF30lFNYKTAcdcW9jaeqiiaMrKzY7NqmGZIM7UpNmiGBURLbZvMeaWm4wib1J29+xQquDMaN670SyM9n2OdHH1EwX3VVP3YeBDYbldIbb9ADb9iwjz8euOSS3h3lNJowoBWCJqQYUZtr1zJKLz2dfs4ZM/pftDQri5P0DRu8V46orgZOOMF3iW6PdHcDy5czfMmTWUiENuRVq4ALLghfspfNBjz5JJVPQYHzRO12hp4++CBw551UDhpNmAhYIYiIALgawHil1AMiUgxgpFLqk1ANRkTOBfAHAGYAf1FKPRyqfWvCT0MDi5MeOsRo0eRkRuz9+c8U6nfe6b8UkBFpuXYtLTZmM4M7zjyTdejKy5kSYCgXi4XKID+fpTOCZvduZyEmb8TFMSxx+3ZqpnDwwQfUeO5x8CYTT66tDXj8cWYGftFyTqMJLcGsEJ4AYAewGMADAFoBvAxgVigG4qig+jiAswBUAtgoIsuUUjtDsX9NeOnuZpKc0bPelZwcKotf/YoTXW95PTYbS8+/9x7N9UZ4/fLlwOuv02py4onMXzDKYsTHs1PkOef4TjL1SkdHYNuJ0IwTDux2mony871Hs6SlOfsx6IQzTZgIRiHMUUqdJCKbAUAp1SgiAyyb1ovZAPYrpT4HABH5L4ClYL6DJsbZsoWl5L217czJYfLY++8zss8Tr75KYe9uyk9P50rghReA227jKuTIESqL3NwB5h0E2v0snLWBjh7lCflbPqWlARs3aoWgCRvBxNL1OGbxCgBEJA9cMYSKIgCHXP6udLzXCxG5WURKRaS03pgmaqLOu+/2rYLqjtEj3lMuZGsr8OabzDXwZMpPSOD3X3yRFhyjrfKAk9COPZbLDF99Dmw2DmrKFO/bDASjGbY/zOaB1zLXaHwQzArhjwBeAZAvIj8HcBmAe0M4Fk9PRB/RoZR6CsBTADOVQ3h8zQCoq3OatltaaOs/dIiyNC0NmDCBQryjg+Yldwfzli20nPjqUJiWxtJGn38ewrptyclMRHr1Vc+1gQynxqJFdISEg8xMHtfI1PZGWxsd3MFgs9FHEhcX2hTu7m76VKqqqFAnTuSPHMoErs5O3kzx8b17XWvCRkAKweFQXgdgE4AzQOF9kVJqVwjHUgnA9W4fDaAqhPvXhJGkJE5eq6oo3E0mZ2Mvi4Xh/Hv20BzkKZz/yJHAKguL0FEdUr70JTolPvyQjoicHB6osZGvk04CrrwyxAd1ISWFSXnr1nkX+EY/hkDNRU1N9MyvXOn0k0ybRmfL5Mn9F65K8Tr95z/cr9HiTimO/ZZbGLE1EGpqWELiww+5X7ud5rTzzwdmzdKKIYwEpBCUUkpEXlVKnQx2TAsHGwFMEpFxAA4DuBLAV8J0LE2ImT8f+Pvf2aQrPb33RDcxka/aWu85VikpnMwGwkAnuobJ6gu5EhcHfP3rwNy5tGnt3s2NJkwArr2WMbPhbq5+3nms693Q0PciufZjCETYVlUBv/wlZ9cFBc4eqEai3SWXAEuX9k+wrlvHKqyFhbThGShFX8gvfgHcd1//q8MeOMDoA5uN+4iLcxaYe/RRNtq45hqtFMJEMHf5RyIySym1MRwDUUpZReRbAN4Gw07/ppTaEY5jaULPqacygsjoNOiOzeZUDLt3s0mZK8cfTwtBbS33kZnZt9ObxcLVxYQJwY+vsxP45BPK++pqjnH2bFZHHT8ezuJw06d70BgRID+/dz+GpCQO0ujHsGhRYIKwpwf4/e/5HdfiT0b4ak4O8PLLnHEHW/Spo4NhYEVFfW1+IvTwV1cDL70EfOtbwe0boGnrD3+gGc/VPGeU/c7I4Ipn0iQqb03ICUYhLAJwi4hUAGgHzUZKKRWyEoBKqTcBvBmq/Wkih8nECd3Bg3QQG+YipShHLBbK2tRUhty7KoSKCuD557m6aG3lakGE8mzyZCoGpRjFdMEFgQcGGTQ2Mny/spJVKYqLKS9LSzmWK65w66UQbkVgt3PGqxSFnKFBi4uBhx8Gdu7k4Do7KXznzg18xr19Ox063sK94uKoFF5/nTG8wZzrp59S4fjKMCwo4HaeVjr+2LKFqxpvYzeZ+AMuX86mGXqVEHKCUQjnhW0UgxBDQLW3U4AVFQ3v+mcdHZwgjh9PwV7pUgGroIA+x9xcmrZdi9Xt30/rRnw8Z+vr11MmpKSw4OTRo1QkTU2UX0uXBjcuu52Whvr63pUf4uKY1dzTA/z3v/z/ieHu7NHdTQ30xhvO/qfp6bTrL1zIk46Lo62/v6WW16/v277OnawsauGGBmfdpkAoK/NvrzOZKKjr6oJXCB9/7L+JR2YmZx1Hj/KG0oSUgBWCUqoinAMZLCjFydsrr3B1bLRjzc+nsDrllOExcWlqYmLt1q3O3i2trVSMJ59MId7TQ5+jq+mnu9tpDTB6kqSmOt+bP5+yav9+Xsfqak6if/hDXttg68vt20c55q10thHAsmwZXQU+fzuLhbPfFSsYeRQXx05Eixd777Rl0NnJBIpdu3qXpujo4PJowwae5EBzHdra+tra3BHhjdvVFdy+jSVfIPTnIejs9P8DG2PX4bdhIZjSFT/29L5S6oHQDSf2efNNzihzc/lMG/d9Swvwpz8xQOLii4e2UvjwQzqQbTZnxGRZGf2BjY0U3HFxnn0JXV3OKtc7d3J7V2GdkEAT8YQJlL89PbSunHxy/4qNfvSR/+9lZdFsX1/f20/ai9ZWCvQDB6hBioo4Eygt5QW5+GJGK3n74V98kdrJXXGkpHDpUlnJi/qd7wR/kq7k5zMu11eIrN3OV7At9Y47jpmDvjByKor6pBD5Z9Qojt1XQosReZCQwG3tdjrN/SXBaAIiGJNRu8v/kwBcAPZZHjaUlzNbtri4r5DJyOCz/dprdJAec0xUhhh2tm1jDbbCwt6mZMMUvnIl5cz06X2/W1XFa7R/P5XB9u18nj1hMnH/SUlUGocOUR4FS1OT/6J6xqTTaxULpXjSFRW97U4mE4WY1UpH7ciRtG2709rK6JyiIu8Ko7CQEUB1dT60UgDMm8dwU6MPsyfq67kcClaInnACf9zWVu8rmepqLvP60791/nxg9WrfY6+s5MP3f//nVD52O/0sS5dy9aXpN8GYjH7r+reI/AbAspCPKIZZvZoTE28zzrg4OjzffXdoKgSluDoaMcKzkB05ktFGH39MBTF6NK9JWxv9LZWVfG/ZMr6/ezejijo7Key9+WCMZ74/5OSw+qq/87LbfZjeDx6k9vJmd4qL45LxlVcYJ+9+Ivv2cWbrK3TVsD3u3j0whTBxIi/m3r0U2hUVzsJPubn8kbq76Z0PloQE4BvfoIe+p6d3spjRwyEvj6ul/jBuHBXV1q29l98GR47ws4kT+TJMYzYbS3ps20ZFoVuI9puBuEFTAAyr/n2bNvF+90VeHs3M/kytra3AZ58xsOLQocBNs9Hk4EGnTd8bEyfSOTtqFP1+Bw/yebXbKaeOPZbPemEhzUKJiUxY8ya0DeuGv+vujblzKf98Xd/GRlpyvB6jtJTOEF92wIwMzu4PH/Z8gOpqYM0a+h/WruWFcbeDm0zUjgPBZKLQ7uhgNM7evZxJWyz0X7z3Hm1y3iJ5/DF1KvCjH1EZVFTwPA4e5HnPns3P+mu+EWFi2/Tp3Hd1NW2xDQ38e9cuKo0pU3r7Scxmrr6UolNqMDxMMUowPoTP4CwlYQaQB+DBcAwqVunp8R9JZDI5e8R7kh9tbQzT/uADCjqj8cu4cey/Essri6YmZxCJL7KyqBSuuILX4aWXmHjqnlM1apRzVXXgAGWUuyWivp4muP4qhAkTeE3LyjzndFksPK+bbvJxXo2NgWXDifS1O1VU0Ddw4ACXK93d3F9FBZXIokVOAapUaLqjbdhAIXnGGTxOayvHP2kSNfHu3fxBzj+/f/s/5hjg/vupCOrreaySEu9lbIMhORm4/Xb6B9ato6JJSmLU1b//7bt0R24ux3TgQAhrmwwvgvEhuK4xrQBqHb2Vhw1jxviPpmtp4TPnSXG0tzMJs7KS2xgWBNckzzvvpKk2FjHyAfxhtTrNLxYLe8t4CqNPSKCw37KFsvTgwd75CU1NVJZf/nL/x2wyMUfqd7+jUsjOptKx2SjLrFYmI/uM8szOpiD3h1K97U6NjcCvf83vJyRQOLvS0sIIo4suohY1LshA6O6mI6u4mErAk/kkIYF2u8WLg0/qMDASRbyZ0QaCCDW5awbiZ59R8fgyu4nwtWuXVgj9JBiT0TeVUhWO12FHZvEvwzayGOSccyikfAnFhgYmOXli+XKah4qLe9/XIlQyI0awmUwgsicaGHWIfBUGNezxhlxrauLKylskpGE2ttk4sauu5qSwooLP/913B1/PzZ3MTJqWv/EN/r+2lrJ4/nzggQfYfMcnM2c6l33eaGmh7d81uubDD7li6Oriy2bjhTC85WlpvDgrV3LWvnRp/wW0wa5d/hv+JCTwR9w5iCrLB1rXRFeEHRDBrBDOAnCX23vneXhvyDJjBm3gBw5QSLmaGJSiT62khPLDnc5Omm99tXhMS3P6zWbPDvnwB0xSEkvJLFvG8/RkYqmt5cTOMFH7C10Xof0+LY3bzZ9PZTllCrOU/RW86+zkNROhPPameBIT6U/oV8WD4mIu23bupN3J/cStVg7ittt6Lw1XrqSmX7uW0S8pKU4HrzEjMIroJSd7n0kEQ2trYNspFfi2scCIEc4ier5sllZrP/qoagz8KgQR+QaAbwIYLyLbXD5KB7A+XAOLReLjad586ikGNMTFUQB1d3MCM3kyZ6GeInCqqnzPlA2Sk+lgjUWFAAAXXsjZ++bNlHFGdGF3N3MwcnJ4DYxnNieH1pD2dt+dH7u6WFD07LMDG0dTE83gq1c7J+9JScBZZ3El5y9ZNyhEgJtvZo2g/ftpAsrI4IGPHOHJX3pp7x/NbneWbu7p4cmPGEF7VXMzLwjgrAQYH09tOlBhFkzlv1CWww43o0dzluGrJEZ3N6+jp5hnTUAEskL4D4C3APwCwN0u77cqpRrCMqoYJi0N+O53afopLaXtPyuL0YZjx3qfvPib2BgYTuZYJSEB+Pa32fnszTdp9xfhc3jhhfRjugaZmM30Xf7jHzQPeboGXV2cWHsK4ffEkSP0tzQ00ERuhAF3ddF8vmUL8IMf9C8U3ivp6cBdd1ETvvWW/0xlEWqo+vreq4aEBHrIDS95Tw+Vh9kcGoVgxO/66q1gNPyZPHlgx4okIsBXvsIfPiGh749rsdA5d+21IZ4NDC/8KgSlVDOAZgBXiUg2gElgYhpEBEqpdeEdYuwhQiuCv46HruTlUSnYbL7NIJ2dvXOfYpH4eMrAhQtp7bDbPVcnNTj9dMrR7dspwI3tjKrGDQ3ArbcGFq2oFPD004zWcvdnJiVxEnnoEH21N944kLP0QEICtVYgmksEWLAA+NvffNvM2tspmAOdMfigqQnYuDEDSeYFKHr/PaRPHYvcPOm9W6UoOOfPD1/Dn3BxzDGcjT35JGdiSUm8Zp2dVHDXXBOAQ0jji2DCTm8CcDvYuGYLgLkANgBYHJ6hDS0yM2lR+PRT73kzFgvv61mzIju2/mIUn/RHfDwrMixbxsoHhs9PKfpgb7gh8FpulZXMW/AV3FJYyBpvl10W5YoGCxeyZEV1tefPLRbODgoLqRX76T232ZgT98Yb/DvV/GUs6KxF4dufYXdOLqafls7ckdZWLq+mTGGM82Bk2jSGjG3ZwhmGzUan1axZvhNkNAERjFP5dgCzAHyklFokIscB+Gl4hjU0ufhiRs8dOdK3UGN3N4XdNdcMzfs6IYEC+oILGP5psXCC6ikh1Rd79/JfX98xm6lsDhwIvuR/SBk1imFSN9zAHz0nx5moYvgQ5syhr2HWrH7nILz4IpXB2LGGlSgJu0bdgZbDH6No65vYseIgZswAkscWMOFi7lz/zqxYJjEx8JWaJiiCUQhdSqkuEYGIJCqldovIsWEb2RCkoIDhj088wbpIJhOFl9XKWfS118b+itdqZbDNhg2UaXl5LJ/jzT/gTlLSwEzX3d2BlRlXimONOrNns+rhPffQzBEf70zkGjuWtq+MjH636KyrY9OfkpLepki7OR6Hik/DoTHzUFveif2nAl+9OXloV13UDJhgFEKliGQBeBXAOyLSCN3zOGiKithZ7MABhp5bLLQYTJ8e+76w6moW/KyrYzRUfDzD3t99lyv5W2/1HUkUCvLzA69rFIrE2ZAwfz696s8+y7pGcXHUak1NLON61VX9Xh2sX08Z79UvJYIRY1KwdiNw8dX0W5eW0nqUk0N95avmnmZ4IaofdT9EZAGATAArlFI+0pTCy8yZM1VpaWm0Dj+saGoCfvITzrrdy0goRUfusccC3/++/9yBgdDVxdDfnBzvUZOtrRzDww/HWNMio6tSXZ0zMiGYBjUeeOQRmuD86ZMDB2jBqqvrGy594olsKR3rExJN6BCRTUqpPhlTAT8uQq4RkR8rpdaCjuUZoRykJnZZs4aC1lNNIRH6Q3fu5KonlHR3c1K9cydXKImJwOWX09/iKSG1q4uRnldeGWPKAOCFGj2ajo0TTxywMgC4SvO3YurpYd7M4cO0Uo0ezZXWmDH8e+tWdpWLCRObJqoEYzJ6AoAdjCp6AEArgJdBR7NmCGOzMTrIV1VmEZqLVq3qXY+ov1gszHNYuZJC3iiBPWECaxtdeSVbEADOLOe2Ns5+b701ys7kCDJjBqvw+uomuX8/r+eECX1NQ4Yy37GDL53TNbwJRiHMUUqdJCKbAUAp1SgigzhUQRMoHR0M9fZXcTQ93XP152CxWIA//pERWYWFTkWkFG3gDz3EcPTf/IbObaPd5vHHM1hnoF0oPdLTwzDHTZu4bCkqYmu4KDdkOekk4F//8p4Jbrdz2JMn++43kZFB5asVwvAmGIXQIyJmOEpgi0geuGLQDHHi4iiMAykjEwo79Jo1NHG4Ry6JOJvzPPEE7ef9reAcFGVl1FCNjTx4XBwVw6uvMjX7qqt8V+EMIykpwDe/yWvR2cnrY1wzI5Q5Odn/qi0jg1nnmuFNMHfxHwG8AiBfRH4O4DIA94ZlVJqYIjmZFREOHfJtmmhsZAbzQLDZaCoaOdK78klNZVh/aSmzoMNKdTW900YatCt2O6fVSjFmOEpMm8ZwZqNts1FQMCHB2ebZXxSR14ZubW3AJ5/wZbHQAbFgQd9SHZohQSDF7f6plPoqgFwAPwRwBgABcJFSalj1VB7OnHceTTTZ2Z6jiDo6+P4ppwzsOHV1zNPyVxYkLY3lMMKuEF5/ndLVUwyrycRlzHvvsapeFKtsTppEpVBTQ8UcF+dsiVBZSUXhywd09CgXO7347DPg8ce51DCaZh8+zFpOqal82e0873POoUNjMBXM0/QhkDiMk0VkLIAbANQCeA4seFcrIiFo76QZDJxwArOMy8tZf8iIVrbbKcTr6tj9cKANv2y2wCaeJlMEyt63trJBtC8/gZFduD42Cv+OHEl/waRJTtl89tmc6Hsrmmix8LNeyrWsjHaotDSGIhmdhSwWpot/+CG1TE4OtckTT9C509wc9nPUhI9AFMKfAawAcByATQBKHS/j/wNGRC4XkR0iYhcRD90ENNFGhOGe3/oWJ4aHDnHmeeiQs07T3r2cLA9EJhjVHfyFQLa10WoRVhob+a+/xIrU1Jg2wE+ZAixZQmXu2gJBKeaXHDrEQqKuvX3w2mvUKK5VRevrWYwrPZ3e/ro6Lg2zs7lSqqpi/GqgmYOamCOQaqd/BPBHEfmTUuobYRrHdgCXAHgyTPvXhAARlo+ZPZuyoLKSVUWrq50JT93djHpZsoS1m4JNUktJYWLvunWeeyADnM3abCyZEVYMb7o/jE5oMYoI+1sXFbHAYEUFla7dzmt83XVMi/iCxkZ69d1/gD17qCQMZ4PJxJvAqCBYVMQMuH37mKWoGXQE4kMQRbwqA2Ob/g7C8EWIdlINCkQo//75TyoA9z4QVisnmBYLZ57BsmQJsHGj5yKANhtnukuWRCDis6CAws5fd5/2dpagiGFEqGjnzaMM7+riKRUWejDRtbTwTdc4VaM1nWv52Pj4vl3XEhJoPvOnELq6uNr48EMu9/LyWB02kDZ5mrARSJTRahF5GcBrSqkv1sWOHITTAFwHYDWAZ8IyQk1M8vbb3p2/cXEMyFm5Eli0KHhfa24u8KMfAY89RuFv1IPr7qac+tKXqBA2bGAdpSNHuLI4ctdV9wAAHlpJREFU/XQ6tUNW5t9s5oGefdZ7VE1LCyXrjMGRtG8yBdDHIz6+78rIqM3ueg1sNmd3IoOkJPoUfFFRQf9EUxOd1QkJNEeVltL0dPvtg69XwxAhEIVwLuhQfk5ExgFoAhvkmAGsBPCIUmqLv52IyLsARnr46B6l1GuBDlhEbgZwMwAUB9OhRhMyurrYunKkp1/TgdnM1wcf0PcQLIWFwM9+Rr/E1q08ZmEhG5RZrSwQWFNDuZGSQgfzCy9wZfKd79BuHhIWLWIFv40bqdmMRAvDm261sj2bp76pg5WRI6mVW1udWX6GknBNRrFa3RwPoNb21YTi6FHgV79yzhoMDF9FZSVbld5zT19lowk7gfgQusCyFU+ISDwYftqplGoK5kBKqZAUdlZKPQXgKYDF7UKxT01wNDZSFvgzm6enA59/3v/jmEzMfzjuOOd73d0sstfU1FueGP7Plhbg/vuBr36VFp9x4/xnWPskLo6ZX+++y3DLI0ecBvgZM4ClS3136xmMmEwMKXv6aa5+TCYmo2Rl0YmcnEwTUkpK34vb3e3bubNmDbW7t2ZARUVcFm7f7ubY0ESCoNIrlVI9ALy0f9IMFwI18RqtgkPJtm1cGbjniAGcsG/bxn9rali7B6BcueaaAYTExsUB557LZhVVVdSG2dkRr69dU0N9ZDbT7BPWUuOnnUZtvmoV058zM+kX+OADCnSjSY3rD1xTQ0e0N/+BzUbF6ishAuBM4t13tUKIAtHJt3dDRC4G8CiAPABviMgWpdQ54TiWxcLKmc3NvKePOWbgsfPDDUM++PO1traG/pl+7z3PHeVqauhTSE7myqC9nbJJhPlVDz0E3HdtBTI3r6H5p6eHs9Szz+ZMP5AoISPbK8J8/jnNYXv2OLOQzWb6YC+6KEyKwWRi9vXkycDy5QyrNSrhtbdTIxtVBdvbqany82n/9zYL6OzkCsJf8lpKCotWaSJOTCgEpdQrYFmMMB6Dk53//Y+rXsBpCp07lzPIcDd3GSqYzcxc/uc/vXdK6+yk/Jw9O7THPnqUQt8Vq5WlhVJTnXLdbuf7iYlAUaFC5kcrUPuNF5A5JY5mDpOJjszHH2cW1+23h6kq3sDYs4cm96Sk3u1Ge3o4id63D/jhD8PUy8BkcsYZNzXxgmZk0M6/ahXLWVit9Ddcey239fUQJSTwQbTbfdcmD1VRLE3Q9EshiMhIpVRNqAcTTl5/nbVeiop6r1htNt7XNTV8sNyFjcYzCxfS2WtUJDUmfd3ddAQfPsyon48+ojwJVbP79PS+SqG2lgLSkEVGXpQxUR1V/SnmVP0H+8zFGJsbj0RjMZCdTbt4WRnw5JPA974XU/V5LBbqq8zMvqui+Hi6LsrLeW9fcYWfnXV0UGtu3+6sIz5nTmBmL5He202YwNfXvx6cXTAhgUvGXbt8xww3NLAUhibi9LeFyJshHUWYqakBXnmFD5B7MIjZzFXwgQNMhtIERnw8o3mWLqWT+eBBypuXX+a1HDeOMui554A772Q/hf5nqjiZP5+TVVeOHu0tk9rbqaTi4gAoheN2v4LO1BGwSnyfsPkvmtZs385wyBhi+3Y6yT2ZyAwKC2lG6+rysaNNm1gv/G9/40537WJG4fe+x0qC/f1hfPbu9MI55/DG8FZ3pK3N6Z/QRJz+KoTYmUYFwIcfcoXqq0LxyJEMIvFW70XTl4QE4JJLGCV4wQV874wzqCSOPZaWhOJiTgaffTY0Ctfod+CqFNwjIa1WYOJE/p3eVo30lsPoTvSxRBHhzbFx48AHGEK2bfMfzZqQwPOtrPSywfbtLN2dmckZUV4ef5ixYxlG+9xztD1FimOPZXejQ4dosjOUkc1Gh31jI/Dtb4duSakJiv4qhKdDOoows2OH//srJYWzMV2bK3gSEujQnTSJwt/d6pKQQFPd88/TpOSNo0c5mf3kE5pCPE1cU1O54uju5qqku5vWDIuFTuy2NiYNGxaOeEs7YDLDrjgo19I8vUhMpKkihrDZAmsDanST64NSFPjZ2Z5t8vHxXB29+CKdPpHivPOAu+/mMvLgQSqHqiquCn7609C03NP0i/46lf8X0lFoBjX79zPIxFc4flISTXc7dvRtb9nQAPznP1QGIs78pzFjmE9wzDG9tx83jklra9dyctvTQ4VQWMicBdckV2t8MmC3o61VoahIvM+4/SVURYGSEkZ5+sJu58tjrkV5ubORsjcSE3nxtm5ldEWkmDKFr7Y2Zx0N7cCLOsPChzB1qv+Zf0cHbbUxJhMGBYFOrEVoJXClsZEhoVu3UgEUF1N+jR3LFdvDD1OJuDNiBM1Vjz3G/Kmnn2awgHtya3NaEeqkAGloxeTJXgamFO0uoQ6JGiCzZvkv811b23tF1IuGhr7lJjwRF0dtHQ3S0mjC0sogJhgWPoR585xhiN6oqeFKVtfVCp5Au0fa7X3D/V96if6AoqLe5hEjsCU7mwFA3oSiycTVx/z5wG23cbJZXu58VRwUHJ59EeZOqEdqopcboKqKy5Bx4wI7kQiRkQFceimtKhZL388bGnidLr7Yyw4C/WFsNt3YRgOg/yajQeVDGDmSD40RdupqNrDZuKqeODEC3beGKBMmUDDZbN4VquEPmDTJ+V5zM30P7uVwXElPp2DfscN//bg5c2iO2rWLM+e4OI5tzOjZkGWXMtQsKYlLCZOJToejR7ksue22mAo5NTjvPA715ZepFBMTeZ2tVp7G975HU5lHxo93NpfwphwM+5xrfRDNsKVfCkEp9USoBxJuLryQZsqXX6awMASUCCtkXn21XrX2l+xsCuNPPvFeoqa2lkmvrsL/8OHAIhfj45mgFUhB0fh49hjujTD0aepUJlSVllKqjhoF3HQTbTMxWpxOhFUzTjuNPpbDh7nKmjKFATs+r116Or+4bp33DOu6Oq6MPNUC0Qw7YiJTORKIMCRy/nzOIJub+WDp0hWh4aqraNooL+eM1TAN9fSwgU5WFmWv6yQ80PB3r1E0wSDC5cmkSc5ZcSAhPDFCWhp72wfN5Zcz8a68nArQMA1Zrfxh0tKAW2+NydWRJvIMG4VgkJAATJ8e7VEMPTIy2MPgrbcY+WP4a0wmYPFi4Pzz+zo+CwqcUTK+ZLPFEuJ2mSLDRwCmpgJ33QWsWMEfpqvLef6nn84EkhEjoj1KTYwQSMe0QObP9mDLYWsCQylnFnVlJRXanDm0cniNqY8SaWmckF54odMsl5fnvbxNbi7NQDt3em+i09lJa45W4gMgJYUhWeedx25mGzdSy7a3O1tgBuqA1gxpxF/nSxHpAlAF35FFZqVUxMtAzpw5U5WWlkb6sBGjq4vhlKWlFIppaZx5t7Rw5X/bbcAJJ0R7lAOjupo5BQAVhOvEvaOD0V/f+Ab9PJoB0NTEjOXPP+fNlJREpdDRwaiL7343Aj1Je1NbywAvo3qIe7tUTfgQkU1KqZl93g9AIWxWSvksYhzINuFgKCsEpYA//Qn4+GP6+9wtHG1tDDv88Y8Hvz+wspKhpYcO8W+j/0x6OqvQ6rI2A8RiAX7+c0pfTyFddXVUED/9aUQqvh4+zETEHTucpkKluFq86ir/7RI0A8ebQghknRjI3EzP30LMoUOM2vGkDACuFtrbgWXLWGRuMDN6NPDAA5y8HjjgDACaOlV3UQwJW7awcJ+3mUN+Pj9fvz7sVUYrK6mbACYfGve23U7T4YMPAvfeG/HFisaB3zALRwtNj4jI1/xto+kf69dTGPryfebl8Vl3r/45GBFhzsDZZ9PUPWOGVgYh4513/Kfg5+UxIiAUJWm9oBRNoCZT35pXJhMnAT09LISoiQ4Djbv7aUhGoelDdbX/vAijKoEuyKfxiRFe6ouUFM4sfNXJGCDl5VyI+PIVFBTQlFStG/VGhUCijLZ5+wiAXtiFiZQU36U2DDyVg9BoepGYyJvJ15LLSDMPY7RReTn/9bXqNSJiDx70Hnk2LFGKduT16+mNT0lh7a2pU0P6mwWypwIA5wBodHtfAKwP2Ug0vZg9m2UdfNHaSvOvtrdqfHLqqWyE4y2NHKBj2aimFyaCsUYNOBFxKNHZCfzlL0xVj4+n6aCnh41e8vKAO+7wXf8lCAL59ZcDSFNKVbi9ygGsCckoNH04/njmC3mrJGq38xm+4IJBlXCriQbz5/Nfbz0PenoY43zmmWEdxsiR/vMBjSRyPclxYLczBG/TJnrhi4pYWqGggEECnZ0sCXz0aEgOF4hT+UallMeq7Eqpr4RkFJo+xMdT8dtsXCkapl2laOotK2MpjnnzojtOzSAgP5/9j2trOYswpt9KUZAcPMguZkabuTBx7LHMVu/TxtSFpiZGncVY4dnosX8/sHlz75AsV3JzGW743nshOZyeW8YwY8YA999PwV9fT8Vw8CADRr71LeDaa/XqQBMgc+cC99zDWk6HDjlvplGjgB/+kKFdYcZsBr72NTZTamvr+3lLC5XFddcNn8oiflmzhjkivi7IyJEs2hiCgIBAEtM+VUqdNNBtwsFQTkxzp7ubD0x8PBWCfmA0/aa5mbPK5GQvnXXCy9atwF//yvvZqNZqs9EScvPNuhJ3L+67j2Yhf1FiBw8Cv/1twL/nQBLTJvuINALoXNZ9xsJMYqKXNomaoU17O7B9O1vLJSezhvhAU3kzM6PaGnD6dMqunTsZeWQy0UQ0ebJuUNWHhATPyylXDMdLCKKNAtlDIPraNtCBaDQaF+x24PXXgeXLaQow6nmIADNn0q4Sa9UNgyA+nopBFy30w5w5wH//61uBNzVRo4bgfvCrEJRSFa5/i8gDAMwAtgDYopTaN+BRaDQaJ0pRCLz1FhvbuOYP2O3Ap5/SqXTXXbqr01Bnzhz2mW1v91w22G5nKOJXvxoSO3LQLkml1I8B/BFAK4BLRWTA7TRF5NcisltEtonIKyKSNdB9ajSDlkOHgJUrGVbonkxmMjHaoKwM+MBj8J9mKJGZCXzzm/TEu0aIAfTAl5UBZ50FnHxySA4XsEIQkd+LUAUppWqVUiuUUg8rpb4egnG8A+B4pdQ0AHsB/CgE+9RoBidr1tAe7Mugnp/PFYTO4Br6zJjBin/uEWLx8cAtt7AkcIiiTILxQrQBWCYiVyql2kXkbAA/UUoNOBJeKbXS5c+PAFw20H1qNIOWPXv8R4ukplIotLWxXZ1maDNhAntWNDYyPCshgclpIY47D1ghKKXuFZGvAFgjIt0A2gHcHdLRkBsAPB+G/Wo0gwORwOo8KKXjj4cb2dlhDRUOxmR0BoCvg4ogD8B3lFLvB/H9d0Vku4fXUpdt7gFgBfBvH/u5WURKRaS0vr4+0MNrNIOHE07gTNAXLS2cIQ7iSCNN7BGMyegeAPcppT4QkRMAPC8idyqlAsqZVkr5LJQiItcBuADAGcpHtpxS6ikATwFMTAt49BrNYOH004EVKxhu6qlCqVJ0Mt5449BfIbS3A9u2scSGkYdRWBjtUQ1ZgjEZLXb5/2cich6AlwGcOtBBiMi5AO4CsEAp1THQ/Wk0g5pRo4BLLwWef57/T0lxftbTwx6U06axiulQxW5nDsbrr/OczWamMwNcQd14I5ClgxFDjd/SFT6/LJKslPJSQjGo/ewHkAjAKNn3kVLqVn/fG06lKzTDDKWAdeuA//2P5iHDr2A2A4sXU2EkJkZ7lOHjhReoDNzzMJRib+i8POD//k+bzPrJQEpXeCUUysCxn/CWWdRoBhsiwIIFLGe7dy+VQmIicMwxnhOUhhKHD7N/Q0lJ39BbEZaALitjeO4FF0RjhEOW8LVH0mg0AycuDpgyJdqjiCzvv+8/D2PkSPpZzjlHN98OIbp4skajiS327fOfW5GcDHR0cOWkCRl6haDR+KGzk2ZrpZggrPPAwkygeRjGtpqQoRWCRuOFtjb6NVevZo96Q07NnQtcdNHAq1BrvDBtGvDqq74rfLa1MUErimW8hyLaZKTReKC1FfjFL1hjLjeXwS5jxtCfuXEj8OCDQHV1tEc5RDHCaS0Wz58rxUJvS5boBgohRisEjcYDL7wA1NSwlW1CgvN9s5lKwWYD/vznwC0bmiDIzQWuvhqorOzbHKanhzWcpk4F5s+PzviGMNpkpNG40dwMrF/vOyE2Lw+oqGD04/jxkRvbsGHxYobXvvACL7ThKzCbgTPPZB6Gq6bWhAStEDQaN8rKnDlg3hDha8+eYaQQOjuBzz/nLD0zk3kC4XLqitBZM2sWcOAAtXR8PDBxok5GCyNaIWiihs3G6r2xFihiC7AhrMnk3cw9pLBYgGXLgHfeoTIAqDELCoArrgBOPDF8xzabmYyniQhaIWgiSkMDG3298w4dt0lJTMhduJBle2KBESNYSsdfdWmbLXbGHDZ6eoBHH2WBuaKi3maalhbgkUeAm25iQT7NoEcrBE3EKCsDfvMbWh7y8yl4LRZg1Srg3XeB224DTjop2qOkI3nMGPYu91Z6vrublSSmTYvs2CLO++8DW7bQLuauHTMyqCCefZYF58JYp18TGXSUkSYitLYCv/sdzcDFxVwZAJQno0dTOTz+OMvYRBsR4Ctfodm6vb3v5xYLA2CuvNJ5HkMSu501hQoKvC+VkpK43YYNkR2bJixohaCJCJ98wghCbxWLU1Jok1+1KrLj8sbkyexY2NYGlJcDtbUMfa+o4L/XXUcz15CmsZE2Pn9O3MxMriI0gx5tMtJEhNWrgZwc39sUFNBCcfXVsZFvNH06TeRbtgA7dtCnMH48A1/S06M9uggQaItOkcA98ZqYRisETURobe3d58UTcXEsEWGxsHZZLJCUxOjHuXOjPZIokJlJR4nhMPFGSwu1pGbQo01GmoiQlQV0dfnexmKhT2Eo930ZVMTHA2edxZRtb9hs1OI6ymhIoBWCJiIsXsyoHV/U1QGLFtGXoIkRzjiDpSSqq/vW6bBa6WA56yzd53iIoB89TUSYNYurhCNHPH/e2kpT9KJFkR2Xxg8ZGcDddzMWt6KCr8OH/7+9+4/1qq7jOP58CcgPUZl4mYok2hil5mCaM3UNJia5pmFlOio3Xc6ZkySbGk1NcqOxtVazZZss28jGJhnTsotkqS1KMFQU8UeTgRKCv/ACegXe/XE+V75cv/fe871wv+ec7309tju+53vP93te4uX7vp/POefzzv7cvDlb9vWyy8p3d6H1ywH1VC6aeypXy2uvwcKF2Uhh7NjsPEFnZ1Ykhg2DOXMGX3OwyojIFpV79tls7q+tLbtD2c0hKqmnnsouCNZUHR3Z8tHLl++7onHatGzF476uQjKzg6OnguCrjKypRo/OpoU8NWRWPj6HYGZmgAuCmZklLghmZga4IJiZWeKCYGZmQEkKgqT5kp6RtEZSuyTf9mhm1mSlKAjAwog4LSKmAA8CtxYdyMxssClFQYiI7TWbhwHVvVvOzKyiSnNjmqQ7gW8B7wK+bcnMrMmaNkKQ9IiktXW+LgaIiHkRMQFYDFzXy/tcLWmVpFVbt25tVnwzs5ZXurWMJJ0APBQRp/a1r9cyMjNrXE9rGZXiHIKkSTWbFwEvFJXFzGywKss5hAWSJgN7gQ3ANQXnMTMbdEpRECLiK0VnMDMb7EoxZWRmZsVzQTAzM8AFwczMEhcEMzMDXBDMzCxxQTAzM8AFwczMEhcEMzMDXBDMzCwpxZ3K1hwffghvvQUSHHUUDPX/fTOr4Y+EQaCjA1asgPZ2eP/97LnRo2HmTJg+HUaMKDafmZWDC0KLe/ddWLAANm+GY46Btrbs+Z074b77YPVqmDsXRo0qNqeZFc/nEFrcokWwbRtMnLj/SGDUKDjxRHjlFViypLB4ZlYiLggtbPNmePppOO64+t+XYPx4ePxx2L69/j5mNni4ILSw9euzP6We9xk6FPbuhZdeak4mMysvF4QW9sEHvReDLlJ2BZKZDW4uCC1s7FjI0zI7AsaMGfg8ZlZuLggt7JRTYPhw6OzseZ8dO7JiMGlSz/uY2eDggtDCRo6EWbNg40bYvfvj3+/shC1b4NJLYciQ5uczs3LxfQgt7oILYNcuWLYMDjkEjjgimyLquqroiivgrLOKzWhm5eCC0OKkbJRwzjnwxBPw4ovZczNmwNlnZ0tYmJmBC8KgMW4cXHJJ0SnMrMx8DsHMzAAXBDMzS1wQzMwMcEEwM7NEkedW1pKStBXYMMCHORrYNsDHGAjO3VxVzQ3Vze7c/XdCRLR1f7LSBaEZJK2KiDOKztEo526uquaG6mZ37oPPU0ZmZga4IJiZWeKC0LdfFx2gn5y7uaqaG6qb3bkPMp9DMDMzwCMEMzNLXBBykDRf0jOS1khql9RDl+JykbRQ0gsp+x8kVaINjqSvSXpO0l5Jpbwao5akmZLWS3pZ0s1F58lL0iJJb0haW3SWvCRNkPSopHXpZ2RO0ZnykjRC0r8lPZ2y/6joTN15yigHSUdExPb0+Hrg5Ii4puBYfZL0BeCvEbFb0k8AIuKmgmP1SdKngb3A3cCNEbGq4Eg9kjQEeBE4H9gEPAlcHhHPFxosB0mfBzqA30bEqUXnyUPSscCxEfGUpMOB1cCXK/L3LeCwiOiQNAx4ApgTESsLjvYRjxBy6CoGyWFAJapoRLRHRFdrnJXA8UXmySsi1kXE+qJz5HQm8HJE/DciOoHfAxcXnCmXiHgMeKvoHI2IiM0R8VR6/B6wDhhfbKp8ItORNoelr1J9lrgg5CTpTkkbgdnArUXn6YcrgT8XHaIFjQc21mxvoiIfUFUnaSIwFfhXsUnykzRE0hrgDWB5RJQquwtCIukRSWvrfF0MEBHzImICsBi4rti0+/SVO+0zD9hNlr0U8uSuCNV5rlS/9bUiSaOB+4HvdhvBl1pE7ImIKWSj9TMllWqqzg1ykoiYkXPX3wEPAbcNYJzc+sot6QrgS8B5UaITRg38fZfdJmBCzfbxwOsFZRkU0vz7/cDiiFhadJ7+iIh3JP0NmAmU5qS+Rwg5SJpUs3kR8EJRWRohaSZwE3BRROwsOk+LehKYJOlESYcClwHLCs7UstKJ2XuAdRHx06LzNEJSW9eVfpJGAjMo2WeJrzLKQdL9wGSyK182ANdExGvFpuqbpJeB4cCb6amVFbk6ahbwC6ANeAdYExEXFJuqZ5IuBH4GDAEWRcSdBUfKRdJ9wDSy1Te3ALdFxD2FhuqDpHOBx4Fnyf49AvwgIv5UXKp8JJ0G3Ev2c3IIsCQi7ig21f5cEMzMDPCUkZmZJS4IZmYGuCCYmVnigmBmZoALgpmZJS4IZmYGuCCYmVnigmAtQdJESbvSwmFdz32sT4GkkamvRaekow/wmCMl/T0tgY2k69M6/Q2tGSVpjKRrDyRLjmN8rPeBpEMlPSbJS9gY4IJgreWVtHBYV5+Cu4AvAicDl0s6OSJ2pX0OxnpDVwJLI2JP2r4WuDAiZjf4PmPSaxuiTN5/w78hWzfnI2m57hXA1xs9trUmFwSrjNQp6/z0+MeSft7L7s3oUzAb+GPK8yvgJGCZpBskfSN1x1oj6e6aUcQDklanjllXp/dZAHwy7bswjXZqf5O/UdLt6fHENAr5JfAUMKGnY9XqpffBA+m/w8wFwSrlNmCepNlk6+Df0Mu+A9qnIC1kd1JEvAqQ1oh6HZgOPEz2W/c5aTSyh30fuldGxOnAGcD1ksYCN5NGNxHx/RyHn0zW5WwqMKqXY+WxFvhsA/tbC/PcoVVGRDyWVrucC0zrmqqRNJ9sBcxaA92n4GiyhffqOQ84HXgyi8tIsoYokBWBWenxBGAS8L8Gj72hpu1ib8fqU0TsSedTDk8dyGwQc0GwypD0GeBYYFvXh5ekY6j/c9xwnwJJ3wG+nTYvBGbVbkdE7et3ASN6eivg3oi4pdv7TyNb8vhzEbEzrYdf7z12s//ovfs+O/o6VoOGA+8fwOutRXjKyCpBWXP1xWTnAXZI6loOeyqwps5LGu5TEBF3pWmbKRHxevftbvu+DQyRVO8DfQXwVUnjUvajJJ0AHAm8nYrBp4Cz0v7vAYfXvH4LME7SWEnDyRoc9aSnY+WSpqy2RsSHeV9jrcsFwUpP0ihgKfC9iFgHzAduT9+eQp2CEBG7yVqd/oWsEfuSiHjuIEdrB86tc+zngR8C7ZKeAZaTjWweBoam5+YDK9P+bwL/SC1EF6YP5zvIegU/SC9NVHo51n5S74N/ApMlbZJ0VfrWdKD0vQSsOdwPwSpN0j1k0zqfAB6MiFw9aiW9CpwREdsO4NhTgbkR8c3+vkfRJC0FbomI9UVnseJ5hGCVFhFXRcResqtrjqy9Ma2erhvTgGHs67jV32P/B3i03mWeVZCm0h5wMbAuHiGYmRngEYKZmSUuCGZmBrggmJlZ4oJgZmaAC4KZmSUuCGZmBrggmJlZ4oJgZmYA/B84qOy0PfvCMwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_dataset('Scatterplot of the training data', train_x, train_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 1.3382818 -0.98613256]\n", + " [ 0.5128146 0.43299454]\n", + " [-0.4473693 -0.2680512 ]\n", + " [-0.9865851 -0.28692 ]\n", + " [-1.0693829 0.41718036]]\n", + "[1 1 0 0 0]\n" + ] + } + ], + "source": [ + "print(train_x[:5])\n", + "print(train_labels[:5])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Подход\n", + "\n", + " * Задаём функцию потерь (loss function) $\\mathcal{L}$\n", + " * Определяем модель $f_{\\theta}$ с параметрами $\\theta$\n", + " * Подстраиваем $\\theta$ для минимизации $\\mathcal{L_{\\theta}}$ на обучающей выборке\n", + "$\\theta = \\mathrm{argmin}_\\theta \\mathcal{L_\\theta}(X,Y)$\n", + " * Проверяем качество модели на тестовой выборке\n", + "\n", + "Результат: $f_{\\theta}$, которая делает предсказания на новых данных: $\\hat{Y} = f_{\\theta}(X_{new})$" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Функции потерь\n", + "\n", + "* Определяют (формализуют) цель обучения, т.е. фразу _\"точно предсказать\"_\n", + "* Выбор обусловлен требуемыми свойствами (непрерывность, дифференцируемость)\n", + "\n", + "**Часто используемые функции для регрессии**\n", + "\n", + "Абсолютная ошибка: $\\mathcal{L}_{abs}(\\theta) = \\sum_{i=1}^n |y_i - f_{\\theta}(x_i)|$\n", + "\n", + "Среднеквадратичная ошибка: $\\mathcal{L}_{sq}(\\theta) = \\sum_{i=1}^n (y_i - f_{\\theta}(x_i))^2$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "# helper function for plotting various loss functions\n", + "def plot_loss_functions(suptitle, functions, ylabels, xlabel):\n", + " fig, ax = plt.subplots(1,len(functions), figsize=(9, 3))\n", + " plt.subplots_adjust(bottom=0.2, wspace=0.4)\n", + " fig.suptitle(suptitle)\n", + " for i, fun in enumerate(functions):\n", + " ax[i].set_xlabel(xlabel)\n", + " if len(ylabels) > i:\n", + " ax[i].set_ylabel(ylabels[i])\n", + " ax[i].plot(x, fun)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAADZCAYAAADsdQBUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3hUZdrH8e+dSknoPYGE3iEUKVbEhhUQQbD31dUVlLWvvXfBsrvWtQIiiNhFBUGlSAmhQyiB0EJPKOn3+8cM+2ZjIJlkZs6U+3Nd52Imc8pvJuHMc57zFFFVjDHGGGMCXYTTAYwxxhhjKsIKLcYYY4wJClZoMcYYY0xQsEKLMcYYY4KCFVqMMcYYExSs0GKMMcaYoGCFFmOCiIioiLTx07Eai8hsEckRkRf9ccwSxz4oIq38eDwRkfdEZJ+ILPDXcQONvz93YzxlhRYTMkTkMhFZ6D7xbheRb0XkZKdzBbGbgN1ALVUd66uDiMgsEbmh5M9UNU5VN/jqmGU4GTgLSFTVPn48bkBx4HM3xiNWaDEhQUTuBF4BngIaAy2AN4DBTuYKcknASg2PESiTgE2qesjTDUUkyhvrePuYxoQkVbXFlqBegNrAQWD4cdaJxVWo2eZeXgFi3a8NADKBu4EsYDswBDgPWAvsBe4vsa9HgMnAR0AOsAxoB9zn3n4LcHaJ9ZsB0937SQduLLWvT4EP3PtaAfQ+zvtQoE2J9/0BsAvIAP4BRLhfawP8AhzAVVsyyf1zAV525zwApAFdyjjOf4ACIN/92Z7p/tkTJdYZAGSWeL4J+Lt7nweASUC1Eq8PBlKBbGA9MAh4EigCct3Hec3D93kN8CvwArAP2AicW+KY1wAb3J/tRuDyMt7r9e7jF7kzPOr++Y3u39de9++vWanfw63AOmBjGftMdq9zPbAZmO3+eT/gd2A/sBQYUGKblsBsd9YfgdeBj6qwvzLf+7H+Nrz5udtii68WxwPYYktVF/eXXyEQdZx1HgPmAY2Ahu4T/ePu1wa4t38IiHZ/We0CPgHigc7uL7VW7vUfcT8/B4hyn9g3Ag+U2H5jiWP/gqvWpxqQ4t73GaX2dR4QCTwNzDvO+yj5pfIB8IU7YzKuAtb17tcmuPNEuI97svvn5wCLgDq4CjAdgabHONZ/+N9CSunnA/hzoWUBrkJaPWAVcLP7tT7uL8mz3JkSgA7u12YBN1TyfV6Dq3B1o/vzuwVXoVSAmrgKSO3d6zYFOh/jvV4D/Fri+UBcX+g9cRV4X8VdUCiRb4b7fVYvY3/J7nU+cOeo7n7Pe9y/6wj3Z7EHaOjeZi6uQkAMrttV2fy50FKh/R3vvR/rb8Nbn7vT5wNbQntxPIAttlR1AS4HdpSzznrgvBLPz8F1OwBcX75HgEj383j3ybtvifUXAUPcjx8BZpR47UJcV+ilt68DNMd1BR9fYv2ngf+U2NePJV7rBBw5zvtQXFfKkUAe0KnEa38BZrkffwC8iauNRsntB7q/fPrhvmo+zrH+g+eFlitKPH8O+Jf78b+Bl49xnFkco9BSgfd5DZBe4rUa7m2b4Pri3g8Mo4yCRanjXcP/FlreAZ4r8TzO/SWdXCLfwOPsL9m9TqsSP7sH+LDUet8DV+O6nVkI1Cjx2kf8udBS0f0d870f62/DW5+7r/6f22KLqlqbFhMS9gANyrnP3wxXFfdRGe6f/XcfqlrkfnzE/e/OEq8fwfXFxTFe213G9nHuY+xV1ZxSx04o8XxHiceHgWoVaLPQANcVeen3dHS/d+OqbVggIitE5DoAVf0ZeA3XrYedIvKmiNQq51ieKP1ejn5mzXEVHD1V3vv8n2Oq6mH3wzh1tU+5FLgZ2C4iX4tIhwoe93/+XlT1IK6/s5LH3VKB/ZRcJwkYLiL7jy64alSa8v9/J4ePsa1H+yvnvZf5t1FKpT/3MvZljNdYocWEgrm4brEMOc4623Cd5I9q4f6Zr20D6olIfKljb63ifnfjuvIv/Z62AqjqDlW9UVWb4bpCfuNoV2lVHa+qvXDd9moH3FXBYx7CdUV9VBMP8m4BWh/jNT3Odsd9n+VR1e9V9SxcBYPVwFsV2Y5Sfy8iUhOoX+q4x8td1jpbcNWM1Cmx1FTVZ3C1o6onIiU/3+ZV2N8x3/vx/jZKqNLnboyvWKHFBD1VPYCrPcrrIjJERGqISLSInCsiz7lXmwD8Q0QaikgD9/of+SHbFlztZ54WkWoi0g1XY8qPq7jfIlwNeJ8UkXgRSQLuxP2eRGS4iCS6V9+H68uuSEROEJG+IhKNqxBytAFqRaQC54lIPRFpAozxIPI7wLUicoaIRIhIQokr/51AmWODlPc+j8c9zsxF7gJHHq5beBV9r5+486aISCyuXmnzVXVTBbcvy0fAhSJyjohEuv8eBohIoqpmAAuBR0QkRkT647rtWKn9He+9H+tvo+SOq/K5G+NLVmgxIUFVX8J1Uv0HroauW4DbgGnuVZ7A9aWQhqu3z2L3z/xhFK42CduAz4GHVXWGF/b7N1wFjw24enJ8Arzrfu0EYL6IHMTV82W0qm4EauG64t6Hq7p/D67GnxXxIa4eKpuAH3D1DqoQVV0AXIur59IBXI2Tj17FjwMucQ/sNt7D93k8EcBYXJ/7XuA04K8VzPsT8CAwBVctSGtgZEW2Pc4+t+DqQXU///83ehf/fx6+HOiP63fyBK7PN6+S+zveez/W30Zplf3cjfEZUa1IDacxxhh/EpFJwGpVfdjpLMYECqtpMcaYAOC+ddfafftsEK5alGnlbWdMOLFRFY0xJjA0AabiavCbCdyiqkucjWRMYLHbQ8YYY4wJCnZ7yBhjjDFBwQotxhhjjAkKVmgxxhhjTFCwQosxxhhjgoIVWowxxhgTFKzQYowxxpigYIUWY4wxxgSFoB9crkGDBpqcnOx0DGOC1qJFi3arakOnc4QKOycZUzXHOycFfaElOTmZhQsXOh3DmKAlIhlOZwgldk4ypmqOd06y20PGGGOMCQp+K7SISHMRmSkiq0RkhYiMLmMdEZHxIpIuImki0tNf+Ywx5lhEJFJElojIV05nMSac+bOmpRAYq6odgX7ArSLSqdQ65wJt3ctNwD/9mM8YY45lNLDK6RDGhDu/FVpUdbuqLnY/zsF1Akgotdpg4AN1mQfUEZGmVTlucbGydMv+quzCmKC0ZPM+pyOEBBFJBM4H3vbG/gqKilm+9YA3dmVM0Ch0/91XdZJmR9q0iEgy0AOYX+qlBGBLieeZ/Llg45FXf07nkn/9bicJE1a+W76DoW/8zvSl25yOEgpeAe4Gir2xs+e/X8PF//yd7NwCb+zOmKDw2/o9XPDqr8xas6tK+/F7oUVE4oApwBhVzS79chmb/KlYJiI3ichCEVm4a9fxP4Cr+idRr2YMoycu4Uh+UaVzGxMsdhzI5d6paXRNqM2gzk2cjhPUROQCIEtVF5WzXoXPSed2aUJ+YTHfLdvhzajGBLRpS7ZSq1oU/VvXr9J+/FpoEZFoXAWWj1V1ahmrZALNSzxPBP50qaiqb6pqb1Xt3bDh8YeXqFszhheHp7B+1yGe+HplFdIbE/iKi5Wxk1PJKyhm3MgUYqKsg2AVnQRcJCKbgInAQBH5qPRKnpyTUprXIbl+DT5fstUngY0JNIfyCvlu+Q7O79aUatGRVdqXP3sPCfAOsEpVXzrGatOBq9y9iPoBB1R1e1WPfXLbBtx4Sks+nr+ZH1furOrujAlYb/+6gd/S9/DwhZ1o1TDO6ThBT1XvU9VEVU0GRgI/q+oVVdmniDCkRwLzNu5h2/4jXslpTCCbsXInRwqKGJJSpdYegH9rWk4CrsR1pZLqXs4TkZtF5Gb3Ot8AG4B04C3gr946+N/PaU+nprW4e0oaWdm53tqtMQFj+dYDPP/9Gs7p3JhLT2he/gbGMUNSElDF2hyZsPD5kq0k1KnOCcn1qrwvf/Ye+lVVRVW7qWqKe/lGVf+lqv9yr6OqequqtlbVrqrqtWElY6MiGT8qhcP5hYydvJTi4qq1YDYmkBzJL2L0xCXUqxnDMxd3w1WxabxJVWep6gXe2Fdyg5r0aFGHzxfbLSIT2nbl5PFr+m4GpzQjIqLq56WwuuHdplE8/zi/E3PW7ea93zc5HccYr3ni65Vs2H2Il0akULdmjNNxTAVc3COBNTtzWLHNejaa0PVF6laKipWhPap+awjCrNACcHnfFpzZsRHPfruaVdtLd14yJvjMWLmTj+dv5qZTWnFSmwZOxzEVdEG3ZkRHClOttsWEsKmLt9ItsTZtG8d7ZX9hV2gREZ4d1o3aNaK5fcIScgusG7QJXlnZudwzJY3OzWpx59ntnI5jPFC3ZgxndGjMF6lbKSzyyhAwxgSUVduzWbk9m2E9E722z7ArtADUj4vlheHdWZd1kKe/sZG5TXBydW9eyuH8QsaNTCE2qmpdCY3/Xdwzgd0H85mzbrfTUYzxus+XbCUqQriwezOv7TMsCy0Ap7VryHUnteT9uRn8vNq6QZvg897vm5izbjf/OL8TbRp5p+rV+NeA9o2oWyOazxZnOh3FGK8qKlamLdnK6R0aUc+L7ezCttACcPeg9nRoEs/dn6WxKyfP6TjGVNiq7dk8++1qzurUmMv7tnA6jqmkmKgILurejBkrd3LgiA3rb0LHnHW7yMrJY1hP7zTAPSqsCy3VoiMZP6oHObmF3P3Z0ipP5GSMP+QWFHH7hCXUrhHNMxd3te7NQe6SXs3JLyzmqzQbs8WEjs8WZVK3RjQDOzT26n7DutAC0K5xPPef15GZa3bxvnWDNkHg6W9WsS7rIC+N6E79uFin45gq6pJQi/aN4/lskd0iMqHhwOECfli5k8EpCV6fSiTsCy3gmlTx9PYNeerb1azdmeN0HGOO6efVO3l/bgbXn9ySU9oef44bExxEhEt6JbJk837Ssw46HceYKvsybRv5hcVc0st7vYaOskILrpPGc5d0p1a1KOsGbQLWrpw87pqc5mqHNai903GMFw3u0YzICGGKNcg1IeCzRZl0aBJP52a1vL5vK7S4NYyP5flLurN6Rw7Pfrfa6TjG/A9V5e+Tl3Iwr5Dxo3pY9+YQ0yi+GgPaNWTq4kwbs8UEtXU7c0jdsp9hPRN90t7OCi0lnN6hEdecmMx7v21i1posp+MY81/v/76JX9bu4v7zOtLOSyNLmsAyvHdzdmbn2ZgtJqhNXpRJVIQw1Mu9ho6yQksp957bgXaN4/j75DT2HLRu0MZ5a3bk8NS3qxnYoRFX9U9yOo7xkYEdGlG/ZgyfLtzidBRjKqWgqJipizM5o2MjGviok4AVWkqpFh3JuJE9yD5SwD1T0qwbtHFUboFr9uZa1aJ57hKbvTmUxURFMLRHAj+u2mkXTCYozVydxe6D+Yzo3dxnx7BCSxk6Nq3FPed24MdVWXw8f7PTcUwYe+67NazekcPzw7v57MrFBI4RJzSnoEj5fIlNomiCz6cLt9AoPpbT2vmuZ6PHhRYRqSkiId8K8NoTkzm1XUOe+Hol6VnWDdr43y9rd/Hubxu55sRkTm/fyOk4xg/aNY4npXkdPl24xWp5TVDZmZ3LzDW7uLhnIlGRvqsPKXfPIhIhIpeJyNcikgWsBraLyAoReV5E2vosnYMiIoQXhnejRkwUt09IJa/QukEb/9lzMI+xny6lfeN47j23g9NxjB+NPKE5a3ceZMmW/U5HMabCPluUSVGxcukJvrs1BBWraZkJtAbuA5qoanNVbQScAswDnhGRK3yY0TGN4qvx3LBurNyezQvfr3E6jgkTqsrdn6WRnVvAuFEpVIsO+YpNU8IF3ZtRIyaSSQusQa4JDsXFyqcLt9CvVT1aNqjp02NVpNBypqo+rqppqvrfAQRUda+qTlHVYcAk30V01pmdGnNFvxa8NWcjv1pXROMHH83fzE+rs7jv3A50aOL9wZlMYIuLjeLCbs34Mm0bB/MKnY5jTLnmbdxDxp7DjDzB95O3lltoUdUCABEZLiLx7scPishUEelZcp1Q9cB5nWjTKI47P01l36F8p+OYEJaelcMTX63ktHYNuebEZKfjhIRgbId3aZ/mHM4v4sulNomiCXwTF2yhVrUoBnVp4vNjedJa5kFVzRGRk4GzgfeBf/omVmCpHhPJuJEp7Ducb92gjc/kFRZx+4RU4mKjeH64dW+urFBoh9ejeR3aN45nwgLrvWgC295D+Xy3fAdDeyT45Va2J4WWoy1Rzwf+qapfADHejxSYOjerzd3ndOCHlTuZ+Ifdazbe98L3a1i5PZvnLulGo/hqTscJZkHfDk9EGNWnOWmZB1i+9YDTcYw5pqmLM8kvKuayvv4Z+NKTQstWEfk3MAL4RkRiPdw+6F1/cktObtOAx75cyfpdNhur8Z5f1+3mrTkbubJfEmd0bOx0nGAXEu3whvZMJDYqgk+stsUEKFXlkwWb6ZVUl/ZN/DO9iCeFjhHA98AgVd0P1APu8kmqABURIbw4ojux0RGMnriE/EKb2MxU3d5D+dz5aSptGsXxwPkdnY4T9EKlHV7t6tFc0K0ZXyzZag1yTUCat2EvG3YdYlQf3zfAParChRZVPayqU1V1nfv5dlX9wXfRAlPjWtV4dlg3lm/N5qUZa52OY4KcqnLPlDT2Hy5g/Mge1r3Zu4K+Hd5lfVtwKL+I6anWINcEngkLNlOrWhQXdGvqt2NWuNBS6qrlHyWvWiq4/bsikiUiy4/x+gAROSAiqe7loYru29/O6dyEUX1a8O/Z6/l9vXWDNpU3YcEWZqzcyd2D2tOpmXVv9rIqt8MTkWoiskBElrob8j7q9ZTH0bNFHTo0iefj+RnWAcAElN0H8/h2+XYu7pno14utyvYeOgfPr1r+AwwqZ505qpriXh7zYN9+9+AFHWlZvyZ3TlrK/sPWDdp4bv2ugzz+1UpOaduA605q6XScUOSNdnh5wEBV7Q6kAINEpJ+Xcx6TiHB5vyRWbMsm1UbINQHk04VbKChSrujnv1tD4MfeQ6o6G9jrwfECWo2YKMaP6sGeQ3ncN3WZXQUZj+QXFjN64hKqRUfwwvDuRERY92YfKN0Ory4etsNTl6Ot7qPdi1//sw/tkUDNmEg+mmcNck1gKCpWPpm/mX6t6tGmkX8a4B4VaL2H+rurYb8Vkc7HWklEbhKRhSKycNeuXV6OUHFdEmoz9uz2fLt8B5MXZjqWwwSfF2esYfnWbJ4Z1o3Gtax7s4+cD8xQ1XUi8g/gDcDj+7kiEikiqUCWe3/zy1jHZ+ekuNgohvRI4Ku0bVarawLC7HW7yNx3hCv6+aebc0mB1HtoMZDkroZ9FZh2rBVV9U1V7a2qvRs29N0U2BVx0ymt6N+qPo98uYKNuw85msUEh9/Td/Pm7A2M6tOcczr7fgTJMFbVW9oAqGqRqqYAiUAfEelSxjo+PSdd3jeJvMJiPltkF0fGeR/NzaBBXCxnd/L/+cuj3kPAeuAcEbkNaOTN3kOqmn20GlZVvwGiRaSBt/bvKxERwkuXdic6MoIxE5dQUGTdoM2x7T+cz52fLqVlg5o8eEEnp+OEOq8OiOm+WJtF+W3zvK5Ts1r0TqrLh/MyKC62W9HGOVv2HubnNVmM6tOcmCj/D9XmSe+h0cDHQCP38pGI/M1bQUSkibjHLReRPu5se7y1f19qWrs6z1zclaWZB3jlR+sGbcqmqtw3dRl7DuUxfmQPasREOR0p1FX5lraINBSROu7H1YEzcU0L4HdX9k8iY89hflnn3C1xYz6al0GECJf19W8D3KM8+Q98PdBXVR9S1YeAfsCNFd1YRCYAc4H2IpIpIteLyM0icrN7lUuA5SKyFBgPjNQgat16btemjOidyBuz1jN/Q1CUtYyfTV6YybfLdzD27PZ0SajtdJxw4I1b2k2BmSKSBvyBq03LV96NWTHndmlKg7hYPpyb4cThjSG3oIhJC7dwdqfGNK1d3ZEMnlzqCf9f3Yr7cYW7PKjqqHJefw14zYM8AefhCzuzYONe7piUyrejT6V2jWinI5kAsXH3IR75cgUntq7PTae0cjpOWFDVwyJy9Jb2ObiGVPDolraqpgE9fBLQQzFREVzWpzmvzkwnY88hkurXdDqSCTPTl25j/+ECruzv/wa4R3lS0/IeMF9EHhGRR3BNPPaOT1IFqZqxUYwb2YOsnDzun2bdoI1LQZGre3N0ZAQvjrDuzf7i61vaTrisbxKRIlbbYvxOVXn/9020bRRH/1b1HcvhSUPcl4DrcI21sg+4VlVf8VWwYNW9eR3uOKsdX6dtZ+rirU7HMQHg5RlrScs8wDMXd3WsSjVMVemWdiBqUrsag7o0YdLCLRyy+YiMHy3M2MeKbdlcc1Iy7uanjvCoUZqqLlLV8ao6TlWX+CpUsLv5tNb0aVmPh75YTsYe6wYdzuZt2MM/f1nPiN6JnNvVf/NzGKCKt7QD1bUnJZOTW8jnS+yiyPjPf37fRK1qUQztkeBojnILLSKSIyLZZSw5IpLtj5DBJjJCePnSFCIihDGTUq0bdJg6cLiAOyelkly/Jg9feMyxEo3vhOQt7Z4t6tI1oTb/+X2T3YI2frH9wBG+W76DkX1aON7rsdxCi6rGq2qtMpZ4VbUZ3o4hoU51nhralSWb9/PqT+ucjmP8TFW5f9oysnLyeOXSFGrGWvdmfwvVW9oiwjUnJpOedZA562zCVuN7H851Tdh5pQMj4Jbm/5FhwsiF3ZsxrGcir81M549NITPtkqmAKYu38nXadu44qx3dm9dxOk7YCtVb2hd0d3V/fve3jU5HMSHuSH4RnyzYzNmdmtC8Xg2n41ihxdceHdyZxLo1GDMxlezcAqfjGD/I2HOIh79YTp+W9bj5tNZOxwk74XBLOzYqkiv7JTFrzS7Ssw6Wv4ExlTR1SSb7Dxdw3cmBMRO9FVp8LC42ildGprAjO5eHpi13Oo7xMVf35tT/tmuKtO7Nfhcut7Qv79eCmKgI3rPaFuMjxcXKu79upGtCbU5Irut0HMCzYfxFRK4QkYfcz1u4h9s35ejZoi6jz2jLtNRtTLMW/yHt1Z/WkbplP09d3JWEOta92fhOg7hYhqQ0Y8riTPYdstmfjff9sm4X63cd4rqTne3mXJInNS1vAP2BoyPb5gCvez1RiPrrgNb0TqrLg9OWs2XvYafjGB/4Y9NeXpuZzsU9E7igWzOn45gwcP3JrcgtKObj+TbYnPG+t+dsoHGtWM7vGjjnM08KLX1V9VYgF0BV91GFGVPDTVRkBC9fmgLAmEmpFFo36JCSnVvAmImpJNatwWODuzgdx4SJ9k3iOa1dQ/7zewa5BUXlb2BMBa3YdoDf0vdw7UktHZnN+Vg8SVIgIpGAgmv2U8C+eT3QvF4NnhjahUUZ+3h95nqn4xgvenDacnZk5/LKyBTirHuz8aMbT2nF7oN5TE/d5nQUE0LenrORmjGRjOrjzGzOx+LJ2XU88DnQSESexDUr84M+SRXCBqckMHN1FuN/XsfJbRvQKykwGjeZypu2ZCtfpG7jzrPa0bOF/T6dJiJ3Hu919/gtIeOkNvXp2LQWb83ZwCW9Em1uK1Nl2w8c4cul27iqfzK1qwfWxL+ezD30MXA38DSwHRiiqp/6Klgoe2xIF5rWrsaYSUvIsW7QQW3L3sP8Y9pyeifV5a8DrHtzgIh3L72BW4AE93Iz0MnBXD4hItx0akvWZR1k5posp+OYEPDOnI0orikjAo0nvYeeVdXVqvq6qr6mqqtE5FlfhgtVtapF88qlKWzdd4SHp69wOo6ppMKiYsZMSkWAly9NISoycO77hjNVfVRVHwUaAD1VdayqjgV6AYnOpvONC7o1I6FOdf79ywano5ggd+BwARMWbObCbk0DYjC50jw5y55Vxs/O9VaQcNM7uR63DWzL1MVbmb7U7kUHo9dnrmdRxj6eGNolIP9zG1oAJfsC5wPJzkTxrejICK4/uSULNu1lUcY+p+OYIPbR/AwO5Rdx06mBWXNckQkTbxGRZUAHEUlzL8tEZCOwzPcRQ9ftA9vQo0UdHvh8GZn7rBt0MFmUsY/xP69jaI8EBqc4O+upOaYPgQXuCRMfBuYDHzicyWdG9mlOnRrR/OsXa+RvKie3oIj3ftvIae0a0qlZYI7DWJGalk+AC4EvgAvcjy8Aeqnq5T7MFvKiIiMYd2kPVOHOSUspKrYZW4NBTm4BYyYtoWntajw62GZvDlSq+iRwLa7JEvfjmjDxKWdT+U6NmCiu6p/MjJU7Wbczx+k4JghNXpTJ7oP5/OW0Vk5HOaaKzPJ8QFU3AauBa4Cr3cttR0fHNZXXon4NHr2oMws27eWfs9KdjmMq4OEvVrB13xHGjUyhVrXAallv/p+4hvDsBNRW1XHAnlAfxfvaE5OpHh3JP2dZbYvxTGFRMf/+ZT09WtShf6v6Tsc5Jk/atBwEDrmXIlztWZJ9kCnsuEZQbcorP7qGgDeB64vUrUxdspW/DWxLr6R6Tscxxxd2o3jXrRnDZX1b8MXSbTbytvHIl2nbyNx3hFsHtAmYIfvL4kmX5xdLLE8CA3B1IzRVJCI8ObQrjWtVY/TEJRzKK3Q6kilD5j5X9+aeLerwt4FtnI5jyheWo3jfeEorIgT+PdtqW0zFFBcrb8xcT4cm8Qzs0MjpOMdVlT6aNYDAvfEVZGpXj+alEd3Zsvcwj1g36IBTVKzcOWkpqvDKpT2se3NwCMtRvJvUrsYlvRL5dGEmO7NznY5jgsAPK3ewLusgtwxoHfCDE3oyTsuyEr2HVgBrgHG+ixZ++raqz18HtGHyoky+TtvudBxTwj9npbNg014eG9yZFvWte3OQKD2K969AyDbELemW09pQVKy8OdvGbTHHp6q8+nM6LRvUDIqJXj0Zxv+CEo8LgZ2qavcxvGz0mW2Zs24X901No0eLOjSrU93pSGEvdct+Xv5xHRd0a8rQHnZHNBi4G+HOBhYBZwCCaxTvVY4G85MW9WswOKUZH8/P4JYBrWkQF+t0JBOgfl6dxYpt2Tx/STciA7yWBTxr05JRYtnqaYFFRN4VkSwRWX6M10VExotIurs2p6cn+w8V0ZERjBvZg8Ji5c5PU60btMMO5RUyeuISmtSqxpNDu/oks9oAACAASURBVAZ0AzXz/1RVgWmlR/H2dD8i0lxEZorIKhFZISKjfRDXJ249vQ15hcW8NcdqW0zZVJXxP6eTWLc6Q4Lkgqwig8vliEh2iSWn5L8eHOs/wKDjvH4u0Na93AT804N9h5TkBjV55KLOzNuw16p3HfbI9BVs2XuYl0Z0D7iJw0y55onICVXcRyEwVlU7Av2AW0UkKOYvat0wjou6N+PDuRnsOZjndBwTgGat3cXSLfu59fQ2RAdJO72KjNMSr6q1SizxJf+t6IFUdTaw9zirDAY+UJd5QB0RaVrR/Yea4b0SOa9rE178YQ1pmdYN2glfp21n8qJM/jqgDX0DeNwCc0ynA3NFZH2JkbzTPNmBqm5X1cXuxznAKoKo1+TfBrblSEERb83Z6HQUE2BUlVd+XEdi3eoM6xk8U3J5VLQSke4icpt76eblLAnAlhLPMznGyUFEbhKRhSKycNeuXV6OERhEhKeGdqVhfCxjJqZyON+aD/nTtv1HuG9qGt0TazP6zLZOxzGVcy7QGhjI/4/kfWFldyYiyUAPXNMBlH4tIM9JbRq5als+mLvJalvM/5i1xlXLctvpbYiJCo5aFvCs99Bo4GOgkXv5WET+5sUsZTUWKLNBh6q+qaq9VbV3w4YNvRghsNSpEcOLI7qzcc8hHv9qpdNxwkZRsXLHpFQKi5VxI3sETbWp+V+qmgFkA42BpBKLx0QkDpgCjFHVP90WD+Rz0t8GtiW3oIh/261m46aqvPzjWlctS6/gqWUBz2parsc1WNNDqvoQrvu7N3oxSybQvMTzRCDspz8+sXUD/nJqayYs2MJ3y60btD+8OXsD8zfu5ZGLOpPcoKbTcUwlicgNuHoQfQ886v73kUrsJxpXgeVjVZ3qzYz+0KZRHENSEvhg7iaybNwWA8xYuZO0zAPcPrBt0F2UeZJWcA3ff1QRZdeOVNZ04Cp3L6J+wAFVtW9p4M6z2tE1oTb3Tl3GjgN20vGltMz9vPjDGs7r2oThQXYFYv5kNHACkKGqp+O6tePRvRt31+l3gFWq+pL3I/rH6DPbUlCkvGFzEoW94mLlpRlradmgJhf3DJrmWf/lSaHlPWC+e5r3R4F5uP4zV4iITADmAu1FJFNErheRm0XkZvcq3wAbgHTgLeCvHmQLaTFREbwyMoW8gmLGTk6l2LpB+8Th/EJGT0ylYXwsT1n35lCQq6q5ACISq6qrgfYe7uMk4EpgoIikupfzvB3U15Lq12RE70Q+mb+ZrfuPOB3HOOjrZdtZvSOHMWe2DcqRvSs8uJyqviQis4CT3T+6RlVTPdh+VDmvK3BrRfcXblo3jOOhCztx39RlvPPrRm481WZQ8LbHv1rJpj2H+PiGvtSpEfJT1ISDTBGpA0wDZojIPjy85ayqv+LdGmXH3DawLVMWbWXcj2t57pLuTscxDigsKublGWtp3zieC4Ng9NuyeNIQdziwTlXHA7WBh0Skh8+SmT8ZeUJzzuncmOe+X83yrQecjhNSvlu+gwkLtvCXU1tzYusGTscxXqCqQ1V1v6o+AjyIq2Z4sLOpnJNQpzpX9Evis0WZpGcddDqOccDkRZls2H2Iv5/TPuDnGDoWT+qGHlTVHBE5GTgLeB/4l29imbKICM9c3I16NWMYPXEJR/KLyt/IlGvHgVzunZpG14Ta3HlWO6fjGC8RkYeOLsBpQApwn8OxHHXr6a2pHh3JC9+vcTqK8bMj+UW88uNaeraow5kdA3sm5+PxpNBy9BvyfOBfqvoFYTDNe6CpWzOGF4ensH7XIZ78xrpBV1VxsTJ2cip5BcW8MjIlqMYrMOU6VGIpwjVuS7KTgZxWPy6WG09txXcrdrBk8z6n4xg/en/uJnZm53HPoA5B3V7PkzP0VhH5NzAC+EZEYj3c3njJyW0bcOMpLflo3mZmrNzpdJyg9vavG/gtfQ8PXdiJ1g3jnI5jvEhVXyyxPAkMIIhGs/WVG05pRYO4GJ75djWupoQm1O07lM/rM9M5vX3DoB/d25NCxwhc4xwMUtX9QD3gLp+kMuX6+znt6dS0Fnd/ttTGXqik5VsP8Pz3azinc2NGntC8/A1MsKsBhH0L9rjYKEaf0Zb5G/fy8+osp+MYP3h9ZjqH8gq599yOTkepMk9meT6sqlNVdZ37+XZV/cF30czxxEZFMn5UCkcKihg7eal1g/bQkfwiRk9cQr2aMTxzcbegri41ZTs615B7WQGsAcY5nSsQjOzTglYNavLMt6spLCp2Oo7xoS17D/PB3AyG92pO+ybxTsepMk96D1UTkTtFZKqITBGRO0Skmi/DmeNr0yief5zfiTnrdvPe75ucjhNUnvh6Jet3HeKlESnUrWlNs0LU0bmGLgTOBpqp6mvORgoM0ZER3D2oA+uyDjJp4ZbyNzBB69nvVhMRAXeESCcDT24PfQB0Bl4FXgM6Ah/6IpSpuMv7tuDMjo149tvVrNz2pylRTBlmrNzJx/M3c9OprTipjXVvDlWqmlFi2aqqNutoCed0bkyflvV46Ye1ZOcWOB3H+MCijL18lbadm05tTZPaoVHHUOHB5YD2qlpyRKKZIrLU24GMZ0SEZ4d1Y9C4OYyeuIQv/3Yy1aIjnY4VsLKyc7lnShqdmtZi7NmhceVhyiYidx7v9WAelt8bRIQHz+/Eha/9yhsz13PvuR2cjmS8qLhYeeyrVTSKj+Xm00KnKZcnNS1L3HMCASAifYHfvB/JeKp+XCwvDO/OuqyDPP3NKqfjBCxX9+alHM4vZPyoFGKjrHAX4noDt+DqMZQA3Ax0AuLdS9jrmlibi3sm8O6vG9m857DTcYwXTV+6jaVb9nPXOe2pEeNJ/URgK7fQcrQxG9AX+F1ENonIJlzzCJ3q43ymgk5r15DrTmrJ+3Mz+Hm1dYMuy3u/b2LOut08cH4n2jSy76ww0ADoqapjVXUs0AtIVNVHVfVRh7MFjHsGdSAqUnjiaxv3KVQcyivk6W9X0TWhNsN6htbErxWpaTnamG0Q0BLXyJKnuR+f77toxlN3D2pPhybx3DU5jV05eU7HCSgrt2Xz7LerObNjY67o28LpOMY/WgD5JZ7nE+aDy5Wlca1q3Hp6G35YuZM56zyaBNsEqDdmpbMzO49HLuoctMP1H0u5hZaSjdmAbKAxkFRiMQGiWnQk40f14GBeIXd9ttQGjnLLLXB1b65dI5pnh9nszWHkQ2CBe2b6R4AFuDoUmFKuP7klLerV4NEvV1JgXaCDWsaeQ7w1eyNDeyTQK6mu03G8zpMuzzcAs3ENMPeo+99HfBPLVFa7xvHcf15HZq3ZxQdzM5yOExCe/mYV67IO8sLw7tSPi3U6jvET9yi41wL7gL3A1ar6lLOpAlO16EgeuqAT6VkHee+3jU7HMZWkqjwyfQXRkRKyDas9aYg7GjgByFDV04EegNUlBqCr+idxevuGPPnNKtbsyHE6jqN+Xr2T9+dmcN1JLTmtXUOn4xg/KjEz/ThsZvpyndmpMWd0aMQrP65j+4EjTscxlTBj5U5mrtnFHWe1o3Gt0OjiXJonhZZcVc0FEJFYVV0NtPdNLFMVIsLzw7tTq1oUoycuIbcgPGeD3pWTx12T0+jQJJ67B9mfahiymek99PCFnSkqVp742nohBpsj+UU8+uVK2jWO4+oTk52O4zOeFFoyRaQOMA2YISJfANt8E8tUVYO4WJ4f3p3VO3J4PgynoVdV7vpsKQfzChk/qoeNXROebGZ6D7WoX4NbT2/D12nbmbXG5iUKJuN+WsfW/Ud4fHAXoiNDdy5jT+YeGqqq+1X1EeBB4B1giK+Cmao7vX0jrjkxmXd+3cgva8PrTt77v29i1ppdPHB+R9o1tu7NYcpmpq+Ev5zWilYNa/LgF8s5kh+etbTBZvWObN6es4ERvRODfhbn8lRknJY/dbVQ1V9Udbqq5h9rHRMY7j23A+0bx/P3yUvZczA8ukGv2ZHDU9+uZmCHRlzZzzq4hTGbmb4SYqMieWpoV7bsPcK4n9Y5HceUo7hYuX/qMmpVj+a+EJjFuTwVueqYKSJ/E5H/GdxCRGJEZKCIvA9c7Zt4pqqqRUcyblQKB44UcM+UtJDvBp1bUMTtE5ZQq1oUz11iszeHo6MXUcebmd4utI6vX6v6DO+VyFtzNrBi2wGn45jj+HBeBos37+cf53cMi8lfK1JoGYTr3vAEEdkmIitFZAOwDhgFvKyq//FhRlNFHZrU4t5BHfhxVRYfzd/sdByfeva71azZmcPzw7vTwLo3hyu70PKCB87vSN0aMdwzJY1CG7slIGXuO8yz363mlLYNGNojwek4flGRweVyVfUNVT0J12ByZ+AaGjtJVW9U1VSfpzRVds2JyZzariFPfr2S9KzQ7AY9a00W7/22iWtOTOb09o2cjmOcYxdaXlCnRgyPDe7M8q3ZvDXHxm4JNKrKA58vB+CpoeEzaKZHjdJUtcBdxbrfV4GMb0RECC8M70aNmChun5BKXmFoNbDbfTCPv09Oo33j+JAdVMlUjF1oec+5XZpwTufGvPzj2pC92AlWkxdl8svaXdx1Tnua16vhdBy/sZb0YaRRfDWeG9aNlduzeSGEukGrKvd8lkZ2bgHjRqVY92bzX3ahVTUiwhNDulIzJpKxk+02UaDYtv8Ij3+5kj4t63F1/2Sn4/iVFVrCzJmdGnNFvxa8NWcjv67b7XQcr/ho/mZ+Wp3FvYM60KFJLafjGBNSGsbH8tjgLizdsp9/z97gdJywp6rcMyWNIlVeuKR7yE2IWJ5KFVpEZKD736YiUuHLWhEZJCJrRCRdRO4t4/VrRGSXiKS6lxsqk88c3wPndaJNozjGTk5l36H88jcIYOlZOTzx1UpOa9eQa09KdjqOCUEi8q6IZInIcqezOOWCbk05v2tTXvlxLcu3Wm8iJ30wN4M563Zz33kdaVE/fG4LHVXZmpZBIpKIa0jslyuygbtw8zpwLtAJGCUincpYdZKqpriXtyuZzxxH9ZhIxo1MYd+hAu6dGrzdoPMKi/jbhFRqxkbx/HDr3myOTUSqMuLWf3A17g1brttEXahbI4Yxk1LDdmoQp6Vn5fDUN6sY0L4hV/RtUf4GIaiyhZY6wD3A3UBFRyzrA6Sr6gb3oHQTgcGVPL6pos7NanP3oPZ8v2InE//Y4nScSnn+uzWs2p7N85d0o1F8aE4OZrzmSxH5QkTeFJG/i8iAim6oqrNxzRId1urWjOGF4d1JzzrI09/Y3ET+lldYxJhJqdSIieS5YeF7kVbZQstjwCxVXcP/z+9RngSg5LdjpvtnpQ0TkTQR+UxEmpe1IxG5SUQWisjCXbvCa3h6b7rupJac3KYBj325kvW7DjodxyNz1u3i7V83cmW/JM7o2NjpOCbw/aKqg4G/A82BJt7cebick05t15DrTmrJ+3Mz+HHlTqfjhJXnv1vD8q3ZPDusG41CdAbniqhsoeV54Cz3IE3fVHCbsoqFpe9LfAkkq2o34Edcs7L+eSPVN1W1t6r2btiwYUUzm1IiIoQXR3SnWnQEYyamkl8YHD0D9h7KZ+ynS2nTKI77zwv9YauNV8SJSE/gCFBPVSd6c+fhdE6659z2dGpai7s+W8qOA7lOxwkLM9dk8favG7mqfxJnd/ZqeTvoVLbQskpVb1bVq4GRFdwmE9cVzlGJlJolWlX3qOrR201vAb0qmc9UUONa1XhmWDeWbT3ASzPWOh2nXEdbzu8/XMC4kSlUj7HuzaZCpgP9ga+A7xzOEtRioyJ59bIe5BYUc/uEJdYN2se2HzjC2E+X0r5xvF2kUYlCi4i8hauW5Xb3feGKjn/wB9BWRFqKSAyuws70UvtuWuLpRYDdOPWDczo3YVSfFvx79np+Xx/Y3aAnLNjCjJU7uXtQezo3q+10HBM8zgK+AHKBvg5nCXqtG8bx1MVdWLBpb1Bc7ASrgiJXwTCvoIjXL+9pY1BRiUKLqt6I6wTwB9AdqFCrfFUtBG7DNevqKuBTVV0hIo+JyEXu1W4XkRUishS4HbjG03ymch68oCMtG9TkzklL2X84MLtBp2cd5LGvVnBymwZcd1JLp+OY4FKy84BH9zREZAIwF2gvIpkicr0P8gWdoT0SGdWnBW/MWs9Pq6x9iy88//0a/ti0j6cu7kqbRnFOxwkIlb09dA/wENANSKvoRqr6jaq2U9XWqvqk+2cPqep09+P7VLWzqnZX1dNVdXUl8xkP1YiJYvzIHuw5lMf9ny8LuG7Q+YXFjJm0hOrRkbw4IvwGVDJV9hjwhbvzgEf3M1R1lKo2VdVoVU1U1Xd8EzH4PHxhJzo3q8WYSals3H3I6Tgh5au0bbw5ewNX9kticEp4TIZYEZUttNQF5gFPAO29F8c4qUtCbcae3Z5vlu1g8qJMp+P8jxdnuFrOPzOsG43DuOW8qRxVzQS2uh//aWBLUznVoiP51xW9iIoQ/vLhQg7lFTodKSSs2ZHD3Z+l0SupLg9eUNZwZuGrsoWWvUAkkIWNXxBSbjqlFSe2rs8j01ewKUCunH5P382bszdwWd8WnBPmLedNxYjIXSLyu4i0KfHjTBG52bFQIap5vRqMH9WD9KyD3DEpleLiwKqlDTZ7D+Vzwwd/UDM2ijcu70lMlM22U1KlPg1VfQzXaLjjARvTOYQc7QYdHRnB6IlLKHC4Z8D+w/nc+elSWjaoyT/Ot5bzpsLaAHdQoqOAquYAFzqWKISd0rYhD5zfiR9W7uTlH61hbmXlFxZzy0eL2Jmdx5tX9rJa5TJUpQhXC7hBVSs0jL8JHk1rV+eZi7uyNPMA435c51gOVeW+qcvYcyiPcZf2oEZMlGNZTND5CVeHgYKjPxCRBsBJjiUKcdedlMzIE5rz6s/pTF0cWLeXg4Gq8o9py5i/cS/PDetGjxZ1nY4UkCpUaDlGVetW4C++iWWcdm7Xpozoncjrs9KZv2GPIxkmL8zk2+U7GHt2e7omWvdmU3Gq+imuWuB0EflDRJ4ETgTWOJssdIkIjw3uwomt63PPlLSAHz4h0Lw+M51PF2Zy+8A2DOlhDW+PpaI1LVbVGoYevrAzSfVqcOenSzlwpKD8Dbxo4+5DPPLlCk5sXZ+bTmnl12Ob0KCqrwItgIdxtcH7O5DjaKgQFxMVwT+v6EVy/Zr85cNFrN6R7XSkoDBlUSYv/LCWi3skcMdZ7ZyOE9AqWmixqtYwVDM2inEje7AzO5cH/NgNuqComNETlxAdGWHdm02VqOoR91AL96rqqcDjTmcKdbWrR/PetSdQMyaKq95ZwJa9h52OFNB+Xr2Tu6ekcVKb+jw9rGvYToRYURUqtFhVa/jq3rwOd5zVjq/StjN18Va/HPOVH9eSlnmAZy7uStPa1f1yTBMeVPUXpzOEg8S6NXj/uj7kFhRx5TvzycqxOYrKsmDjXv768WI6Na3Fv6/sTWyUjXhbngo3xLWq1vB182mt6duyHg9PX8HmPb69apq3YQ9vzFrPiN6JnNu1afkbGGMCUvsm8bx37Qlk5eRxxdvz2XsoMEfadsqSzfu49r0FJNSpznvXnkBcrHU0qAiPeg9ZVWt4iowQXro0hQiB0ZN8N0HagcMF3DkpleT6NXn4ws4+OYYxxn96JdXj7at7k7HnMJdbweW/lm7Zz9XvLqB+XCwf39CPBnGxTkcKGlUatcaqWsNHQp3qPHVxV5Zs3s/4n9O9vn9V5f5py8jKyeOVS1OoaVcdxoSEE1s34M2rerN+10Eue2seew7mOR3JUYs37+OKt+dTq3o0n9zYlya1bSwWT9hQe6bCLujWjGE9E3nt53Us3OTdgZCnLN7K12nbueOsdnRvXser+zbGOOu0dg159+oT2Lj7EJe+OY/tB444HckRc9fv4ap3FlAvLoZJf+lPYt0aTkcKOlZoMR555KJOJNatwZhJqWTneqcbdMaeQzz8xXL6tKzHzae19so+jTGB5eS2DXj/uj7sOJDLJf+cy4ZdB52O5Fc/rNjB1e8toGntaky6qT8JdayTQWVYocV4JL5aNK+MTGH7gVwemra8yvtzdW9OJTJCePnSFCKte7MxIatfq/pMuLEfRwqKGPbP31mUsc/pSH7x4bwMbv5oER2b1uLTv/S3W0JVYIUW47GeLeoy+oy2TEvdxrQlVesG/epP60jdsp8nh3a1Kw9jwkDXxNpMueVEaleP5rK35vFV2janI/lMUbHy1DereHDacga0b8QnN/Slbs0Yp2MFNSu0mEr564DW9E6qy4PTlld68Kg/Nu3ltZnpDOuZyIXdm3k5oTEmULVsUJMpt5xIl4Ta3PbJEl78YU3IzQ594EgB17//B2/O3sCV/ZJ488pe1sHAC6zQYiolKjKCly9NAeCOSaked4M+cKSAMRNTSaxbg0cHW/dmY8JN/bhYPrmxLyN6J/Lqz+lc//4f7AuRLtErt2Uz5PXf+HXdbp4Y0oXHh3QhKtK+br3BPkVTac3r1eDxIV1YmLGPN2at92jbh75Yzo7sXMaNTLFBlYwJU7FRkTw7rBuPD+nCb+l7OH/8HK/3TPQnVeWT+ZsZ+sZvHM4v5JMb+3FFvySnY4UUK7SYKhnSI4EhKc0Y99O6Cjeq+3xJJl+kbmPMGW1t+nVjwpyIcGW/JKbcciJRkRGM+Pdcnv9+NfmFvhnE0ld2H8zjxg8Wcv/ny+jTsh5f334KfVrWczpWyLFCi6myx4Z0oUmtaoyZtISccrpBb9l7mAenraB3Ul3+enobPyU0xgS6rom1+Wb0KVzSK5HXZ67nwld/JXXLfqdjlUtVmbo4kzNf+oXZ63bz0AWdeP/aPjbKrY9YocVUWS13N+it+47wyPSVx1yvsKiYMZNSEbDuzcaYP4mLjeK5S7rzztW9OXCkgKFv/Mb9ny8L2LYua3fmcPnb87nz06W0alCTb24/metObmkz0/uQNSYwXnFCcj1uO70N439OZ0D7hmX2Bnp95noWZexj3MgUmtezkSCNMWU7o2Nj+rSsx0sz1vLB3Ay+TtvOXwe05uoTk6kW7fxMyDuzcxn/0zom/rGFuNgoHh/cmcv6JtmFmB9YocV4ze1ntGVO+m7u/3wZPZPq/s+4K4sy9jH+53UMSWnG4JQEB1MaY4JBfLVoHr6wMyNPaMHT367i6W9X8+5vG7nxlFZc1rcFNWL8//W1bf8R3pqzgQkLNlNYpFzetwVjzmxHPRt7xW9ENbj7xvfu3VsXLlzodAzjlrHnEOeNm0PnhNpMuLEfkRFCTm4B542fgyp8M/oUalWLdjqmKUFEFqlqb6dzhAo7J/nGvA17eOXHtczbsJfa1aMZ0TuRy/smkdygpk+Pq6oszNjHB3Mz+HbZdhQYnNKM0We0Jam+b48dro53TvJrUVVEBgHjgEjgbVV9ptTrscAHQC9gD3Cpqm7yZ0ZTNUn1a/LY4C6MnbyUf/2ynltPb8PD01ewdd8RJt/c3wosJuiUd94y/tGvVX0m3tSfRRl7effXTbz72ybemrORXkl1uah7M87q1JhmXhpVW1VZsS2bH1bu5IvUrWTsOUx8tSiu6p/MtScl2+1tB/mt0CIikcDrwFlAJvCHiExX1ZItN68H9qlqGxEZCTwLXOqvjMY7Lu6ZwMw1Wbw8Yy05uYVMXbyV0We0pVeSdf8zwaWC5y3jR72S6tErqR47s3OZungrny/J5OHpK3h4+graNY6jf6v69GhRly4JtUiqX5PoCgzqdiS/iLU7c1i29QCLMvYxd/0edmTnIgL9W9XnttPbcH63po7ckjL/y5+/gT5AuqpuABCRicBgoOR//sHAI+7HnwGviYhosN/DCjMiwpNDurI4Yx//+mU9PVvU4W8DrXuzCUoVOW8ZBzSuVY1bBrTmlgGtWb/rIDNW7uS39N18ujCT9+dmABAZISTUqU7D+Fjq1oihekwkURFCflExh/MK2Xsonx3ZuezMzvvvfhvExdC3VX1ObduAMzo2tq7LAcafhZYEYEuJ55lA32Oto6qFInIAqA/sLrmSiNwE3ATQokULX+U1VVC7RjSvXtaDF75fy7PDutkQ1iZYVeS8Zeckh7VuGEfr0+K4+bTWFBYVk77rIMu3ZrNp9yEy9h5md04emfsOk1dYTGFxMdGREVSPjqR+XCxtG8eTXL8GrRvG0SWhNol1qyNivYAClT8LLWX9FZSuQanIOqjqm8Cb4Gr0VvVoxhd6JdVjwk39nI5hTFXYOSnIREVG0KFJLTo0qeV0FOMD/rz8zQSal3ieCJSek/y/64hIFFAbCN6JKIwxwa4i5y1jjJ/4s9DyB9BWRFqKSAwwEpheap3pwNXux5cAP1t7FmOMgypy3jLG+Infbg+526jcBnyPq+vgu6q6QkQeAxaq6nTgHeBDEUnHVcMy0l/5jDGmtGOdtxyOZUzY8mv/LVX9Bvim1M8eKvE4Fxjuz0zGGHM8ZZ23jDHOsC4dxhhjjAkKQT+Mv4jsAjLKWa0BpbpNB5lgzh/M2SE88iepakN/hAkHdk4KeMGcHYI7f0WzH/OcFPSFlooQkYXBPLdKMOcP5uxg+Y1vBPvvJZjzB3N2CO783shut4eMMcYYExSs0GKMMcaYoBAuhZY3nQ5QRcGcP5izg+U3vhHsv5dgzh/M2SG481c5e1i0aTHGGGNM8AuXmhZjjDHGBDkrtBhjjDEmKIRNoUVEnheR1SKSJiKfi0gdpzN5QkSGi8gKESkWkaDo7iYig0RkjYiki8i9TufxhIi8KyJZIrLc6SyeEpHmIjJTRFa5/2ZGO53J/Jmdk/zPzknO8OY5KWwKLcAMoIuqdgPWAvc5nMdTy4GLgdlOB6kIEYkEXgfOBToBo0Skk7OpPPIfYJDTISqpEBirqh2BfsCtQfbZhws7J/mRnZMc5bVzUtgUWlT1B1UtdD+dh2uK+aChqqtUdY3TOTzQB0hX1Q2qmg9MBAY7nKnCVHU2rkk7g46qblfVxe7HOcAqIMHZVKY0Oyf5TpGZpQAABAtJREFUnZ2THOLNc1LYFFpKuQ741ukQIS4B2FLieSb2xel3IpIM9ADmO5vElMPOSb5n56QAUNVzkl9nefY1EfkRaFLGSw+o6hfudR7AVVX1sT+zVURF8gcRKeNn1r/ej0QkDpgCjFHVbKfzhCM7JwUUOyc5zBvnpJAqtKjqmcd7XUSuBi4AztAAHKCmvPxBJhNoXuJ5IrDNoSxhR0SicZ0cPlbVqU7nCVd2Tgoodk5ykLfOSWFze0hEBgH3ABep6mGn84SBP4C2ItJSRGKAkcB0hzOFBRER4B1glaq+5HQeUzY7J/mdnZMc4s1zUtgUWoDXgHhghoikisi/nA7kCREZKiKZQH/gaxH53ulMx+NuYHgb8D2uRlefquoKZ1NVnIhMAOYC7UUkU0SudzqTB04CrgQGuv/WU0XkPKdDmT+xc5If2TnJUV47J9kw/sYYY4wJCuFU02KMMcaYIGaFFmOMMcYEBSu0GGOMMSYoWKHFGGOMMUHBCi3GGGOMCQpWaDHGGGNMULBCizHGGGOCghVajF+IyIsislJE3hKRX9zTxFdkuxgRmS0iITXlhDHGWXZOCk5WaDE+JyKtgJNUtROQCkxV1aKKbOueQv4n4FIfRjTGhBE7JwUvK7SYMolIVxH5rcTzniLycyX20x74BUgSkSXADcAXJV6fKSJnuR8/ISLjy9jNNOByT49tjAkddk4yYMP4m2MQkQhcM6AmqGqRiMwExqrq4krs6wlgE/ABsFlVm5R47VTgMeAt4DJck8cVldo+Etihqg0r+36MMcHNzkkGrKbFHIOqFgMrgM4iMgzXf+z/OTmIyI8isryMZXCp3XUFlgINgP2ljjMbEOBOYOTRk4OIPF5inSIgX/6vvftniSOKwjD+HIT4B1LZaJ3SBGKfJiBol09gFWws06ULJE36tBYp0ppUFja2Qoo0guYDWClYiBKxeFPsCIuSweg2d/f5dbN39nAHlpczdy6zVU9HfJmSGmEmCcCNROqzz+DfOTeBtduDSVbuWWeJQdhMAzPDA1X1AlgETpOcd58tcPe3OQ38+Z/JSxo7ZtKEc6VFffaBT8D3JMcPKdDdiVwnuUxyBkxV1Uw3tgh8A94AF1W12n1tmcHmuJsa88BJkuuHX4qkMWAmTTibFvU5Aq6Az4+o8Rw4GDreBV5V1RywzeCZ9CHwEfjQnfOSoYAAXgM7j5iDpPFgJk04N+Lqn6rqC/AzydcR1lwG3iVZ7zlnC9jonmFTVdvA+yS/RzUPSe0xk+RKi+6oqmdVdQTMjjIcAJL8Avb6XuSU5O1QODwBfhgO0uQyk3TDlRZJktQEV1okSVITbFokSVITbFokSVITbFokSVITbFokSVITbFokSVITbFokSVIT/gLBqJo0q8gj7wAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "x = np.linspace(-2, 2, 101)\n", + "plot_loss_functions(\n", + " suptitle = 'Common loss functions for regression',\n", + " functions = [np.abs(x), np.power(x, 2)],\n", + " ylabels = ['$\\mathcal{L}_{abs}}$ (absolute loss)',\n", + " '$\\mathcal{L}_{sq}$ (squared loss)'],\n", + " xlabel = '$y - f(x_i)$')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Функции ошибки для классификации\n", + "\n", + "**0-1 loss**\n", + "\n", + "Предполагается решающая функция вида $\\hat{y} = \\mathrm{sign}(f_{\\theta}(x))$:\n", + "\n", + "$\\mathcal{L}_{0-1}(\\theta) = \\sum_{i=1}^n l_i \\quad l_i = \\begin{cases}\n", + " 0 & y_i f_{\\theta}(x) > 0 \\\\\n", + " 1 & в\\ противном\\ случае\n", + " \\end{cases} \\\\\n", + "$\n", + "\n", + "**логистическая функция ошибки**\n", + "\n", + "$\\mathcal{L}_{log}(\\theta) = \\sum_{i=1}^n \\frac{1}{\\log(2)} \\log(1 + e^{-y_i f_{\\theta}(x)})$" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# define and vectorize zero-one loss\n", + "def zero_one(d):\n", + " if d < 0:\n", + " return 1\n", + " return 0\n", + "\n", + "def logistic_loss(fx):\n", + " # assumes y == 1\n", + " y = 1\n", + " return 1 / np.log(2) * np.log(1 + np.exp(-y * fx))\n", + "\n", + "zero_one_v = np.vectorize(zero_one)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAADZCAYAAADsdQBUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd5iU5dXH8e9vZpfeewcFRAGRXtQYG7GLxobYUJRYo69Go4lRYolJTOxYQBGxERULMRg1EcWGAlKUoiKgNBUFKQLSzvvH8ywZ12V3dndmnpnd87muuZiZp53ZGe45c1eZGc4555xz2S4WdQDOOeecc8nwpMU555xzOcGTFuecc87lBE9anHPOOZcTPGlxzjnnXE7wpMU555xzOcGTFueyiCST1CFD12oqaYqk9ZL+nolrJlx7g6TdM3g9SXpY0hpJ76f5WmMl3ZTG8+/820mqLumfktZKelrSaZJeScM1fybp41Sf17nSyos6AOeSJWkIcDmwJ7AemAXcbGZvRRpY7hoOfAPUsTRO2CTpdeAxM3uw4Dkzq5Wu6+3C/sBAoJWZfZ/ha6dUob/diUBToKGZbQufe7y815BkQEczWxhe802gU3nP61x5eU2LywmSLgfuAP5EUEi3Ae4FBkUZV45rC8xLZ8KSRdoCS8qSsEjK5h93bYFPEhIW5yo2M/Ob37L6BtQFNgAnFbNPVYKkZkV4uwOoGm47EFgGXAV8DawEjgOOBD4BVgO/SzjXCOBp4DGCGp0PgT2Aa8LjlwK/SNi/BTAxPM9C4LxC53oKGBeeay7Qu5jXYUCHhNc9DlgFfA5cC8TCbR2AN4C1BLUl/wifF3B7GOdaYA7QtYjrjAW2AlvCv+2h4XM3JexzILAs4fES4DfhOdcC/wCqJWwfRFD7tQ74DDgcuBnYDmwOr3NPKV/nUOAt4G/AGmAxcETCNYcCi8K/7WLgtCJe67Dw+tvDGP4YPn9e+H6tDt+/FoXeh4uAT4HFu3iv9gfeAb4LPxNDE/62N4X36wMvhq9tTXi/VUnx7+r9TfzbAX8M37+t4esaVvD3Sti3C/Bq+Bq/IvycA32Bd8PYVwL3AFXCbVPCa3wfnveUIj4LewGvh8fPBY4t9NkaCfwrfF3vAe2jLkf8VjFukQfgN7+VdAu//LYBecXscwMwFWgCNA6/TG4Mtx0YHn8dkB9+Wa0CngBqhwX7ZmD3cP8R4ePDCJpQx4VfKL9POH5xwrXfIKj1qQZ0D899SKFzHQnEgVuAqcW8jsQv83HAC2GM7QgSrGHhtifDeGLhdfcPnz8MmAHUI0hg9gKa7+JaY/lxklL4ceEvqiXA+wRJWgNgPnB+uK0vwRfswDCmlsCe4bbXgXPL+DqHEnwpnxf+/S4gSEoF1CRIkDqF+zYHuuzitQ7lx1/mBxMkAz0JEt67gSmF4ns1fJ3VizhfG4Iv5FPDz0RDoHvhv2P4/AlAjfD1PQ08H27bZfy7en+L+NuNIGh6+8nrDK+3ErgiPEdtoF+4rRfQn+Dz3S58Ly8r6hqFPwvh610I/A6oEv4t1ye8jrEESVLf8PyPA+OjLkf8VjFu3jzkckFD4Bsrvgr8NOAGM/vazFYR/Ao9I2H7VoL+L1uB8UAj4E4zW29mcwl+LXZL2P9NM3s5vObTBInQnxOObyepnqTWBL+4f2tmm81sFvBgoWu/ZWaTzGw78CiwT0kvWFKc4BfuNWGMS4C/J5x3K0HTQIvwum8lPF+boN+PzGy+ma0s6XqlcJeZrTCz1cA/CZI0CH7ljzGzV81sh5ktN7MFJZ0sidcJ8LmZjQ7/fo8QfLk3DbftALpKqm5mK8P3MhmnhfF+YGY/ENSiDZDULmGfW8xstZlt2sXx/zGzJ81sq5l9G773PxI+P8HMNprZeoKap58n7LKr+Hf1/pbG0cCXZvb38Bzrzey9MK4ZZjbVzLaFf/MHCsVVnP5ALYL/D1vM7DWCGqRTE/Z51szeD///PM7/PifOlYsnLS4XfAs0KqFvQQuCpoUCn4fP7TxH+KUHUPAl9FXC9k0EBTG72PZNEcfXCq+xOvxCSrx2y4THXybc3whUS6KfRCOCX7GFX1PBea8iqG14X9JcSecAhF8g9xBUz38laZSkOiVcqzQKv5aCv1lrgiah0irpdf7omma2Mbxby4L+KacA5wMrJf1L0p5JXvdHnxcz20DwOUu87tJijk/q9UqqIekBSZ9LWkfQ9FJPUryE+It8f0tplzFK2kPSi5K+DOP6E8F7kYwWwFIz25HwXEmf+Ux3vHYVlCctLhe8S9DEclwx+6wg+GVaoE34XLqtABpIql3o2svLed5v+N+v7Z+c18y+NLPzzKwF8Cvg3oKh0mZ2l5n1Imj22gO4Mslrfk/QjFGgWSniXQq038W24jr6Fvs6SxLWhg0kqH1ZAIxO5jgKfV4k1SSo0Uu8bnFxF/d6E11BMOqmn5nVAQ4ouGRx8Rf3/pZCcTHeF16vYxjX7wpiSsIKoLWkxO+PVHzmnSuRJy0u65nZWoL+KCMlHRf+es2XdISkv4a7PQlcK6mxpEbh/o9lILalBP1nbpFUTVI3gqaScg07DWt1ngJullRbUluC4d6PAUg6SVKrcPc1BF+w2yX1kdRPUj5BElLQATUZs4AjJTWQ1Ay4rBQhPwScLekQSTFJLRNqDb4CipyTpaTXWZxwnpljw4TjB4JOo8m+1ifCeLtLqkpQ0/Be2FSSjMeBQyWdLClPUkNJRTWB1CaomftOUgPg+mTi39X7m2RsBV4Emkm6TFLV8O/bLyGudcCG8H26oNCxu3zPCDrWfg9cFf4/PBA4hqDZ1Lm08qTF5QQzu43gy+xago6uS4GLgefDXW4CphOMbPkQ+CB8LhNOJejMuAJ4DrjezF5NwXkvIfhyWEQwguYJYEy4rQ/wnqQNBCNfLjWzxUAdgl/rawiq7L8lGHmTjEeB2QQdbl8hGB2UFDN7HzibYOTSWoLOyQU1GXcCJ4YTu91VytdZnBhBTcYKgo6fPwcuTDLe/wJ/ACYQdFZtDwxO5tjw+C8IOldfEV57FkX3VboDqE5QozQV+HeS8e/q/U1a2GQ5kCCh+JJgJNRB4ebfAEMIOtCO5qfv9QjgEUnfSTq50Hm3AMcCR4Sv617gzGT6MDlXXjKrDFM0OOeccy7XeU2Lc84553KCJy3OOeecywmetDjnnHMuJ3jS4pxzzrmc4EmLc84553KCJy3OOeecywmetDjnnHMuJ3jS4pxzzrmc4EmLc84553KCJy3OOeecywl5UQdQXo0aNbJ27dpFHYZzOWvGjBnfmFnjqOOoKLxMcq58iiuTcj5padeuHdOnT486DOdylqTPo46hIvEyybnyKa5M8uYh55xzzuWEjCUtksZI+lrSR7vYLkl3SVooaY6knpmKzTnnnHPZL5M1LWOBw4vZfgTQMbwNB+7LQEzOOeecyxEZ69NiZlMktStml0HAODMzYKqkepKam9nK8lz3mw0/sHbT1vKcwpVBnWr5NK5dNeownMs6G7ds44vVG9mzWZ2oQ3Eu52RTR9yWwNKEx8vC58qVtIycvJCH315SnlO4MojHxNRrDvHExblCfvP0bN5fvJqJF+9Pi3rVow7HuZySTUmLinjOitxRGk7QhESbNm2KPekve7Sie+t65Q7OJW/mF98x9p0lrN20xZMWl7UkVQOmAFUJysJnzOz6QvtUBcYBvYBvgVPMbEl5rnv5wE4cP/Jthj86nad/tS/Vq8TLczrnKpVsSlqWAa0THrcCVhS1o5mNAkYB9O7du8jEpsDereqyd6u6qYrRJaFKPMbYd5awbUexb41zUfsBONjMNkjKB96S9JKZTU3YZxiwxsw6SBoM/AU4pTwX7dCkFncM7s6546bz2wlzuHNwd6SifrM55wrLpiHPE4Ezw1FE/YG15e3P4qIRiwUF8HZPWlwWs8CG8GF+eCv8oR0EPBLefwY4RCnIMA7Zqym/+UUnJs5ewQNTFpX3dM5VGhmraZH0JHAg0EjSMuB6gkICM7sfmAQcCSwENgJnZyo2l1p5nrS4HCEpDswAOgAjzey9Qrvs7GtnZtskrQUaAt8UOk/STdYFLjywPfNWruMv/15Ap6a1OWjPJuV6Lc5VBpkcPXRqCdsNuChD4bg0Kqhp8eYhl+3MbDvQXVI94DlJXc0scS6ppPralabJeueJJW49sRuLV33Pr5+cyXMX7UeHJrXK8CqcqzyyqXnIVRAFNS07PGlxOcLMvgNe56dzSe3saycpD6gLrE7VdWtUyWPUmb2okhdj+LjpPj2DcyXwpMWlXNxrWlwOkNQ4rGFBUnXgUGBBod0mAmeF908EXgtrhVOmVf0a3Hd6L5au2cglT85k2/YdqTy9cxWKJy0u5eLyPi0uJzQHJkuaA0wDXjWzFyXdIOnYcJ+HgIaSFgKXA1enI5C+uzXgxkFdmfLJKm55qXDe5JwrkE1Dnl0FkRf3pMVlPzObA/Qo4vnrEu5vBk7KRDyD+7ZhwZfreeitxXRqWpuT+7Qu+SDnKhmvaXEpF48FHytPWpwrnWuP2ov9OzTi989/yLQlKes641yF4UmLSzlvHnKubPLiMUYO6Umr+jU4/9EZLF29MeqQnMsqnrS4lPOOuM6VXd0a+Tx4Vm+2bN/BeeOms+GHbVGH5FzW8KTFpZz3aXGufNo3rsXIIT359OsNXDZ+lk8f4FzIkxaXcrGC5qHUjgx1rlI5YI/G/OGovfjP/K/468sfRx2Oc1nBRw+5lPvfNP4+34Rz5XHWvu349OsN3P/GZ3RoUosTe7WKOiTnIuU1LS7ldvZp2e41Lc6VhyRGHNuF/To05Jpn5/D+Yh9R5Co3T1pcyhUkLTu8eci5csuPx7h3SC9a16/Brx6dzhff+ogiV3l50uJSLs9HDzmXUnVr5PPQ0D7sMDjnkWm+RpGrtDxpcSkXi/noIedSbbdGNbn/9F4s+eZ7Ln7iA7b6GkWuEvKkxaVcnictzqXFgPYN+dPxe/Pmp98wYuJcUrx2o3NZz0cPuZSLe9LiXNqc3Kc1n32zgQfeWMRujWpy7s92jzok5zLGkxaXcj4jrnPp9dvD9uTzbzZy86T5tG1Yk4Gdm0YdknMZ4c1DLuW8psW59IrFxO2ndKdby7r8+smZfLhsbdQhOZcRGU1aJB0u6WNJCyVdXcT2NpImS5opaY6kIzMZn0uNPF/l2WU5Sa3Dsma+pLmSLi1inwMlrZU0K7xdF0Wsu1K9SpzRZ/WmQc0qDHtkGiu+2xR1SM6lXcaSFklxYCRwBNAZOFVS50K7XQs8ZWY9gMHAvZmKz6VOWNHiSYtLO0k1w7KltLYBV5jZXkB/4KIiyiOAN82se3i7oVzBpkGT2tUYM7QPm7Zs55yx01i/2YdCu4otkzUtfYGFZrbIzLYA44FBhfYxoE54vy6wIoPxuRSRRDwmT1pcykmKSRoi6V+SvgYWACvD2pJbJXVM5jxmttLMPgjvrwfmAy3TF3n6dGpWm5GnBYsrXvTETB8K7Sq0TCYtLYGlCY+X8dNCYgRwuqRlwCTgkqJOJGm4pOmSpq9atSodsbpyisfkHXFdOkwG2gPXAM3MrLWZNQF+BkwF/izp9NKcUFI7oAfwXhGbB0iaLeklSV2KOUekZdIBezTm5uO6MuWTVVz3wkc+FNpVWJkcPaQiniv8P+tUYKyZ/V3SAOBRSV3N7Ec/HcxsFDAKoHfv3v6/MwvFJZ/G36XDoWb2kzYQM1sNTAAmSMpP9mSSaoXHXWZm6wpt/gBoa2Ybwv51zwNF1uRkQ5k0uG8blq7ZyMjJn9G6QQ0uPLBDFGE4l1aZrGlZBrROeNyKnzb/DAOeAjCzd4FqQKOMROdSKi8mXzDRpVxRCUtZ9gEIk5sJwONm9mwR51lnZhvC+5OAfElZXR795hedGNS9BX/998e8MGt51OE4l3KZTFqmAR0l7SapCkFH24mF9vkCOARA0l4ESYu3/+SgeFxs3+Ft6y49JJ0kqXZ4/1pJz0rqWYrjBTwEzDez23axT7NwPyT1JSgvvy1/9Okjib+e2I2+uzXgyqfnMHVRVofrXKllLGkxs23AxcDLBJ3enjKzuZJukHRsuNsVwHmSZgNPAkPNG2dzUlxiu791Ln3+YGbrJe0PHAY8AtxXiuP3A84ADk4Y0nykpPMlnR/ucyLwUVge3QUMzoXyqGpenNFn9KZNwxoMHzedT75aH3VIzqVMqfu0SKoJbDaz7aU9NqxinVTouesS7s8jKExcjvPRQy7NCsqfo4D7zOwFSSOSPdjM3qLofnaJ+9wD3FPmCCNUt0Y+Y8/uw/H3vsPQMe/z3EX70bROtajDcq7cSqxpSdUQQ1e5eJ8Wl2bLJT0AnAxMklQVn+H7R1rVr8HDQ/uwdtNWhj7sc7i4iiGZ/+QpH2LoKr5YzJuHXFqdTNDUfLiZfQfUB66MNqTs07VlXe47vReffrWeCx77gC3bvJ+Zy23JJC2HmtmNZjYnceixma02swlmdgLwj/SF6HJRnjcPufQ6CnjVzD6VdC3B7NnfRBxTVjpgj8b8+YRuvLXwG656ZjY7/P+ly2ElJi0FwwcL9db/Q2Jv/WSHGLrKI+aTy7n0Km9H3ErlxF6tuPKwTjw/awV//veCqMNxrsxK0wacWEj8Ai8kXDHyYvJfdC6dftIRF6gSYTxZ78ID23PmgLaMmrKIB99cFHU4zpVJaZIWLyRc0uKxmNe0uHTyjrilJInrj+nCEV2bcdO/5vvkcy4nleY/uRcSLmnxmK/y7NKqcEfcBnhH3BLFY+L2U7rTb7cG/Obp2bz5qc/d6XJLaZIO763vkhaPxTxpcWljZhuBz4DDJF0MNDGzVyIOKydUy48z+qzedGhSm189OoPZS7+LOiTnklaapMV767uk+eghl06SLgUeB5qEt8ckFbkqvPupOtXyeeTsPjSsVYWzx07js1Ubog7JuaSUtSOu99Z3xYrLkxaXVsOAfmZ2XTirdn/gvIhjyilN6lRj3Dn9iAnOfOh9Vq7dFHVIzpXIO+K6tPBp/F2aif+VSYT3i52W3/3Ubo1qMvbsvqzbtJUzH3qfNd9viTok54rlHXFdWuTFxTZf5dmlz8PAe5JGhGsOTSVYtdmVUteWdRl9Vm8+X72RoWOn8f0P26IOybldKk9HXO+t73YpJuFLD7l0MbPbgHOA1cAa4GwzuyPaqHJX/90bMnJITz5avpbhj05n89ZSr4frXEYknbR4b31XGkFHXK9pceljZjPM7C4zu9PMZkYdT64b2Lkpfz2hG28v/JZfPzmTbdv9/6/LPkknLd5b35VG3Fd5dmkgab2kdUXc1ktaV8pztZY0WdL8cNX6S4vYR5LukrRQ0pyCpUsqqhN6teL6YzrzyryvuGrCHJ/V2mWdvFLsW9Bb/3sASX8B3gXuTkdgLrfFY2KHr/LsUszMaqfwdNuAK8zsg3BdtRmSXjWzeQn7HAF0DG/9CEZM9kthDFnn7P12Y/3mbdz26ifUrprHiGO7IHkfZ5cdSpO0eG99l7S4L5jospyZrQRWhvfXS5oPtAQSk5ZBwDgzM2CqpHqSmofHVliXHNyB9Zu3MvrNxdSsmsdVh+8ZdUjOAaVLWgp66z8XPj4O763vdsGHPLtcIqkd0AN4r9CmlsDShMfLwud+lLRIGg4MB2jTpk26wswYSfzuyL34fst27n39M2pWzeOigzpEHZZzpeqIW67e+pIOl/Rx2DZ89S72OVnSvLB9+Ylkz+2yjyctLldIqgVMAC4zs8L9YoqqTf7JB9vMRplZbzPr3bhx43SEmXGSuGlQV47v0ZJbX/6YMW8tjjok50pV04KZzQBmlPYikuLASGAgwS+VaZImJrYdS+oIXAPsZ2ZrJDUp7XVc9vBp/F06SXoEuDScfgFJ9YG/m9k5pTxPPkHC8riZPVvELsuA1gmPWwEryhZ17onFxK0ndmPz1u3c8OI8quXHGdIv92uSXO4qsaYlRb31+wILzWyRmW0BxhO0FSc6DxhpZmsAzOzr0rwQl128T4tLs24FCQtAWG70KM0JFPQufQiYH9YkF2UicGY4iqg/sLai92cpLC8e487BPTioU2N+//yHPDNjWdQhuUqsxJqWFPXWL6pduHAP/D0AJL0NxIERZvbvok5W0dqPK6J4TD5c0qVTTFL9gh85khpQyppjYD/gDOBDSbPC534HtAEws/uBScCRwEJgI3B2CmLPOVXyYtx3ei/OfWQ6Vz0zm/y4GNS9ZdRhuUqotP/JyyqZduE8gmGFBxJUwb4pqWvir6mdB5qNAkYB9O7d278Zs1BeLOY1LS6d/g68I+kZgrLkZODm0pzAzN6ihBGQ4aihi8oaZEVSLT/O6DN7M/Th97n8qdnkx2McuXfzqMNylUym1g5Kpl14GfCCmW01s8XAxwRJjMtBMXlNi0sfMxsHnAh8BawCfmlmj0YbVcVXvUqcMUP70KN1PX795Exenvtl1CG5SiZTScs0oKOk3SRVAQYTtBUneh44CEBSI4LmokUZis+lWLBgoictLn3MbK6Z3WNmdxeaEM6lUc2qeTx8dh/2blWXix7/gFfnfRV1SK4SyUjSYmbbgIsJFlycDzxlZnMl3SDp2HC3l4FvJc0DJgNXmtm3mYjPpZ4PeXbpIOmt8N/CAwRKPY2/K7va1fJ55Jy+dGlZlwsfn8F/PHFxGVKupEVS0p3SzGySme1hZu3N7ObwuevMbGJ438zscjPrbGZ7m9n48sTmohWX2O7T+LsUM7P9w39rm1mdhFttM6sTdXyVSZ1q+Yw7py+dm9fhgsdneI2Ly4jy1rT8MSVRuAqnoKbFPHFxaRCufVbicy696lbPZ9ywfnRuEdS4eB8Xl27JzNMyZxe3D4GmGYjR5aC8WDAow5uIXJoMLOK5IzIehaNu9XweHdaXLi2CPi4vfVipprFxGZbMkOemwGEEU/cnEvBOyiNyFUKsIGkxy9i4elfxSboAuBBoL2lOwqbaeHkUmTrVgsTlrDHvc/GTM7ljh3HMPi2iDstVQMl8n7wI1DKzWYU3SHo95RG5CsFrWlyaPAG8BNwCJK5htt7MVkcTkoOgc+64Yf045+FpXDp+Jlu37+CXPVtFHZarYEpsHjKzYeEkTDtJahZuG5KuwFxui4dJiw97dqlkZmvNbAnwLLDazD4nmNX2QUmlmsbfpV6tqnmMPacP/XdvyBVPz2b8+19EHZKrYMraEXdSSqNwFU5B0uITzLk0+YOZrZe0P0Hz9SPA/RHH5IAaVfIYM7QPB3RszNXPfsjYt311aJc6ZU1aip362rk8r2lx6bU9/Pco4D4zewGoEmE8LkG1/DijzuzFLzo3ZcQ/5zFy8sKoQ3IVRFmTltEpjcJVODHv0+LSa7mkBwjWHJokqSqZm+HbJaFqXpyRp/VkUPcW3Pryx/z13wt8CgRXbmUa2GFm96Y6EFexeEdcl2YnA4cDfzOz7yQ1B66MOCZXSH48xm0nd6dGlTzuff0z1m/exh+P7bLzR41zpZVU0iJpT2AQ0JJgRdUVwEQzm5/G2FwOi8eCH72etLh0MLONBJ1xCx6vBHyCkCwUj4k/Hd+VOtXyeGDKItZv3sqtJ+1DftwrxlzpJTO53G+B8QT9WN4nWPxQwJOSri7uWFd5FZRH3qfFpVIq1x6SNEbS15I+2sX2AyWtlTQrvF2XitdQGUni6iP25MrDOvH8rBWc/+gMNm/dXvKBzhWSTE3LMKCLmW1NfFLSbcBc4M/pCMzlNq9pcemQuPZQCk43FrgHGFfMPm+a2dEpuFalJ4mLDupAner5XPfCR5zx0Hs8eFYf6lbPjzo0l0OSqZ/bARQ1tWHzcJtzP+F9Wly2M7MpgE9Il2Fn9G/L3af2YNbS7zjlgXf5at3mqENyOSSZmpbLgP9K+hRYGj7XBugAXJKuwFxui8mTFpc+ki4v4um1wIyiZu8uhwGSZhP04/uNmc3dRTzDgeEAbdq0SeHlK6aju7WgXvUq/OrR6fzy3ncYN6wv7RvXijoslwOSqWl5GdiDYEXnl4FXgBFAJzN7CUCSdwV3P+I1LS7NegPnEwwOaEmQMBwIjJZ0VYqu8QHQ1sz2Ae4Gnt/VjmY2ysx6m1nvxo0bp+jyFdv+HRsxfvgANm/dzon3vcMHXxRe3s65n0omaZkMXASsMLMJZvaMmU0F4pIOlvQIcFZao3Q5Jx4vmFzOWxBdWjQEeprZFWZ2BUES0xg4ABiaiguY2Toz2xDenwTkS2qUinO7wN6t6jLhgn2pUz2fIaOn8uq8r6IOyWW5ZJKWwwlmn3xS0gpJ8yQtAj4FTgVuN7OxaYzR5aB4WPm2wyeTcunRBtiS8HgrQa3IJuCHVFxAUrOCWmRJfQnKy29TcW73P+0a1WTCBfvSqWltfvXodB6d+nnUIbksVmKfFjPbDNwL3CspH2gEbDKz79IdnMtdO6fx3+5Ji0uLJ4Cpkl4gmILhaIIfVjWBecmcQNKTBE1KjSQtA64H8gHM7H7gROACSduATcBg8yld06JRrao8Obw/lzwxkz88/xHL1mzkt4ft6ZPQuZ8o1Yy44bDnMk/gJOlw4E4gDjxoZkUOl5Z0IvA00MfMppf1ei46ce/T4tLIzG6UNAnYnyBpOT+hrDgtyXOcWsL2ewiGRLsMqFEljwfO6MWIf87lgTcWsWz1Jv5+8j5Uy49HHZrLImWaxr8sJMWBkcBAYBkwTdJEM5tXaL/awK+B9zIVm0u9nUmL/zB16bONYNoFI2gecjkuLx7jxkFdadOgBre8tIAVazcx6ozeNK5dNerQXJbI5DzKfYGFZrbIzLYQzLI7qIj9bgT+Cvjg/RwW91WeXRpJuhR4nKC5ugnwmCSfgqECkMTwA9pz32k9mb9yHceNfJuPv1wfdVguS2QyaWnJ/+Z5gaC2pWXiDpJ6AK3N7MXiTiRpuKTpkqavWrUq9ZG6cttZ0+J9Wlx6DAP6mdn1ZnYd0B84L+KYXAod3rU5T/1qAFu37+CE+95h8oKvow7JZYFyJy3h2kRJ7VrEczu/0STFgNuBK0o6kc+JkP28ecilmQhGNRbYTtFljMth3VrV44WL96NtwxoMe2QaD765CO8LXbmVuk+LpKcSHwLdgb8kcegyoOZsYccAABTQSURBVHXC41YEs0wWqA10BV4PRxk2AyZKOtY74+aePF97yKXXw8B7kp4jKIeOA8ZEG5JLh+Z1q/P0+QO44qnZ3PSv+Sz4cj03H9+VqnneQbcyKktH3HVmdm7BA0n3JXncNKCjpN2A5cBgYEjBRjNbS9A+XXDe1wmmzfaEJQf5Ks8unczstrCM2I8gaTkrxdP3uyxSo0oeI4f05M7/fsqd//2URas2cP/pvWhSp1rUobkMK0vScnOhx79P5iAz2ybpYoKlAOLAGDObK+kGYLqZTSxDLC5LFazyvMOTFpdCktaT0KxMQpOQJDOzOpmPymVCLCb+b+AedGpWmyuems3Rd7/F/Wf0omeb+lGH5jKo1EmLmS0u9DjpVVLDqbAnFXruul3se2BpY3PZI89HD7k0MLPaUcfgonXk3s3ZrVFNhj86ncEPTGXEsV0Y0s8XqawsMjl6yFUiBTNZek2LS6VkFmf1BVwrvr2a1+GfF+9P//YN+d1zH/LbZ+aweev2kg90Oa/MSYsvHOaK4zUtLk0mS7pE0o9+Wkuq4gu4Vi71alTh4aF9uOig9vxj+lJOuv9dlq7eGHVYLs3KU9PiPfXdLv1vGn9f5dmllC/g6naKx8SVh+3J6DN7s+Tb7zn67rd4bYGvFF2RlSdp8SpYt0sFqzz7kGeXSma22czuNbP9gLbAIUBPM2trZuf5CKLKaWDnprx4yf60rFedc8ZO588vLWDbdv/BVBGVJ2nxbyO3S/G4Nw+59DKzrWa20lecdwBtG9bk2Qv35dS+bbj/jc84dfRUVq7dFHVYLsW8psWlRZ6v8uycy7Bq+XFu+eXe3HFKd+auWMcRd77Jf+Z5c1FFUp6k5ZqUReEqnJh8Gn+X3SSNkfS1pI92sV2S7pK0UNIcST0zHaMrm+N6tOTFS/anRd3qnDtuOiMmzvXRRRVEmZMWMyvyP7pzkFDT4gsmujSTVLeMh44l6Ni7K0cAHcPbcCDZ2b9dFti9cS2eu2hfzt6vHWPfWcJxI9/mk698tehcV+qkRdIQSeMlPSbpCUmnpiMwl9viPuTZZc4LkiZIGinpXElVkjnIzKYAxU2OOQgYZ4GpQD1JzVMRsMuMqnlxrj+mC2OG9mbV+h845u63GPv2Yl90MYeVpabl52Y22MxON7MhwP6pDsrlPknEBDu8cHDp97aZnQBcBfQEbkrReVsCSxMeLwuf+wlJwyVNlzR91apVKbq8S5WD92zKvy87gH3bN2TEP+dx5pj3+XLt5qjDcmVQlqSlqqSjJHWTdCRQPdVBuYohLxbzmhaXCfUl9Qa2AnVI3cjGogYbFHluMxtlZr3NrHfjxo1TdHmXSo1rV2XM0D7cdFxXpi1ZzWF3TOGFWcu91iXHlCVpuRCoDxwJNAAuTmlErsKIxXz0kMuIywhqfO8HXgRS1d9uGdA64XErYEWKzu0iIInT+7dl0q9/xu6Na3Lp+Flc9MQHfLPhh6hDc0kqy4KJG4HHJO1lZvPTEJOrIPJiMU9aXNqZ2RbgjjSceiJwsaTxQD9grZmtTMN1XIbt3rgWz5y/L6OmLOL2Vz9h6qIp3DCoC0ft3Rxfuiq7JVXTIulKSe9I6pDw9DJJ56cpLlcBxGPypMWlnaQbJT0taaykTqU47kngXaCTpGWShkk6P6FcmwQsAhYCowlqmV0FEY+JCw5sz4u/3p/W9atz8RMz+dWjM/h6nfd1yWbJ1rR0AP4P2DnzpJmtl3QMQZWscz/hSYvLkPpmdlI4auh24KJkDjKzYkc+WtDZIalzudy1R9PaTLhgXx56azG3vfoJh9z2Br8/ci9O6dPaa12yULJ9Wv4LDCTo6AbsXOV5v3QE5SqGeEzeEddlwg/hxG8G1Iw6GJd78uIxfvXz9vz7sgPo0qIOVz/7IaeMmsrCr31el2yTVNJiZk8Ba4GFkqZJuhnYF/g4ncG53JYXk6/y7NJK0sNAXeBQYDzwj2gjcrlst0Y1efK8/vzlhL35+Mv1HHHnm9z68gI2bfHZdLNF0qOHzOxuoA1wPRAHfgMknYZKOlzSx+GU2FcXsf3ycJn5OZL+K6ltsud22Skm4QutunQys7OBS4C3gTeB46ONyOU6SZzSpw3/veLnHNOtBSMnf8Yv7njD1zDKEqUa8mxmm8xskpldbWYHADcmc5ykODCSYFrszsCpkjoX2m0m0NvMugHPAH8tTWwu++TFvabFpZ+ZbQJWm9kdZjY86nhcxdCoVlVuO6U744f3p1penHPHTeecsdNY8s33UYdWqZVnwUTM7I0kd+0LLDSzReHwxPEEU2QnnmtyOJwaYCrBnAguh3mfFpcOPprRZVL/3Rsy6dKfce1Re/H+4tUMvP0NbnlpPus3by35YJdy5UpaSiHp6bBDw4CXdrXRp8zODXHJp/F36VDkaEbgmMgichVafjzGuT/bndd+83MGdW/JA28s4qC/vcE/pn3hIyQzLFNJS9LTYUs6HegN3Lqrk/mU2bkhHhPbfJVnl3o+mtFFokntavztpH144aL9aNewBr+d8CFH3fUmUz7xH8+ZkqmkJanpsCUdCvweONbMfF7lHBf0afGkxaWWj2Z0UdundT2ePn8AI4f05Pst2zhzzPuc/uB7fLR8bdShVXiZSlqmAR0l7RZOADWYYIrsnST1AB4gSFi+zlBcLo3iEtu9ecilQXlHMzpXXpI4qltz/nP5z/nD0Z2Zu2ItR9/9Fhc/8QGLvbNu2pR67aGyMLNtki4GXiYoYMaY2VxJNwDTzWwiQXNQLeDpcBbCL8zs2EzE59LDZ8R16RSOGpoU3pD082gjcpVR1bw4w/bfjZN6t2L0lEU8+OZiXvroS07o2ZJLDu5I6wY1og6xQslI0gJgZjsLl4Tnrku4f2imYnGZ4X1aXCaVYjSjcylXp1o+V/yiE2cOaMd9r3/GY+99znMzl3Nir9ZcfHAHWtarHnWIFUKmmodcJRSPefOQc65yaVy7Ktcd05k3rjyQwX3a8MyMpRx462SunjCHz7/1ZqPy8qTFpU1eLObNQ865Sql53erceFxX3rjyIE7t24ZnZy7noL+9zqXjZ7Lgy3VRh5ezMtY85CqfmE8u55yr5FrUq84Ng7py8UEdePCtxTw29XNemLWCgzo1ZvgB7em/ewNfTboUvKbFpU1eTOzwpMVlqSTWQxsqaZWkWeHt3CjidBVDkzrV+N2Re/HO1QdzxcA9mLNsLaeOnsqx97zNC7OWs9UXakuKJy0ubXwaf5etklwPDeAfZtY9vD2Y0SBdhVSvRhUuOaQjb199MDcf35Xvf9jGpeNnsf9fXuOe1z7l2w0+RVlxvHnIpU1cXtPistbO9dAAJBWshzYv0qhcpVEtP85p/dpyap82vP7J1zz89hL+9son3PXaQo7euzmnD2hLj9b1vOmoEE9aXNrE42Kbr/LsslNR66H1K2K/EyQdAHwC/J+ZLS1iHyQNB4YDtGnTJsWhuoosFhMH79mUg/dsysKv1/Pou58z4YPlPDtzOXs1r8OQfm0Y1L0FdarlRx1qVvDmIZc2eT65nMteyayH9k+gnZl1A/4DPLKrk/l6aC4VOjSpzR8HdWXq7w7h5uO7AvCH5z+i383/5YqnZvPeom+xSj6NhNe0uLTxafxdFitxPTQz+zbh4WjgLxmIyzlqVc3jtH5tGdK3DXOWrWX8tC/45+yVTPhgGW0b1uCXPVrxy54tK+Vsu560uLSJx8R2nxHXZaed66EBywnWQxuSuIOk5ma2Mnx4LDA/syG6yk4S+7Suxz6t6/GHozsz6cMvmTBjGbf/5xNu/88n9GlXn0HdW3Lk3s1pULNK1OFmhCctLm3y4j56yGWnJNdD+7WkY4FtwGpgaGQBu0qvRpU8TuzVihN7tWLZmo28MGsFz81czrXPf8SIiXPZr0Mjju7WnF90bkbdGhW3/4snLS5tYhI7vHnIZakk1kO7Brgm03E5V5JW9Wtw0UEduPDA9sxfuZ4XZi/nxdkrufKZOfwu/iH7dWjE4V2acWjnpjSqVTXqcFPKkxaXNnk+T4tzzqWNJDq3qEPnFnW4+vA9mbX0O/790ZdM+mglVz/7IXruQ3q1qc/Azk05ZK+mtG9cM+eHUHvS4tImHot5nxbnnMsASfRoU58ebepz9RF7Mm/lOl6Z+xWvzPuKW15awC0vLaBdwxoctGcTDurUhL67NaBafjzqsEvNkxaXNvEYPnrIOecyTBJdWtSlS4u6/N/APVi2ZiOvLfia1xZ8zRPvfcHDby+hWn6Mfrs15GcdG/Gzjo3Zo2mtnKiF8aTFpU08FvPmIeeci1ir+jU4c0A7zhzQjk1btjN18be88fEq3vx0FTf9az4wn0a1qjKgfUP2bd+Q/rs3pF3DGlmZxHjS4tImHsMnl3POuSxSvUqcgzoFTUQAy7/bxNsLv+GtT7/h3UXf8s/ZwXRFTetUpU+7BvTdrQG92zagU7PaxGPRJzGetLi0icdibN9hmFlWZuzOOVfZtaxXnZN7t+bk3q0xMz5btYGpi1bz3uLVTFu8mhfnBFMV1aqaR4829YJ+M63r0b11PepHMDdMRpMWSYcDdxLMi/Cgmf250PaqwDigF/AtcIqZLclkjC518sKsfIdB3HMW55zLapLo0KQ2HZrU5vT+bTEzlq3ZxPTPVzN9yRo++OI77nntUwoq0Ns0qEG3VnXp1qouXVsGfWjqVk/vHDEZS1oSloIfSDCF9jRJE80scVXVYcAaM+sgaTDBtNmnZCpGl1oFVYnbd1hWVCs655xLniRaN6hB6wY1OL5HKwC+/2Ebc5atZc6y75i19DtmfvHdztoYgNYNqtO5eR32Krg1q0Or+tWJpeg7IJM1LcksBT8IGBHefwa4R5Kssq8QlaMKEpWrnplNPOZrc0bhysM60axutajDcM5VEDWr5jGgfUMGtG+487lvN/zA3BXr+GjFWuauWMe8Fet4Zd5XFHxz16gSp2PT2uzRpBZXHtaJJnXKXiZlMmlJZin4nfuE02yvBRoC3yTu5MvA54Z9WtWjbcMaTFuyJupQKq1NW7dHHYJzroJrWKsqB+zRmAP2+N8K59//sI1PvlrPx1+uZ8GX6/nkq/VM/ngV1x7duVzXymTSksxS8Mnsg5mNAkYB9O7d22thstSA9g1548qDog7DOedchtWsmrdzsrtUymSdfYlLwSfuIykPqEuwUJlzzjnnKrlMJi07l4KXVIVgKfiJhfaZCJwV3j8ReM37szjnnHMOMtg8lORS8A8Bj0paSFDDMjhT8TnnnHMuu2V0npYkloLfDJyUyZicc845lxt8HKpzzjnncoJyvcuIpFXA5yXs1ohCw6ZzTC7Hn8uxQ27Hn2zsbc2sccm7uWR4mZT1cjl2yO34y10m5XzSkgxJ082sd9RxlFUux5/LsUNux5/LsVd0uf7e5HL8uRw75Hb8qYjdm4ecc845lxM8aXHOOedcTqgsScuoqAMop1yOP5djh9yOP5djr+hy/b3J5fhzOXbI7fjLHXul6NPinHPOudxXWWpanHPOOZfjPGlxzjnnXE6oNEmLpFslLZA0R9JzkupFHVOyJJ0kaa6kHZJyZqibpMMlfSxpoaSro46nNCSNkfS1pI+ijqW0JLWWNFnS/PBzc2nUMbmf8jIp87xMikYqy6RKk7QArwJdzawb8AlwTcTxlMZHwC+BKVEHkixJcWAkcATQGThVUudooyqVscDhUQdRRtuAK8xsL6A/cFGO/e0rCy+TMsjLpEilrEyqNEmLmb1iZtvCh1OBVlHGUxpmNt/MPo46jlLqCyw0s0VmtgUYDwyKOKakmdkUgkU7c46ZrTSzD8L764H5QMtoo3KFeZmUcV4mRSSVZVKlSVoKOQd4KeogKriWwNKEx8vwL86Mk9QO6AG8F20krgReJqWfl0lZoLxlUkZXeU43Sf8BmhWx6fdm9kK4z+8Jqqoez2RsJUkm9hyjIp7z8fUZJKkWMAG4zMzWRR1PZeRlUlbxMiliqSiTKlTSYmaHFrdd0lnA0cAhlmUT1JQUew5aBrROeNwKWBFRLJWOpHyCwuFxM3s26ngqKy+TsoqXSRFKVZlUaZqHJB0O/BY41sw2Rh1PJTAN6ChpN0lVgMHAxIhjqhQkCXgImG9mt0Udjyual0kZ52VSRFJZJlWapAW4B6gNvCpplqT7ow4oWZKOl7QMGAD8S9LLUcdUkrCD4cXAywSdrp4ys7nRRpU8SU8C7wKdJC2TNCzqmEphP+AM4ODwsz5L0pFRB+V+wsukDPIyKVIpK5N8Gn/nnHPO5YTKVNPinHPOuRzmSYtzzjnncoInLc4555zLCZ60OOeccy4neNLinHPOuZzgSYtzzjnncoInLc4555zLCZ60uJSS9HdJ8ySNlvRGuBx8ScdUkTRFUoVaVsI5Fz0vkyoWT1pcykjaHdjPzDoDs4BnzWx7SceFy8T/FzglzSE65yoRL5MqHk9aXKlI2lvS2wmPe0p6TVIn4A2graSZwLnACwn7TZY0MLx/k6S7Cp36eeC09L8C51xF4mVS5eLT+LtSkRQjWBm1pZltlzQZuMLMPpB0E7AEGAd8YWbNEo47ALgBGA0MIVgkbnvC9jjwpZk1ztyrcc7lOi+TKhevaXGlYmY7gLlAF0knEBQEH4Sb9wZmA42A7wodNwUQcDkwuKBwkHRjuH07sEVS7Yy8EOdcheBlUuXinYxcWUwlWLXzQuDwhOe7EBQeVYFqiQdI2htoDnxjZuvD55rx489gVWBz+sJ2zlVQXiZVEl7T4spiKnAT8JyZLQcIf41sNbONZrYGiEuqFm5rDjwODAK+l3RYeJ4eBJ3jkNQQWGVmWzP7UpxzFYCXSZWEJy2uLBYAPwB/SXiuK/BRwuNXgP0l1QCeJWhjng/cCIwI9+lOWEAABwGT0hizc67i8jKpkvCOuK7UJN0DTDOzR4rZpwdwuZmdUcw+DwHnmdkOSc8C15jZx6mP2DlXkXmZVHl4TYtLmqT2khYA1YsrHADMbCYwubiJnMxsWFg4VAGe98LBOVcaXiZVPl7T4pxzzrmc4DUtzjnnnMsJnrQ455xzLid40uKcc865nOBJi3POOedygictzjnnnMsJnrQ455xzLid40uKcc865nPD/YbIc/OI+cmIAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_loss_functions(suptitle = 'Common loss functions for classification',\n", + " functions = [zero_one_v(x), logistic_loss(x)],\n", + " ylabels = ['$\\mathcal{L}_{0-1}}$ (0-1 loss)',\n", + " '$\\mathcal{L}_{log}$ (logistic loss)'],\n", + " xlabel = '$y f(x_i)$')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Строим нейросеть\n", + "Рассмотрим решение нашей задачи при помощи простейшей однослойной нейросети такого вида:\n", + "\n", + "\n", + "При этом модель будет описываться как\n", + "$$\n", + "f_\\theta(x) = W\\times x + b\n", + "$$\n", + "где параметры $$\\theta = $$" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[ 1.77202116, -0.25384488],\n", + " [ 0.28370828, -0.39610552],\n", + " [-0.30097433, 0.30513182],\n", + " [-0.8120485 , 0.56079421],\n", + " [-1.23519653, 0.3394973 ]])" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class Linear:\n", + " def __init__(self,nin,nout):\n", + " self.W = np.random.normal(0, 1.0/np.sqrt(nin), (nout, nin))\n", + " self.b = np.zeros((1,nout))\n", + " \n", + " def forward(self, x):\n", + " return np.dot(x, self.W.T) + self.b\n", + " \n", + "net = Linear(2,2)\n", + "net.forward(train_x[0:5])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Переходим к вероятностям\n", + "Расширяем нейросетевую модель с помощью функции **softmax**: $\\sigma(\\mathbf{z}_c) = \\frac{e^{z_c}}{\\sum_{j \\in J} e^{z_j}}$ для $c \\in 1 .. |C|$\n", + "\n", + "\n", + "\n", + "Можем рассматривать $\\sigma(\\mathbf{z})$ как распределение вероятности на классах $C$: $q = \\sigma(\\mathbf{z}_c) = \\hat{p}(c | x)$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.88348621, 0.11651379],\n", + " [0.66369714, 0.33630286],\n", + " [0.35294795, 0.64705205],\n", + " [0.20216095, 0.79783905],\n", + " [0.17154828, 0.82845172],\n", + " [0.24279153, 0.75720847],\n", + " [0.18915732, 0.81084268],\n", + " [0.17282951, 0.82717049],\n", + " [0.13897531, 0.86102469],\n", + " [0.72746882, 0.27253118]])" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class Softmax:\n", + " def forward(self,z):\n", + " zmax = z.max(axis=1,keepdims=True)\n", + " expz = np.exp(z-zmax)\n", + " Z = expz.sum(axis=1,keepdims=True)\n", + " return expz / Z\n", + "\n", + "softmax = Softmax()\n", + "softmax.forward(net.forward(train_x[0:10]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Ещё один взгляд на архитектуру сети\n", + "\n", + "![Архитектура нейросети](https://raw.githubusercontent.com/shwars/NeuroWorkshop/master/images/Cross-Entropy-Loss.PNG)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Cross-Entropy Loss\n", + "\n", + "* Повсеместно применяется в глубоком обучении\n", + "* Основная идея:\n", + " - трактуем выход модели как распределение вероятностей появления того или иного класса\n", + " - минимизируем вероятность неправильной классификации\n", + "\n", + "Два подхода к пониманию Cross-Entropy Loss:\n", + " * Цена ошибки, которую мы платим за неправильную классификацию, т.е. $-\\log p_y$, где $y$ - правильный класс\n", + " * Разница между двумя распределениями вероятностей. Энтропия $p$ + KL-расстояние между $q$ и $p$:\n", + "$\\begin{align} H(p, q) = & ~\\color{red}{H(p)} + \\color{blue}{D_{KL}(p||q)} \\\\\n", + " = & ~\\color{red}{-\\sum_{c \\in C} p(c) \\log p(c)} + \\color{blue}{\\sum_{c \\in C} p(c) \\log \\frac{p(c)}{q(c)}} \\\\\n", + " = & ~-\\sum_{c \\in C} p(c) \\log q(c)\n", + "\\end{align}$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "def plot_cross_ent():\n", + " p = np.linspace(0.01, 0.99, 101) # estimated probability p(y|x)\n", + " cross_ent_v = np.vectorize(cross_ent)\n", + " f3, ax = plt.subplots(1,1, figsize=(8, 3))\n", + " l1, = plt.plot(p, cross_ent_v(p, 1), 'r--')\n", + " l2, = plt.plot(p, cross_ent_v(p, 0), 'r-')\n", + " plt.legend([l1, l2], ['$y = 1$', '$y = 0$'], loc = 'upper center', ncol = 2)\n", + " plt.xlabel('$\\hat{p}(y|x)$', size=18)\n", + " plt.ylabel('$\\mathcal{L}_{CE}$', size=18)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": true, + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAADfCAYAAADm4+qPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd5hU5dnH8e9NXbAhHXZZiqA0ARUVRVCxoGgswYbR2KImaiyxx+QlGqPGrolGsUQJisZIQIxgrLGCgrogKAIKsoBSVECalOf9455xtsIsO7tnyu9zXec6M3POzNx7dmbu8zznKRZCQERERDJPnagDEBERkW2jJC4iIpKhlMRFREQylJK4iIhIhlISFxERyVBK4iIiIhmqXtQBVFXz5s1Dhw4dog5DRESkVkydOnVZCKFFRdsyLol36NCBKVOmRB2GiIhIrTCz+ZVtU3W6iIhIhlISFxERyVBK4iIiIhkq466Ji9SWDRs2UFxczLp166IORdJMXl4eBQUF1K9fP+pQJMcpiYtUori4mB122IEOHTpgZlGHI2kihMDy5cspLi6mY8eOUYcj6SQEeOghOOgg2HXXWnnL3K5Ov/lmX0QqsG7dOpo1a6YELqWYGc2aNVMNjZS3aBGcfz68+GKtvWVuJ/F334VRo6KOQtKYErhURJ8LqVBRka979661t8ztJN67N8yaBWvXRh2JiIhkungS79Wr1t5SSXzTJpgxI+pIREQk0xUVQfv20KRJrb1lbifxPn18HT97EsliZ599Ni1btqRnz55Rh5LWJk6cyG677Ubnzp255ZZbog5HMklRUSKv1JLcTuKdOsEuu4AaqEgOOPPMM5k4cWLUYaS1TZs2ceGFFzJhwgRmzpzJ6NGjmTlzZtRhSSZYuxY++6xWr4dDrifxOnVgzhy48MKoIxGp0PTp0+nfv/+P9z/44AMGDRq0Ta81cOBAmjZtmqrQ0kqqjtN7771H586d6dSpEw0aNOCUU05h3LhxqQxVstXHH8PmzbWexNVPXCRZBx1U/rGTToILLoA1a2DIkPLbzzzTl2XL4IQTSm97/fWtvmWPHj2YO3cumzZtom7dulx++eXccccdpfYZMGAAq1atKvfc22+/nUMPPXSr75FSl14KH32U2tfs0wfuvnuLu6TqOC1cuJB27dr9uK2goIDJkyen4I+QrBdBy3RQEoeXX4aLL4YJE7xBgkgaqVOnDj169GDGjBnMnj2bwsJC9txzz1L7vPnmmxFFlz5SdZxCCOUeU3cyScpHH8H220MtDwCkJL7ddvDJJ4lWhSKV2VLJuXHjLW9v3jypkndF+vXrx9tvv839999f4TXttCqJb6XEXJNScZwKCgpYsGDBj9uKi4tp27ZtzQUt2aOoyLuW1andq9RK4rvvDmZ+FnXMMVFHI1JOv379OPPMM7nwwgvJz88vt10lcZeK47T33nsze/ZsvvjiC/Lz83nqqad48sknayJcySYhwLRp8LOf1fpb53bDNvDqj86d1c1M0lbXrl1p2LAhV199dbVeZ9iwYey3337MmjWLgoICHnnkkRRFmB5ScZzq1avHX//6VwYPHky3bt046aST6NGjRwqjlKw0bx6sXFnr3ctAJXHXuzd8+GHUUYhU6J577uHmm29mu+22q9brjB49OkURpadUHachQ4YwpKJGiiKViahRG6gk7o48EgYO9O4BImli7ty5dO3albVr13LGGWdEHU7a0nGSyBUV+WXZCAZSUkkc4OyzfRFJI7vssguffvpp1GGkPR0niVxREXTp4g2la5lK4nEheF9fERGRqigqiqQqHdIkiZtZXTP70MyejyyIzp3hsssie3sREclAK1fC55/ndhIHLgE+iTSCdu3UQl1ERKpm2jRfR9AyHdIgiZtZAXAU8HCkgfTuDdOn+9SkIjEVjeAlos+F/CjClumQBkkcuBu4Coi2aXifPn5NfO7cSMOQ9JGXl8fy5cv1gy2lhBBYvnw5eXl5UYci6aCoCJo2hQoGGKoNkbZON7OjgSUhhKlmdtAW9jsPOA+gsLCwZoKJn0UVFcGuu9bMe0hGKSgooLi4mKVLl0YdiqSZvLw8CgoKog5D0kG8UVtEY+xH3cWsP3CMmQ0B8oAdzWxUCOG0kjuFEEYAIwD69u1bM8Wi7t3h2mtht91q5OUl89SvX5+OtTyZgYhkkE2b/DLs+edHFkKkSTyEcC1wLUCsJH5F2QRea/Ly4KabInlrERHJQNOnw9q1UGbGvNqUDtfE08fGjT7T1IoVUUciIiLpbtw4n7XsiCMiCyFtkngI4fUQwtGRBjF1Khx8MIwfH2kYIiKSAcaOhf79oUWLyEJImySeFvbeG9q08X+MiIhIZb74wqewPu64SMNQEi+pTh049liYONGvc4iIiFRk3DhfH3tspGEoiZd1/PGwejW88krUkYiISLoaOxZ23x122SXSMJTEyzroINhxR5gwIepIREQkHS1bBm++GXlVOkTfTzz9NGgA776rAV9ERKRizz8Pmzcriaet7t2jjkBERNLV2LE+adYee0QdiarTK3XddXDbbVFHISIi6WTNGvjvf70UHtFQqyUpiVfmww/hgQdAk1+IiEjciy9676U0qEoHJfHKHXecT/T+8cdRRyIiIuli7FjYeWcYMCDqSAAl8codc4xXlWjgFxERAR+ae/x4OPpoqF8/6mgAJfHKtW4N++8Po0Z5K0QREcltEybAt9/6eCJpQkl8Sy6+2IdiXbUq6khERCRqd93lrdJ/8pOoI/mRuphtyUkn+SIiIrmtqAheew3+/Geolz6pUyXxZBQV+WD3IiKSm+6+Gxo3hnPPjTqSUpTEt2blSthvP7jppqgjERGRKHz9NTz5JJx1lrdMTyNK4luz447w85/DP/4BS5dGHY2IiNS2v/0NfvjB20mlGSXxZFxyCaxfDw8+GHUkIiJSm9at8yR+9NFpOaeGkngyunWDI46A++7zszEREckNo0fDkiVw6aVRR1IhJfFkXXYZfP89TJsWdSQiIlIbQvAGbb16waBBUUdTofRpJ5/uDjsMiothp52ijkRERGrDhAlecHvkkbSY7KQiKokny8wTeAg+prqIiGSvDRvgiiugSxc47bSoo6mUknhVXXyxdzlbuTLqSEREpKaMGAGffOJTUjdoEHU0lVISr6ozzvBGDuo3LiKSnb79FoYPh4MP9smw0likSdzM8szsPTMrMrMZZnZ9lPEkpW9f7zd+110axU1EJBvdeCN88w3ceWfaXguPi7okvh4YFELoDfQBjjCzfhHHtHU33eRj5159ddSRiIhIKs2eDX/5C5x9NvTpE3U0WxVpEg/u+9jd+rElRBhScvLz4aqrYMoU+O67qKMREZFUueoqaNjQS+MZIOqSOGZW18w+ApYAL4UQJkcdU1KuugpmzoQmTaKOREREUuG//4WxY+Haa6F166ijSUrkSTyEsCmE0AcoAPYxs55l9zGz88xsiplNWZou45c3agR5ebB2rfclFBGRzPXtt16F3q2bD+6VISJP4nEhhO+A14EjKtg2IoTQN4TQt0WLFrUe2xZdf71PEP/ee1FHIiIi2+rCC322slGjvJCWIaJund7CzJrEbjcCDgU+jTKmKrv2WmjbFk4/HdasiToaERGpqqee8jHShw+HPfeMOpoqibok3gZ4zcymAe/j18SfjzimqtlpJ3j8cfjsM79OLiIimWPhQvjVr6BfP7jmmqijqbJIx04PIUwD9ogyhpQ4+GC/hnLXXV61Pnhw1BGJiMjWbN4MZ53ls1OOHOldhzNM5kWcrm66CZYtgw4doo5ERESScfvt8NJLPl94ly5RR7NNlMRTJS/Pz+TAz+42bkzr8XZFRHLaCy949fmJJ8L550cdzTaL+pp49gkBhg3zrgoh/cetERHJOZ9+6r/TvXvD3/+e9kOrbomSeKqZ+QTyTzzhs9+IiEj6+PZbn9SkYUMYNw622y7qiKplm5O4mWXGyGpR+O1v4eSTvarmhReijkZERMAvcw4bBvPmwZgxUFgYdUTVVp2SePeSd8xsVpn7J1fjtTObGTz6qA+ef8opPsa6iIhEJwQf0OXFF+H+++GAA6KOKCWqk8TLXvBtWeb+Q9V47czXuDGMH+8tHjdujDoaEZHcFQJceSWMGOEDdP3iF1FHlDKpbJ1eNqlnbkuBVMnP91J4vNHEqlWwww7RxiQikmtuvBHuuAMuugj+9Keoo0mp6pTEzcy2lJHUNBsSCfyee7zBW3FxtPGIiOSSu++G//s/OOMM/x3O4JboFalOEq8LfGdmC8xsIpBnZseZWX6KYssuBxwAy5fDoYf6MH8iIlKz/vY3H01z6FB4+GGok30dsqrzF20P9AauA2YBU4F/AF+a2SIgc6aBqQ177eUt1Rct8oQ+Z07UEYmIZK9bboELLoCjj/Yuvxk4pGoytvmvCiFsBj6OLSPB69eBrsBeQGZNBVMbDjgAXnvNx1YfOBBmzdI1chGRVArBG6/9+c9w6qnw2GNQv37UUdWYbU7iZjYIODyE8OO0LyGEAHwSW0ZVP7wstNde8OabMGmSEriISCpt2uTdyB580Gcm++tfs7IKvaTq/HVXADMr2mBmw8ws83vR15Ru3XzmHICJE+Hf/442HhGRTLd6tY+D/uCDXhK/776sT+BQvSS+B/BMJdtaA3+sxmvnhhB8+tKf/hRuvlljrYuIbIsFC/xy5bhx3hr9ppuyrhV6ZaqTxBsB6yrZNgYYWI3Xzg1mMHasDwP429/Cz38O6yo7pCIiUs5778E++8DcufD883DJJVFHVKuqk8Q/xxuwlRNCmA80q8Zr545Gjbzl5I03wqhRMGiQVwuJiMiWjRoFBx7ov6PvvgtHHhl1RLWuOkl8FPAXM8sru8HMWgLfVeO1c4sZXHcdPPOMT43XuHHUEYmIpK+1a+G88+D002Hffb003qNH1FFFojpJ/C/AGmCqmf3UzOoCmFkD4DbgtRTEl1tOOMEHJzCDmTPhhhu8taWIiLjZs2G//eChh7wB28svQ/PmUUcVmW1O4iGEDcAQYCLwBLDazL4AvgX2BX6bkghz1TPPwPDhcPjhPkCMiEiuGz3au+kuWAD/+Y83YMvSQVySVa329yGE9SGEy4EC4AzgbuBUYI8QgsYWrY7hw30603ffhZ494emno45IRCQa33zj0zqfeqr/Hn70EQwZEnVUaSFVneh2A14PIdwTQhgXQlibotfNbWedBUVFsOuu/gF+8smoIxIRqV0vvgi77w7PPusNgN94A9q1izqqtLHVJG5mO5vZr8zsTDOrbP/3gN/Hhl2VVOrSBd56ywcuGDrUH1u2LNqYRERq2ooV8MtfwhFHQJMm3njtuutyvvq8rGRK4hOAvwKPAOdWtEMIYSPwQmXbpZrq1fOB/Bs2hO+/92tCJ54IixdHHZmISOqNGeMjWz70EFx+OUydCnvsEXVUaSmZJL4DsAvwDt43vDKTgfOr8uZm1s7MXjOzT8xshpnlVi/9bdGwIZx/Powf7x/yESNg8+aooxIRqb7iYjj+eK91bNUKJk+G22+HvHI9mSUmmSQ+M4QwL4QwIITw0hb26wz0MbOqTBezEbg8hNAN6AdcaGbdq/D83FO/vo/uNm2an5mefz7sv783/BARyUTr1/usY127+nwSf/6zV5/37Rt1ZGkvmSQ+3sxuSGK/y2Lr7ZJ98xDC4hDCB7Hbq/DZz/KTfX5O23VXePVVePxxaN8edt7ZH9+wIdq4RESq4oUXvOHaNdfAoYfCjBlw1VVZPX1oKm01iYcQRgJ7m9ksM7vBzA43s4rm0BwArAohbNNIbWbWAZ9UZXIF284zsylmNmXp0qXb8vLZyczHW3/6ab+9aBF06uTVTxqDXUTS2YwZcNRRvpjBhAk+l0SnTlFHllGS7WI2FJgK/A5v6PaNmU0ysz+aWXz89PXArG0Jwsy2B54FLg0hrCy7PYQwIoTQN4TQt0WLFtvyFrnhhx/8jPbKK71a6okndL1cRNLLokVw7rnQqxe8/TbcdhtMn+6t0KXKkkriIYQ1IYRTgQOAR4FFwD7AdcB7ZvYGsAxYUNUAYtfQnwWeCCGMqerzpYQOHbxq6uWXoVkzOO00H1d4/fqoIxORXPfdd/D733u32ccfh4sv9pnHrrgCGjSIOrqMVaUOdyGEd/BW6vHq7wNLLB3xIVeTFutX/gjwSQjhzqo8V7bgkEPg/fd9iMKiIm/RDl59laOTBIhIRFatgnvugTvu8ER+8sk+XKqqzVOiOmOnzwshPB5CODuEsAt+Pbuqrar6A6cDg8zso9iisfRSoU4d+NnP4NZb/f60aT5c4aBBPuKRiEhNWrXKW5l37Ogl8IED4cMP4amnlMBTKFXDrhJCKAKureJz3gohWAihVwihT2x5IVUxSQldusDdd/vsaAceCAMG+HCGIUQdmYhkk2++geuv914z11wDe+/t3cXGjYM+faKOLuukLIkDhBCmp/L1JIUaNYJLLoHPP4d774V58+CnP4Vvq3QFRESkYgsXwtVXe/L+wx+85P3ee97qfO+9o44ua6U0iUsGaNwYfv1rb1Dy6qvQtKmXxk86ycdnX7066ghFJJNMnw5nnunV5rffDkcf7Zfvxo5V8q4FSuK5qkEDb7kO3thkwQK46CIoLPRJBoqLo41PRNLX5s1ewj7iCO8q9swzPlnJ7NneoHb33aOOMGcoiYuP9vbOOz5b2sCBcMst3l3t1VejjkxE0snKlX45rmtXn8+7qAj+9CcvBNx7rxqsRUBzuokzg/79fZk3Dx5+GPbbz7c9+aS3ND31VNihosH6RCSrTZsGDzwA//iHz6TYr583Xhs6VH28I6aSuJTXoQPceKM3hgN49lmvKmvbFn71K/joo0jDE5FasGYNjBzpEyz17g2PPuozjE2eDO++C8OGKYGnASVx2bp//cu/tEOHwmOP+expv/511FGJSKqF4HN3X3CBn7SfcQYsXw533unDpY4cCfvsE3WUUoKq02XrzLz6rF8//zI/8YRfEwNvAHfttT4Ry6BBULdutLGKSNUtWeKXzR5/3Gva8vLghBPgnHN8XAmzqCOUSiiJS9U0bVq6FD59Ojz/PIwaBfn5Pl77z38O3TUtvEhaW7fOv7sjR3pL840bYa+9vKvpqadCkyZRRyhJsJBhI3b17ds3TJkyJeowpKR162D8eD+LnzjRH/vqK2je3Oc317zAIulh0yZ47TUvdT/7rLc2b9s2cfKtuRXSkplNDSH0rWibSuJSfXl5cOKJvnz9tU8v2Ly5bxsyxGdRO/lkr55r1SraWEVyzebN3qbl6ae9P/dXX3kvk6FDvXHaIYfoMlgGU8M2Sa1WrXw4V/BGMgcf7A1jLrrIz/gPOQSeey7aGEWyXTxx/+Y3PgzqAQfAiBHebfSZZ/xk++9/h8MPVwLPcEriUnPM4Le/9SlQP/44MRLcnDm+fcUKHyBCo8OJVN/GjV5VftFF0K6ddw277z7Yc09vs7J0KYwZ4zVi8e6jkvF0TVxqVwh+Xa5ePR9b+fjj/fG+feG443zp3l2tYUWSsWYNvPSSf5fGj/dar0aNfDjUoUN9HPOddoo6SqmmLV0TVxKXaM2aBf/+t09TOGmSP/bZZz516tKl3kJWDeNEEhYv9lbl48fDyy/D2rX+PTnqKD8JPvJI2G67qKOUFFISl8yweDG88oq3lAXv5vKf/3ip4qijfN2yZbQxitS2zZt9AJYXXvDkHf/9a98efvITT9wDB+pkN4updbpkhjZtEgkcfLSoxo09kf/zn/7YySfDU0/57RBU7S7ZadkyryafONH7cC9dmhh06aabPHn36KHPvyiJSxobPNiXzZt9tqQXXkgMQLFhA+y2m19LHzzYW9m2axdtvCLbasMGH5P8xRd9mTLFT1KbNfMaqCFD/DMe77opEqMkLumvTh0fr32PPRKPrVrl3dcmTvQuM+BJ/dZb4ZhjoolTJFkhwCef+DXtl16C11/32cHq1EnMEDZ4sI+gpi5gsgVK4pKZmjaFRx7xH8MZM/yH8KWXfG508K4211zj/dIHDfIpVtWtRqI0f763+Xj1VV8WL/bHO3f2y0iHHeYnpvHPsEgSlMQls5lBz56+XHZZ4vF4N7bbboObb/YpE/fd10etatMmungldyxY4CeTr7/u63nz/PFWrfzEctAgP8ns2DHKKCXDKYlLdjr0UF9WrYI33/Qf0ffegxYtfPtVV3mXtoEDfZam/faD7bePNmbJXCHA55/DG2/A//7nSzxpN23qn7Hf/MZL2mqQJimkLmaSm+6910ex+uADL7XXrev9a8eP9+2rV6uvrVRu0yafwe/NN315661E9Xjz5n5yGD9B7NXLr3WLbKO07WJmZo8CRwNLQgg9o4xFcszFF/uyahW8847/CDdokNjevbvf79/fl/33h27d9GOcq1au9Nbjb7/ty6RJ3hANvFfEwQfDgAGeuLt1U0lbak2kJXEzGwh8D4xMNomrJC41btMmuPPOxA/2smX++MUXwz33+PZXXoF99tGcy9lo82YfNXDSJJ9E5N13fez/EPwkbvfdEyd2AwZAYWHUEUuWS9uSeAjhDTPrEGUMIuXUrQtXXulLCDB7tv+Q77abb58xw7v/AHTt6l2C9t3XB+DIz48ubtk2S5fC++97STu+fPedb9tpJ///nnCCt5vYd1/Yccdo4xUpQQ3bRLbEDHbd1Ze4zp29f++kSb48/zw89pgPg5mf7wnhiSdg77196dxZ1fDp4vvvvR3E++/7gCqTJ8MXX/i2OnW8l8OJJyZOzHQJRdJc5A3bYiXx57dUnW5m5wHnARQWFu41f/782glOJBkheEvkVq18mNjHHoMLLvCJKcBLc337ekO61q398bw8XTetaWvW+Eh/U6b42ONTpvgAK5s3+/Z27TxR77OPL3vuCTvsEG3MIhVI6wlQkkniJemauGSEjRu92v39932ZNs27HTVoAJdeCiNHetIouXTposS+rVat8oT9wQeesD/4wBP2pk2+vWVLP5GK14707esnXSIZIG2viYtkrXr1oHdvX37xi9LbBg3y0vjUqd5Q7ocffACaRYt8+6OP+nqPPbyVfMOGtRt7uvv6a/joI/jww8QyZ47XiIAn57328tm9+vb12/n5OkGSrBR1F7PRwEFAczMrBoaHEB6JMiaRGnfMMYnx3X/4AWbO9MQUd++9XqoEPxno1g2GDoXhw/2x777LjVbxGzZ4K/GiotLLV18l9unQwU92Tj/d13vtpRH5JKdE3Tp9WJTvLxK5Bg2gT5/Sj33wgZcsi4q8xFlU5Nd3wa/nFhb6tfdevRJL//6wyy61H38qhOAnMdOn+2WHadP89syZsH6979OggddKHHGEH68+ffzv1jjjkuMivyZeVbomLjlt/Xq4/35P7NOn+3X39evhd7+DP/4RVqyAc89NjCffs6cn93SZCWvFCo/54499mT7d1/G++OAl6fjJSe/evu7aFerXjy5ukQjpmrhItmjYsPRELxs3eqk9PkTs4sVekv/XvxLXiPPy4PHH4aSTYMkS7xbXo4dXRddUcl+1ykvSM2aUXoqLE/tst52fZBx3nA+g0rOnJ2zNmS2SNCVxkUxWr56XUuO6dvWkvnq1t86Ol3i7d/ft//ufJ3Pw5N61q2+7/nrvz75mjZd4ky31Ll/u7xNfZs70ZcGCxD7x9znwwNI1BIWF6oMtUk1K4iLZaLvtvGV23zI1cEce6aPPzZiRSLpvv50okT/4oM/w1rmzN6jr2tVHquvXz/vCf/IJfPqpL5984iX7uEaN/DkHHugnBt26eYm/U6f0qc4XyTJK4iK5ZPvtPSH361d+26pV3up9yBCYNctHpfv3v8vvV6+eV3l36uSt7Pv1825z7durZC1Sy5TERXLJ+vU+7/Xs2d59q+QSn0oTPBl37AgHHOCjzMWHIP3f/+A///EkHx929rnnEl3kbr/dxyKPD1XbpYv321YfbZEaoSQukm1++MGrvmfP9uvj8fVnn8H8+YlhR8FL1LvuCocf7tXm8aVz54oHmRk4EH7/e280t2yZJ/Nvv01sf+cdH0t+w4bEYwMGwBtv+O0HHvBq986dPcG3aKEEL1INSuIimWjNGi9Rz53rCTq+njOnfKLecUdPmPvuC6edlighd+kCTZtu2/ubeQJu0aL042PGeIv5+fP95GH27NLjkQ8fXvo6+g47+Ih2d97p9594AgoKvFtc27aqnhfZCvUTF0lHIXi19OefJ5L13LmJ2/EhWuOaNEmUbjt39iQYT9TNm6dPabdkLUH8xKN3bzjnHD8xiXeVA2/V3rGjjzV/3nleun/xRf/bOnTwEr1IDlA/cZF0tHq1J7QvvvDl889Lr7//vvT++fmewAYP9kZlu+ySSNjbWqKubQ0alJ/aNS4vz//2kjULc+d6YzzwY/WTnyT2z8/3JH/ttd4Yb+VKHwSnY0eV4iVnKImL1JR16+DLLxOJuuy6ZLUy+FCqnTr5MmiQJ+f4/VwoecYb03XsCIcdVn57u3bePS5eIxE/2Ykn6ylT4JBD/HaDBt5avkMHuOkm72q3ZInv36GDz2qWLrUTItWgJC6yrVav9iQ9f74n5rLrkq29wQdQKSz0JHXMMYmEFV+UWLYsL6/y7nHgE6BMnJio2YifMMX7qE+YAGeemXiteJL/29/8+M+b55cp2rf3oV9VkpcMoCQuUpF46+v58xOJuuy65HjfkEjS7dv7RB0dO3qSiC9t22rQk5q0885+qaEygwfD+PGerOPL/PmJGo5Ro7zlPfj/sl07/1+OGeNtDoqKvJ1CYaFvy/aaEckISuKSm1av9qFBFyzwpFx2/eWXXh1eUuPG/qNeWOhTXsZLcu3bJ0pvStLpq3VrOProyrefdZb/X0vWqBQXJ1rX338/jBiR2L9lS7/U8c47XoPy8sve3a5dO19at9bnQWqcWqdL9lm7FhYu9IRcXJxI1iWXkn2bwX+E27RJlLIKC0sv7dt74zFVd+euRYu8VX38JG/+fD/RGznStw8Z4lX2cfXqebe+t97y+48+6iePBQWJRN+ihartZavUOl2yx6pVnpgXLvR1Rcvy5eWf17RpIjn375/4EY0n7fx8bziTFtEAAA50SURBVAwlUpm2bX2pzJNPlq/VKVnlft99PsNcSQMH+ih4AFde6ev8fE/0BQV+SaZVq9T+HZJVlMQlPWzc6K2HFy4svSxalEjYCxd6Ei+rWbNEUt5/f/8RbNcu8UNYUOBV4SI1qUkTX3r1qnj7++97O4qStUMluwa+9ppPTFPyMs4pp8Do0X570CDYaSf/fLdt6+u99vIZ4SRnKYlLzdq82UvGixZ5a+1FixJLPEkvWgRffVV6lDHw6sg2bfzHqkcPHxq0oKB0SaVtW29pLJLu6tTx6+gtW8Kee5bfPmWKN6j85ptErVKzZr5twwavKZozx0vu8ctBV1wBt93mYwq0bp1I8PHl2GN9/PsNG7x2oE0bndBmGSVx2TabN3tL3cWLSy8lk3X8sZLjaMc1a5YoTey+u69LljDy8/3HTtcLJZeY+XejWTMfyS6ufn3vPhe3Zo1/x+LV9Rs3wrnn+onx4sU+Mc3ChV4jdcABnvzjc8o3aeLfszZt4Jpr4NBD/bv86quJx1u3TgyyI2lNSVxKW7fOS8WLF/s6fjt+P377669h06byz995Z/8RaNvWJ9Jo0yZRmo6XDlq3VulZpDoaN/bR+uKaNIG77iq9TwiJ72jLlvD3vyeSfPxEO779o4+86r6k7bf37nWHHebbH3ss8X1u3drXXbrouxwxJfFcsHGjn2nHk/LXX1d8e/FiWLGi/PPj1YDxL26vXokvc3xRchZJL2Z+SQq8ZB8f6KYiBxwA06eXr1nr0MG3z5kDjzxSfijgKVP8uvzo0T4NbevWiaVVKzjjDL+Ov3Kln1TsuKN6eKSYknim2rDBG4J9/fXWl2XL/AtU1g47+BetTRuv0j7ssMT9+Nl269beDaaePioiWatRI28gV1kjuRNO8OX770uf9MfHwN9+e/+tWLzYS+3xmrqTT/YkfscdcMMNfpLfqpUvrVt78m/c2Kv/v/wysa1VK69dUMLfKvUTTxch+BcknpiXLEks8fsl1998U/HrNGpU+kw4/mWpaF1yxigRkVSJN2ht1sxr8iZP9v7y8dq/eOHi/fd9+y9+4SX9knbayRvwmflUtdOne41gq1a+zs+Hgw9OvF8Wt59J637iZnYEcA9QF3g4hHBLxCGlztq1Xo29ZEn5dUXL+vUVv06TJokPbvfu/sGNJ+j4hzq+qDGKiEStTp3Sc83vu68vlbn1VrjkktI1iOvXJ0riX3wBr7zij//wgz/WubMPvgM+pO7UqYnW/y1bQp8+8Lvf+fbXXvPXatnS42raNGtG04u0JG5mdYHPgMOAYuB9YFgIYWZlz4msJB6Cj7a0dGnyS9nrR3ENGyY+aC1aJJJx2futWvl9DUIiIuK/wytWeKFn7dpEC/4HH/SSerymculSr+ofO9a3d+0Ks2YlXqdOHTj+ePjXv/z+hRf6ukWLxLLbbonX37Qp0qSfziXxfYA5IYTPAczsKeBYoNIknlLxkvGyZYklfr/kOn677FjacQ0blv7nd+ni63hSLpmgW7Twa9G61iMiUjVmiUF1Sjr//C0/b8wYv15fska0Y8fE9smTvbRf8jLlGWd4i/wQvIazUSP//W7e3NdDh8Lpp3uCf+IJf7x580QX2VoSdRLPBxaUuF8MbKHOJcX69Ck/XSR4C8r4P6ttWz8bi9+P/wNLLkrKIiLpq3v3RD/5isRrdzdu9Gv5S5cm+uBv2uT96UsW7L74IpE7vv3WE37cMcfAuHE183dUIOokXlHmK1e/b2bnAecBFBYWpu7d77gjce0mnqCbN1f1tYhILqpXL9G+qORjw4dX/pydd/YuePEkX7aWoIZFncSLgXYl7hcAi8ruFEIYAYwAvyaesncfNixlLyUiIjmobl3YZRdfIhB1m/z3gS5m1tHMGgCnAM9FHJOIiEhGiLQkHkLYaGYXAS/iXcweDSHMiDImERGRTBF1dTohhBeAF6KOQ0REJNNEXZ0uIiIi20hJXEREJENl3NjpZrYUmF+Nl2gOLEtROLlOxzI1dBxTR8cydXQsU6e6x7J9CKFFRRsyLolXl5lNqWz4OqkaHcvU0HFMHR3L1NGxTJ2aPJaqThcREclQSuIiIiIZKheT+IioA8giOpapoeOYOjqWqaNjmTo1dixz7pq4iIhItsjFkriIiEhWyMokbmZHmNksM5tjZtdUsL2hmT0d2z7ZzDrUfpSZIYlj+Rszm2lm08zsFTNrH0WcmWBrx7LEfieYWTAztQyuRDLH0sxOin02Z5jZk7UdY6ZI4jteaGavmdmHse/5kCjiTHdm9qiZLTGzjyvZbmZ2b+w4TzOzPVPyxiGErFrwMdjnAp2ABkAR0L3MPhcAD8RunwI8HXXc6bgkeSwPBhrHbv9Kx3Lbj2Vsvx2AN4BJQN+o407HJcnPZRfgQ2Dn2P2WUcedjkuSx3IE8KvY7e7AvKjjTscFGAjsCXxcyfYhwAR8Cu5+wORUvG82lsT3AeaEED4PIfwAPAUcW2afY4HHY7f/BRxiZhXNbZ7rtnosQwivhRDWxO5OwqeTlfKS+VwC/BG4FVhXm8FlmGSO5bnAfSGEbwFCCEtqOcZMkcyxDMCOsds7UcF00QIhhDeAb7awy7HAyOAmAU3MrE113zcbk3g+sKDE/eLYYxXuE0LYCKwAmtVKdJklmWNZ0jn4maaUt9VjaWZ7AO1CCM/XZmAZKJnP5a7Armb2tplNMrMjai26zJLMsfwDcJqZFeOTVf26dkLLOlX9PU1K5LOY1YCKStRlm+Ans49U4TiZ2WlAX+DAGo0oc23xWJpZHeAu4MzaCiiDJfO5rIdXqR+E1w69aWY9Qwjf1XBsmSaZYzkMeCyEcIeZ7Qf8I3YsN9d8eFmlRvJONpbEi4F2Je4XUL7658d9zKweXkW0pWqQXJXMscTMDgWuA44JIayvpdgyzdaO5Q5AT+B1M5uHXzN7To3bKpTsd3xcCGFDCOELYBae1KW0ZI7lOcA/AUII7wJ5+FjgUjVJ/Z5WVTYm8feBLmbW0cwa4A3Xniuzz3PAGbHbJwCvhljLAyllq8cyVgX8IJ7Add2xcls8liGEFSGE5iGEDiGEDnj7gmNCCFOiCTetJfMdH4s3usTMmuPV65/XapSZIZlj+SVwCICZdcOT+NJajTI7PAf8PNZKvR+wIoSwuLovmnXV6SGEjWZ2EfAi3vLy0RDCDDO7AZgSQngOeASvEpqDl8BPiS7i9JXksbwN2B54JtY28MsQwjGRBZ2mkjyWkoQkj+WLwOFmNhPYBFwZQlgeXdTpKcljeTnwkJldhlf/nqlCT3lmNhq/fNM81n5gOFAfIITwAN6eYAgwB1gDnJWS99X/QkREJDNlY3W6iIhITlASFxERyVBK4iIiIhlKSVxERCRDKYmLiIhkKCVxERGRDKUkLpKFzGw3M/uDme0WdSwiUnPUT1wky8Rm5HsT6IVPLTlQg3OIZCeVxEWyzzlAB6AP0JHEEMMikmWUxEWySGyc8JuBs0IInwNnA7eaWdMK9u1pZhvN7LBqvN/rZvb6Nj73ODP7wcw0MYnINlJ1ukiOMrP/Ag1DCNs8fWw8gYcQDtrG508F5ocQfrqtMYjkMpXERXJQbF7ow4A7Iw7lHuB4M+sRcRwiGUlJXCQ3XQAsx2dWitIYfEanX0Ych0hGUhIXyRJmdr2ZBTMbZGajzexrM1tjZu+Z2cAS+9UDjgNeCiFsKPMaD8Reo20Fr79b7Br2PVuJo5GZFZvZl2bWsMy2h81sk5mdAhBC+B5vSX/itv/lIrlLSVwke/TB585+CmgE/B6vLu8OTDCzNrH99sLngH+vgtd4N7bep4JtdwErgT9sKYgQwlp8LuV2eIkfADO7GW85/+sQwlNl3rOVmXXd0uuKSHlK4iLZow9QF7glhHBcCGFECOF3wEVAY+Ck2H7dY+u5FbzGpNi6VBI3s6OAI4H/CyF8m0QsjwEzgGvNbHszuxS4BhgeQri/zL7xOHRdXKSKlMRFsoCZ7QwUAm+FEMo2Vnsltu4QW7eIrb8p+zohhFmxx39M4mZWHy/Rfww8mEw8IYRNeNJuAYyNPf8vIYQbKth9eWzdMpnXFpGEelEHICIpsUds/XAF2+In69/H1vF+pVbJa00C+puZxUZ6uwTYFTg0lpyTEkJ43sw+AA7Bq/gvqWTXeBzq7ypSRSqJi2SHPrH1lAq27RtbfxhbL42tyw0AEzMJ2AnYzcxa4tfWx4YQXqlk/wqZ2Ukl4lq1haFf43EsrWS7iFRCJXGR7BBPlhsr2PYbvIr8v7H7H8fWlY2UVrJx20CgIXB5VYIxs8OBfwD/BjYAZ5vZXSGETyrYvXOZuEQkSSqJi2SHeHV6qdHXzOwcvCR+Y6w7F3iJfCXQr5LXmgxsxluSnwXcHRvCNSlmti/e//tt4GfA72Kvd3MlT+kHfB27Hi8iVaCSuEiGi/XF7oon57vMrD0wDzgIGAb8E7g7vn8IYZOZjQGONbOGIYT1JV8vhLDKzGbipfCvgD9VIZZuwH+Az4DjYq8918weAX5pZv1DCG+X2H97YADwaJX/cBFRSVwkC/TET8jvBK4GTgX+ipfOLwOGVXA9+m/AzsDRlbxmvA/5tSGEVckEYWaFeJX9CuDIEMLKEptvANYCt5Z52lC8+1tSrd5FpDRNgCKS4WJV5g8DvUMI06rwvInAdiGEAWUerw98Sqyr2ZbmItcEKCLRUklcJPPtgTce+7SKz7sc2C/WCK2kK/B5yH+9pQReXWZ2HLA7XnsgIttA18RFMl8fYFYI4YeqPCmEMIPYb0BsvvHBQC/gSuDOEMKkLTy92kIIY4EGNfkeItlOSVwkg5mZ4Yn3+Wq+1GDgSWAJPkb6NdV8PRGpBbomLiIikqF0TVxERCRDKYmLiIhkKCVxERGRDKUkLiIikqGUxEVERDKUkriIiEiGUhIXERHJUEriIiIiGUpJXEREJEP9P5VmBCAoNxpqAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "def cross_ent(prediction, ground_truth):\n", + " t = 1 if ground_truth > 0.5 else 0\n", + " return -t * np.log(prediction) - (1 - t) * np.log(1 - prediction)\n", + "plot_cross_ent()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1.429664938969559" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class CrossEntropyLoss:\n", + " def forward(self,p,y):\n", + " self.p = p\n", + " self.y = y\n", + " p_of_y = p[np.arange(len(y)), y]\n", + " log_prob = np.log(p_of_y)\n", + " return -log_prob.mean()\n", + "\n", + "cross_ent_loss = CrossEntropyLoss()\n", + "p = softmax.forward(net.forward(train_x[0:10]))\n", + "cross_ent_loss.forward(p,train_labels[0:10])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Задача минимизации\n", + "Описав нейронную сеть как модель $f_\\theta$ и функцию ошибки $\\mathcal{L}(Y,f_\\theta(X))$, можем рассмотреть $\\mathcal{L}$ как функцию $\\theta$ на всем множестве обучающей выборки $\\mathcal{L}(\\theta) = \\mathcal{L}(Y,f_\\theta(X))$\n", + "\n", + "В этом случае задача обучения сети будет формулироваться как задача минимизации $\\mathcal{L}$ по $\\theta$:\n", + "$$\n", + "\\theta = \\mathrm{argmin}_{\\theta} \\mathcal{L}(Y,f_\\theta(X))\n", + "$$\n", + "\n", + "Минимизацию можно осуществлять разными методами, например, стохастическим градиентным спуском (stochastic gradient descent, SGD)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Реализация нейронных сетей\n", + "\n", + " * Вручную\n", + " * С использованием готовых фреймворков\n", + " - PyTorch\n", + " - Tensorflow\n", + " - Chainer\n", + " - [Microsoft Cognitive Toolkit](http://cntk.ai)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Вычислительный граф\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.429664938969559\n" + ] + } + ], + "source": [ + "z = net.forward(train_x[0:10])\n", + "p = softmax.forward(z)\n", + "loss = cross_ent_loss.forward(p,train_labels[0:10])\n", + "print(loss)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Обучение сети\n", + "\n", + " * Для обучения сети необходимо предъявлять ей примеры, считать ошибку и подстраивать коэффициенты\n", + " * В соответствии с принципом градиентного спуска, необходимо расчитывать изменение коэффициентов в соответствии с градиентом функции $\\nabla f_\\theta$\n", + " * Итерация обучения выглядит так:\n", + " $$\\begin{align}\\def\\L{\\mathcal{L}}\n", + " W^{i+1}&=W^i-\\eta\\frac{\\partial\\L}{\\partial W}\\cr\n", + " b^{i+1}&=b^i-\\eta\\frac{\\partial\\L}{\\partial b}\n", + " \\end{align}\n", + " $$" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Обратное распространение ошибки\n", + "\n", + "\n", + "\n", + "$$\\def\\zz#1#2{\\frac{\\partial#1}{\\partial#2}}\n", + "\\begin{align}\n", + "\\zz{\\L}{W} =& \\zz{\\L}{p}\\zz{p}{z}\\zz{z}{W}\\cr\n", + "\\zz{\\L}{b} =& \\zz{\\L}{p}\\zz{p}{z}\\zz{z}{b}\n", + "\\end{align}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Обратное распространение ошибки\n", + "\n", + "\n", + "\n", + " * Не повторяем одинаковые вычисления\n", + " * Вычисляем ошибку на каждом узле начиная с конца\n", + " * Обратное распространение ошибки\n", + " * Все вычисления фреймворк берёт на себя" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Реализация обратного распространения\n", + "\n", + "* К каждому узлу добавляем функцию `backward`, которая вычисляет производную и значение ошибки\n", + "* После вычисления производных, реализуем обновление весов в соответствии с формулой выше\n", + "\n", + "Например, для линейного узла $z = x\\times W+b$:\n", + "$$\\begin{align}\n", + "\\frac{\\partial z}{\\partial W} &= x \\\\\n", + "\\frac{\\partial z}{\\partial b} &= 1 \\\\\n", + "\\end{align}$$\n", + "\n", + "Соответственно, если на вход пришла ошибка $\\Delta z$, то изменения весов вычисляются так:\n", + "$$\\begin{align}\n", + "\\Delta x &= \\Delta z \\times W \\\\\n", + "\\Delta W &= \\frac{\\partial z}{\\partial W} \\Delta z = \\Delta z \\times x \\\\\n", + "\\Delta b &= \\frac{\\partial z}{\\partial b} \\Delta z = \\Delta z \\\\\n", + "\\end{align}$$\n", + "\n", + "**ВАЖНО:** Вычисления производятся не для одного элемента обучающей выборки, а сразу для целой последовательности, называемой **minibatch**. Необходимые значения градиентов $\\Delta W$ и $\\Delta b$ вычисляются по всей выборке, а вектора имеют соответствующую размерность: $x\\in\\mathbb{R}^{\\mathrm{minibatch}\\, \\times\\, \\mathrm{nclass}}$" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "class Linear:\n", + " def __init__(self,nin,nout):\n", + " self.W = np.random.normal(0, 1.0/np.sqrt(nin), (nout, nin))\n", + " self.b = np.zeros((1,nout))\n", + " self.dW = np.zeros_like(self.W)\n", + " self.db = np.zeros_like(self.b)\n", + " \n", + " def forward(self, x):\n", + " self.x=x\n", + " return np.dot(x, self.W.T) + self.b\n", + " \n", + " def backward(self, dz):\n", + " dx = np.dot(dz, self.W)\n", + " dW = np.dot(dz.T, self.x)\n", + " db = dz.sum(axis=0)\n", + " self.dW = dW\n", + " self.db = db\n", + " return dx\n", + " \n", + " def update(self,lr):\n", + " self.W -= lr*self.dW\n", + " self.b -= lr*self.db" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Аналогичный образом функции обратного распространения `backward` добавляются к другим составляющим вычислительного графа:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "class Softmax:\n", + " def forward(self,z):\n", + " self.z = z\n", + " zmax = z.max(axis=1,keepdims=True)\n", + " expz = np.exp(z-zmax)\n", + " Z = expz.sum(axis=1,keepdims=True)\n", + " return expz / Z\n", + " def backward(self,dp):\n", + " p = self.forward(self.z)\n", + " pdp = p * dp\n", + " return pdp - p * pdp.sum(axis=1, keepdims=True)\n", + " \n", + "class CrossEntropyLoss:\n", + " def forward(self,p,y):\n", + " self.p = p\n", + " self.y = y\n", + " p_of_y = p[np.arange(len(y)), y]\n", + " log_prob = np.log(p_of_y)\n", + " return -log_prob.mean()\n", + " def backward(self,loss):\n", + " dlog_softmax = np.zeros_like(self.p)\n", + " dlog_softmax[np.arange(len(self.y)), self.y] -= 1.0/len(self.y)\n", + " return dlog_softmax / self.p" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Теперь напишем цикл обучения модели на нашем датасете. Будем рассматривать один проход по модели - т.н. **эпоху**" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Initial accuracy: 0.725\n", + "Final accuracy: 0.825\n" + ] + } + ], + "source": [ + "lin = Linear(2,2)\n", + "softmax = Softmax()\n", + "cross_ent_loss = CrossEntropyLoss()\n", + "\n", + "pred = np.argmax(lin.forward(train_x),axis=1)\n", + "acc = (pred==train_labels).mean()\n", + "print(\"Initial accuracy: \",acc)\n", + "\n", + "batch_size=4\n", + "for i in range(0,len(train_x),batch_size):\n", + " xb = train_x[i:i+batch_size]\n", + " yb = train_labels[i:i+batch_size]\n", + " \n", + " # forward pass\n", + " z = lin.forward(xb)\n", + " p = softmax.forward(z)\n", + " loss = cross_ent_loss.forward(p,yb)\n", + " \n", + " # backward pass\n", + " dp = cross_ent_loss.backward(loss)\n", + " dz = softmax.backward(dp)\n", + " dx = lin.backward(dz)\n", + " lin.update(0.1)\n", + " \n", + "pred = np.argmax(lin.forward(train_x),axis=1)\n", + "acc = (pred==train_labels).mean()\n", + "print(\"Final accuracy: \",acc)\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Для удобства опишем класс, который позволяет объединять узлы вычислительного графа в единую сеть, и применять функции `forward` и `backward` сразу ко всей сети последовательно:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "scrolled": true, + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "class Net:\n", + " def __init__(self):\n", + " self.layers = []\n", + " \n", + " def add(self,l):\n", + " self.layers.append(l)\n", + " \n", + " def forward(self,x):\n", + " for l in self.layers:\n", + " x = l.forward(x)\n", + " return x\n", + " \n", + " def backward(self,z):\n", + " for l in self.layers[::-1]:\n", + " z = l.backward(z)\n", + " return z\n", + " \n", + " def update(self,lr):\n", + " for l in self.layers:\n", + " if 'update' in l.__dir__():\n", + " l.update(lr)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ещё раз пробуем создать и обучить нашу нейросеть:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Initial loss=0.6212072429381601, accuracy=0.6875: \n", + "Final loss=0.44369925927417986, accuracy=0.8: \n", + "Test loss=0.4767711377257787, accuracy=0.85: \n" + ] + } + ], + "source": [ + "net = Net()\n", + "net.add(Linear(2,2))\n", + "net.add(Softmax())\n", + "loss = CrossEntropyLoss()\n", + "\n", + "def get_loss_acc(x,y,loss=CrossEntropyLoss()):\n", + " p = net.forward(x)\n", + " l = loss.forward(p,y)\n", + " pred = np.argmax(p,axis=1)\n", + " acc = (pred==y).mean()\n", + " return l,acc\n", + "\n", + "print(\"Initial loss={}, accuracy={}: \".format(*get_loss_acc(train_x,train_labels)))\n", + "\n", + "def train_epoch(net, train_x, train_labels, loss=CrossEntropyLoss(), batch_size=4, lr=0.1):\n", + " for i in range(0,len(train_x),batch_size):\n", + " xb = train_x[i:i+batch_size]\n", + " yb = train_labels[i:i+batch_size]\n", + "\n", + " p = net.forward(xb)\n", + " l = loss.forward(p,yb)\n", + " dp = loss.backward(l)\n", + " dx = net.backward(dp)\n", + " net.update(lr)\n", + " \n", + "train_epoch(net,train_x,train_labels)\n", + " \n", + "print(\"Final loss={}, accuracy={}: \".format(*get_loss_acc(train_x,train_labels)))\n", + "print(\"Test loss={}, accuracy={}: \".format(*get_loss_acc(test_x,test_labels)))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "def train_and_plot(n_epoch, net, loss=CrossEntropyLoss(), batch_size=4, lr=0.1):\n", + " fig, ax = plt.subplots(2, 1)\n", + " ax[0].set_xlim(0, n_epoch + 1)\n", + " ax[0].set_ylim(0,1)\n", + "\n", + " train_acc = np.empty((n_epoch, 3))\n", + " train_acc[:] = np.NAN\n", + " valid_acc = np.empty((n_epoch, 3))\n", + " valid_acc[:] = np.NAN\n", + "\n", + " for epoch in range(1, n_epoch + 1):\n", + "\n", + " train_epoch(net,train_x,train_labels,loss,batch_size,lr)\n", + " tloss, taccuracy = get_loss_acc(train_x,train_labels,loss)\n", + " train_acc[epoch-1, :] = [epoch, tloss, taccuracy]\n", + " vloss, vaccuracy = get_loss_acc(test_x,test_labels,loss)\n", + " valid_acc[epoch-1, :] = [epoch, vloss, vaccuracy]\n", + " \n", + " ax[0].set_ylim(0, max(max(train_acc[:, 2]), max(valid_acc[:, 2])) * 1.1)\n", + "\n", + " plot_training_progress(train_acc[:, 0], (train_acc[:, 2],\n", + " valid_acc[:, 2]), fig, ax[0])\n", + " plot_decision_boundary(net, fig, ax[1])\n", + " fig.canvas.draw()\n", + " fig.canvas.flush_events()\n", + "\n", + " return train_acc, valid_acc" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "import matplotlib.cm as cm\n", + "\n", + "def plot_decision_boundary(net, fig, ax):\n", + " draw_colorbar = True\n", + " # remove previous plot\n", + " while ax.collections:\n", + " ax.collections.pop()\n", + " draw_colorbar = False\n", + "\n", + " # generate countour grid\n", + " x_min, x_max = train_x[:, 0].min() - 1, train_x[:, 0].max() + 1\n", + " y_min, y_max = train_x[:, 1].min() - 1, train_x[:, 1].max() + 1\n", + " xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),\n", + " np.arange(y_min, y_max, 0.1))\n", + " grid_points = np.c_[xx.ravel().astype('float32'), yy.ravel().astype('float32')]\n", + " n_classes = max(train_labels)+1\n", + " while train_x.shape[1] > grid_points.shape[1]:\n", + " # pad dimensions (plot only the first two)\n", + " grid_points = np.c_[grid_points,\n", + " np.empty(len(xx.ravel())).astype('float32')]\n", + " grid_points[:, -1].fill(train_x[:, grid_points.shape[1]-1].mean())\n", + "\n", + " # evaluate predictions\n", + " prediction = np.array(net.forward(grid_points))\n", + " # for two classes: prediction difference\n", + " if (n_classes == 2):\n", + " Z = np.array([0.5+(p[0]-p[1])/2.0 for p in prediction]).reshape(xx.shape)\n", + " else:\n", + " Z = np.array([p.argsort()[-1]/float(n_classes-1) for p in prediction]).reshape(xx.shape)\n", + " \n", + " # draw contour\n", + " levels = np.linspace(0, 1, 40)\n", + " cs = ax.contourf(xx, yy, Z, alpha=0.4, levels = levels)\n", + " if draw_colorbar:\n", + " fig.colorbar(cs, ax=ax, ticks = [0, 0.5, 1])\n", + " c_map = [cm.jet(x) for x in np.linspace(0.0, 1.0, n_classes) ]\n", + " colors = [c_map[l] for l in train_labels]\n", + " ax.scatter(train_x[:, 0], train_x[:, 1], marker='o', c=colors, s=60, alpha = 0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "def plot_training_progress(x, y_data, fig, ax):\n", + " styles = ['k--', 'g-']\n", + " # remove previous plot\n", + " while ax.lines:\n", + " ax.lines.pop()\n", + " # draw updated lines\n", + " for i in range(len(y_data)):\n", + " ax.plot(x, y_data[i], styles[i])\n", + " ax.legend(ax.lines, ['training accuracy', 'validation accuracy'],\n", + " loc='upper center', ncol = 2)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "application/javascript": [ + "/* Put everything inside the global mpl namespace */\n", + "window.mpl = {};\n", + "\n", + "\n", + "mpl.get_websocket_type = function() {\n", + " if (typeof(WebSocket) !== 'undefined') {\n", + " return WebSocket;\n", + " } else if (typeof(MozWebSocket) !== 'undefined') {\n", + " return MozWebSocket;\n", + " } else {\n", + " alert('Your browser does not have WebSocket support. ' +\n", + " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", + " 'Firefox 4 and 5 are also supported but you ' +\n", + " 'have to enable WebSockets in about:config.');\n", + " };\n", + "}\n", + "\n", + "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", + " this.id = figure_id;\n", + "\n", + " this.ws = websocket;\n", + "\n", + " this.supports_binary = (this.ws.binaryType != undefined);\n", + "\n", + " if (!this.supports_binary) {\n", + " var warnings = document.getElementById(\"mpl-warnings\");\n", + " if (warnings) {\n", + " warnings.style.display = 'block';\n", + " warnings.textContent = (\n", + " \"This browser does not support binary websocket messages. \" +\n", + " \"Performance may be slow.\");\n", + " }\n", + " }\n", + "\n", + " this.imageObj = new Image();\n", + "\n", + " this.context = undefined;\n", + " this.message = undefined;\n", + " this.canvas = undefined;\n", + " this.rubberband_canvas = undefined;\n", + " this.rubberband_context = undefined;\n", + " this.format_dropdown = undefined;\n", + "\n", + " this.image_mode = 'full';\n", + "\n", + " this.root = $('
');\n", + " this._root_extra_style(this.root)\n", + " this.root.attr('style', 'display: inline-block');\n", + "\n", + " $(parent_element).append(this.root);\n", + "\n", + " this._init_header(this);\n", + " this._init_canvas(this);\n", + " this._init_toolbar(this);\n", + "\n", + " var fig = this;\n", + "\n", + " this.waiting = false;\n", + "\n", + " this.ws.onopen = function () {\n", + " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", + " fig.send_message(\"send_image_mode\", {});\n", + " if (mpl.ratio != 1) {\n", + " fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n", + " }\n", + " fig.send_message(\"refresh\", {});\n", + " }\n", + "\n", + " this.imageObj.onload = function() {\n", + " if (fig.image_mode == 'full') {\n", + " // Full images could contain transparency (where diff images\n", + " // almost always do), so we need to clear the canvas so that\n", + " // there is no ghosting.\n", + " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", + " }\n", + " fig.context.drawImage(fig.imageObj, 0, 0);\n", + " };\n", + "\n", + " this.imageObj.onunload = function() {\n", + " fig.ws.close();\n", + " }\n", + "\n", + " this.ws.onmessage = this._make_on_message_function(this);\n", + "\n", + " this.ondownload = ondownload;\n", + "}\n", + "\n", + "mpl.figure.prototype._init_header = function() {\n", + " var titlebar = $(\n", + " '
');\n", + " var titletext = $(\n", + " '
');\n", + " titlebar.append(titletext)\n", + " this.root.append(titlebar);\n", + " this.header = titletext[0];\n", + "}\n", + "\n", + "\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", + "\n", + "}\n", + "\n", + "\n", + "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", + "\n", + "}\n", + "\n", + "mpl.figure.prototype._init_canvas = function() {\n", + " var fig = this;\n", + "\n", + " var canvas_div = $('
');\n", + "\n", + " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", + "\n", + " function canvas_keyboard_event(event) {\n", + " return fig.key_event(event, event['data']);\n", + " }\n", + "\n", + " canvas_div.keydown('key_press', canvas_keyboard_event);\n", + " canvas_div.keyup('key_release', canvas_keyboard_event);\n", + " this.canvas_div = canvas_div\n", + " this._canvas_extra_style(canvas_div)\n", + " this.root.append(canvas_div);\n", + "\n", + " var canvas = $('');\n", + " canvas.addClass('mpl-canvas');\n", + " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", + "\n", + " this.canvas = canvas[0];\n", + " this.context = canvas[0].getContext(\"2d\");\n", + "\n", + " var backingStore = this.context.backingStorePixelRatio ||\n", + "\tthis.context.webkitBackingStorePixelRatio ||\n", + "\tthis.context.mozBackingStorePixelRatio ||\n", + "\tthis.context.msBackingStorePixelRatio ||\n", + "\tthis.context.oBackingStorePixelRatio ||\n", + "\tthis.context.backingStorePixelRatio || 1;\n", + "\n", + " mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n", + "\n", + " var rubberband = $('');\n", + " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", + "\n", + " var pass_mouse_events = true;\n", + "\n", + " canvas_div.resizable({\n", + " start: function(event, ui) {\n", + " pass_mouse_events = false;\n", + " },\n", + " resize: function(event, ui) {\n", + " fig.request_resize(ui.size.width, ui.size.height);\n", + " },\n", + " stop: function(event, ui) {\n", + " pass_mouse_events = true;\n", + " fig.request_resize(ui.size.width, ui.size.height);\n", + " },\n", + " });\n", + "\n", + " function mouse_event_fn(event) {\n", + " if (pass_mouse_events)\n", + " return fig.mouse_event(event, event['data']);\n", + " }\n", + "\n", + " rubberband.mousedown('button_press', mouse_event_fn);\n", + " rubberband.mouseup('button_release', mouse_event_fn);\n", + " // Throttle sequential mouse events to 1 every 20ms.\n", + " rubberband.mousemove('motion_notify', mouse_event_fn);\n", + "\n", + " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", + " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", + "\n", + " canvas_div.on(\"wheel\", function (event) {\n", + " event = event.originalEvent;\n", + " event['data'] = 'scroll'\n", + " if (event.deltaY < 0) {\n", + " event.step = 1;\n", + " } else {\n", + " event.step = -1;\n", + " }\n", + " mouse_event_fn(event);\n", + " });\n", + "\n", + " canvas_div.append(canvas);\n", + " canvas_div.append(rubberband);\n", + "\n", + " this.rubberband = rubberband;\n", + " this.rubberband_canvas = rubberband[0];\n", + " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", + " this.rubberband_context.strokeStyle = \"#000000\";\n", + "\n", + " this._resize_canvas = function(width, height) {\n", + " // Keep the size of the canvas, canvas container, and rubber band\n", + " // canvas in synch.\n", + " canvas_div.css('width', width)\n", + " canvas_div.css('height', height)\n", + "\n", + " canvas.attr('width', width * mpl.ratio);\n", + " canvas.attr('height', height * mpl.ratio);\n", + " canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n", + "\n", + " rubberband.attr('width', width);\n", + " rubberband.attr('height', height);\n", + " }\n", + "\n", + " // Set the figure to an initial 600x600px, this will subsequently be updated\n", + " // upon first draw.\n", + " this._resize_canvas(600, 600);\n", + "\n", + " // Disable right mouse context menu.\n", + " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", + " return false;\n", + " });\n", + "\n", + " function set_focus () {\n", + " canvas.focus();\n", + " canvas_div.focus();\n", + " }\n", + "\n", + " window.setTimeout(set_focus, 100);\n", + "}\n", + "\n", + "mpl.figure.prototype._init_toolbar = function() {\n", + " var fig = this;\n", + "\n", + " var nav_element = $('
');\n", + " nav_element.attr('style', 'width: 100%');\n", + " this.root.append(nav_element);\n", + "\n", + " // Define a callback function for later on.\n", + " function toolbar_event(event) {\n", + " return fig.toolbar_button_onclick(event['data']);\n", + " }\n", + " function toolbar_mouse_event(event) {\n", + " return fig.toolbar_button_onmouseover(event['data']);\n", + " }\n", + "\n", + " for(var toolbar_ind in mpl.toolbar_items) {\n", + " var name = mpl.toolbar_items[toolbar_ind][0];\n", + " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", + " var image = mpl.toolbar_items[toolbar_ind][2];\n", + " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", + "\n", + " if (!name) {\n", + " // put a spacer in here.\n", + " continue;\n", + " }\n", + " var button = $('');\n", + " button.click(method_name, toolbar_event);\n", + " button.mouseover(tooltip, toolbar_mouse_event);\n", + " nav_element.append(button);\n", + " }\n", + "\n", + " // Add the status bar.\n", + " var status_bar = $('');\n", + " nav_element.append(status_bar);\n", + " this.message = status_bar[0];\n", + "\n", + " // Add the close button to the window.\n", + " var buttongrp = $('
');\n", + " var button = $('');\n", + " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", + " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", + " buttongrp.append(button);\n", + " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", + " titlebar.prepend(buttongrp);\n", + "}\n", + "\n", + "mpl.figure.prototype._root_extra_style = function(el){\n", + " var fig = this\n", + " el.on(\"remove\", function(){\n", + "\tfig.close_ws(fig, {});\n", + " });\n", + "}\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function(el){\n", + " // this is important to make the div 'focusable\n", + " el.attr('tabindex', 0)\n", + " // reach out to IPython and tell the keyboard manager to turn it's self\n", + " // off when our div gets focus\n", + "\n", + " // location in version 3\n", + " if (IPython.notebook.keyboard_manager) {\n", + " IPython.notebook.keyboard_manager.register_events(el);\n", + " }\n", + " else {\n", + " // location in version 2\n", + " IPython.keyboard_manager.register_events(el);\n", + " }\n", + "\n", + "}\n", + "\n", + "mpl.figure.prototype._key_event_extra = function(event, name) {\n", + " var manager = IPython.notebook.keyboard_manager;\n", + " if (!manager)\n", + " manager = IPython.keyboard_manager;\n", + "\n", + " // Check for shift+enter\n", + " if (event.shiftKey && event.which == 13) {\n", + " this.canvas_div.blur();\n", + " event.shiftKey = false;\n", + " // Send a \"J\" for go to next cell\n", + " event.which = 74;\n", + " event.keyCode = 74;\n", + " manager.command_mode();\n", + " manager.handle_keydown(event);\n", + " }\n", + "}\n", + "\n", + "mpl.figure.prototype.handle_save = function(fig, msg) {\n", + " fig.ondownload(fig, null);\n", + "}\n", + "\n", + "\n", + "mpl.find_output_cell = function(html_output) {\n", + " // Return the cell and output element which can be found *uniquely* in the notebook.\n", + " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", + " // IPython event is triggered only after the cells have been serialised, which for\n", + " // our purposes (turning an active figure into a static one), is too late.\n", + " var cells = IPython.notebook.get_cells();\n", + " var ncells = cells.length;\n", + " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", + " data = data.data;\n", + " }\n", + " if (data['text/html'] == html_output) {\n", + " return [cell, data, j];\n", + " }\n", + " }\n", + " }\n", + " }\n", + "}\n", + "\n", + "// Register the function which deals with the matplotlib target/channel.\n", + "// The kernel may be null if the page has been refreshed.\n", + "if (IPython.notebook.kernel != null) {\n", + " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", + "}\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "res = train_and_plot(30,net,lr=0.01)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Важное замечание\n", + "\n", + "Простая линейная модель \n", + "* высокий training loss - \"недообучение\", выразительности модели не хватает, чтобы разделить данные\n", + "* valiadation loss и training loss примерно совпадают - модель хорошо обобщается\n", + "\n", + "Сложная многослойная модель\n", + "* низкий training loss - почти идеально приближает обучающую выборку (но может переобучиться)\n", + "* validation loss >> training loss и может возрастать - плохо обобщает данные" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Выводы\n", + "\n", + "* Простые модели с небольшим числом параметров (\"low capacity\") менее склонные к переобучению\n", + "* Более сложные модели (high capacity) могут переобучиться (надо следить за validation error)\n", + "* Для более сложных моделей необходимо иметь больше данных\n", + "* \"bias-variance trade-off\" - необходимо достичь компромисса между недообучением и переобучением (обучением на распознавание нерелевантного шума во входных данных)" + ] + } + ], + "metadata": { + "celltoolbar": "Slideshow", + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + }, + "livereveal": { + "start_slideshow_at": "selected" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/3-NeuralNetworks/05-Frameworks/IntroKerasTF.ipynb b/3-NeuralNetworks/05-Frameworks/IntroKerasTF.ipynb new file mode 100644 index 00000000..94dd075c --- /dev/null +++ b/3-NeuralNetworks/05-Frameworks/IntroKerasTF.ipynb @@ -0,0 +1,1433 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "celltoolbar": "Slideshow", + "kernelspec": { + "name": "python3", + "display_name": "Python 3", + "language": "python" + }, + "language_info": { + "mimetype": "text/x-python", + "nbconvert_exporter": "python", + "name": "python", + "file_extension": ".py", + "version": "3.7.4-final", + "pygments_lexer": "ipython3", + "codemirror_mode": { + "version": 3, + "name": "ipython" + } + }, + "livereveal": { + "start_slideshow_at": "selected" + }, + "colab": { + "name": "IntroKerasTF.ipynb", + "provenance": [], + "collapsed_sections": [] + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "En2vX4FuwHlu" + }, + "source": [ + "# Введение в нейронные сети\n", + "\n", + "## Эпизод 2а: Многослойный персептрон на TensorFlow и Keras\n", + "\n", + "Дмитрий Сошников | dmitri@soshnikov.com\n", + "\n", + "http://github.com/shwars/NeuroWorkshop\n", + "-> Notebooks -> IntroKerasTF.ipynb" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fVjlmWBwwHl2" + }, + "source": [ + "## Нейросетевые фреймворки\n", + "\n", + "Мы видели, что для обучения нейросетей нужно:\n", + "* Быстро умножать матрицы (тензоры)\n", + "* Считать производные для вычисления градиента для метода обратного распространения ошибки\n", + "\n", + "Что позволяют делать нейросетевые фреймворки:\n", + "* Оперировать с тензорами, как на CPU, так и на GPU\n", + "* Автоматически вычислять производные (они вручную прописаны для всех элементарных функций)\n", + "\n", + "Опционально:\n", + "* Конструктор для нейросетей (описание сети как набора слоёв)\n", + "* Простые функции для обучения (`fit`, как в Scikit Learn)\n", + "* Набор алгоритмов оптимизации\n", + "* Набор абстракций для работы с данными" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8cACQoFMwHl3" + }, + "source": [ + "## Основные фреймворки\n", + "\n", + "* Tensorflow 1.0 - первый, получивший широкое распространение (Google). Позволял определять статический computation graph, и затем в явном виде выполнять вычисления\n", + "* PyTorch - Facebook\n", + "* Keras - надстройка над Tensorflow/PyTorch для унификации (Francois Chollet)\n", + "* Tensorflow 2.0 + Keras - динамический вычислительный граф, код получается похожим на обычные вычисления в numpy\n", + "\n", + "Мы рассмотрим Tensorflow 2.0 и Keras. Вам необходимо убедиться, что у вас установлена версия 2.x.x Tensorflow:\n", + "```\n", + "pip install tensorflow\n", + "```\n", + "или\n", + "```\n", + "conda install tensorflow\n", + "```\n", + "или выполняйте код в [Google Colab](https://colab.research.google.com/)" + ] + }, + { + "cell_type": "code", + "metadata": { + "tags": [], + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "xwqVx9-bwHl3", + "outputId": "2aa591b4-b647-441f-9c8e-4e0da2d517a0" + }, + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "print(tf.__version__)" + ], + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "text": [ + "2.4.1\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6tp2xGV7wHl4" + }, + "source": [ + "## Основные понятия в TensorFlow\n", + "\n", + "**Тензор** - это многомерный массив произвольной размерности. Удобно использовать при обучении нейросетей, например:\n", + "* 400x400 - чёрно-белая картинка\n", + "* 400x400x3 - цветная картинка\n", + "* 16x400x400x3 - minibatch из 16 картинок, используемый для одного шага обучения\n", + "* 25x400x400x3 - секунда видео\n", + "* 8x25x400x400x3 - minibatch из 8 1-секундных видео" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qG2bsaR7wHl4" + }, + "source": [ + "### Простые тензоры" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ybpnk08HwHl4", + "outputId": "fad9ed4a-df82-44a0-84ea-324bc71ea46f" + }, + "source": [ + "a = tf.constant([[1,2],[3,4]])\n", + "print(a)\n", + "a = tf.random.normal(shape=(10,3))\n", + "print(a)" + ], + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tf.Tensor(\n", + "[[1 2]\n", + " [3 4]], shape=(2, 2), dtype=int32)\n", + "tf.Tensor(\n", + "[[ 1.1846514 0.7508411 -1.2235912 ]\n", + " [ 0.9710432 -0.15184928 -0.10070106]\n", + " [ 0.01476252 -0.19518584 0.43259642]\n", + " [ 1.8912938 0.6803918 -0.41853315]\n", + " [-1.0234195 -0.46338254 -1.6521171 ]\n", + " [-2.6472843 0.72314113 -0.8514363 ]\n", + " [-0.2650735 -0.22901714 -0.4702833 ]\n", + " [ 0.4691955 -0.6513927 -0.22865944]\n", + " [-1.0068002 2.3035717 -0.8701199 ]\n", + " [ 0.35074237 0.192767 -1.013126 ]], shape=(10, 3), dtype=float32)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AXFMsV3r09Ux" + }, + "source": [ + "С тензорами можно производить обычные вычисления, которые производятся поэлементно (как в numpy). При этом тензоры автоматически дополняются до нужной размерности. Можно извлечь numpy-массив из тензора при помощи `.numpy()`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "e5Nu5Xgj1DnQ", + "outputId": "0dfc8758-4ffd-4968-c7bf-6ba8d435df2e" + }, + "source": [ + "print(a-a[0])\r\n", + "print(tf.exp(a)[0].numpy())" + ], + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tf.Tensor(\n", + "[[ 0. 0. 0. ]\n", + " [-0.21360815 -0.90269035 1.1228901 ]\n", + " [-1.1698889 -0.9460269 1.6561877 ]\n", + " [ 0.7066424 -0.07044929 0.80505806]\n", + " [-2.2080708 -1.2142236 -0.42852592]\n", + " [-3.8319356 -0.02769995 0.3721549 ]\n", + " [-1.4497249 -0.9798582 0.75330794]\n", + " [-0.7154559 -1.4022338 0.99493176]\n", + " [-2.1914515 1.5527306 0.35347128]\n", + " [-0.83390903 -0.5580741 0.2104652 ]], shape=(10, 3), dtype=float32)\n", + "[3.2695467 2.1187813 0.29417184]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uQ5zN6cVyrG7" + }, + "source": [ + "## Переменные\r\n", + "\r\n", + "Переменные могут содержать какие-то значения, которые мы затем можем модифицировать с помощью методов `assign` и `assign_add`. \r\n", + "\r\n", + "Например, вот глупый способ посчитать сумму всех строк тензора `a`" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7pu0UZ-_yqfB", + "outputId": "6708c83e-02e6-4442-8757-45918eb1fbc2" + }, + "source": [ + "s = tf.Variable(tf.zeros_like(a[0]))\r\n", + "for i in a:\r\n", + " s.assign_add(i)\r\n", + "\r\n", + "print(s)" + ], + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rIh1EHcezlNo" + }, + "source": [ + "Умный способ:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aQIdWZ1kzn6P", + "outputId": "1c123d9a-ecd2-4f2e-828e-5ade85ac8f63" + }, + "source": [ + "tf.reduce_sum(a,axis=0)" + ], + "execution_count": 8, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 8 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U-auwezDwHl6" + }, + "source": [ + "## Вычисляем производные\r\n", + "\r\n", + "Для обратного распространения ошибки, нам нужно уметь вычислять градиенты. Это делается с помощью `tf.GradientTape()`:\r\n", + " * Оборачиваем интересующие нас вычисления в `with tf.GradientTape`\r\n", + " * Помечаем интересующие нас тензоры вызовом `tape.watch` (переменные отслеживаются автоматически)\r\n", + " * Проводим вычисления\r\n", + " * Получаем градиенты через `tape.gradient` " + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "m8vFOXr7wHl6", + "outputId": "860ac72e-50c7-4ff2-f258-747f27194f90" + }, + "source": [ + "a = tf.random.normal(shape=(2, 2))\n", + "b = tf.random.normal(shape=(2, 2))\n", + "\n", + "with tf.GradientTape() as tape:\n", + " tape.watch(a) # Start recording the history of operations applied to `a`\n", + " c = tf.sqrt(tf.square(a) + tf.square(b)) # Do some math using `a`\n", + " # What's the gradient of `c` with respect to `a`?\n", + " dc_da = tape.gradient(c, a)\n", + " print(dc_da)" + ], + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tf.Tensor(\n", + "[[0.9086961 0.954213 ]\n", + " [0.85484374 0.9623889 ]], shape=(2, 2), dtype=float32)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8sfjBMBu59B5" + }, + "source": [ + "## Пример 1: Линейная регрессия\r\n", + "\r\n", + "Попробуем с помощью полученных знаний решить классическую задачу линейной регрессии. Для этого сгенерируем небольшой синтетический датасет:" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "j723455WwHl7" + }, + "source": [ + "import matplotlib.pyplot as plt\n", + "from sklearn.datasets import make_classification, make_regression\n", + "from sklearn.model_selection import train_test_split\n", + "import random" + ], + "execution_count": 10, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "WJNK_J6v6I-Z", + "outputId": "eb4a66a6-6b9a-4c8a-bc24-d81eeb2d3f27" + }, + "source": [ + "np.random.seed(13) # pick the seed for reproducability - change it to explore the effects of random variations\r\n", + "\r\n", + "train_x = np.linspace(0, 3, 120)\r\n", + "train_labels = 2 * train_x + 0.9 + np.random.randn(*train_x.shape) * 0.5\r\n", + "\r\n", + "plt.scatter(train_x,train_labels)" + ], + "execution_count": 11, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 11 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAb2ElEQVR4nO3df4xlZXkH8O+zwwUuP8pVmVoYXZemzZIKZZedEJo1RtZYbFHcgFEb0xbbZpM2bYVQ2tE/ivYfJjFWbW1qJpQWW6pLXdhQUJFkN6FuFDvDLgKyNChuZbBllB0QZsrODE//uPcsd86c97zvuec957zvvd9PsnF25+yZ9+yV5773eZ/3eUVVQURE4drU9ACIiCgfAzURUeAYqImIAsdATUQUOAZqIqLAnVLFTc8991zdsmVLFbcmIhpKc3NzP1HV8azvVRKot2zZgtnZ2SpuTUQ0lETkmOl7TH0QEQWOgZqIKHAM1EREgWOgJiIKHAM1EVHgKqn6ICIaJfsPz+NT9z+JZxeXcX6njZuu3Ird2ye83Z+BmoiohP2H5/Gxux7F8soaAGB+cRkfu+tRAPAWrJn6ICIq4VP3P3kySCeWV9bwqfuf9PYzGKiJiEp4dnG50J8PgoGaiKiE8zvtQn8+CAZqIiJ0c807pw/ggqn7sHP6APYfnnf6ezdduRXt1ti6P2u3xnDTlVu9jY2LiUQ08sosCCbfZ9UHEVGF8hYEXQLu7u0TXgNzGlMfRDTy6lgQLMMaqEVkq4gc6fv1oohcX8fgiIjqUMeCYBnWQK2qT6rqNlXdBmAHgCUAd1c+MiKimtSxIFhG0Rz1OwF8X1WNDa6JiGJTZEGwf7v4Oe0WRIDFpZVKFhEToqruF4vcBuBhVf18xvf2ANgDAJs3b95x7BhjORENl3R1SFq7NYZbrrl4oGAtInOqOpn1PefFRBE5FcDVAP4t6/uqOqOqk6o6OT6eeewXEVHw8uqps6pD+vneOp4okvr4DXRn0//rfRRERAGw1VO7VIFUUSlSpDzvtwB8yfsIiIgCYWuw5FIFUkWliFOgFpEzAbwLwF3eR0BEFAhbPXVWdUi/qipFnFIfqvoygDd4/+lERAE5v9PGfEawTmbJ6eqQuqo+uIWciKjnpiu3bqjqSM+Sq94unoWBmoiop44GS4NgoCYi6tPEjNmGTZmIiALHQE1EFDgGaiKiwDFHTUQjob+ZUiiLhK4YqIloKOQF4kGP2kruOb+4jDERrKliooEgz0BNRIWEODPNC8QAcOOdj2At1SnUdtRW+p7J3y9ynqIvhdqcupqcnNTZ2Vnv9yWiZmW1+SzT2tOXndMHMncUdtotvLL6am7Hu4lOO/NNx3TP/r93aGpX+cH3eGlzSkRka1rUFFOPjsXlldwgLejOkBWvzZSTtqa2Lnh1nqfIQE1EzkI9BHaQjnUCIJ1PKNIpr87zFBmoiciZKTid024Zm+3XwXTm4evOaGVePyayIUgnXDrl1X2eIgM1ETnLCl6tTYKXT6waUwhVSJ/CAgC3XHMxJjptCLr541uuuRg3v/etmQH80x+4BBOWk8d3b584eU+gG9zRd29WfRBRkLKaFi2dWMXxpZV119kqKsowVXjccs3FxsU9U5VKiJ3ysrDqg4hKuWDqvsw0ggB4evoq7z/PVI0xSBVGSKWGeVUfnFETUSm2Zvu++VzQDGXGbMMcNRGVYlrIq2qxzfQGUGcVRt0YqImolP5Ft/6FvKpmqnW/MYSAqQ8iKq3OFEKop7BUySlQi0gHwK0ALkK3Rvz3VPVbVQ6MiMgkltyyL64z6s8B+Lqqvl9ETgVwRoVjIiLKFFKVRp2sgVpEzgHwdgDXAYCqngBwotphERGtN2ir0mHgsph4AYAFAP8oIodF5FYROTN9kYjsEZFZEZldWFjwPlAialZ6N2Dd28RDbQhVB5dAfQqASwH8vapuB/AygKn0Rao6o6qTqjo5Pj7ueZhE1KRkNlvnNvH+n53XcrTphlB1cMlRPwPgGVV9qPf7ryAjUBPR8DLNZj9xz+O5p6qUzSdn9b9OG+b66YQ1UKvq/4jIj0Rkq6o+CeCdAL5X/dCIKBR5/Z4Xl7t9PtKnqvjIJ2e9QfQb9vrphGvVx58AuKNX8fEDAB+pbkhEFBrTNvG0/pyxKZ9cJFDnpTWaOLuwKU6BWlWPAMhsFkJEw++mK7daUxCJvOBaNJ9seoPwfQxW6LiFnIissraJm5ryn99pO/XjcKkiGcXt4lm4hZyInKR3A5oOuk2CaN73XGqik8XI5ZU1jIlgTXWk0h39GKiJRliZygyXnhum7+XVRO/ePrEhkK+pngz0oxakAR4cQDSyTDPiOo6ZMh02AHTTKqaFy6zc9LBsK887OIA5aqIR1eROP1MOW4Dc6pL0YmSTG3HqxEBNNKJ8npRSVNYioQDGWXYiHeBHZVs5AzXRkDNVV5hmtQpU3ssjXUXSabesQTqr2qPJN5s6cTGRaIjlVVfk1Ub77kxnyiP3LxzmMVV71H1eY1MYqImGUBIYs4JYkhpIFuVs15UN1LZSvLxt4rbFzaw3m3ZrDFdcOI6d0weiX2BMMPVB5EHTLUDTY0kW2EyS1MDu7RM4NLULYrmuDFseOe9n2CpQsjbiXLtjAvvm5odqgZEzaqKSQmtob2tkBGxMDfhMIaTTHLb2pHnbxF3+/dIbcXZOH/DSZyQknFETldRE5UHeDN42C85alPO1VTurXM40W0/eBHxvE3dZYAzpE5ALzqiJSqq78sA2g8+bxY6JrHsTSWaYvk72znrTUmSX3i2dWMX+w/PeTxW3fToI7ROQCwZqopLqrjywbb/OWmBrbRJAgJW1brjMCk4+TvY2vTkpuiV4Se9qADi+tLJuDL6CpGmBMZmh2/79QsTUB1FJdXd4s83gsxbYzjr9lJNBOmFKz5RJC5jenCY6bZx52sZ5YRUpoqzn71+UjLH2mjNqopJ8f3S3cZnBp2eoF0zdl3kv05bsQdMCebPZG/YecRqDD3kz9BhrrzmjJvIgKXN7evoqHJraVelH6EFm8C79oYHyC6N5s1nXMVQtxh7XnFETRWaQGbwtb5vwkRYwzWZdx1C1uj8B+cBATRShootvrsGpyrRASAHS5+JlHdiPmihQTfRZbrJH9ajL60fNGTVRgJqq9Q1p1kuvcQrUIvJDAD8DsAZg1RT1iciPJmt9Y0sLjIIiM+orVPUnlY2EiE6KsdaXqsPUB1EA0vnozhktHF9a2XBdyLW+VB3XOmoF8A0RmRORPVkXiMgeEZkVkdmFhQV/IyQaclmNjF76v1W0xta3Mwq91peq4zqjfpuqzovIzwN4QESOquqD/Reo6gyAGaBb9eF5nERDIz17XjqxuiEfvfKqotNu4czTTuGiHrkFalWd7/3vcyJyN4DLADyY/7eIKC2rmsPkheUVHLn51+saGgXMmvoQkTNF5OzkawC/DuCxqgdGNIxcmvonmI+mhMuM+o0A7haR5Pp/VdWvVzoqoiHlWrXBfDT1swZqVf0BgEtqGAvR0DNt0WY+mvKwPI+i1MT2ah9MjYk+cfVbjePvf9Zz2i2IAItLK1E9N5XDQE3RqXN7te0NoegbRtEt2uln7T8hJeu5Y30Do3xsykTR2Tl9wHhq9aGpXd5+jq1BUR0NjEzP2i95bjZUilteUyYeHEDRqWt7ta2Jfh2nj7s8U3JNE6ehUz0YqCk6dZ0UYntDqOMNw+WZkmvYH2R4MVBTdOo6Ssn2hlDHG0bWs/brf+5Qjroi/xioKTq2U6Z9sb0h1PGGkX7WTruF153RynzuKsZT5kRy8oeLiUQ5fFd9ND3eovfi4mR98hYTGaiJIlfVm0Vd1TXUxaO4iCJRNOhWWVPOxclwMEdNlNJUXjarL/XH7no09+dXWZLHxclwMFBTVKoOooMES9f72sY9SNCtctZbV3UN2TFQUzSqCqL9qpihuo57kKBb5ay3ruoasmOOmqJRx8ncVcxQXcdt6qyXF3RNTZ58zXp5InkYGKgpGnXtBCwaLG1M45tfXMbO6QMnFw6vuHAc++bmCwXdok2eKE4M1BSNKoJoWhUzVNO4Ba8dxTW/uIx9c/O4dscEDh5dKBR0OesdfgzUFI2qP+YD+TPUQeuVs8YtANI7GJZX1nDw6EItNcqhbdShfNzwQlFpKsBk7dIDgNed0cLN781u+p/X8N/UulQAPD19VRWPsG5c3HEYHm54oaFh+5hfVSA3HUp7fGklc4NJVsP/dmsMn/ngNuzePmHc9VdHjXIdi7LkF8vzyIsQmvdUWb6Xt2CZVb5nK/NrskaZOw7j4xyoRWRMRA6LyL1VDojiU0d9swtTcLzxzkdKj8U2000HOVswbLJGmTsO41Mk9fFRAE8A+LmKxkKRCuWjtCk4rqmW7n+RtSDYrz/I7T88j00iWMtY/+m/rqlqjToWZckvpxm1iLwJwFUAbq12OBSjUD5K580Iy+4uTGbAnXZrw/f6g1zy6SIrSIcSDLnjMD6uM+rPAvhzAGebLhCRPQD2AMDmzZvLj4yiUUd9swvbrLfsG0cyA85bsDQtOo6JBBUMWXsdF2ugFpH3AHhOVedE5B2m61R1BsAM0C3P8zZCCl4oH6WTwHPjnY9Y0w5lf44pyJneDF5VZWCkgbmkPnYCuFpEfgjgywB2ici/VDoqikpIH6V3b5/Apz9wSWMVFVyooyoU2vDSm1H/maq+J+86bnihpoW0MYabScgFN7zQyGkqB8smSVQFbiGnxjTZb4K9Lig0nFFTcHye9efrnMHZY88X7lxHVAfOqKkRpl4XQHcx0jVIFskJJwE9ryFS/38NzC1TnfJm1Oz1QY3Iq2kusgXd5eis/Yfnse2T38D1e48YgzSQ3XbUxyGxRGUxUFMjbOVqrkHStisymXEvLq8UH2TO/YnqxEBNznx2yMvqHpfmEiRtdcumnYJpUvD+RHVioCYnvjvk9W+SMXEJkrZ2oS7BfqLTxocv39zYJhkiG1Z9kJMqOuT1984YdAu6rW457zSV9GLh5Ftez5I9ChIDNTmpskNe2U0ieZtbTI2aso7QYqMiChUD9RCoY/NG1R3yqgqS3ClIw4CBOnI+N47kCaVD3iA4U6bYcTExci51xD6E1CGPaNRwRh25Ok9XqWNm2p/GOafdgkj3pO+x3tFWRXYtEg0LBurIhXK6io1LHj2dxunfpJIcBFBVaocoZEx9RM5WRxwC1xps180p3NpNo4aBOnIx5I5NefTr9x5Zt8OxSLqGW7tplDD1MQRCr2pwacAE5G9OSQsttUNUJQZq8i6dj+6c0cLxJXNTpCSVYTtFPBFaaoeoagzU5FVWXXdrk6A1JlhZM/c+f3ZxecPmFFZ9EHUxUJNXWfnolVcVnXYLZ552ijG1kaQy6k7j8EguigEXE8krUz76heUVHJrahc9+cFswVSq+OwISVcUaqEXkdBH5jog8IiKPi8gn6xgYvcZnH+iq2fpDh1SlUteuTqKyXFIfrwDYpaoviUgLwDdF5Guq+u2Kx0aor5eHLy49QUKpUqlzVydRGdZArd3Tb1/q/bbV++X/RFzK5LMPdB352Ji61cWyq5PI6RRyERkDMAfglwD8nar+RcY1ewDsAYDNmzfvOHbsmOehjqYLpu7LfFcUAE9PX+V8n6zm/Mmp21mVFKOwyFbkBHOiquWdQu5U9aGqawC2iUgHwN0icpGqPpa6ZgbADABMTk5yxu2Jr1lf1sw8eZHS6RRbuiWrcdLi0kp0AT2m2T+NtkLleaq6KCIHAbwbwGO264dVnbNNX32gbXnX/nSKbZHN1Dgp9Px5llDy5UR5XKo+xnszaYhIG8C7ABytemChqruky1eVhMsMPAnmeYtstsZJrJog8s9lRn0egNt7eepNAO5U1XurHVa4qjjk1cbHrM9le3YSzE3plk0iTr04WDVB5JdL1cd3AWyvYSxRiLGkK0nVLK+sndyKnSwkJvrTKaagnvX3siQBfxQWJInqwJ2JBdk2dISmP1UDdINtuzWGD1++2ZhOSdItYyIb7qfoVouYJAGfu/6I/GGvj4JCOuTVZcZqStUcPLqAQ1O7jPfevX0CN+w9kvm9pKQvr+pj5/SB2lNERMOKgbqgUEq6XHcslknVmHLVE512bpAv+3OJaD0G6gGEUNLluqhZpg67zKcH7voj8oc56opV1VDJdcZa5kzFMqWBMZzlSBQLzqgrVGVDJdcZa9lUzaCfHkJJERENA6deH0VNTk7q7Oys9/vGZuf0gcxgOiaCT3/gklJBi30qiIZL6V4fNBhTemJNtfTMmjNWotHBQF2hvFO1fZSq+V7U5AYVojBxMbFCWQtq/UIqVeMGFaJwMVBXKG+HHxBWqRqPpSIKF1MfFUtSB3XvZiyaxuAGFaJwMVDXwGXhz2d+eJCyQG5QIQoXA3VN8hb+fNdbD9KKNaQeJkS0HnPUAfCZH95/eN5YaZKXxvB1QAER+ccZdQB85YeTmbmJLY0RQg8TItqIM+oA+OpxnXdMFtMYRPHijLokH4uApvzwFReOY+f0gdKVGwCYxiCKGAN1Cb4WAbOqQq64cBz75ua9VG5MdNoM0kQRG9pAXXamW+b0FNet4Xk/Y5ATUli5QTScrIFaRN4M4IsA3ojuKUwzqvq5qgdWRtmZbh2np9h+xiD3HqRRE/t7EIXPZUa9CuBGVX1YRM4GMCciD6jq9yoe28DKznTrOD3F9jMGvXeRyo0q+2UTkT/Wqg9V/bGqPtz7+mcAngAQ9H/FZcvd6jg9xfYz6jghhf09iOJQqDxPRLYA2A7goYzv7RGRWRGZXVhY8DO6AZUtd3P9+2U2idh+Rh0bUNjfgygOzouJInIWgH0ArlfVF9PfV9UZADNA94QXbyN0kM6zpismgGKz0SKLcoNuEnH5GVVvQGF/D6I4OM2oRaSFbpC+Q1XvqnZIxWT1Ud43N49rd0wMPBsdZDZb9BDbELZs8wBaojhYz0wUEQFwO4DnVfV6l5vWeWai6VzCiU4bh6Z2lb6/S1VEzOcXsuqDKAx5Zya6BOq3AfgPAI8CeLX3xx9X1a+a/k6VgTodWEwNiATA09NXFb5ff6DKCsCCbo3iRN+1Vb9ZENHwK3W4rap+E9341LiscrIkcKa55Flt5WlZVRHJz+q/lotyRFSlqJoymQJn+l3ENc9qK0+zBdrkWl9NlYiIskQVqE2BM0lFCIBOu4XTW5tww94j1kU920zYJdA+u7jMRTkiqlRUgdoUOJNc8Gc+uA2vrL6K40srTidp22bCtlPEk2tDqOAgouEVVVMmW+1x0a3jtvv1987Iyoenr2VgJqIqRBWobU2Hii7quTQx6g/AIZSyhTAGIqqXtTxvEHXWUfcb9jK5mOu1iShfXnleVDlqm1gW9YruYkywiRLRaIoq9WEzSD/mupVpLcp6baLRFHWgNuVrBw3MvvK/efcp0yubTZSIRlO0gdp303tf9yt7cktekOdRW0SjKdocte98ra/72e6TV7ud1Qmwvw6c9dpEoym4GbVr+sF3vtbX/VxObjHNil3SIqzXJho9QQVqU9pg9tjzOHh0YV3w9p2v9XU/233yFjxv2Hsk855cLCQabUEFatOM8o5v//eGrnXX7pgodYpLmq/8b5mTW7hYSERZgspR5zVd6re8soaDRxe85mt95X/L3CeWOnAiqldQOxNNOwuzuB4MEBtuEScaTaUODqhTVtqgzMEAMeJiIRGlBZX6yEobfPjyzUwHENFIC2pGDWTPKCff8nqmA4hoZAUXqLMwHUBEo8waqEXkNgDvAfCcql5U/ZCqw4U6IoqRy4z6nwB8HsAXqx3KRj4Dq+/eIEREdbEuJqrqgwCer2Es69j6XhTFXs5EFKugqj76+Q6s7OVMRLHyFqhFZI+IzIrI7MLCQun7+Q6sthPHiYhC5S1Qq+qMqk6q6uT4+Hjp+/kOrNyeTUSxCjb14TOwJouSyytrGBMBAHTaLZze2oQb9h4pdG4hEVHdrIFaRL4E4FsAtorIMyLy+9UPy1+TpP5FSQBYU0Vrk+DlE6s4vrTiZaGSiKhKQTVlqkKRRk8TnTYOTe2qeERERBvlNWUKNvXhS5HFR1aAEFGIhj5QF1l8ZAUIEYVo6AN11qJka5OgNSbr/owVIEQUqiiaMpVhOqMw68+4lZyIQjT0i4lERDGI4oQXdrYjIsoWRKBmZzsiIrMgFhPZ2Y6IyCyIQM3OdkREZkEEana2IyIyCyJQs7MdEZFZEIuJplpnLiQSEQUSqAGeNE5EZBJE6oOIiMwYqImIAsdATUQUOAZqIqLAMVATEQWuku55IrIA4NiAf/1cAD/xOJwmDcuzDMtzAHyWEA3LcwDlnuUtqjqe9Y1KAnUZIjJravUXm2F5lmF5DoDPEqJheQ6gumdh6oOIKHAM1EREgQsxUM80PQCPhuVZhuU5AD5LiIblOYCKniW4HDUREa0X4oyaiIj6MFATEQWusUAtIu8WkSdF5CkRmcr4/mkisrf3/YdEZEv9o7RzeI7rRGRBRI70fv1BE+O0EZHbROQ5EXnM8H0Rkb/pPed3ReTSusfoyuFZ3iEiL/S9Jn9Z9xhdicibReSgiHxPRB4XkY9mXBP8a+P4HFG8LiJyuoh8R0Qe6T3LJzOu8Ru/VLX2XwDGAHwfwC8COBXAIwB+JXXNHwH4Qu/rDwHY28RYPTzHdQA+3/RYHZ7l7QAuBfCY4fu/CeBrAATA5QAeanrMJZ7lHQDubXqcjs9yHoBLe1+fDeC/Mv4/Fvxr4/gcUbwuvX/ns3pftwA8BODy1DVe41dTM+rLADylqj9Q1RMAvgzgfalr3gfg9t7XXwHwThGRGsfowuU5oqCqDwJ4PueS9wH4onZ9G0BHRM6rZ3TFODxLNFT1x6r6cO/rnwF4AkC6cXvwr43jc0Sh9+/8Uu+3rd6vdFWG1/jVVKCeAPCjvt8/g40v2slrVHUVwAsA3lDL6Ny5PAcAXNv7SPoVEXlzPUPzzvVZY/FrvY+uXxORtzY9GBe9j8/b0Z3B9Yvqtcl5DiCS10VExkTkCIDnADygqsbXxEf84mJi9f4dwBZV/VUAD+C1d1lqzsPo9lW4BMDfAtjf8HisROQsAPsAXK+qLzY9nkFZniOa10VV11R1G4A3AbhMRC6q8uc1FajnAfTPLN/U+7PMa0TkFADnAPhpLaNzZ30OVf2pqr7S++2tAHbUNDbfXF6zKKjqi8lHV1X9KoCWiJzb8LCMRKSFbnC7Q1XvyrgkitfG9hyxvS4AoKqLAA4CeHfqW17jV1OB+j8B/LKIXCAip6KbbL8ndc09AH639/X7ARzQXmY+INbnSOUKr0Y3NxejewD8Tq/C4HIAL6jqj5se1CBE5BeSfKGIXIbufwehTQIAdCs6APwDgCdU9a8NlwX/2rg8Ryyvi4iMi0in93UbwLsAHE1d5jV+NXK4raquisgfA7gf3cqJ21T1cRH5KwCzqnoPui/qP4vIU+guDH2oibHmcXyOPxWRqwGsovsc1zU24Bwi8iV0V93PFZFnANyM7iIJVPULAL6KbnXBUwCWAHykmZHaOTzL+wH8oYisAlgG8KEAJwGJnQB+G8CjvZwoAHwcwGYgqtfG5TlieV3OA3C7iIyh+2Zyp6reW2X84hZyIqLAcTGRiChwDNRERIFjoCYiChwDNRFR4BioiYgCx0BNRBQ4BmoiosD9P3fFq1WlCzEwAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ng4rZmGc6oxk" + }, + "source": [ + "Линейная регрессия вычисляется как $f_{W,b}(x) = Wx+b$, где $W, b$ - параметры модели, которые необходимо найти. Функция ошибки на наборе данных $\\{x_i,y_u\\}_{i=1}^N$ может быть определена как среднеевадратичное отклонение\r\n", + "$$\r\n", + "\\mathcal{L}(W,b) = {1\\over N}\\sum_{i=1}^N (f_{W,b}(x_i)-y_i)^2\r\n", + "$$\r\n", + "\r\n", + "Опишем модель и функцию ошибки:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "QxhI4GlB6aiH" + }, + "source": [ + "input_dim = 1\r\n", + "output_dim = 1\r\n", + "learning_rate = 0.1\r\n", + "\r\n", + "# This is our weight matrix\r\n", + "w = tf.Variable([[100.0]])\r\n", + "# This is our bias vector\r\n", + "b = tf.Variable(tf.zeros(shape=(output_dim,)))\r\n", + "\r\n", + "def f(x):\r\n", + " return tf.matmul(x,w) + b\r\n", + "\r\n", + "def compute_loss(labels, predictions):\r\n", + " return tf.reduce_mean(tf.square(labels - predictions))" + ], + "execution_count": 14, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JUxwj3367gD2" + }, + "source": [ + "Обучать модель будем на сериях примеров - minibatches. Для обучения используем градиентный спуск, подстраивая парметры в соответствии с формулой:\r\n", + "$$\r\n", + "\\begin{array}{l}\r\n", + "W^{(n+1)}=W^{(n)}-\\eta\\frac{\\partial\\mathcal{L}}{\\partial W} \\\\\r\n", + "b^{(n+1)}=b^{(n)}-\\eta\\frac{\\partial\\mathcal{L}}{\\partial b} \\\\\r\n", + "\\end{array}\r\n", + "$$" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "-991PErM7fJU" + }, + "source": [ + "def train_on_batch(x, y):\r\n", + " with tf.GradientTape() as tape:\r\n", + " predictions = f(x)\r\n", + " loss = compute_loss(y, predictions)\r\n", + " # Note that `tape.gradient` works with a list as well (w, b).\r\n", + " dloss_dw, dloss_db = tape.gradient(loss, [w, b])\r\n", + " w.assign_sub(learning_rate * dloss_dw)\r\n", + " b.assign_sub(learning_rate * dloss_db)\r\n", + " return loss" + ], + "execution_count": 13, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "idr2VEWb9rr0" + }, + "source": [ + "Теперь приступаем к обучению: делаем несколько проходов по всему датасету (эпох), разбиваем его на minibatches, и вызываем функцию обучения:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nOuu0qpx-wAp" + }, + "source": [ + "# Shuffle the data.\r\n", + "indices = np.random.permutation(len(train_x))\r\n", + "features = tf.constant(train_x[indices],dtype=tf.float32)\r\n", + "labels = tf.constant(train_labels[indices],dtype=tf.float32)" + ], + "execution_count": 12, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3zdIf6c_85Ht", + "outputId": "43b04684-8b90-4c65-d5ff-20ebac61c73c" + }, + "source": [ + "batch_size = 4\r\n", + "for epoch in range(10):\r\n", + " for i in range(0,len(features),batch_size):\r\n", + " loss = train_on_batch(tf.reshape(features[i:i+batch_size],(-1,1)),tf.reshape(labels[i:i+batch_size],(-1,1)))\r\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 15, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 94.5247\n", + "Epoch 1: last batch loss = 9.3428\n", + "Epoch 2: last batch loss = 1.4166\n", + "Epoch 3: last batch loss = 0.5224\n", + "Epoch 4: last batch loss = 0.3807\n", + "Epoch 5: last batch loss = 0.3495\n", + "Epoch 6: last batch loss = 0.3413\n", + "Epoch 7: last batch loss = 0.3390\n", + "Epoch 8: last batch loss = 0.3384\n", + "Epoch 9: last batch loss = 0.3382\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "US6q0nCBD-LL", + "outputId": "65a79620-a3eb-445b-aafb-60a60575ab0e" + }, + "source": [ + "w,b" + ], + "execution_count": 16, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(,\n", + " )" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 16 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "_e6xRMZFDnyI", + "outputId": "d202b7fe-4383-4d82-b98e-a20f3180093e" + }, + "source": [ + "plt.scatter(train_x,train_labels)\r\n", + "x = np.array([min(train_x),max(train_x)])\r\n", + "y = w.numpy()[0,0]*x+b.numpy()[0]\r\n", + "plt.plot(x,y)" + ], + "execution_count": 17, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 17 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhU1Z3/8ffp7uqFrRulQWhoQFZFZOsxGowKiBhN3KNJRo3JTJiJGbcYlGQWZ/L8ZgTcMyYTiZplYhKNInFiZFFQXFFWUeluEGRp9qWbpffq8/uju9qmum7Vra5b1beaz+t5eNJ03br3XMp876lzvud7jLUWERHxr4zOboCIiESnQC0i4nMK1CIiPqdALSLicwrUIiI+l5WMk/bp08cOGTIkGacWEemSVq9efcBaWxjptaQE6iFDhrBq1apknFpEpEsyxmxzek1DHyIiPqdALSLicwrUIiI+p0AtIuJzCtQiIj6XlKwPEZGTycK1FTywuIxdlTUMKMhj1oxRXDWhyLPzK1CLiCRg4doKfrRgAzUNQQAqKmv40YINAJ4Faw19iIgk4IHFZa1BOqSmIcgDi8s8u4YCtYhIAnZV1sT1+45QoBYRScCAgry4ft8RCtQiIjSPNU+es4yhs19m8pxlLFxb4ep9s2aMIi+QecLv8gKZzJoxyrO2aTJRRE56iUwIhl5X1oeISBJFmxB0E3CvmlDkaWAOp6EPETnppWJCMBExA7UxZpQxZl2bP0eMMXemonEiIqmQignBRMQM1NbaMmvteGvteGASUA28mPSWiYikSComBBMR7xj1NOBTa61jgWsRkXQTz4Rg2+Xi+XkBjIHK6oakTCKGGGut+4ONeRpYY619PMJrM4GZAMXFxZO2bVMsF5GuJTw7JFxeIJP7rxnboWBtjFltrS2J9JrryURjTDZwBfCnSK9ba+dba0ustSWFhRG3/RIR8b1o+dSRskPa8nrpeEg8Qx9fprk3vdfzVoiI+ECsfGo3WSDJyBSJJz3vG8AfPG+BiIhPxCqw5CYLJBmZIq4CtTGmOzAdWOB5C0REfCJWPnWk7JC2kpUp4mrow1p7HDjV86uLiPjIgII8KiIE61AvOTw7JFVZH1pCLiLSYtaMUe2yOsJ7ycleLh6JArWISItUFFjqCAVqEZE2OqPHHIuKMomIeOCjiiqeWZmchX7qUYuIJOCzA8d5cEkZf/lwN3175nDNhIHkZTtnhnSEArWISAfsO1rLf7+2mT+8v51AZga3TR3Ody843fMgDQrUInKSaFtMKZFJwqO1DcxfsYWn3tpKfWMTXz9nELdPG0HfnrlJaHUzBWoR6RKiBeKObrUVOmdFZQ0ZQBOQYaDJwlfO7s8PLxnFkD7dk31rCtQiEh+veqZet8kpEAPc/dx6gmGVQmNttRV+zqaW3zdZyMnK4OIz+qUkSEOcZU7dKikpsatWrfL8vCLSuSKV+UyktKdXJs9ZFnFFYUFegLrGpqgV74oK8iI+dL54/2vsqqqN+r63Z09NvPEtopU5VY9aRFxLdBPYZHGq0VFZ0xD1fQZaA3zbXnjxqd2iBulo10wGBWoRcc2vm8A61eiIxgDh4wk1DUFmL/iQ2oam1rHoaNdMFS14ERHXnIJTfl7Asdh+Kjjtedi7WyDi8ZnGtAvSIbUNTdw9fST3Xz3WsVJeqvdTVKAWEdciBcRAhuF4fSMVlTVYPh9CSGawDt+FBeD+a8ZSVJCHoXn8+P5rxnLfV8dEDOAPXT+OIoeHzmm9crlt2ghuOKe49ZzQHNxpc+5UDvVo6ENEXItUtKi6vpHD1SeOBSdz3Nopw+P+a8Y6Tu6FZ6nMGHMar27c2264JC+Qyewvj279u1/qfijrQ0QSMnT2yxGHEQywdc7lnl/PKcPDTRZGY7CJP63eyaOvlrP3SB1jBvRi/9E69h+t6/RUQ2V9iEjSxCq277WOTGhaa1n88R7mLS5jy/7jTCwu4L+/MZFzhp6SlDZ6TYFaRBLipti+l+J9MLz76UHmLipl3Y5KRvTtwfybJjH9zH6YljHndKBALSIJSXWxfbcPhk92HWHuolLeKN9P//xc5l13NtdOHEhmRvoE6BAFahFJWCon3WI9GHYcquahJWX8ef0ueuUG+PFlo7n5vCHkRtmU1u9cBWpjTAHwJHAWzTni37HWvpvMhomIOIn0YDhwrI7Hl23mmZXbyMww/OOFw/jHC4eRnxc5lzqduO1RPwYsstZeZ4zJBrolsU0iIhFFKgh18Zn9ePLNLfxyxRZqG5u4vmQQd148gn69kld2NNViBmpjTD5wAXALgLW2HqhPbrNERE4UKX961p/WkxPI5FhdI18+6zR+OGMUwwp7dHJLveemRz0U2A/8yhgzDlgN3GGtPd72IGPMTGAmQHFxsdftFJFO1tnlTSMVhGposphgEwu/P5nxgwpS1pZUc7OEPAuYCPyPtXYCcByYHX6QtXa+tbbEWltSWFjocTNFpDOFerOpXCbe9tpOi1wA6hubunSQBnc96p3ATmvtypa/P0+EQC0iXZdTedN/f+njqLuqJNoDj1T/OpxTzY6uJGagttbuMcbsMMaMstaWAdOAT5LfNBHxi2j1nkM1n8N3VenI1lfh/uuvG6MG6VRXsessbrM+bgOeacn42AJ8O3lNEhG/cVvvOVSMKfRzpNfcBOq9R2p59NVN7Dta53hMkU+2AUsFV4HaWrsOiFgsRES6vkirAZ1Eq7kRa4OBqpoGnnjjU55+eyvBJkv37EyO17e/ptfbYPmdViaKSExuy5vC5zU3YtXjaDuG3T8/l5Ihp/BG+X6O1DZw5bgB/GD6KNZsP5zSOiJ+pUAtIq6ErwZ02ug2FESjvRb+3l1Vtby0fhejT+vJ77/7BcYMyG8N5DUNQTKNIWjtSTXc0ZYCtchJLJHMDDfFmJxem7eoNOIwytHaxtYg3TaQB61tDfQnW5AGbRwgctJy6hEne5up97ce4vonnEsFFUWZuIw0Nt3ZC3G8oo0DRKQdp9zoZG2hVbrnCA8sKuO10n2OO3wbIo9th4RPRjptywXxpQH6nQK1yEmqIzuldMTOw9U8vLScF9dW0CMni3svHc2p3bO576WPT3hQGHDcGTwkfHOAVD9sOosCtUgX5zQ04JQbbWnelzDRIYRDx+v52fLN/O+728DAzC+dzvcuGkZBt2wAsrMyWtuVnxdoXTjjJFK2R6oeNp1NgVqkC4s2NBAtNzqRIYTq+kaeenMr81ds4Xh9I9dNGsgZ/XvxZMvv2j4srppQ1NrGaJyyPVK9X2NnUaAW6YJCvehIQSw0NBCalIt1nNtA3RBs4o8f7OCnr21i/9E6LjmzH7NmjOLjXUeijiNHGr4IiTW56bQt15TRhUyesyztJxhDFKhFPOCnzAM3hYxCQwOhXu3Q2S9HHB92M4TQ1GR5ecNuHlpSxmcHqzlnyCn84sZJTBrcG4BbfvVB1HHkaNeIlYESKUVwyuhCXlhd0aUmGBWoRRLkt8yDaD3UkPChgY4OIby16QBzF5WyoaKK0af15OlbSqiqbuD2P6xtDZxOWRyhAO10TFFBnqt/v/CFOJPnLOtyE4xu6lGLSBTRMg+SJVSjeejsl5k8Z9kJdaFj9YIjTcrNmjGKvLDNX6Mt1d6ws4obn1zJjU+t5NDxeh6+fhwv3/4ljtQ08uMXPzqhbrXTnt+hh0C8147FzQRjtH8/P1KPWiRBqc48iNWDj9aLzTTmhIdIqIfpZpUhwGcHjvPgkjL+8uFuencL8K9fOZMbzy0mJyuz9f3hDy1L5NS76vpGFq6tcH1tt2J9O/DbNyA3FKhFEpTqzINYucORJtgCGQYMNASbw2Wk4BRpZ++QfUdr+elrm/jj+zsIZGZw29ThfPeC0+mVe+IO304PJwsUhKXgHa5uOKENXgVJpwnGUA89HXOvFahFEhQrMHgtVg/ebaU7p+DUdmL0tPxcxhbl8+amAzQEm/jGOcXcNm04fXtG3uE72ngz0C5XOhkBMlYPPR1zrxWoRRLk9Vf3WNz04MN7qENnvxzxXLGWZO+uqmV3VS0TBhXwyA3jGdKne9S2RXto3fXsOldt8EK0Hno65l5rMlHEA1dNKOLt2VPZOudy3p49NalfoTsy+eYUhMJ/71TVbt/RuphBGpr/He6/ZixFBXkYmnvSoRQ7t21INq8nL1NBPWqRNNORHnys4RlrLcvL9rGrqjbi++Pp9Tr1ZlM9ROQk1d+AvKBALZKG4p18ixacVm87zNxXSnn/s0NkZhiCEcraedHr9VOA9HLyMhVUj1rEp5K92nHT3qPMW1zG0k/2UtgzhzumjSAvkMm/LPwo5TWqRfWoRdJOMnN9d1XW8Oir5Ty/eifds7P44SUj+c75Q+mW3RwOMjOML3q98jlXgdoY8xlwFAgCjU5RX0S8kYxc38rqen7++qf8+p3PwMJ3Jg/l1inDOaV79gnHpduwwMkgnh71FGvtgaS1RERaeZnrW1Mf5FfvbOV/Xv+UY3WNXDNhIHdNH8HA3t0SbaakiIY+RHwgfDy6oFug3QIViG9SrzHYxHOrdvLYa+XsPVLHtNF9mXXpKEaf1svLpksKuA3UFlhijLHAE9ba+eEHGGNmAjMBiouLvWuhSBcXaTw6kGEIZJrWJd/gPpXNWsuij/bwwJIytuw/zqTBvXn8mxP5myGnJO0eJLncBurzrbUVxpi+wFJjTKm1dkXbA1qC93xozvrwuJ0iXUZ477m6vrHdeHRDk6UgL0D3nKy4JvXe+fQAcxeVsX5HJSP69uCXN5dw8Rl9Mcaphp2kA1eB2lpb0fK/+4wxLwLnACuiv0tEwkXqPTupqmlg3X2XuDrvx7uqmLeojDfK9zMgP5d5153NtRMHkpmhAN0VxAzUxpjuQIa19mjLz5cAP0l6y0S6IDdF/UPcjEdvP1jNQ0vL+PO6XeTnBfjny87gpvMGkxu2RFrSm5sedT/gxZavTlnA7621i5LaKpEuym3WRqzx6APH6nh82WaeWbmNzAzDrRcN4x8uHEZ+XsDxPZK+YgZqa+0WYFwK2iLS5TlVbnM7Hn2srpFfrtjCk29uobaxietLBnHnxSPo1yty2VHpGpSeJ2nJT5vJxsOpMNG/XzHGsf0L11Ywb1Epu6pqT9gppW/PHL4w9BQF6ZOAArWknVRupRTrgRDvAyPewkQL1uzk3hc+bE3Ta5tOte9oXbv7TtcHmESnokySdibPWea4i8jbs6d6dp3wBwKcWKAo1uuJsNbyRvl+vvvbVSfkUkcSuu9ktkeSL1pRJm0cIGknVVspxdpdPFm7j6/bUck3fvket/zqg5hBGj6/787YDV1SQ4Fa0k6qdgqJ9UDw+oHx6f5jfO93q7nqZ2+zed8x/uOKMQzIjz3+HLrvdNwLUNxRoJa0k6qtlGI9ELx6YOypquVHCz7kkkdWsKJ8P3ddPJLXZ03hW18cwj2Xjm53r221vW+/bHUl3lOglrQTbV8+L8V6ICT6wKiqaWDuolIuenA5z6/eyU3nDuaNe6Zwx8Uj6JHTPM8ffq8FeQF6dwtEvO9kPMAWrq1g8pxlDJ39MpPnLGPh2ooOn0s6TpOJIlF4nfUBUNsQ5DfvfMbPX/+UI7UNXDW+iB9MH8mgUxIvO+pl1ocmJ1Mr2mSiArVIijQGm1iwpoJHXi1nd1UtF40q5J4ZozlzQGJlR5OVkpeq7Bpppq24RDqRtZYln+zlgcVlbN53jHGDCnj4+vGcN+zUdsfGG3STvWVXPL+X5FGgFgnjZQ/1/a2HmPPKRtZsr+T0wu784saJzBhzWsSyox0JusnYsivEabm7JidTT5OJklaSPbkVCpYVlTVYPg+W8V6ndM8RvvPrD7j+iXepqKzhhpJB1NYH+d7v1nD+3OURz9eRPOhk9npTlV0jsalHLWkjFUvHE+2h7jxczcNLy3lxbQU9c7K499LRnNo9m/te+jhmuzsSdJPZ6413ubskjwK1pI1kfs0P6WgP9dDxeh5ftpnfvbcNY2DmBadz64XDye8WYPKcZa7a3ZGg61Tkyater3Yk9wcFakkbqZjcijdYVtc38tSbW3lixRaq6xv52qRB3Dl9BP3zPz/eqX0VlTVMnrOstbc6ZXQhL6yuiCvoqtd7clCglrSRiskttz3UhmATf3x/O4+9tpkDx+q45Mx+3HPpKIb37em63YbPt+KqqKzhhdUVXDupiOWl++MKuur1dn0K1JI2kv01H6L3UNvWhc7MMASbLOcMPYUnbprEpMG942p327rSITUNQZaX7k9JjrLKoaYXBWpJG6n6mh+ph7pwbQX3PP8h9cEmAIJNzWG2fM8Rdhyqjhio2wbD/LwAuYEMKqsbHHvYkJoc5VTW8xZvKFBLWon1NT8ZPcUPd1Zy7wufB+m2KmsaIwa58GBYWdNAXiCTR24Yz1UTihxX/aUiRzkVk7LiLeVRiyf8ULzHqxzokK0HjvP936/hisffpq6xfZAOiZTrHCsnujNzlLXiMP247lEbYzKBVUCFtfYryWuSpBu/fJV2Co53P7c+rrbsO1rLT1/bxB/f30F2Vga3Tx3On1bvZHdVreN7woNcrGDYmdkaWnGYfuIZ+rgD2AgkVkFGuhy/fJV2Co5Ba109OI7UNjD/jS089dZWGoJNfPMLxdw2dQSFPXM4vbBHuwnBttoGuYVrK8gwhmCEgmdtj+usbI1UTMqKt1wFamPMQOBy4D+BHyS1RZJ2/PJVOtokXbQHR21DkN+9t42fLd/M4eoGvjpuAHdPH8mQPt1bjwm9799f+pjKmoYT3t82yIW+XUQK0n4Jhsq9Tj9ue9SPAvcA7ZNEWxhjZgIzAYqLixNvmaQNv3yVjtRTbCv8wRFssry4toJHlpZTUVnDl0b04d5LR3NWUX7E94d6wNEmLCN9uwDINMZXdZyVe51eYgZqY8xXgH3W2tXGmIucjrPWzgfmQ3M9as9aKL7nl6/SocBz93Prow47WGtZVrqPeYvKKNt7lLMH5jPvurOZPLyP6+s4BTmnbxFN1iowSoe56VFPBq4wxlwG5AK9jDG/s9bemNymSbrw01fp0DWdHhyrtx1iziulfPDZYYac2o2ffXMil42NXHa0I/zy7UK6lrh2eGnpUf8wVtaHdniRzhY+PHHjucWs3lbJqxv3UtgzhzumjeCGvxlEINPbDFVtXyUdpR1e5KQTGp7YVVnDI0vLeWBxGd2zs5g1YxTfnjyEbtnJ+U/fT98upOvQnonSaZJZb6Kyup6fv/4pv37nM7Bw83mD+f6U4fTunp30a4t0hHrU4jteLpJpG3RPy89lYnFvVmzaz7G6Rq6dOJC7po+kKCzPOdK1V207FHflOpFUUI9aOoVTrQto3uXabZCMNCYMMGZALx6+fjyjTut5wrEPLC5zvG54RTuNLUsqqUctvhNtMUw8vet5i0oj5i1XVje0BumFaysiLlQJF6nsqAoViR+oKJN0iljparE2dQV459MD7HKovxF6EIR63LGCtBMVKhI/UI9aXPNyAi7WKkJwDpIfVVQxb3EZK8r3kxmjpobTSsFwkQr5tz2PSGdSoBZXvK6Q1zaNzWnMODxIbj9YzYNLynhp/S4KugX458vOoKBbgH/788eOqyLd9IiLOrhfoUiqKFCLK8mokNe2dka0Jej7j9bx+LJN/P797WRmGL4/ZRgzLxhGfl4AgEBmhmNPP1qhpvDJwpLBpyhlT3xJgVpcSWaFPKdFIhef2Y+Hl5bz5JtbqGts4oa/GcQd00bQr1duu/c7BVSnIZbe3QLc99UxJ7xPhYrErxSou4BULN5Idg2LtkGyrjHI71du58J5yzl4vJ7Lx/bn7ktGcnphjw6dF7RSUNKbAnWaS9XuKqmokNfUZHlp/S4eXFLGzsM1fHHYqdx76WjGDSpI6LzqKUu6U6BOc6naXSWZPVNrLa+X72feojI27j7Cmf178dvvjOVLI/p4VtVOJJ0pUKe5VO6ukoye6drth5nzSikrtx6i+JRu3HTuYF7buJdvPf0++XkBjIHD1Q2taXjxrFoU6SoUqNNcutQ/Dh9Hv/m8wazdXsmij/fQp0c2P7lyDN0Cmfxrm1S7totUQrnSnbVxrkhn0srENDdrxijyApkn/M5v+b+hcfSKyhoszcH2/ldKWV62j7suHskbs6Zw83lDeOTVTa4Wp7hZtSjSlShQp7mrJhRx/zVjKSrIw9C8eMNvhYScVgfWNTbx3KodLP1kLxDfcI2WdsvJREMfXYCfsxpqG4KOC07gxKGMaItTwvltaEckmRSoxXML11Ywb1Epu6pqyXCRtBEaynBT/wP8N7QjkmwK1OKpF9fs5N4XNlAfbAKgyUKmgYwMQ0PQufb5rsqadimAyvoQaaZALZ5ZueXgCUE6JGihZ3YW3XOyYhZgSvUwjrbkknSgQC0J27j7CPMWlbK8bL/jMVU1Day775KYBZhSKVWrOkUSFTNQG2NygRVATsvxz1tr70t2w+Rzfu317ThUzSNLy3lxXQU9c7KY/eXR/PadzyIW82/bYwZ/1N5I1apOkUS56VHXAVOttceMMQHgLWPMK9ba95LcNsGfvb6Dx+p4fPlmnnlvO8bAzAtO59YLh5PfLcBpvXJj9pj9kqWSylWdIomIGaht8+63x1r+Gmj54/2OuBKRl72+RHvmx+saeeqtrcxfsYXq+kauLxnEHRePoH/+56lyfuoxx5IuqzpFXO1CbozJBFYDw4GfWWvvjXDMTGAmQHFx8aRt27Z53NST09DZL0d8Khpg65zLXZ8n0thwaPupSJkUbYN6//xczht2Km+UH+DAsTpmjOnHrBmjGN63Z/sLpRGn8XK/LRiSk0PCu5Bba4PAeGNMAfCiMeYsa+1HYcfMB+YDlJSUqMftEa96fZF65qEPKXw4JTyA7aqq5YU1FQwr7M78myex/WA133r6gxNS6CqrG3zde44knXr/cnKLK+vDWltpjFkOXAp8FOv4riqVk3te1YGONe7adjjFacl3TX2Q7QerT2hP28JJfhg/j5dfxstFoolZ68MYU9jSk8YYkwdMB0qT3TC/ilRg6EcLNrBwbUVSrudVLQ83PfBdlTV8uLPSMdd5d1VtzF29VTBJxHsxx6iNMWcDvwEyaQ7sz1lrfxLtPSUlJXbVqlWeNdJPJs9ZFjGQFRXk8fbsqZ3QIncijceGywtkUtMQJMM0rygMF1odGEu84+cikuAYtbX2Q2CC561KU+mY0hUaqqlpCLYG29BEYltBa7l92ggG5OfyH//3Sbug7vS+cKHeu1/zv0XSjVYmxindUrrCe9JBa8kLZPLVcf3564Y9HKtrBOD84X145IbxFPbMASA3kMndz61v14O2EDVYh8bP/Zj/LZKuVI86Tn4q1L9wbQWT5yxj6OyXmTxnWcRxcqc87D+t3smxukauGDeAN2ZdxO/+/gutQRqag2mTwzBHKKXPAAV5AXp3C7QbP4+W/y0i8VGPOk5+Sely22N1GpKxFv5y2/mcVZTveA2nbw9uxuPTcYhIxK8UqDvADyldblcs9s/PjVh7o6ggL2qQhsRSA9NtiEjEzzT0kWRuhic6wk2PddVnh8gJG6YB98E2kdRAPw0RiaQ79aiTKJkTatF6rOV7jzJvURmvbtxL3545fG3SQN7efIDdVbVxD9V09NuDX4aIRLoCV7U+4tWV86jj4ZRznWkMD10/LqGgFSkvOicrg3EDC1i17RDds7P4x4uG8e3JQ+iWreexiN8lXOtDOsZpeCJobcI967Y91orKGnrkZFHbEGTdzkr+7vyh3HrRcHp3z+5Yw0XEVxSokyjartpeFKi/ZEw/Kipr+MXrn3K8vpFrJg7krukjKerghJ0WqIj4kwJ1EsXaVbujqWoNwSaeW7WDx17dxL6jdVx8Rj/uuXQUI/t1vOyoFqiI+JcCdRKFAlykFX4Qf6qatZa/btjDg0vK2HrgOCWDe/Pzv51IyZBTEm6rtqUS8S8F6iQLBblES5W+s/kAcxeVsn5nFSP79eDJm0uYdkZfjDERj493GEMLVET8S4E6BdykqjkF1o8qqpi7qJQ3Nx1gQH4uD35tHFdPKCIzI3KADp0r3mEMLVAR8S+l5/mAU6rdmAG9WLO9koJuAf5pynBuPHcwuREWsITrSClWbUsl0rmUnudzkcaH6xqbWLu9ku9PGcY/XDiMXrkBV+dauLbCMdMk2jCGFqiI+JcCtQ84Fk4CZs0Y7fo8oV6xk1jDGH6oYSIi7anWRyerawzSKy9ybznefOho22SpzoZI+lKPOkEdXSTS1GT58/oKHlpSTlVNQ7vtr/ICmUwZXcjkOcsSztwANNYsksYUqBPQkewKay2vl+1n7qJSSvccZcyAXvzX1WM5eKyOB5eUtwblKaMLeWF1hSeZG0UFeQrSImmsywbqRJdDu3l/vItE1mw/zNxXSlm59RCDT+3GzecN5tVP9vKtp99vd43Jc5bFvQAlkfrRIuJfMQO1MWYQ8FugH83zW/OttY8lu2GJSHQ5dKK7p4T/fvO+YzywuJTFH++lT49sfnLlGLoFMvnXP3/seI2OLEDpSOaG6nuI+J+bHnUjcLe1do0xpiew2hiz1Fr7SZLb1mGJLod2+/5Yi0T2VNXy6KvlPLdqB92ys/jB9JH83flD6Z6TFbPH3NEFKPFkbqi+h0h6iJn1Ya3dba1d0/LzUWAj4Ov/Fye6HNrt+512Mfn+lGHc/8pGLnxgOS+s2cm3vjiEN2ZdxO3TRtA9J8vVNVKxQ4o2oBVJD3GNURtjhgATgJURXpsJzAQoLi72oGkdl+hyaLfvDx9q6J+fy6TBvZnzSilH6xq5enwRd00fyaBTusV9jVQsQFF9D5H04DpQG2N6AC8Ad1prj4S/bq2dD8yH5iXknrXQhfBx1vCMCYivNxrPpNxVE4r4ytn9eX71Th59dRP/9+Fupowq5J5LR3NG/14JXSPZC1BU30MkPbha8GKMCdAcpJ+x1i5IbpPiExpnraiswdI8zvrC6gqunVTUoU1Zwf2mrtZaFn20hxmPrmD2gg0cPF4HQPneY5TtOerJNZJJG9CKpIeYRZlMcx3N3wCHrLV3ujlpKosydaQAUTycsiLe23KQuYtKWbu9kr49c6isbjgHR4kAAAn7SURBVKA+2NT6vnQpaKSsDxF/iFaUyU2gPh94E9gAhCLRj621f3V6TzIDdXhgcSpAZICtcy6P+3xtA1WkinJt5ecF+PFlo3ns1U3sqqpt97pXDwsR6foSqp5nrX2L5rjX6SKlkxmak7vDuRlnjZWeFq12BkBdQ5CcrEx2RwjSoEk5EfFGWhVlihQ4Le2fIm7HWWOlp8UKtLWNTTywuMzxoaBJORHxQloF6mjlQEOTcgV5AXIDGdz17Domz1nGwrUVcZ9vV2UNx+sa6ZEbOylmV2WNJuVEJKnSKlA79VBDY8GP3DCeusYmDlc3tGaA/GjBBsdg7XS+XnkBLnxgOUdrG4my41XrOfyQwSEiXVdaBepYPdd4V9pFOp8BqmoaGFbYgwW3fpGHrx/fWhc62hDLVROKeHv2VLbOuZy3Z09VkBYRz6RV9bxYq/XiXWl31YQirLX8v5c3cvB4PQD983P5z2vGctHIQowxTCzufUIWSGensvmhDSKSWmkVqCH6ar14V9qt31HJn1bv5ODxegb2zuPuS0Zy5bgiMhzGOzp7qyoVURI5OaVdoI7G7dLvLfuP8dCScl7esJtTumdz31fP5JtfKCYnK/YO317oaK840aqAIpKeulSgjjU0su9ILY++tolnP9hBTlYGt08bwXe/NJSeLnf49kIivWIVURI5OaV1oHbqmYYHvCO1DTzxxqc89dZWgk2WG79QzD9NHUFhzxxX5/OqXZBYr1hFlEROTmkbqN30TGsbgvzvu9v42eubqaxu4IpxA7j7kpEMPrV7h87nRbti9YqjBXlttSVyckrbQB2tZ/rVcQNYsGYnjywtZ1dVLReMLOSeGaM4qyi/Q+eLJ1DHOk+0XnGsIJ+KGtUi4j++C9Ruhx+ceqYVlTV8+bEVlO89xriB+Tz4tXF8cXifmNf1avzXzc4tTr1iNw+Lzs48EZHU81WgdupRrtp2iOWl+08I3tEq5zUGLT//24l8+azTaK7SGptX47+J7Nxy17PrIp5Tk4UiJzdfBWqnHuUz721vrZAXCt7XTipqt4sLwPUlA/nPq8cSyIxv0aVX47+J7NyiyUIRicRXS8ijFV1qq6YhyNJP9jK2zZhzr9ws5l17NvOuGxd3kAbvdlxJ5Dwq7iQikcTcOKAjOrpxgNNuLU6yszK45YtD+N6Fw+jdPTvu6/mRloiLnJwS2jgglSINGzhtDNAtO5OlP7iwtWBSV6HJQhEJ56uhj0jDBt84ZxCBsNobOVkZ/NfVY7tckBYRicRXPWr4vEdpreXlDbt5aEk5DU2W7MwM6oNNFGk4QEROMr4L1ABvbz7A3EWlfLizipH9evDkzSVMO6Ov61Q7EZGuJGagNsY8DXwF2GetPSuZjfmoooq5i0p5c9MBigryePBr47h6QhGZsbZZcUkTdSKSjtz0qH8NPA78NpkNqapp4Gu/eJecQAb/cvkZ3HjuYBZ9tIcL5i33JLCqlrOIpKuYgdpau8IYMyTZDcnPC/DETZMYX1xAr9yA54FVtZxFJF35KuvjgpGF9GqpDR3v/oexqJaziKQrzwK1MWamMWaVMWbV/v37Ez6f14HVaRm2lmeLiN95FqittfOttSXW2pLCwsKEz+d1YNXybBFJV74a+mjLy8AayvaoaQiS2ZLiV5AXIDeQwV3PrmPynGUsXFvhSbtFRLwWM1AbY/4AvAuMMsbsNMb8XfKb5V2RpNCkZKiGSNBaAhmG4/WNHK5uwPL5RKWCtYj4ka+KMiVDPIWeigryeHv21CS3SESkvWhFmXw79OGVeCYflQEiIn7U5QN1PJOPygARET/q8oE60qRkIMMQyDxxWboyQETEr3xZlMlLTnsURvqdViiKiB91+clEEZF0kBY7vKiynYhIZL4I1KpsJyLizBeTiV4XYBIR6Up8EahV2U5ExJkvArUq24mIOPNFoFZlOxERZ76YTHTKddZEooiITwI1NAdrBWYRkfZ8MfQhIiLOFKhFRHxOgVpExOcUqEVEfE6BWkTE55JSPc8Ysx/Y1sG39wEOeNicztRV7qWr3AfoXvyoq9wHJHYvg621hZFeSEqgToQxZpVTqb9001XupavcB+he/Kir3Ack71409CEi4nMK1CIiPufHQD2/sxvgoa5yL13lPkD34kdd5T4gSffiuzFqERE5kR971CIi0oYCtYiIz3VaoDbGXGqMKTPGbDbGzI7weo4x5tmW11caY4akvpWxubiPW4wx+40x61r+/H1ntDMWY8zTxph9xpiPHF43xpifttznh8aYialuo1su7uUiY0xVm8/k31LdRreMMYOMMcuNMZ8YYz42xtwR4RjffzYu7yMtPhdjTK4x5n1jzPqWe/mPCMd4G7+stSn/A2QCnwKnA9nAeuDMsGNuBX7R8vPXgWc7o60e3MctwOOd3VYX93IBMBH4yOH1y4BXAAOcC6zs7DYncC8XAX/p7Ha6vJf+wMSWn3sC5RH+G/P9Z+PyPtLic2n5d+7R8nMAWAmcG3aMp/Grs3rU5wCbrbVbrLX1wB+BK8OOuRL4TcvPzwPTjDEmhW10w819pAVr7QrgUJRDrgR+a5u9BxQYY/qnpnXxcXEvacNau9tau6bl56PARiC8cLvvPxuX95EWWv6dj7X8NdDyJzwrw9P41VmBugjY0ebvO2n/obUeY61tBKqAU1PSOvfc3AfAtS1fSZ83xgxKTdM85/Ze08V5LV9dXzHGjOnsxrjR8vV5As09uLbS6rOJch+QJp+LMSbTGLMO2AcstdY6fiZexC9NJibf/wFDrLVnA0v5/CkrnWcNzXUVxgH/DSzs5PbEZIzpAbwA3GmtPdLZ7emoGPeRNp+LtTZorR0PDATOMcaclczrdVagrgDa9iwHtvwu4jHGmCwgHziYkta5F/M+rLUHrbV1LX99EpiUorZ5zc1nlhastUdCX12ttX8FAsaYPp3cLEfGmADNwe0Za+2CCIekxWcT6z7S7XMBsNZWAsuBS8Ne8jR+dVag/gAYYYwZaozJpnmw/aWwY14CvtXy83XAMtsyMu8jMe8jbKzwCprH5tLRS8DNLRkG5wJV1trdnd2ojjDGnBYaLzTGnEPz/w/81gkAmjM6gKeAjdbahx0O8/1n4+Y+0uVzMcYUGmMKWn7OA6YDpWGHeRq/OmVzW2ttozHmn4DFNGdOPG2t/dgY8xNglbX2JZo/1P81xmymeWLo653R1mhc3sftxpgrgEaa7+OWTmtwFMaYP9A8697HGLMTuI/mSRKstb8A/kpzdsFmoBr4due0NDYX93Id8D1jTCNQA3zdh52AkMnATcCGljFRgB8DxZBWn42b+0iXz6U/8BtjTCbND5PnrLV/SWb80hJyERGf02SiiIjPKVCLiPicArWIiM8pUIuI+JwCtYiIzylQi4j4nAK1iIjP/X8W5zmilIONHQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0giuwC9GHzi8" + }, + "source": [ + "## Вычислительный граф\r\n", + "\r\n", + "Для проведения вычислений Tensorflow строит внутри себя вычислительный граф, который, в т.ч., может вычисляться на GPU. Однако в нашем случае, поскольку мы использовали пользовательские Python-функции, они не включались в вычислительный граф, и при вычислениях на GPU производилась бы передача данных между GPU и CPU и обратно.\r\n", + "\r\n", + "Для ускорения высчислений и построения единого статического графа, необходимо отметить все функции соответствующим декоратором:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "HK7HPLz3Hyrl" + }, + "source": [ + "@tf.function\r\n", + "def train_on_batch(x, y):\r\n", + " with tf.GradientTape() as tape:\r\n", + " predictions = f(x)\r\n", + " loss = compute_loss(y, predictions)\r\n", + " # Note that `tape.gradient` works with a list as well (w, b).\r\n", + " dloss_dw, dloss_db = tape.gradient(loss, [w, b])\r\n", + " w.assign_sub(learning_rate * dloss_dw)\r\n", + " b.assign_sub(learning_rate * dloss_db)\r\n", + " return loss" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "J7HusxWkGjLX" + }, + "source": [ + "## Dataset API\r\n", + "\r\n", + "Для работы с данными в Tensorflow присутствует удобное API, которым мы в данном случае можем воспользоваться:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "oYro9Lbr8q0M", + "outputId": "78c0a6de-71bd-4eef-8819-439495b28672" + }, + "source": [ + "w.assign([[10.0]])\r\n", + "b.assign([0.0])\r\n", + "\r\n", + "# Create a tf.data.Dataset object for easy batched iteration\r\n", + "dataset = tf.data.Dataset.from_tensor_slices((train_x.astype(np.float32), train_labels.astype(np.float32)))\r\n", + "dataset = dataset.shuffle(buffer_size=1024).batch(256)\r\n", + "\r\n", + "for epoch in range(10):\r\n", + " for step, (x, y) in enumerate(dataset):\r\n", + " loss = train_on_batch(tf.reshape(x,(-1,1)), tf.reshape(y,(-1,1)))\r\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 18, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 173.4585\n", + "Epoch 1: last batch loss = 13.8459\n", + "Epoch 2: last batch loss = 4.5407\n", + "Epoch 3: last batch loss = 3.7364\n", + "Epoch 4: last batch loss = 3.4334\n", + "Epoch 5: last batch loss = 3.1790\n", + "Epoch 6: last batch loss = 2.9458\n", + "Epoch 7: last batch loss = 2.7311\n", + "Epoch 8: last batch loss = 2.5332\n", + "Epoch 9: last batch loss = 2.3508\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A10prCPowHl7" + }, + "source": [ + "## Пример\n", + "Рассмотрим пример двухмерной задачи классификации на 2 класса. Примером такой задачи может быть классификация опухоли на 2 типа - доброкачественная и злокачественная, в зависимости от её размера и возраста.\n", + "\n", + "Сгенерируем тестовые данные случайным образом:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "id": "j0OTPkGpwHl7" + }, + "source": [ + "np.random.seed(0) # pick the seed for reproducability - change it to explore the effects of random variations\n", + "\n", + "n = 100\n", + "X, Y = make_classification(n_samples = n, n_features=2,\n", + " n_redundant=0, n_informative=2, flip_y=0.2,class_sep=1)\n", + "X = X.astype(np.float32)\n", + "Y = Y.astype(np.int32)\n", + "\n", + "split = [ 70*n//100, (15+70)*n//100 ]\n", + "train_x, valid_x, test_x = np.split(X, split)\n", + "train_labels, valid_labels, test_labels = np.split(Y, split)" + ], + "execution_count": 19, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "id": "c-_BjSHPwHl8" + }, + "source": [ + "def plot_dataset(features, labels, W=None, b=None):\n", + " # prepare the plot\n", + " fig, ax = plt.subplots(1, 1)\n", + " ax.set_xlabel('$x_i[0]$ -- (feature 1)')\n", + " ax.set_ylabel('$x_i[1]$ -- (feature 2)')\n", + " colors = ['r' if l else 'b' for l in labels]\n", + " ax.scatter(features[:, 0], features[:, 1], marker='o', c=colors, s=100, alpha = 0.5)\n", + " if W is not None:\n", + " min_x = min(features[:,0])\n", + " max_x = max(features[:,1])\n", + " min_y = min(features[:,1])*(1-.1)\n", + " max_y = max(features[:,1])*(1+.1)\n", + " cx = np.array([min_x,max_x],dtype=np.float32)\n", + " cy = (0.5-W[0]*cx-b)/W[1]\n", + " ax.plot(cx,cy,'g')\n", + " ax.set_ylim(min_y,max_y)\n", + " fig.show()" + ], + "execution_count": 20, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 283 + }, + "id": "tq0vFchQwHl8", + "outputId": "9a5aa6a0-c92f-4d72-9e78-c0f615804bff" + }, + "source": [ + "plot_dataset(train_x, train_labels)" + ], + "execution_count": 21, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd5iU5dX/v2dmtrINdpfOsoBKV9AFsQRFNIgx1thiNJZojDH6mkSNKSbRn0l8TaKmGDUxRo2vFbEiYgsIEaWIIFUEgaXsLmzvOzP374/vPM6WqbvT93yua67dnXn2mTPzzJxz36eKMQaKoiiKYou3AIqiKEpioAZBURRFAaAGQVEURfGgBkFRFEUBoAZBURRF8aAGQVEURQEAOOItQF8oKioypaWl8RZDURQlqVizZs1BY0xx9/uT2iCUlpZi9erV8RZDURQlqRCRXb7uV5eRoiiKAkANgqIoqYgxQHs70NERb0mSiqR2GSmK0gecTmDjRuDNN4GdOwG7HZg+HZgzBxgzBhCJt4Th09YGrFoFLFoE7NvH+w47DDjjDOCoo/gaFb+oQVCiijFAYyN/5uQANt2TJgatrcCDDwKffALk5gKFhYDbDXz4IfD++8BZZwHnnptcRqGpCbj/fmDrVqCoCBg9mvdXVgL33QccfzxwzTWAQ9WeP/SdUaJCRwfwwQdcqFVUUK/k5wPz5wNf+QqQlRVvCfs5TzwBbNgAlJZ2VfrDh3PnsHAhlers2XETMWwefxz4/POeu5tBg4CBA4GVK4Fhw4BzzomfjAlOwqzXRGSUiLwnIptEZKOI3BRvmZTe0d4O/PnPwN//zt9LSnhzOIB//xv43e+AhoZ4S9mPqayktR41yvcOwOEAhg4FXnoJcLliL19vqKqiq2jkSN+vSYTGbvFi7o4UnySMQQDgBPAjY8wkALMAfF9EJsVZJqUXLFwIrF8PjB0L5OV5v58DBvC+8nLgscfiK2O/5uOP+TOQ/27AAKCmhrGFZGDDBvolA72mjAyuUD77LHZyJRkJ4zIyxuwHsN/ze4OIbAYwAsCmuAqmhEVTE/D228CIEf7dzyNHUidVVABDhsRWPgVAbS2Qlhb8OJuNFzRW1NRwlb9vH5CZCUydCkyYEFoguKkp9ICx7hD8kjAGoTMiUgpgOoAP4yuJEi7bt9MFHUjfiHAxt3EjMHgwvRJ2e3LFL5OavLzQ0jGNiU2wx+Wie+r11/mcmZn8EC1ezA/ID35A91YgCgpCc29Z2Q2KTxLOIIhIDoAFAP7HGFPv4/FrAVwLACUlJTGWTglGW1toit3lApYtA155hQtWhwM49ljg1FMZE1SiyLRpwLPPUjn6u1gtLcw+Gjs2+vIsWAC8+iqzgrpnAB06xKDTHXcE3k5OncodjbW68IX1mg4/PHKypxiJFEOAiKSBxuApY8yLvo4xxjxijCkzxpQVF/doxaHEmbw86plANDQAa9YAH30EpKcz0WXoUHoLfv1r4N13YyJq/2XYMODooxnM8XWx3G66bc48M/opmlVVwBtv8EPg67kKC7nKeP11733G0O3TeUdQUADMnQvs2uX7NTmdfE3nnqtppwFImHdGRATAowA2G2P+GG95lN5x2GFML21uBrKzez7ucgHLl/P3adNoEAB+R4cPZ8zvX//i7xMmxEzs/sfVVzNnf9s2pmValvzgQRaOfPWrVLDR5oMPuLIP5P8fOhRYsQI47TQGn5YsYcxABJg5k/ePGwdceCFXGx98wA9fQQFf06FDdJGdcw5w8snRf01JjJhgy7kYISInAngfwAYAbs/dPzXGLPL3P2VlZUab2yUe778PPPIIU027xxLKy4H33gPKyoCJE33/f1UVPRU/+lH0Ze3XtLcDq1dzhV5eTgU7dSqNwaRJsQnqPPAAA0+FhYGP27yZsQW7nXGFzEyuLioruYO47DIaMLebRu6tt1iTIAIceSSrr0eP1kCVBxFZY4wp635/wuwQjDHLAejVSgFOPBGorwdeeIGLv4ED+T2srfWmo44f7///CwsZcG5ooMtXiRLp6azePf54KlKR2CtMh4PPHQinE/j0U+4GOscN7Xa6v9rbWZQ2ciQ/WBMm6PaylySMQVBSBxHga1+jm3r5choBY2goLAUfKF3cZuM5rBigEgPi1VNk+nQGjwLFA3ftotHwl2mUns7MoUWLAq80lKCoQVCixrBhwAUX8Gbxl78AmzbRZe0Pt5sGxFcMQkkxpk/nhW5s9J0Oagw/MIcfHthoFRVx5eHvPEpIJFSWkZL6nHQSA86BqKwEjjlGv9f9gqws4PrrGfg9eLBrhlBbG/DFF/Q5jhsX+DzWtrKtLaripjpqEJSYMnEid/779/t+vKWFGYVnnBFbuZQ4MmUK8POf84OxaxewZw+wezeDTmefzQ9De3vgc7hcNAi6rewT6jJSYorDAdx8M/CHP3DxV1DAtjkdHVwg2mzADTdocVrCcegQW0s4HOxLEkrri3AYNw647Tb2M6mu5vOUlLD/0Cef0B0UKM5QWQnMmqVtdPuIGgQl5gwaBPziF8C6dUwpr6hgFuGZZzLwPHhwvCVUvmTXLuDFF6mQbTZvcOf004F58yJvGIYM6VmRPHmyd1s5bFjP/2lq4opi3rzIytIPSZg6hN6gdQiKEkW2bgXuvZdKv7jYG9RtaaFynjaNfYYibRR8UV3NbWV5eddt5aFD3E18//uciKaERMLXISiKkkC0tXGoRV5ez5SwrCz69NatA955h7uFaDNoEPsZffIJi84qK7mtPO881lEMGhT6uZqaaNDcbm5HCwqiJ3eSoQZBiQvxqoNSQmTdOirOoiLfj4uwpcQbb7AjYSz6A2VksDht5sze/X9DA/Dyy8DSpV37IM2cyR5H2otdDYISO+rr2ZLmrbfoAcjMZMxgzhzGKZUE4uOPgwdos7OZCXDgAKuEE5n6enZN3b+fjbIsN5fLxU6Ln34K/PSnfKwfo2mnSkzYu5eB5Oee42Jy9Gi2qFi6lPevXBlvCZUutLeHNnBGJDnGbD7zDLMXRo/uGvOw27kaMQZ46KHgrXpTHDUIStRpbWU80OXi9zE7m3okLY3fxcGDgYcfZhqqkiCUlASfluZyUYEOHBgbmXpLbS1XHIFW/0VFrH9IlpGhUUINghJ11q6li8ifOzozk+1oliyJrVxh4XJxbu8//gHcdx/w5JPsppmqK8rjjuNrC7T6r6hg7n+gPiSJgLXSCLTjsYJZ27ZFXZxERmMIStRZtowzEgIxeDDw4YfAt7/N2GFCUVHBNs3WrN/0dPbXeecd5sh/73up12djyBDOGVi8mNu67sq0poZK9Otfj4984RCqS8tmC220aAqjBkGJOrW1wZW83c4FaUtLghmE+nrgnnvo9yot7fqYMezT/6c/AbfemnqTuC66iEryzTep/LOyqFxbWxkAuv1234ViiUZhobdjYqC0Nrc7OV5PFEmxT7CSiAwcyKByoDYzTqdX5yQUy5ZxNTx6dM/HRFhBu3UrdwxHHhl7+aKJ3Q5cfDHTSleu5EXMyGCH0smTk8cAjh7N61Rb6z/e0drK1zZ1amxlSzCS5IoqycycOcCDD/qvHWpsZNq7zcaF9ogRHNo1ZUqcdY7bzcBGoF4aInQXvfVW6hkEi6Ii9hVJVkSAb30L+O1vmcnQ3b3X1kZj953vJNj2NPZoUFmJOkcdRZ1aWdnzsV276JHYvZuT1LKzmexx333A//4vjUXcaG1lMVOwbUtuLuMLSuIyfjzwwx/SJ7lrF2snKioYcD54ELjySmD27HhLGXd0h6BEnYwMfhfvvZffv8JCKv59+zh/ecAA7iKGDvUeP2gQR+0+9BBnK8elotnanrjdgYezuFwMNCuJzZFHAn/8I9tfbNrE6zp2LAd8p1pSQC9Rg6DEhKFDgTvv5LTEt97i4mzzZg7CmjKFRqEzlnv+009pROLSDjs9ncLt2BG49fKhQ+zb3wusrFVt4REj+tr+IsVRg6DEjAEDgJNP5q2qivGCkhL/ytAqXvvgg14ahMZGYONGZgoNGABMmhR+I7PTT2eW0cCBvgMaLS3cPZxwQsindDpZ0vDGG9wFiXBw0Lx5jNXGa7yxoqhBUOJCYyOTWIKtjDMz6eINC6eTPfyXLOHvFjYbZ3hefHHowcNJk7j6f+kl+rry8ym0MRSssZF1CP6q7rrR1gb87W8s1svP97YA2rmTLrXZs4ErrkieBB4ltdCPnRIXMjO9nQ8CGYX29jALYd1u4LHHGJwoKemqWV0u4L336OK58cbQtK4IWyyXlACvvMLot83G57GMxRFHhCzes88yo2rMmK6vu7iY9mbZMv4899wwXrOiRAg1CEpcGDqUrWXq6wNXMbe3A8ceG8aJt24Fli9nEVl334vdzpz0devYzXPGjNDOKcJjy8qYKtXWxiBkOD34wTT4//yHsRFfRtBm445h8WJg/nwaTUWJJeqtVOKCCHDWWYwldPbqdMbqqjx+fBgnfvttpjD5c8SLMB6waFHYMkOELR1KSsI2BoA3sSVQS530dBrBLVvCF09R+ooaBCVuzJxJb8zu3Vx4u928v6mJmUW5ucBNN4UZZN26NXj3zYICOu1j3La5oSH0bKKWlujKoii+SCiXkYj8E8CZACqNMVPiLY8SXUToK584kcVpn3zC+wsKgG9+kxk3tbVAXR3dSyG1tQhV48YhzzMvL/TmqIHafChKtEgogwDgXwD+AuCJOMuhxAgr5XLiRC7YnU4agZdf5kwTK6HH4QBOOYUdFALWEE2ezBiBVeXmi+pqFkCEMgAmgkyezKd0Ov3Hs9vamAAVlptMUSJEQrmMjDHLAFTHWw4lPtjtTAC66y4WsI0YwQBsSQmzOhcvZklAQ0OAk8ydS3+LP3eQMdxyzJ8fldcQiLw8GrU9e3zvFNxuoLwc+NrXNKCsxIeEMghK/8YY5ugbQ2PQeQGfns7Eob17geefD3CSww5jH/8vvuByuzMdHYwdzJoVt0Z0F17I2MnOnd6AutPJyu1du2jPzjgjLqIpSsK5jIIiItcCuBYASkpK4iyNEkk+/5wr5ECXdfhwYMUK4Bvf8FOfIMIAxMCBwKuvdjUKaWlMbTr77Ji7izqLcN11LEB7803gs88o8qRJtGMTJmgbCyV+JJ1BMMY8AuARACgrK0vR+YX9EyvVMpBCdDi4g/jiiwCLfJuNfpdTT+VJGxsZpR0/PiGitXY72+7389b7SgKSdAZBSV06OkJfuPurXehCRgZ7byuKEhIJFUMQkacBfABgvIiUi8jV8ZZJiR3DhwdX9MYw+FpYGBuZFKU/kVA7BGPMJfGWQYkM9fXeRqO5uUy5DNSiAqALKCPDm3oJAHZXO4bsX4fDPn8TefXlaOlw4OCYGShxnwygNMqvQlH6FwllEJTkp6ODWUDvvMPMT6uOwGYDvlpWjfOG/Rfp2zfz4IkTgeOP/7INRFYW57o/9hjTTXPQiGM/uB+F1Z+hNbMANWmD0druwpyMFZBfvceD58/XKKyiRAg1CErEcLmAv/8d+PBDKvQvi6+MQen2tzD8vmewbSAw6dhctqPYvJltqi+6iEOURTBnDt1Gzz1rcOT6h5HZuBPl2aVwtgnS0oCZJ9pRMGQELc/TT3M2Z1lZPF+2oqQMahCUiLFpE7ByZc/WziW738f0DU+idsgobKtPR2E7MGwYmBra3g78+9/cHsyeDRHahuNH7EbDDzdg/9DRyLYJiotZfPylkUlLY7XaSy8BxxyjuwRFiQAJFVRWkpslSxgv6Kybba4OTN74PBpyh8M40pGZySlhX5Kezmjyc89x1e8hZ/MqDBtpx9HHCKZPZ9fTHu0e8vI4mFkH3CtKRFCDoESMbdt6NhotOrgF6e0NcKaxM11WFlsJdWndkJXFWoHNm733VVcH798gwuBEU1PfhW9sZN+M1ta+nyuWVFez7HnfPm+7WEXpJeoyUiKGzdazR09WSzXS25swuGID0tsb4LKlQWwjANcQwNGt6KC21vt7fn7P1hPdMYaBi74Um23eDLz+Ov1dIrwdfzwHHI8Y0fvzRpvPPwcWLmQql/XGFxUBX/865zvrYGalF6hBUCLGpEnUq0OG8G+HsxVHbH0FQw98jLaMPLhtaXB2uDHJsRfydiZ7CnUeep+W5v195kxOoQ80Y7OhgU/WW8W9eDHwf/9HP5c1xszpZCDkv/8FfvhDvqhEY+1a4M9/piEcNcqr/BsagEceAXbsAC67TI2CEjb6iVHC4uBB4LXX2ITu0Uepm9rb+dhpp9F743YDMAbT1/wduQ170Z6eg/b0XHQ4stAiA5A9vICKfsWKTv8AtqS2KC1lYx9/8QGnk93hzjmndwHlrVuZpTRqFAcaW+dwOBjTyM8H/vQndkZNJOrqgIceoszFxV2Vfm4uI/rvvAOsWRM/GZWkRXcISki4XMALL3BRLcLFqcvFWfb5+cAPfsBWQaecArz7LjA1eweG71+D6sLxyGyrx4D6/ahHPvLzgQEDAEgWq9a2b2fZ8YwZdHlYiADf+x7wxz/SR15YSIXndtMQtLRw3FpYA5c78eabfBGddyUADdWhQ3yOigrm0V53XZAhDDHkww9pDP25yWw274jQsjLNvlLCQg1CClBVxWZvbjfT8ktLI68Hnn+eOmb06J79hurqgP/9X+AXv6CnorAQqP7jMtS1pKMWghr7UTjc2YiitFoMyM6B9bFz2jNg1m1Ey/zzkP3Ny3t+GPPzgdtvB1avpvto924+eVkZ+0QfdljvXmhbG/Dxx9wddH8hq1YxwGyzMevpySfZIO+882JfBFdfT3k++ojbsJISYMMGP21eO1FQwF7aDQ3Bj1WUTqhBSGIOHmQKvzV6UoRGYeRI4PLLgSOOiNzzLF7s2xgA1NstLSwJ+MEPGNfsWF2OA5tysP4LoL4lA58POxGNTTswuPxzHNrTBIcDaHc50Jxegservov2X+Vi/nzq+S6L9sxM4MQTebMi1n1Vyu3t3gwli8ZGYPly3m/FNTo6+JxDhtC95HZzZFssWLeOfrn2dr7BdjvwwQeMbwwezPiLw0GlX1/P/8nJoQGwguMhdQBUFC9qEJKUQ4eA3/yGeqxzXNEYJuv89rfALbdEJia6ciX1S6BOpIMHc9FdU0OPRYdkYOd2F9w2FpSJpKO1aAK2NR2OAztb4GwDikZmYnTaXhSNyYVUb8MX/+9DvP9wDWafPRCOE2b13AFEanWelcX6h/Z2/gS4C3C5uq6oOzqojNPSuDp/8UUaps6B8GiwfTvwwAN0oQ0Y4L0/N5dWeetW73tRVcWfVo+QgQN50dPSeLwSHk4nd2GLF3t3pEcfDcyZE52td4KhQeUk5bnnuDgcPrzrQleEOmHgQODhhyOzSNyzJ3hmp83GW7VnAOp/mmfC1ljfpVDN7QbK99vhzMyBLS8HTZWNaEgvQtlHf8Npq3+Dsub34dz8GfY9swy4+25avGgEdR0ObkUOHODfra0cxdY9TtDeDowdy9/T0qhwP/oo8vJ058UX+YZ3NgYWpaWUf+1aoLKSBquggD/z8xmkf/ttDlvoHh9RAtPcDPzhD8D99zOZobCQC4SVK4Ff/5oDl3zNPk0h1CAkITU1dC0HmiOfl0ddunFj358vPT20mie32+vFePXADKTnZSK9vfHLx5uavAPm7eJGfkcV9h1MQ379btQVlKIxbziksBAba0bAPWo0AyN/+EN0isXmzKE7qqaGiqC7C6m+ngp28GDvfdnZDHBHk4MHWRvROcDemUGDuJOxZm/62kFlZHAUW4orr4jzz39yp1haSmPgcHgr6UeOZCBt5cp4SxlVwjYIIjJAROIzf1ABwMUsEHyYjMPRrU1EL5k+nTGCQLS0cEE7fDiTc9ocA7Dq2BuQ2VKDnMYDEOP+Mlab7azHsNad2JAxA1vaxqAxz1tHkJbGhXlDo7C+YNcuroYjTVERcOut/H3vXhqd9nZardpaWtTjjuv6JlsWL5rU1fFN8ueaaGykIcvP57H19TRojY2UOy2Nk+JqayNz8fsL+/YxVbekxPd773BwcbBwYUpXhAc1CCJiE5FvisjrIlIJYAuA/SKySUTuFZHDoi+m0plAtVqdsYLMfWXqVK/+8SfP/v0s7rW8FCJA1eDJWHrSHagYPAV59eUY2FSOYR274LSl45Vh38VHOBbt6b7TOb+Ue+BA+nOjQWkp8LvfATfc4A3GFhXREMye3bN1RktL9OdepqcHXtnX11PO4mLGCsaO5Wp2+HDKPXeuNw6ya1d0ZU0l1q71BuP9kZvLmM3u3bGTK8aEstx5D8DbAG4H8Kkxxg0AIjIIwBwA94jIQmPMv6MnptKZoUOpMN3uwMWoHR3UeX0lLQ248UbgnnuoEwcP9j5vSwuNwZFHskupJZ+V5FJXMBqrZt6AjNY6lG+qx9YdaXAVDoaBIG/3Cgwb2rUPkaULv9TFVgO7aJGVRffRLbfQR+zvDWts5BYo2iM5hw9nTKCpyXcMwVoNuFwMund2afk6VgmN6mpvgkEgItU7K0EJxWV0qjHmLmPMessYAIAxptoYs8AYcz6AZ6MnotKd4mIq4IoK/8c0N9Pl7XcQfZiMGwfccQcnn5WX87Z7N5/nwgtpMKzvU04OF9id9XhbZj6yx4/CQcdQuIwNHU6Bw+7C+OzyLs/T1ESDkpXlucPp9BkcdbkivHM/4wyutnft6tJ1FcbQr3/oEHD99d5RbtHCbmfe7oEDvl9gTg6tcHa2/ziDRSL3Yko08vO7Xnd/GNPpw5l6BN0hGGM6RGQCgBEAPjTGfBklFJHTjTGLjTEhvJNKJLnoIuCuu7iDLSrqutNtaqKx+MEPIqu/Ro7kOWtquKCyujz4SmY56yxm7+3dy9kHNhsXvOPGMWbqcABnT6jFoOZ9aEwfDoD1Ym43O1Z8SVUVm7V5Hl+1ijVqe/fynFOncmcyaVIfMwKzsoAf/5g+4v/8hxYHoECHHQZcfDF/xoLZs9m8bulSXlzLndXeTpdRXh6LTPxtD+vr6UYaPz428qYCRx/Nax/IH9vUxN3b6NGxlS2GiAmyrRSRGwF8H8BmANMA3GSMednz2FpjzNFRl9IPZWVlZvXq1fF6+rhTXs7OCpar2EpFz89nxXC8B4nV1ABPPMEaKwur84TdDkzK2olzN/waNVkj0OZOQ0YGe9oVFnoO7uig5v/1r9EwaDTuu496ctAg6kRr8d7YyPjFJZdEqJ9bUxO3P04nhRk2LPb55243M1oWLfJaP4eD7q0jjmDRWnZ2z37j9fV843/8Y27nlNAwhm1SNm/myqc7Lhe/aFddBZx8cszFizQissYY00NDhGIQNgA4zhjTKCKlAF4A8KQx5gER+dgYMz0aAodCfzcIAD/HO3eywaXLRd01aVL0k2HCobLSK19hIXvYHTzIwltZ/AYmfPw08scMRNG4AjjSxFtdV10NXHopzLzTcd99TKHt3m0C4Hm/+AK48kr2UkoprPfC5WJQ09ry7djB1cCBA11XtYMGUWmpMQifhgb4XHVUVdE3evrp3CmmQHFaXwzCRmPM5E5/54BGYROAU4wx0yItbKioQUgBjOEW4qWXuCq32bg6Li0Fzj4bmDYN5eXAz3/Onbq/72JzMz0qv/99YhnDqOJ2M7V0927+PmIE/W3B8pEV/7S1eXtn7d3LD9y0aWzlO2FCShgDwL9BCOWrUyEi04wx6wDAs1M4E8A/AUQ5B09JeURY6DBtGrcSVjR88OAvv3zr1gVOzQf4LwcPcqcQK1d/3LHZ6D6KVNMqhTuwE07gze0OnoqaYoRiEC4H0KUBgjHGCeByEXk4KlIp/Q8R72SdbtTVhdaFQSR4AZ2ihEw/HDAUSpZReYDHVkRWHEXpycCBoWUEut2+U/eTjpoaNrBra6Mfe+LE4POlFSUC9Bdvq5LEHH00m/kFyghsbGTAOhKFeHGjuZkjPf/73641CJmZnAw3b16/cl8osUcNgpKQGMPEGrudhWqzZnFYmK9WM04n6y6uuy6Jd/ltbcxw2b6daY+dA8NtbcBTT9HqnX++GgUlaoRsEEREAFwKYKwx5k4RKQEw1BgTsX7AInI6gAcA2AH8wxjzu0idW0kO6uuZjvrGG8y2dDhYmzB7NvXhhg3MvszP5yL64EFmF11wAXD88fGWvg+sWEE30ZgxPRV+RgZTrF57jZbRV568okSAcHYIDwJwAzgFwJ0AGgAsADAjEoJ4Oqj+FcBpAMoBrBKRV4wxmyJxfiXxOXCAozhratieo7SUu4Q1a6gvL7mE2X9vvslsIrud/dxOOSVwSmrC43azAK1TZlUPHA7eli4FLr00tvIp/YZwDMKxxpijReRjADDG1IhICN2gQmYmgO3GmB0AICLPADgbrHdQUpyODhaKtrV17Qxgt7PYrr2d7vWf/MTbtTplaGxkEV5JSeDjCgpYSasoUSIcj2uHZxVvAEBEisEdQ6QYAWBPp7/LPfd1QUSuFZHVIrK6yhofqCQcLhdX+jU13rZAgfj0U5YhFBf7fjw9nQk3r70WWTkVRfESzg7hTwAWAhgsIncD+AaAn0dFqgAYYx4B8AjASuVYP78SmNZW4P336QGx5ifk5wPz5/seMWCxfHnwlNHCQg60qqvjOVOGnBy2Smhs7DnGszO1tZx3kAg0NbGid8MGWvxx4+i/+7IRVQRoaGDDLrebNSrBursqfSYkg+AJKC8DsAbAXAAC4BxjTCT3r3sBdO5UM9Jzn5IkNDXR7fPZZ/z+Wh6Qpibg3/9mltAPf+hb8dfXB+/MahWNtrSkmEGw2YCvfQ147DG+Ob7iCE4nFe9XvhJ7+bqzZg3wyCP07+XkUP5PPgEWLGB67Fln9S2gU1fHc63oVObkdnMWxYUXssWuEhVCchkZNjxaZIzZYoz5qzHmLxE2BgCwCsDhIjLGE5u4GMArEX4OJYo8/TQb7Y0d21XpDxjA+3buZPakLwYNCl5l7HYzHbUvxWetrRxt0NDQ+3NEheOPZ6+c3bt7+tja2thp88wz459htHEj8Oc/0yKXlnLVPmgQrf+IEcALLzBFrLfU1gK/+Q23jMOGsZuhdduyBbjzTmDPnuDnUXpFOC6jtSIywxizKhqCGGOcInIDgDfBtNN/GmMiMAuADNAAACAASURBVCJeiQU1NaynCjSTZcQIppRecEHPrs2zZtHN1NLCZJqiop7upaoqtjzKzQ1fvv37mZ20YgX1rTEcF/C1rwFTpiRAhlJGBnDzzbSqy5d7C9NEOKvhW9/yjqSLF8ZQvvx8No/qjsNBw7BwIXDSSb2z3M8+y1zi7gF2m40FKYcOAQ89xGEgSVt0kriElWUE4FIR2QWgCXQbGWNMhGZyAcaYRQAWRep8SuzYupU6LFCjTbudOmXrVhoAgP+zZAl1yJ49fCw7m3qwpITKOi2NhqKlhQo8XLZtYxdUY+jKcjj4+969wL33starr16OsLDmjhrDKLo1Azkri62rzzuPb0R7O63fxInRn9QWCrt3800LlA2VlsaUsY8/Bk48Mbzz19TQrxhoVVFYyN3S9u3a1C8KhGMQ5kVNiiTDGCqvigqv4go02rY/0N4e/rHG0FX8yiv0hMybxx1EfT31ys6d1BFjx9KYfO97jF2GQ1MT8MADdHVbehfgdSss5GJ3wQKed8qUEE/qcvHEDofvlbI/GhuZJvXee4wJWMyaRd+7lWJVUAAce2zo540VNTXB284CvHgHDoR/fmvSU7D23SK08moQIk7IBsEYsyuagiQLO3dyCtjOnd4gpzGcXfytb/lPm0x1OivbQIh4jy0vB15/na5oSwfMnk1d8vnn3krkuXPZlsJPM9SArFrFFkH+rovDQXkWLQrBINTVsTBsyRKe1BhO+znjDAY8AynKhgZW3e3Zw6CoNYDa5eKqeP164Kc/pd88UbG2VsFwuXrXjM9qNx0Mmy20XGYlbMJpXXGHr/uNMXdGTpzEZvt24He/486+c2Ws2816obvvBn72s/5pFCZO5GK5tdW/Lmht5Xs3cSL/XrqUi8nOC0KHg7sFK3ba0MDFeG/f0xUrgmckFRby+gXM+qys5MWvqfGmQBpD188f/kD//qWX+vdrP/cc3S3du+/Z7XyxlZX0jf/qVwkQ0PDDmDGUt6PDfz9yY/iFsC5yOBQXewM8gd4Dp5Nb8i1b+OHIzuYQjFB6pCsBCcdl1NTp90wAZ4JzlvsFLhfw8MNUGAUFXR+z2bjo27eP1bQ33RQfGeNJRgZd348/TmPZfWqZ00mPwDHHMLvS7WbsNFhqeW4uXdfNzYFT9P3R0hJcT4jwGvp1e7nd9Du1tHQto7b8TgMHctcwejS3ON2pqwsecS8u5htkpWklIgMGMFj89tv+28pWVfF96M1rGDmSRufgQf/1DM3NDCw/8QSzryyys5mF9dWvarC5D4TjMvpD579F5PdgRlC/YOtWftYDtVceOpTTvQ4e7J81NHPncpX98sv8TlqGs7aWC2mnkxXJVvLJxo3UtdOn++7pFgmGDgU2bQqc8NLRwYWv32O2bvW9urew2bhife01BlK7K6SdO7nqDeQbt/yPW7dGxiBYw7bffZd1A04nUzdPP52pWum97Dpz/vlsJPXZZ3RvWdvBjg76+nJyGOzpzcUUod/1N7+hFe/uh2xpoWEdMICPZWV5H2ttZU5zZSVw2WWJu8tKcPrS/jobLB7rF+zYEXzhYcXb9uwJbBA6Orjbra/n9/KII1Kj0EqEsdFjj+Xqf5OnC9XgwXytY8d2/Q6PG8dYwbp11JWdF98WDQ18L8OJ3XZmzhwW1AbyQlRUACefHCCR56OPgitQayuzb1/PWoHqan4oNm3iqjYjg2/GqFFd3xC7PbzovD+MoXFasIByFxfzw3noEPDXvzLu8T//07stV1YW8KMfAW+9xTzeqiqvMTvpJKaB9aVaedw44JZbgAcfpOFJT+e529poEHJygLKynlvQzEyuKt5+m49PmtR7Gfox4cQQNsDTxwisEygGcFc0hEpEgrk1Lawgs79zLF3K2p2mJu85RViAevHFXfVDsjJsGGsNAO4AbruNmVjdX9vo0TS0AwZw59B9DIAx1DdXXtl7L8CECdR/O3bw/N2vYV0dn/O00wKcpLExtBW1zdbVjQFwJf3oo1RuhYV8UYcOccuUnc1WrVaAxOmMTFD5ww8Zs+juuxs4kNu2HTtYaXzzzb1bSWdlMU/39NO5Ine7+doiNa5uwgTGZTZu5I7J5eLu7KOPuJLqbgwsbDYajLfeUoPQS8LZIZzZ6XcngArPbOV+wahRwRMbrKEuQ4f6fvy11/g9HT686w7C5QKWLaOO+NGPEiPlPFJ8/jldaL5W/3l5VNZbt/K9q6z06kOXi1lIEyb0bc6B3Q7ceCPwpz9RN2dlUW91dHDXkpPDBWnADKYhQ5hXHwjr4neumquupmIrLqby//xzCmSz8fjmZuD55+n7HjKEF/7IPpb1uN3ASy9xW+ZLcYrww7x+Pd/gUaN6HhMq6enRq5xOS6Nra9o0731PPBF891FYyP5Koa7glC6Es+663hizy3Pb66ksvidqkiUYkyfzu97U5P+YqioqMF+tVg4c4A7e10rZbuf9W7bQ1ZJKHDrk/3spwoXckUfSU7JjBxfSX3xBl/1XvsJFbF8NZG6ut2321Kk0CMOGsQbsnnuYoBKQY4/l6j1QymV1Na1b54KU5cv5whoaqPxtNiq69HS+KGtF/e67XA1ffHHfZyeXl9MHFqicW4QfutWr+/ZcscbtDr5VFOk6flQJi3B2CKcBuK3bffN93JeSpKVRgdx/Pxd83d2v1dVcIH7zm77/f/lyrz7whQh1yaJF9HunSqJEsNR1ESpky202YQL15YQJPbO5fFFdzZvDwSQef++v3U7j0ytPwsiR9EuvXu17Ek9LC7cb3/9+1/vffZd9fpYu5cXNy+M2sKOj6wVuaOAH6qSTeiFcNyzDE4yMDL5xycS4cTR4gXYJdXXRy1DoBwQ1CCLyPQDXAxgrIus7PZQL4L/REiwRmT6dK9bHHuMq1mbzLkaGD2cnT39V/Z9+2rN/T3dycph5WF8fmjJMBqyEGWs+si+sBJwzzgjdA7F7N9tdfPKJ9zpkZ9Ot/dWv9j6JxiciwNVXU5GvW8cnys31Dn2w2ZhZM2FC1/+rr6el6ujgbiAtjT+t4grAuyNIT+fFHzOmb7JmZ4e2Qm5vD/6BTDTmzWM73UGDfCt8Y3g9Lrss9rKlCKHsEP4PwBsAfgvgJ53ubzDGJNkSo+9Mm0a38KZNTCgRodIbNy4yq/pUW9gMGsTODB995N9dXVFBXRooTb8z27ax6DctjQbEet9bWxmj2bKFcYOIGoWsLBaYbNvGlf+uXVS+c+b4nwOQl0ffV+eLarMxpcxKK+vooGGx2Rhs6atBGDmSW9iGBv9uI2PoAisr69tzxZopU7gq++QTrrw6f+HcbmZyTZ3KqnGlVwQ1CMaYOgB1AC4RkYEADgcL0yAiMMYsi66IiYfDQb93OPG/CROYERfItdvURB3Sm26eicwll1B/7tpF372lqDs66EEpKAC+853QjGFbG7sv5+X1TFO3Mg/Xr+d7fcYZEX4hNhsvZPedgD9OOYVpnoF8Zs3NPF+wOoUQaGgAVq2yocl+DiYu/xuyJmRhyHBHz9jy3r1Urn0JKMcDh4M7saee8gbb7HZvtsesWcDll2vFch8IJ+30OwBuAmsP1gGYBeADAKdER7TUYvZsYPFiLsz8Zc1VVLAup496IeHIywNuv53xkXff9fZ1E2H+/5lncicRCuvXU/H5qxETodF54w2mksZVN5xwAvDMM1y5+sp6aWvjxR4xgtH3QFWPATCGmZbPPUfdmJV5HMpzKzB12UvYkZGJ8ScWY/BQO/3rVrfA7343ObejmZl0351zDt13dXX8gB11VP/sGRNhwgkq3wRgBoCVxpg5IjIBwG+iI1bqMXw48PWvMyNw5MiumTNuN2NlY8cmxkCsaJCbC1x0EdPXKyu9nZ/DTV3/+OPgRWpZWcz4OnAgzovgwkLgl79kj6OKCv7tcNAiNjfTGBx3HEu5Z8wI3Sp24513mJFZUmLtvgQVg89BxxFTMHzzO3D/Zw0ypriQP2UUM5mmT0/+3ObCwsQZJ5pChGMQWo0xrSICEckwxmwRkfFRkywFOfdcKrOXXmJMz/IkiFAfXH554hemNTQwHvDppzRkhx/OOoFQdZnVGLC3dE/Q8YfN1rXDdNwYP54Nnm65hZFwh4OrXCs/uaaGRSn+0tOC0NzMncGoUd1iJiKoLjwc1ScejkOHgKVFBr+4Q5JyU6DEjnAMQrmIFAB4CcBbIlIDQFtih4HNxmHzc+Yw7byujgu18eOTo/fRihXAv/5FRZuTQ0O2YQPw4otscXPGGdH3QowaxdY8gXC5aKx6ueCOPJMnMzVtwQL247YmBdXV0Zd47rm97l3yySc0koEW/IMGATu/EOzdy6detYq7tJwcNhuMVEKEkvyE09zuXM+vvxKR9wDkA1gcFalSnMxMfhGTibVr2e11+PCutVODBlEhPfMMldKpp0ZXjuOOY7ppoDTWigruuBKqP9SQIcD113NHYE1WGj68zxkEe/f6j0lZWLVaDz/McIbNxmvY3s64VmkpcMMNybEoUaJLyOsCId8SkTuMMUvBwPK0YP+nJD9uN0fpFhf7LqS10j+ff56pn5GkqYktJ7Zt44K6uJh1Bl984buVSG0tFeBZZ0VWjogxcCDdRePHRySdLCMjtJYqW7dyNzF6tHfC38iRNAYVFazYrq/vszhKkhOOy+hBAG4wq+hOAA0AFoCBZiWFCdSPyCIjg8bg008jk95eX8/RmkuXUuFZTQNnzfLOP37zTe8MepeLzz9wIJvphVrTkOyMHx+8Dq2ykklMJ5/s26U3ZAgN7LJlzPhS+i/hGIRjjTFHi8jHAGCMqRGRSJb+KAlKTU1ox9ntzO7pK3V1HE5WUcEUUit11OWi/3vDBk6mO/VUYOVKuk0yMlg0OHlylFJNnU5WI65fz3TRUaPol4pzte9hh9H4BZrBsW4ddwOB2iQNGUIDO39+6qU9K6ETjkHoEBE7PC2wRaQY3DEoKY7DEVqw2O2OTHXwM8/QsHRvA2Kl7FdW0h9+xx0xWtHu3An85S/s/ZOeTkGWL6egZ57JnPg4RWWtrhm/+Q0r54cM8Sr09nYW/tlswYt3rVTd5ubUK4xUQiccg/AnAAsBDBaRuwF8A8DPoyKVklCMG0eDEKwfkTF0YfSFmhqu+gP1NCoupo7etavXtVyhs28ftyuZmT19Zk4nI9wA54fGiZEjaRwXLuQOyuqubbez/U9xcXC3knX9dHfQvwmlud2TxpjLABQBuBXAXAAC4BxjTL+Zqdyfyc9nrcGKFf6b9x04wLnqffXd79zJn8GmTQIMNEfdILz0En/6cg05HDQSr71GB30c81yHDuVO4ZJLuNK35nxnZfH28suBB6RZBcwh1cE0NzPt7N13GcXPy+PrnzEjckNylLgQyj73GBEZDuAqABUAngYb3lWISKJkeitR5uKL6TbfubNrJlF7O1fqubnANdf0vQ4hWMaMRUwKz2pr2fI60PQcK+dz5cooCxMaBQUsFhw3zqvcTziB18VfBpjbTYMwf34I12/vXuDnPwf+8Q+vC62ujsV3P/0pPwxK0hKKQXgIwDsAJgBYA2C152b9rvQDBgxg9s5557Faec8ettuoqmIg97jjqDu3bOnbfBIrMBqoHxzA5/A3mS5iWNN9gsUHsrISWhEWFQFXXEHvV01N1/e2qYlGfs4c4Oijg5yosRH4/e+5CigtpfXJzOQWcvRonvjee2lIlaQklG6nfwLwJxH5mzHme9EQQkQuAPArABMBzDTGqKFJQLKzgbPP5kqyqor1AS++yFTTjRt5jNWj6NpruVINl9JSZhbV1fmfCdHSQlmmTOn1SwmNUB3qbnfCd9j8ylf4fr74oneWhzHU5VdcEeJQpo8+orL3l388aBDbc6xYAXztaxF+BUosCCWGIIb4NQbWMX2Q41MA5wF4uA/nUGJEejoXi48/TiXT3Y9fW8s47O23hzCeshsi7PhqzTvo7pJua2PmzDXXRHjegS+ssuzW1sA5my0tXWf/JihTp9KI7t/PXV5GBt2AIQeSlywJHicpLmbr1VAMwr59dLXt2cP3t6yMQkb9wir+CCXL6D0RWQDgZWPMbutOTw3CiQC+DeA9AP/qrRBWcFq081ZS4HYDjz5KY+CrPURBAY/517+Au+4KP64waRIn0z38MPPrMzK8PnCHg6NMZ85kRs3771O5DRzICZSTJwdv5RAy6elM01mwgFbP1wupr2dQNZzhGHHE6pjRK2pqus6M9kVmJgtIAs0/7ugAnnySF89mo9V3Omkc8vM53cgatafElFC+OqeDAeWnRWQMgFpwQI4dwBIA9xtjPo6eiF0RkWsBXAsAJf5SXpSosn07XUaBKpcHDqT3YOfO3n23jzoKuO8+FlVt3kz3xtixXETW1TGuWVXFzJmMDOqgtWuZgnnzzYHH7obFvHksSNu8mZrU6iLndtNatbcDt97aP1a1Awbw9QZKRerooFHwZwyMYa/upUtpZLsfV1vLPhq/+hV9h0pMCSWG0Aq2rXhQRNLA9NMWY0xYkSMReRuArzDgz4wxL4d6HmPMIwAeAYCysrK+uKmUXrJ/f/Cgr7WY3r+/94u9jAzg2GN5s6ivpzupo6Orq8oqptq3j7rkqqt4n3dGQC/JyKCFWbSIrpDWVioxl4v+l298o2/9vJOJk05i/mqghVhlZeA5BXv3cmfgyxgA3F42NzOV95pr+iyyEh5hba6NMR0A9vfmiYwxUe6DqcSKcFxAkfYCrljBHYIvHVxVxdqE8nLuTAYP5mJ2/nwu9Hsd983IYIvqM87gyZ1ObkH6W3vQE04AXn+dASRfRQ3NzVwpnHyy/3OsWEGfXqAI9pAhdB9dckng4gkl4kTK25o0VFdz1QrQAxDnVjRJiTWFzNdUSAur8jWSE8uMYb8dX5MS9+5lEkxmJnV1YyPdS62tHCCzYwdw/QVVcHz0X7b9dLmYBjV7NoUMxXJlZDDBv79SVAT84AfAAw/QKg8eTCvrdNJn53IB3/9+YFfPnj3Bi9esKHdtrRqEGJMQBkFEzgXwZwDFAF4XkXXGmHmRfI6KCrZnXrvW+903hsWVF1zQ/xZ7faG0lCv0Q4f8v29VVdS3gVpQhIvLRR3RPauprY1Dc3JyqJ9cLubXAzQQY0oNml99F1VvPEVdVVDAD8HSpcDbb7NL3iWXaN+GUJg6FbjzTs7tXLaMxsBu5+7h1FODrwBC6dcNMEYTsewAJVR69Y6LyFBjzIFICWGMWQj2SYoKBw4Ad99NxTFypHe36nLRQGzdyu6ZOqM7NEQ45/zuu+kyLi72Gtn2dhantbQAJ57IwHJJSWRcR3a7dyRxZ11RXt61FKB7WcCIfR/h6L3/wua8URg6M90rS24uPwRvvknf0vnn913IRMUYrs63bOG2afBgZkYFG1Dti+HDgcsu49jP1lYq+VCV94wZtN6BVmCNjdzmBctoUiJOb03wIgDB6hoTAmOYIul09ky3s7pn7t/PnPof/zg+MiYjJSXM9HnySfrtAb6PO3dSzx5xBNPWlyxhLcK11/b9+y3CiugPP+x6Lffv7zpCsrmZzw8AYtyYvOl5tBcMQUNL+pdFbV9it3O7s2gRJ++kYqvPgweBRx7hhbIqr10uRtvPO48Blt5YbLs9/N5F06ZxK2el63bHGG7nr7pK53rGgd6+40lTMFBezoraQMpo6FBW2x6I2J6nfzBqFIvP7r6b33Oru+bXv86up5Zrqbwc+O1vGb/pK3PnchfS3u69z+326rOODv5uJcIMrNmB7OZDaE+nL9pndpTDQQX5ySd9FzDRqK3lm797Ny/G6NG8cKWlXKU/9RTw6quxkyczk3UG9fX8wnXuc9LUxGDP8ccztqPEnN4ahL9HVIoosnNn4OAnwMdEvJ02ldAR4Yp7/Xpg+vSe+f8iNLh1dUxQCURzM/XWnj1dFX5nSkuBb3+bRqaykvpk4EC6AxsbqVPKyrwL14zWOhgRuFxccPodRu9wRGa6T6KxeDEt8dChPb8E6ek0EAsXchcRK444AvjlLxmPKC/nBd+9m0b5yiu5ndR4TlzorcvoxYhKEUWs8Yuh0JembP0Zq9FnoLTOYcMYgzz//J5u6+pqpp2//773GmRkAKedxh1H9+PnzKHL6PXXOT0tLY3G4PDDeeucOeaypwPGoLGRriu/rm6Xq3f+9ESmtRV4773AWT9WCugHH3BrFytGjWJGUl0ddzEOB42WGoK4kvIxhECdiztjjMawestnnwV3JaeleYt7O9c1VVZy2ldDA/WBZVRaWzmKYP16xna6n3/8eN4aG7mzWLiQKe7d3dLVgw5Dc3sasgZ0YOxYPxbLypGNere8GHPoEINnwSrzcnJ4EeNBfr7v/idKXEj5GML48VwxNjb6P6aujoajP6eY9wW7PXjlskXn3ZoxwEMPUfmPGtV1h5GZCYwZw67Szz/v/3w5OTTkV18NnH466xF276Z7urwc2L43C+UTT8VXxpQjK9OPkPv2sQlSr5v8JCjhbI21j5iCfhBDsNuByy/3zovtTmMje3ZdfrkmNfSWyZO9ef/+aGujG6jzLmzXLsYQA+3ihg/3NrALhMPBLMh77wUuugiYNYvupp/+FLjwqXOQNesoBok6n6ilhb2gi4oiM90n0SguZjqtv8k4Fo2NSdOcT4kuvXIZGWMejLQg0WT6dCY2PPooXRTWDrq9nSvMm2+mUlN6x4wZwP/9H/Wrv75n+/dzlkLnoO6WLd6Avj8cDu4kduwIPigeYFB7Xo+SxnRW2P73v0wv3e1p2jtgAHsRnXxyalbEpqVx2/T889xu+cJqITtzZmxlUxKSflMKeMwxdBGvX0/lAjAAOWVK/2hUGU0GDAC+8x3gr39lu/zOLmGnkx6Z0aN7Kuq2ttB3ZX0el5mWxuZsX/kKUx6NYc1BqlfDzp3L6sudO1l00/n11tczqHPddalZf6GETYp/G7qSkcHV7IwZ8ZYk9Zg5k37/p5+mK8ha9YuwYvmii3om8QwZEryLgRXvjdj8epvN/yi2VCQri1H5F15gmpeVdmcML8DNN3MLrSgIbWJaKF9Fd7jtsJXQsDoObN7MFXVxMV0niZgheeSRTC3fuZOppA4HW1/7Kki1jk9Pp+vO3y6tvp5Zk937FylhkJ3NINm553J73NHBTIsxYzRwpnQhlB3CPs8tUMTNDkCn1USYgwc5Neyzz7p2HEhLY8eB009PvDioCI1AKDMQsrPpwn/ySaaidq9jaGmhYfnxjxPvdSYlubmhBWJiSGOjN5anrtv4E4pB2GyMCbinFJGYTUzrL1gdB5qa6H/vrBDb2xnEbWsDzjknfjJGgtNO4+t50VPqOGAAd0WNjVQQN9zAXYeSOhjDWN7rr3OxY7MxG/Ckk/h50Hqg+CEmSAK5iGR6pqb16ZhoUFZWZlavXh3rp40JzzzDJpz+hnE5ncy5v/fe1GjdXV3Niuft26kcpkzp2oJCSQ2MofF/+WUmHwwaxMVORwd72jkcwG23+U+KUiKDiKwxxpR1vz/UEZr+TnqlMeaxeBiDVCbUjgMizKQ866zYyRYtBg3iQDIltfn4Y1agl5Z27VKRlsbW9LW1wP33c6xyZmbcxOy39DWi9OuISKF0IdSOA7m58es4oCjhYgx7Vg0c6L9lUUEBOwesWxdb2RQSSpbRen8PAQixU5ASDqEmfhijSSJK8lBbywy0kiDpJ7m53PnOmhUbuZKKxkbOtWhvp8/t8MMjWksTypmGAJgHoKbb/QLgvxGTRPmSoiJm4ASq/AXYhUE7DijJQns7FzDBMsbS0oK3Qul3tLay4nzp0q61JHl5TNU78cSIpOKFYhBeA5BjjOmxiROR//RZAqUHaWms6tWOA0oqYRVDu1yBu1w3NQETJ8ZGpqSgvR144AH2eulebd7cDPz971wdRiAIF9ThYIy52hiz3M9j3+yzBIpPTj2Vu8Fdu3q2baivZzuIK6/UjgNK8pCdTTdQRYX/Y4xhOrUOTOvEihXAxo30tXV3D2Vns1Xw888HfmNDRD3QCUpmJvDDH3IYzIED7Me2ezcNRHo6Ow4cf3y8pVSU8Jg/n922fXWvtaryJ07kYkgB36zXX2dxhj+XUFoafXErVvT56UIJKq81xgQchhPKMUr4WB0HzjsP+Pxz7hQKCrTjgJK8jBzJyvMHHmA2XUEBF71NTfR+TJzIQWr6+fZQX88inWCR+IICjg8877w+PV0oMYSJATKNAAaXdeRRFMnJSbiOA0osaWqiYkhLY3/vJO/jMWECCypXrWI2UWsrW53MmcNxyzpFsxOhTp4CIjIDOBSDMCGEY4L0rFQUJWz27wfeeINa0xh+4YcNA848k874JF5G5+TQAMyZE29JEpy8PAYKm5sDd7Ssq2Npfx8JpVJ5V+e/ReROz/+tA7DOGLOtz1IoitKVHTtYrut2c9i0NSmovh7429+YcXLFFUltFJQQsNvZxfKZZ/ynHLpc9CdHIBIf9qfJGHMHgAcA1AE4V0SSZpymoiQFbW3s35CV1TXNUITFSGPGsLfJBx/EV04lNpx0EmMI5eU93UIdHRwDO39+RGaCh2wQROQBETovjTEVxpg3jTH3GGOu6asQInKviGwRkfUislBE+tEEE0XpxiefcCfgb5CPzcbBGK++Gp6PWUlOsrOBW29lFeru3TQA1s+qKuDCC4ELLohZYZpFA4BXRORiY0yTiMwDcIcx5oQ+SwG8BeB2Y4xTRO4BcDuA2yJwXkVJPj78MPiM59xcKoWqKu0X3R/IzeVg+IoKTstqbWVHyClTIjotK2SDYIz5uYh8E8B/RKQdQCOAn0RCCGPMkk5/rgTwjUicV1GSEqsMPRDWxKT29tjIpCQGQ4bwFiXCcRnNBXANgCYARQBuNMa8HwWZrgLwRgA5rhWR1SKyuqqqKgpPryhxZsQINjELhDWMOl8zvpXIEU5Q+WcAfmGMORlcwT8rIqeE+s8i8raIfOrjdnanY34GWx6JlAAADjtJREFUwAngKX/nMcY8YowpM8aUFRcXhyG+oiQJJ5zArJFA8YGKCuDYY7V3iRJRwnEZndLp9w0iMh/AAgAhNVAwxpwa6HERuQLAmQDmmmBj3BQllSkpYdfCDz/kJJnuwcL6ehoLnSikRJheN9I2xuz3uJH6jIicDuBWACcZY5ojcU5FSVpEgKuvptJftYoVyjk53DU0NHCu6C23sA+EokSQPk1WMMa0REiOvwDIAPCWJ7N1pTHmugidW1GSj4wM4Prr2c3w/fc5QDsjgzuHo48OPChDUXpJ5Ebt9AFjzGHxlkFREg4RuoxKS+MtidJP0Lp3RVEUBYAaBEVRFMVDQriMFCVROXCALvyNGxnjnTCBPcSGD0/6LtSK0gM1CIriA2M4qGrBAir+gQN5/zvvAG++yYzPb3xDm40qqYUaBEXxwbJlwLPPsiQgLc17f24ui4RffZWZoFoKoKQSur5RlG50dAAvvEC3UGdjYGG3c675yy8DLZFKvFaUBEANgqJ0Y9s2thIKlOqfns6xBRs3xk4uRYk26jJSlG7U1YV2nAhQUxNdWRKGtjZg3ToGUaqr6S876SRgxozgrbqVpEENghJzqqqArVupYwoKgMmTgczMeEvlJT09tOOM6ScFw5WVwO9/z5/5+bxYdXXA448z6v6jH/kf76gkFWoQlJjR0AA88QSwejUnAYrwlp4OnHsuMG9eYqRyHnEEs4ecTv9jCSz5J0yIrWwxp6WFxqChoWvFdFYWjUNtLXDvvcBddwGFhXETU4kMGkNQYkJzM/XK2rUMyI4ZQ/0yejQHPz31FLBwYbylJHl5rDUoL/ffgXrvXqCsDCgqiq1sMWftWu4M/A1lKSjgQJ9ly2IrlxIV1CAoMWHpUo6AHTWqZ+5+RgYNwyuvsBAsEbjoIq7+d+7sOqumuRnYsYOv49vfjp98MeOdd/zPdrYYMgR46y2d75wCqEFQoo7TCSxeDAwd6v8Yh4PpnO9HYwZfL8jMBG6+GbjiCv69ezdv7e3At74F3HZbP4mlVlUFn9mbkUHXko7zTHo0hqBEnbo6uqCtal9/5OcDW7bERqZQyMgATjkFOPlkzqQBWJhmt8dVrNiSk0NFHyjS7nTyTfFVtKEkFbpDUKJOIgSK+4LNRq9JQUE/MwYAgymHDgU+prISOP547eORAugVVKJOfj5vTU2Bj6utZQqqkkAcdxy3Sg0Nvh9vbWVp96kBJ+QqSYIaBCXq2O3A/PmcC+8v7tjRwcdOPDG2silBKCgAbrqJPrPycl4ogA2d9u1jFsB3vsOmT0rSozEEJSbMns3xwNu3cxRwZ9dLSwuwfz8zewYPjp+Mih8mTgTuvBN4912mizmd9AMeeyxw2mlalJZCiEniVLGysjKzevXqeIuhhEhzM/DMM8CKFSzsssjJYSvp2bOTP96Q8jiddBNlZGgQOYkRkTXGmLLu9+sOQYkZ2dnAVVcB55/PBnLt7YwtjB+vuiVpcDj6Sb5t/0QNghJz8vPZE01RlMRCg8qKoigKADUIiqIoigc1CIqiKAoANQiKoiiKBzUIiqIoCoAEMQgicpeIrBeRdSKyRESGx1smRVGU/kZCGAQA9xpjjjTGTAPwGoA74i2QoihKfyMhDIIxpr7TnwMAJG/5tKIoSpKSMIVpInI3gMsB1AGYE+C4awFcCwAl2lBLURQlYsSsl5GIvA3A18ysnxljXu503O0AMo0xvwx2Tu1lpCiKEj5x72VkjAm1YfpTABYBCGoQFEVRlMiREDEEETm8059nA0igQYqKoij9g0SJIfxORMYDcAPYBeC6OMujKIrS70gIg2CMOT/eMiiKovR3EsJlpCiKosQfNQiKoigKADUIiqIoigc1CIqiKAoANQiKoiiKBzUIiqIoCgA1CIqiKIoHNQiKoigKADUIiqIoioeEqFRWoo/TCezYATQ1AVlZwLhxQFpavKVSFCWRUIOQ4hgDLF0KvPQSUFcHiPC+7GzgzDOBr34VsNvjLaWiKImAGoQUxhjgxRdpDIYNA0aP9j7W2go8/TRw4ADw7W8DNnUeKkq/R9VACrNzJ/DqqzQE2dldH8vMBEpLgffeAzZujIt4iqIkGGoQUpj33gPS0wGHn32gzQbk5gJvvhlbuRRFSUzUIKQw69YBhYWBjykq4g7B7Y6NTIqiJC5qEFIYtzu02IAxvCmK0r9Rg5DCjBsH1NYGPqa+Hhg1SjONFEVRg5DSnHYa0NDgf/VvDHDoEDB/fmzlUhQlMVGDkMJMmgSUlQG7dvWMERgD7NkDjB/PYxRFUbQOIYWx24Hvfhd46ing/fd5n8PBqmVjaAiuvJKZSIqiKGoQUpyMDOCqq4Czz2bWUU0NkJcHHHUUMGRIvKVTFCWRUIPQTygsBObOjbcUiqIkMhpDUBRFUQCoQVAURVE8qEFQFEVRAABikrhEVUSqAOyK4lMUATgYxfNHk2SVXeWOPckqe7LKDcRf9tHGmOLudya1QYg2IrLaGJOUWfrJKrvKHXuSVfZklRtIXNnVZaQoiqIAUIOgKIqieFCDEJhH4i1AH0hW2VXu2JOssier3ECCyq4xBEVRFAWA7hAURVEUD2oQFEVRFABqEIIiIneJyHoRWSciS0RkeLxlCgURuVdEtnhkXygiBfGWKVRE5AIR2SgibhFJuNS87ojI6SKyVUS2i8hP4i1PqIjIP0WkUkQ+jbcs4SAio0TkPRHZ5Pmc3BRvmUJBRDJF5CMR+cQj96/jLVN3NIYQBBHJM8bUe36/EcAkY8x1cRYrKCLyVQDvGmOcInIPABhjbouzWCEhIhMBuAE8DODHxpjVcRbJLyJiB7ANwGkAygGsAnCJMWZTXAULARGZDaARwBPGmCnxlidURGQYgGHGmLUikgtgDYBzEv09FxEBMMAY0ygiaQCWA7jJGLMyzqJ9ie4QgmAZAw8DACSFBTXGLDHGOD1/rgQwMp7yhIMxZrMxZmu85QiRmQC2G2N2GGPaATwD4Ow4yxQSxphlAKrjLUe4GGP2G2PWen5vALAZwIj4ShUcQxo9f6Z5bgmlT9QghICI3C0iewBcCuCOeMvTC64C8Ea8hUhRRgDY0+nvciSBckoVRKQUwHQAH8ZXktAQEbuIrANQCeAtY0xCya0GAYCIvC0in/q4nQ0AxpifGWNGAXgKwA3xldZLMLk9x/wMgBOUPWEIRXZFCYSI5ABYAOB/uu3kExZjjMsYMw3csc8UkYRy1emAHADGmFNDPPQpAIsA/DKK4oRMMLlF5AoAZwKYaxIsWBTGe57o7AUwqtPfIz33KVHE44NfAOApY8yL8ZYnXIwxtSLyHoDTASRMUF93CEEQkcM7/Xk2gC3xkiUcROR0ALcCOMsY0xxveVKYVQAOF5ExIpIO4GIAr8RZppTGE5x9FMBmY8wf4y1PqIhIsZXtJyJZYCJCQukTzTIKgogsADAezHrZBeA6Y0zCrwBFZDuADACHPHetTIbsKAAQkXMB/BlAMYBaAOuMMfPiK5V/ROQMAPcDsAP4pzHm7jiLFBIi8jSAk8FWzBUAfmmMeTSuQoWAiJwI4H0AG8DvJQD81BizKH5SBUdEjgTwOPg5sQF4zhhzZ3yl6ooaBEVRFAWAuowURVEUD2oQFEVRFABqEBRFURQPahAURVEUAGoQFEVRFA9qEBRFURQAahCUFEFESkWkxdMnxrqvR1tqEcnytDJvF5GiPj5nlogs9XQ8hYjcKCKbRSSsNiEiUiAi1/dFlhCeo0eraxFJF5FlIqIdCxQAahCU1OJzT58Yqy31XwHMBzAJwCUiMskY0+I5Zl8Enu8qAC8aY1yev68HcJox5tIwz1Pg+d+wEBLqd/hfYJuEL/F0Z30HwEXhPreSmqhBUJIGz1CU0zy//z8R+XOAw2PRlvpSAC975HkIwFgAb4jIzSLyLc8wlHUi8nCnXcRLIrLGMyDlWs95fgdgnOfYez27nc4r+R+LyK88v5d6dj1PgD1wRvl7rs4EaHX9kud1KIoaBCWp+CWAn4nIpWDL4/8JcGxU21J7+haNNcZ8AQCetiD7AMwBsBhcdZ/g2Y244FW6VxljjgFQBuBGESkE8BN4djfGmFtCePrDATxojJkMIDvAc4XCpwBmhHG8ksKo71BJGowxyzyNzX4I4GTLVSMid4HNzmJJEdhnyRdzARwDYBXFRRbY/x6gETjX8/soULkfCPO5d3WashXouYJijHF54im5nmEzSj9GDYKSNIjIVADDAByylJeIDAUnT3Un7LbUIvJ9ANd4/jwDwLmd/zbGdI47tADI9HcqAI8bY27vdv6TAZwK4DhjTLOI/MfPOZzounvvfkxTsOcKkwwArX34fyVFUJeRkhQI5+g+BcYBGj3tvQFgGoB1Pv4l7LbUxpi/etw204wx+7r/3e3YGgB2EfGl0N8B8A0RGeyRfZCIjAaQD6DGYwwmAJjlOb4BQG6n/68AMFhECkUkA5xp4Q9/zxUSHpfVQWNMR6j/o6QuahCUhEdEsgG8COBHxpjNAO6Cd0iRT4PgmSd9A4A3wZm7zxljNkZYtCUATvTx3JsA/BzAEhFZD+AtcGezGIBDRDaDgeSVnuMPAVjhmRh3r0c53wngI8//+u2ZH+C5uuBpdf0BgPEiUi4iV3semgPg9d68eCX10PbXSlIjIo+Cbp0SAK8ZY0IaSSgiXwAoM8Yc7MNzHw3gZmPMZb09R7wRkRcB/MQYsy3esijxR3cISlJjjLnaGOMGs2vyOxem+cIqTAPjDu5Ax4bw3GsBvOcrzTMZ8LjSXlJjoFjoDkFRFEUBoDsERVEUxYMaBEVRFAWAGgRFURTFgxoERVEUBYAaBEVRFMWDGgRFURQFgBoERVEUxYMaBEVRFAUA8P8BBYGhJSJXix0AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SjPlpf2-wHl8" + }, + "source": [ + "## Обучение простейшего одноуровневого персептрона вручную\r\n", + "\r\n", + "Используем возможности tensorflow по вычислению градиента для обучения одноуровневого персептрона.\r\n", + "\r\n", + "Для начала, задаём архитектуру сети, в которой будет 2 входа и один выход. Соответственно, матрица весов $W$ будет иметь размерность $2\\times1$, а вектор сдвига $b$ -- $1$.\r\n", + "\r\n", + "Функция обучение будет такая же, как в прошлом примере, но функция ошибки будет представлять собой логистическую функцию ошибки. Для этого нам нужно получить на выходе сети значение **вероятности** класса 1, т.е. необходимо привести выход сети $z$ к диапазону [0,1] с помощью передаточной функции `sigmoid`: $p=\\sigma(z)$.\r\n", + "Далее, если для примера с номером класса $y_i\\in\\{0,1\\}$ был получен выход сети $p_i$, то ошибка вычисляется как $\\mathcal{L_i}=-(y_i\\log p_i + (1-y_i)log(1-p_i))$. \r\n", + "\r\n", + "В Tensorflow оба эти этапа (применение сигмоиды и взятие логистической функции ошибки) делается одним вызовом `sigmoid_cross_entropy_with_logits`. Поскольку мы делаем обучение по минибатчам, то необходимо усреднить ошибку по всем компонентам минибатча с помощью `reduce_mean`. " + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "kdDxWeCqwHl8" + }, + "source": [ + "W = tf.Variable(tf.random.normal(shape=(2,1)))\n", + "b = tf.Variable(tf.zeros(shape=(1,),dtype=tf.float32))\n", + "\n", + "learning_rate = 0.1\n", + "\n", + "@tf.function\n", + "def train_on_batch(x, y):\n", + " with tf.GradientTape() as tape:\n", + " z = tf.matmul(x, W) + b\n", + " loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=y,logits=z))\n", + " dloss_dw, dloss_db = tape.gradient(loss, [W, b])\n", + " W.assign_sub(learning_rate * dloss_dw)\n", + " b.assign_sub(learning_rate * dloss_db)\n", + " return loss" + ], + "execution_count": 22, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zAAgw0h6KzUd" + }, + "source": [ + "Далее, разбиваем входные данные на минибатчи по 16 элементов, и по-очереди проводим обучение, подстраивая веса $W$ и $b$" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PfyqjVb2wHl8", + "outputId": "308850b8-fe17-4cda-ac27-8bcda210f113" + }, + "source": [ + "# Create a tf.data.Dataset object for easy batched iteration\n", + "dataset = tf.data.Dataset.from_tensor_slices((train_x, train_labels.astype(np.float32)))\n", + "dataset = dataset.batch(16)\n", + "\n", + "for epoch in range(15):\n", + " for step, (x, y) in enumerate(dataset):\n", + " loss = train_on_batch(tf.reshape(x,(-1,2)), tf.reshape(y,(-1,1)))\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 25, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 0.5500\n", + "Epoch 1: last batch loss = 0.5502\n", + "Epoch 2: last batch loss = 0.5504\n", + "Epoch 3: last batch loss = 0.5506\n", + "Epoch 4: last batch loss = 0.5508\n", + "Epoch 5: last batch loss = 0.5510\n", + "Epoch 6: last batch loss = 0.5513\n", + "Epoch 7: last batch loss = 0.5515\n", + "Epoch 8: last batch loss = 0.5516\n", + "Epoch 9: last batch loss = 0.5518\n", + "Epoch 10: last batch loss = 0.5520\n", + "Epoch 11: last batch loss = 0.5522\n", + "Epoch 12: last batch loss = 0.5524\n", + "Epoch 13: last batch loss = 0.5525\n", + "Epoch 14: last batch loss = 0.5527\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s4_Atvn5K4K9" + }, + "source": [ + "Для демонстрации того, как сработало обучение, построим граничную прямую $W\\times x + b = 0.5$" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 283 + }, + "id": "PgRTHttLwHl9", + "outputId": "e4407e1b-edf5-48e5-fdc2-da28120a3c6b" + }, + "source": [ + "plot_dataset(train_x,train_labels,W.numpy(),b.numpy())" + ], + "execution_count": 26, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd3hU1dbG3z2T3gtJgJCQQhFpAlFERJqCYPeq2CufBa9YsNBROna59nLtF70qliuo9A7SkY4ISYA0SkhPJpPZ3x8v46RMz7Qk+/c884hzzpyzZjKz115dSCmhUCgUCoXG2wIoFAqFwjdQCkGhUCgUAJRCUCgUCsU5lEJQKBQKBQClEBQKhUJxDqUQFAqFQgEA8PO2AI2hVatWMiUlxdtiKDzIieITyCvNw/lx5yPYP9jb4igUTZJt27adklLG1X++SSuElJQUbN261dtiKDxIYUUh0uanITUpFT/f/rO3xVEomiRCiCxzzzdphaBoeUQHR2N8//EYv3w81matxYD2A7wtUtPm1Clg40bgyBFAqwW6dwcyMoDQUG9L5hxSAllZwOrVwMGDgEYD9OwJDBgAtG7tbel8HtGUK5UzMjKkshBaHuXV5egwvwPSotOw9r61EEJ4W6Smh8EA/Pgj8NNPgBBAWBifKysDAgKAhx4Cevf2tpSOUVMDfPEFsHIl4O8PREVRQRQW8r3dfDMwYgTfbwtHCLFNSplR/3kVVFa4jaoq4PffgU8+AT7+mJu2srLGXzfEPwTTBk7D+mPrsejPRY2/YEtk0SJg4UIgMRFITgZiYoBWrYD27YHISGD+fGD/fm9L6Rg//AAsX873k5hIKycsDEhKAtq2BRYsANav97aUPo2yEBRuYdcu4L33gPJyICiIm7KKCm7c7rwTuOyyxm3Uqmuqcf7b5yPILwg7H9oJrUbrOuGbOyUlwJNPAgkJ/IOYo7AQiI4Gpk1rGjvqkhLgiSeANm0APwue8LIyQK8HXn6Z7rEWjM9bCEKIJCHESiHEPiHEXiHE496WSeEc+/YBr73GDVpKCl23CQn8d6tWwEcfNX6j5q/1x8zBM7GnYA8W7FngCrFbDjt20L1iSRkAdLdkZgLHj3tMrEbxxx98T5aUAcAv5NmzwOHDnpOrieEzCgGAHsA4KeX5AC4G8KgQ4nwvy6RwECmBL7/kemIuLhkYyE3cf/4D6HSNu9fNXW9Gr9a9MGXlFOhqGnmxlkRurnVlANAq0GhoKTQFzpyxb9cvBFBc7H55mig+oxCklLlSyu3n/l0CYD+ARO9KpXCUzEwgJ4duaEsEB9N9tGdP4+6lERrMGToHmWcz8f629xt3sZZEUBB30/ZgbcftanJygG++oXn51lsMQFVV2ffakBD73pOU3JUozOKTaadCiBQAvQD8bubYgwAeBIDk5GSPyqWwzalT3ITZcjsLAeTlNf5+w9KHYVDKIMxYMwP3XnAvwgLCGn/R5k7XrsD333NxtPSHqq6mMvBE4Wd1NfD558CaNbyncXHfupVB4bFjgY4drV/j/HPOBFvvyd8f6NDBtfI3I3zGQjAihAgD8B2AJ6SUDWw7KeX7UsoMKWVGXFyDQjuFl9Fq+Zu0hV4P/PUXMHs28PjjwHPPMQPy9GnH7ieEwJyhc1BQVoDXN73unNAtjfR0ZuKcOmX+uJTAiRPAkCFcnN3NF18Aq1ZRpnbtmPEUF8eMJz8/4MUXbccy2rQBevWi3OaQkte4/HLPvKcmik8pBCGEP6gMvpRSLvS2PArHMW4orVnv5eXMQlq1ilZCSIgpLX78eGD7dsfueXG7i3H9edfjpQ0v4VS5hUVOYUIIYMwYau9jx6idjVRUAEePAp06Addf735ZcnJoGaSkMGZRn8hIPv/TT7avdd99VAxHj5rym6VkzODIEeCCCzzznpowPqMQBKuLPgKwX0r5qrflUThHTAxw0UX8nZvDYGDdkFYLdOvG8wMDGYBOSmKm45tvMhbhCDMHz0SprhRz181t9HtoEbRuDUydCgwcSK187BiQnU1tfcstwFNPMdbgbjZs4IJvThkYiY+n+6ioiLL+9BPw/vvMXti/n18qAIiIACZMAO64g0ouO5uPoCAW2v3znyy6U1jEZ+oQhBCXAlgLYDeAc39hTJRSLrb0GlWH4JsUFQFz5gAFBdywGRNaamr4+927Fxg5EggPN//6vDx2G3jkEcfue9+P92HB7gX487E/kRSZ1Lg30ZIoK2OWjkbDxddWBpIref117t5jYqyfl5UFdOnCnGatlrsIvZ5B53btgMceq9uawmAASktNVdhNoZbCg/h8HYKUcp2UUkgpe0gpLzj3sKgMFL5LZCQwcSIwaBCVwrFjdN+eOMG1pm9fy8oAoPt461bHq5qfH/g8JCReWP1Co+RvcRjNs8REzyoDgLv32i4rc0gJHDgAbN7MOENSEhVX27ZAaipTY+fOrZsiq9HQYggPV8rAAXxGISiaFxERwN13cwM4fjzw7LPAq69ykxcba/21xnRyRxVC+6j2GJMxBh/v/BgHTh1wTnCFZ+nTh24qa+TkMA7QpYt511J8PI+vWOEeGVsQSiEo3EpoKDMGO3VisVp4uO3Ucilp8Tvjwp44YCJC/EMwecVk5wRWeJaePbl7KCoyf1xK+hhTU60XniUkAMuWMbVU4TRKISg8yqWX0rVrjdOnmVYeEeH49eNC4/B0v6fx3f7vsOXEFueEVHiOgADmHZeV0RKonZ5WWsr4QlQUdxTWCAzkTkNVITcKpRAUHqVHD7qMLKXAV1ezT9lVVzl/j6f6PYVWIa0wYfkE5y+i8Bzp6Wyil5FBpWDMeAKYSnrxxbaLW6Tkw5OV1c0Q9ekpPEpgIDMa583jbz4+3tRJoaCAm7w77jAVnjpDeGA4Jg+YjCd+ewLLjizD5WmXu+4NKNxDYiLw4IPAbbexAZ2fH78cWi13CV99Zb0fSnExs42cMSsVf6MsBIXHSUwEpk8HbryRXoGsLGYgXXABMHkyMGxY4+/xcMbDSI5Mxvhl4+ErqdUKOwgPZxZRmzammEHfvvx3RYX510hJk/Oqq1RGUSNRFoLCK0RFAVdfzd9wVRWzHV3Zoj7QLxDTB03HvT/ei+/2f4ebzr/JdRdvSRiLR5YtY+5wQAAX6P792cvcE0RFsbDszTdpAcTEmBb+8nJ2bx0wgBWRikbhM4VpzqAK0xTWqDHUoMe7PaA36LF3zF74adT+xyHKyrgI79vH4q6wMCoIYxHb6NH073uKAweA777jPAOtlqloERHcVQwZ0uKH3jiCpcI09QtRNFu0Gi1mD5mN67++Hp/s/ASje4/2tkhNBynZHuLgQfYZqu2KCQuj++add9hrpHNnz8h03nmseCwoYJpqQADjBiqQ7DJUDEHRrLm287W4uN3FeH7V86iotuCDVjQkK4sdCJOSzPvlg4Pp7//xR8/KJQRrDjp1oqJyVBlIyeZ3q1ezqVbtXkgKZSEoPI+Upnb71nqauQIhBOYOnYtBnw7CW1vewtOXPO3eGzYXNm7kH8hakDY2lgvq6dO2y899gWPHOL81M7Pu3IRWrZje2rWrV8XzBZRCUHiMwkJg7VpgyRK6p7Va4JJLgKFD2freXQxMGYgrO1yJ2WtnY3Tv0YgKinLfzZoLBQW0AqxhHLNZUuL7CuHECWDWLH7p2revq+iKi4GXXwbGjWML3haMchkpPEJ2NjBlCgd1hYXxN9m6NbBpE2uS1q517/1nD5mNwspCvLzhZffeqLkQHm67DYSxx0hTGEn55ZdUAnFxDa0eY+bShx/abrTXzFEKQeF2KiqAV17hZrJ9e1OPIj8/Nqxs04aW/F9/uU+GXm164dZut+K1Ta8hr9QFszubOxddBFRWWj+nuJh/vNptp32R3FxmSsXHWz4nPJwFcfv3e04uH0QpBIXb2baNa4ellveBgVQSv/3mXjlmDJ4BXY0OM9fMdO4Cej1THvfsYdC1Cads26RLF2rrggLzx2tqWAx2zTW+XwyWm8vdiC05tVr+XVswKoagcDtr1tjuKGCcgVBZ6b5BXR1iOmB0r9F4b9t7eKrfU0iLTrPvhQYDC7MWLaK/XAg+17o18I9/sAdPc0OrZdO5efO4SMbHM6YgJYPIxcXM//dkHYK7ac4K3k6UhaBwO8XFtt3MxpoiW62xG8uUgVPgr/HH1JVT7XuBwQB88gnw+ed8E8YBLcnJFHb+fEbJmyMJCQzw/OMfrAjOyuKjfXvgmWeAUaN83zoA2CvFGO+whsHANtstGGUhKNxObCyDyiEhls/R62nV20psaSxtw9vi8b6PY976eXjmkmfQs3VP6y/YtQtYtYoLRe0cWSFo9gQFAQsWAN2705/e3IiMZI+RkSMZDPLzaxpB5NokJDB76M8/Lcc7ior4RT3vPM/K5mMoC0HhdgYNoqfFGvn5bI/jiaLTZ/s/i8igSExaMcn2yb/8wkXRUsFEQACPrV7tWiF9DY2G046amjIwcscd/HLl59d1DUnJfOiiInZbbeHtL5RCULidnj25McvPN3/8+HEmgSxZAtx/P/Dcc9yU25qs6CzRwdEY3388Fv25CGuzrOS7VlUBhw7ZHgAfGwvs2OFaIRWupXVrYNIkuruys1mcZnSBhYUBEyZ4rgWHD6Oa2yk8wsmTrP3Jz2fzytBQrre7dvE32aOHaWRuaSnjlm3a0FVtaz12hvLqcnSY3wFp0WlYe99aCHO+8IoKYMwY21VzlZXM2X/lFdcLqnA9J05QKUhJd1JaWtOIhbgQS83tlIWg8AhxccDzzwP/93/sh1ZUxIxGvZ4u6m7daK0LwZTwlBQqhX/9yz2tZkL8QzBt4DSsP7Yei/5cZP6koCAKW1Zm/WJFRRRY0TRITAT69WOZfHp6i1MG1lAKQeExgoMZJ5g2jQt9aqppxro52rZlH7I//3SPPPf3uh8dYjpgwvIJqDHUNDxBCGDECMu5+AB3meXlwOVqKpui6aMUgsIr5OfTarc2FRFgzPb33xtxIykBnc5sjrm/1h8zB8/EnoI9WLBngfnX9+/PFssnTjS8hsFAf1dGhsP+5+pqhh2+/x744Qdg9+4W3zVB4QOotFOFV6ioMLmIrBEYyI4CDnPyJDN/li2jj9/fn+lOgwfT9DjHzV1vxrz18zBl5RTc0vUWBGgD6l4nJISBjHff5YAWjYZayqhkLr0UuOsuh9q2bt8O/PvfjJX4+fEyNTWMrTz8cIvPfFR4EaUQFF4hNJSLYO0uxOaorHSikeahQwzw6vWssA0M5JZ85Uo+Hn0U6NULAKARGswZOgdXfnkl3t/2Pv550T8bXi8qiqlP2dnc1hu7e2ZkWO+PY4adO4E33uDL6k+gLC4GXnyRCS8dOzr4nhUKF6BcRgqvEBfH5I7CQsvnGOcmONQd4exZ4LXXqHGSkkx58/7+dP3ExABvvcX+NucYlj4Mg1IGYcaaGSjVlZq/rhDMNrr+eloEI0c6rAz0ehY9x8VRvPpERDCg/umnqouCwjsohaDwCkIA111HhaDTmT/nxAm6T9LsbDkEANiwgfmsliLVxnLpVatqySIwZ+gcFJQV4PVNrztwM8c4cIAJSWFhls+JimJdRmam28RQKCyiFILCa/ToAdx7Lzfrx49zHdfrqSSOHuWGfswYB7MCV62y7WNKSOB5tbbhF7e7GNefdz1e2vASTpWfcuLd2CYvz/bO3zhzxlIRn0LhTpRCUHiVoUOBGTMY7y0tZSw4NpZu/okTbXdJbUBJCYO+1vD3p/apqZtqOnPwTJTqSjF33VwHb2ofGo19riAp3T9aVKEwh08FlYUQ/wZwNYACKWXLnmXXgmjXDrjzTj7qY1xA7bYSYmJMWUWWqKqi66he35qu8V1xd8+78ebmN/F438eRFJlk503tIyWF78NaIF1KPpKTXXprhcIufG0f8gmAK70thMK7VFdzNsKMGcADD7C6+Y032O/I5g576FDgzBnr5xQUAFdcYXZVfn7g85CQeGH1C86/AQukpjIufcqKRyovj41TfX0ImaJ54lMKQUq5BoCNX7OiOVNRAbz+OscMnDpF66FNG+DgQc5q+fxzG60sLrqI1W6nT5s/XlRE6+Gyy8webh/VHmMyxuDjnR/jwKkDjX9DtRACGD2a8ufl1X0fBgOD6EFBTGJSKLyBTykEheLzz2kJpKayjZBGQ89OfDx318uWAUuXWrlAWBjw9NN8UWYmk/urqxmgyMqiu+jpp60GnicOmIgQ/xBMXjHZ5e8vKQmYPBno0IFlDcbHsWO0DCZPdjibVaFwGT7X7VQIkQLgZ0sxBCHEgwAeBIDk5OQ+WS18Bmpz4uRJ4NlnuWhaCqpWVLB10KuvWg8ToLQU2LIFWL6caUsREaxS7tvXdr8MAC+segHPr34em0dvxoWJFzr3hmyQn09LQQj2W3O4AE+hcBJL3U6bnEKojWp/3bxYtgz48kvb3aazsqg4unRxnywlVSVIm5+Gngk9sezuZe67kULhBVT7a4XPU1xs38Q0Idw3PMdIeGA4Jg+YjOVHl2PZEaUQFC0Dn1IIQogFADYC6CyEOC6EeMDbMik8R2Qk3f32YK71g6t5OONhJEcmY/yy8fA1S1qhcAc+pRCklLdJKdtIKf2llO2klB95WyaF5+jZk7t/a1lEFRVUBunp7pcn0C8Q0wdNx7bcbfhu/3fuv6FC4WV8SiEomheFhWxJYasswEirVhw/YJxuaCSk/BQ6Hfwf+mx6C51Wv487O2+Bv6HKPULX484ed+L8uPMxecVk6A1qYIGieeNTlcqK5sGBA8BPPwH79zP7s6aG7Zyvuw7o2uYMkJPDE9u2bTAw+c47GUvYuROICq/BJbnfoeNfv6CySqBchmBAkgEdtmwADoQBjz3m9sHoWo0Ws4bMwg1f34BPd36KB3orL6ai+eJzWUaOoLKMfI/Vq4GPPmKWZ2ysqVWD7sRJdP7jG1wRvRWJ7c5VCBsMQJ8+wKhR7Al9Dr0e2LMHOPLiN2iz5SecCU9BfBst0tKoP4QAtUZxMTBliu20pEYipUS/j/rhRMkJHPrnIQT7B7v1fgqFu1FZRgq3c+IE+/0nJtL9Y+wMEVaWj+t2z8D5uh3YmtcOZ8OTWGyQlATs2sUeFbXae/r5ARckn8GNQb+g360puOpaLS680KRgAFDj+PsDCxe6/X0JITD38rk4Xnwcb2952+33Uyi8hVIICpexahVdRMaZNAAAKdFr58fQ1uhQHpUIjZ8GR4+eO6bRUHvodMDHH9cNHBgHKddrQFeHuDjgjz8st6lwIYNSBmF4+nDMXjcbRZVFbr+fQuENlEJQuIyNG+t4fgAA4SU5iD11AGWhCQCYIXT8eL0XJiQw8GCMLQDs5RAUZP2GGg0f9katG8nsobNxpuIMXt7wskfup1B4GqUQFC5Dp2tYWBZ1NpP/OOfrEcI0S/lvjH6g2mPCAgMbzCswi5T2VbPZorycaVFVlrOXerfpjVFdR+HVTa8iv9RHJticOcPPLSfHRtc/hcI2KstI4TJat+Z8mtqtgoQ0wE9fiZhThxCgK0WV9EdgeGsItAJQr/107QXtgguAlSut37CyEggOptvJWQ4eBH75hbEMIeiiGjCA7bHbtGlw+ozBM/Dtvm8xc81M/Gvkv5y/b2P56y/ghx8Yfddq+dnFxQFXX83cXTVhR+EE6lujcBnDh9f13mhqqpGUvQ5tc7Yi7tR+hJfmIvJMJroXrQdWrKD2MCJEXX9T165MKSosNH8zKTl7c/hw2xPSLLF0KTBrFnDoEAPcycl0X61ZA0ydSjdWPTrGdsTo3qPx3rb3cKTwiHP3bSw7dwIzZ1IpJCWxR3hyMtOz3n8f+OIL+0azKRT1UApB4RBSMpto9262qa7dU6hPH65NOTk8sefOTxF3ch/KQ+Kg9wtCGUJRExKOkLZR3N2vW8fS4+JiphB16mS6mJ8f6wwqK3nB2u6jigoOXe7WDbjSyXlKf/7JhbNdO/abNu6o/fz4XEQEp/IUFzd46dSBU6HVaDFt1TTn7t0YioqAt99mGldtuQEgPJx9w5cuBbZt87xsiiaPchkp7ObwYeCrr7gxFcI0EH7QIOCGGziV8umnOdymeHc24g6tQ15UKooiwpCStxGBIQJtkwOYOBQaysVt714qg6efbujmSE0Fpk1jldvmzaahxCEhwC23AMOGOW8d/Porr2Pu9VLy+dxcRsqHD69zuG14Wzze93G8uP5FPHPJM+iR0MM5GZzh999pCVhq5qTR0LJatIga2u7ZowqFE4VpQohQAJVSSjsifu5FFaZ5jj17OIMgNLRWcRjYjC4nh2v3M8/QpW8wALkvfY7in9fgdGAigoOB1KBcRGXugKZaZ1r4q6t58scfc9KZNYqK6I/SaunbtzoMwQZVVcDDDzccvGA0fw4fBs6e5cIbFARMnAgMGQJERf19amFFIdLmp+HS5Evxv9v+57wstqip4aAIvZ4Tg157jWm21mY6SMn+H2+8QUtHoaiHpcI0mxaCEEID4FYAdwC4EEAVgEAhxCkAiwC8J6U87GJ5FXZy9iywaRPdyno9J3ENGMCuEK7aHFZWAu+8Q0UQFlb3mL8/C4WPHGFs9sYbz5UX6DKReGE4yrRcm3bktIFBG482/vlICi9ESLDEaUM0KoqrcehgCtJj2LDOosyRkXYNtrELnc5k3hiRkh/i0aPUapGR/ECrq4Gff2ZcYfz4v4cdRwdH47n+z2HC8glYl70OlyZf6hrZjOj1vOeiRYyjGGXNz6dLy9pnYXxvetV7SeEY9riMVgJYBmACgD1SSgMACCFiAAwGME8I8b2U8gv3iakwx/r13FzX1HB90Gg4ZOa33zhr/rbbrNd12cuuXUBZWcMag9q0aQMsWQJcddW5wjR/f5zIrsHWv3g8OBgQWi0yq9pi+59todMBMdES8VXH8PMvWpQs54Z9zBizyT2uJTiYLiGdzuQyysykMoiOrmv+REYyYFtQQF/YjBl/f6hj+47F/N/nY/yy8Vh731oIV2lgvZ4aePNmBrmTk03PHzxIn92wYdTOubm0GKSkxm7blv/292+ovRX2U1PDqXtaLc3iFuJ6s0chXC6lbNClXkp5BsB3AL4TQjTCflc4wx9/AO+9x99/7fqtiAh+l3/7jQvzzTc3/l67d3MNtUZgINfX3FwgJQU4GpuBYzu+RGib6DplAsXF/J0JAaC0BDIxHgmpIeietwlxy1Zj9+KzCB8ZhbCrB7Eftq3iNGfw8+M4zSVLqIUMBmYahYXV/eHrdEBaGv8dH89RbYcO/T2qLcQ/BFMHTsUjix7B4j8X46pOV7lGvuXLqQzS0urK4+fHdNxly/gw1moYP+CsLH4xEhKA++5zPr7SkikpYUOu335jxoTBQIU8ciSQkeGaHZYPYzPLSEpZLYQ4TwgxVAhRZ8shhLjSeI67BFQ0RErg66+5ITS3Xmq1dOP88gtd741Fr7cvrd04y0BK4KsjF0Eb7I8gaUpD0uvpDg8KAoICJfzPnsTB6H4YsnIyLtz2LtoYTsBQXYPMDSeAd99l47rc3Ma/AXMMGcJd9NmzXAQqK+suoCUl1K61J977+wP1YlYP9HoA6dHpmLB8AgzSBYVhej2weDHNJHO70pgYPrKyaMFERVGRhYXx31LSf6diB45z+jQtwG+/ZcKBMRW5qAh4803gww/tK5Zswtj8mQshxgL4EcBjAPYIIa6rdXi2uwRTWCY7m4Fca25kPz8uztu3N/5+qam2R1Yaq49bteLg+IO5Edjd7yGElhUguJwuDWMGZ5CsQNuqo9gTciFOHzgJv+pKFEWloDI4BgFRIfjrTAyqWrfnTV96iSaFq4mL42Dmmhp+oDodH+XlVBJhYUC/fnV3hP7+9J3Vwl/rj5lDZmJ3wW4s2L2g8XLl5FAZhYRYPkdKppgWFtLkqqjgZ3T2LE25wYOB//3P/vFzCn6mb7/NxT8lxWQSC0FFm5ZGH+2SJV4V093YU4fwfwD6SCmvBzAIwBQhxOPnjrUMx5qPUVRkSvu0RkAAXd+NpW9f3sva+pKXB1x8MTemJSVcR/MS+2DdgIkoCW+DyKJshBYeQ9uabATKSiyLvx0rQq5GSYUW5aGm4ITREikvBxftwkK6T9xBWhowbx5wzz10v2g0jCFcfDEwcGBDP1llZV2L4Ry3dL0FF7S+AFNWToGuRtc4mfR663/YwkJ+OImJHDKRlkaZ27alAhs6lC6j0lKm9Crs4+hRxmbOJQ00QAhabYsXN2tFa08MQSOlLAUAKWWmEGIQgG+FEO2hFIJXsDfjUq+3vtG0l6goZg99/TUt6Pr3P3WKyue6c7ZjYKCpUPZ0bCesGzARoaX5OLytCHmn/VEanYQa4Yfw/ZvgF2nez/33mhgbS3/ukCGNfyPmCAkBrr2WebW5ubyfOaSkNXHxxQ0OaYQGc4bOwYgvR+CDbR/g0YsedV6eqCiadgaDeT+d0VTT66mczjvP/HWEcM1uoKWwcyd3MdaUcXAwfZ5ZWUzna4bYYyHkCyEuMP7POeVwNYBWALq7SzCFZVJTuSjrrGxGpeSa0q2ba+551VXA7bcz69HYS+3YMf47PByYMIEbU4BZkVFRdT09ZWEJ0Kd1QpYmFTXCjwJWVyM9sm7rar2ev8u/E2RCQjzS3hr/+AdNm4qKhsek5CJw8cXciZthePpwDGw/EDPWzECprhEurpgYoEcPLjzmMC5YUvKDtoQx00hhH6Wl9n1eQlhtgNjUsUch3A0gr/YTUkq9lPJuAJe5RSqFVYKCWDx74oTlljX5+czrT0lxzT2FAEaMAF5/HXjgAW7Yr7qKimDWrLprk1bLTXd+ft1+da1bM7ah0wGlZQIJQcVIDqjz1UJpKeX+OzNJp3NPplF9OncGHn2UxW9ZWfTNl5XRasjKYtXvffdZfLkQAnOGzkF+WT7e2PRG42S54Qa+79q9noxERvL59u0tp5UavxTp6Y2ToyURF2d9hwWYdlnNOGBv02Ukpazfvb72sfWuFUdhL1dfzd35jh3cmRvXhqoq+vNjYoBHHnF9+nRYGAvfbHHZZYzVLlvGNSwmhhuwbt3YwigyEujXqxJhOQUoiWwHg4FrcFRUPWv85MkGrSPOnqWbSqvlhr3OQHCVzBwAACAASURBVJ7GcOGF9Mtv3Ahs2cIFIiODvTk6dLD5YfZL6ofrOl+HFze8iIczHkZsiAX3ky1SUlj2PX8+raOwML7ZsjKaUP36WX/TBQWUNynJufu3RC68EPjvfy276gAG79q1s26ZNXHUTOUmTHU1sHYt41xnznC98vdn5+bLL3ddYa+zSMmitsWL2UsO4OaqSxcqs8qjubhxx2SUBraC3j8Y7dvz2N/ZnxUVXBBnzgRat0ZuLjs+b9liWpuN73fkSNu1Ep5gb8FedH+nO8b1G4eXhr3UuItVVFDjb91K5dS+PVtbBwcDc+ZQK7ZtazKnampo0YSGst2G0YensI9PPmHL9ZSUhsq/qoom+bhxrI9p4lhqXaEUQjPAYGDyicFAJeCL9UiVlaYgt7FH3fHjQPnqzYj79h2ExwTAPzGeO+GaGu5ydTq6cTIykJ3NNbCmhuucMRu0qorxjPR09sdzRRC9sdz7w734as9XODz2MNpFuGk3WVQE/Pgj21sYf8NS0nq44Qbm/yocQ6cD/v1vWohBQdy9GH9cQgB3383ss2ZAoxWCYF3+HQDSpJTThRDJAFpLKd2UE2gbpRCaCZmZzO/+/XcuakIwgDtsGNC+PQwGthGqqLCcBJSZSc/Sbbd5UnDzZJ7NROc3O+PuHnfjg2s/cO/NSktpFUhJTelts7CpIyXTT1etYiqqnx/Quzcts2akZF2hEN4BYAAwRErZRQgRDWCJlPJC14pqP0ohNDOqqrjqBwfX8ZHv389yAWsB8upqhhtef903rIQnfn0C/9r8L+wdsxfntbKQGqpQeAlLCsGRATl9pZSPAqgEACllIQAfdE4omiyBgYwq1wuY7t1re2yyvz9dUllZbpTPASYOmIgQ/xBMWTnF26IoFHbjiEKoFkJoAUgAEELEgRaDQuFWdDr7eooJ4TutZuJD4zGu3zh8u+9bbDmxxdviKBR24YhCmA/gewDxQohZANZB9TJSeIDERPtTxC3FGLzBU/2eQquQVpi4YqK3RVEo7MKuEZrnAsprAGwDMBRsWXG9lHK/G2VTKAAwpvf554wTWComPX2aJQRun6XgABGBEZg0YBKe/O1JLDuyDJenXW77RUVFnHi0di3bVMTHsz9Rz56+mT6maFY4ElTeLaX0qVYVKqjccli8GFiwwHwvpZISZgZOmuR7xbmV+kp0frMz4kPjsXn0ZutDdPbtYzFaVZWpkq+8nBV7iYnAU0/5lgmkaLK4Iqi8XQjh1owiIcSVQoiDQojDQojx7ryXwrfR602dnQG2zRg1ytRJ4uRJU1+lqioW9vqaMgCAIL8gvDDoBWzN2YqF+xdaPjEnh/OSQ0NZgBYezlz4mBimV506xePNuNOmwvs4YiEcANABQBaAMtBtJKWUPVwiCAPWhwBcAeA4gC0AbpNS7rP0GmUhND9OnwZWrODQMJ2OsYGuXakQzj+fHpXNm00p4t2705visvYVbqDGUIMe7/ZAjaEGe8bsgZ/GjKf200/pJrLWFiEzE3j8caBXL7fJqmgZWLIQ7IohnGO47VMaxUUADkspjwCAEOIrANcBsKgQFM2L7GzgxRfpJWndmi5zKbn4z5vHAtzrr2e9WlNCq9Fi1pBZuOHrG/Dpzk/xQO8H6p6g01EZWOrFbyQsjK0VlEJQuAm7XUZSyixzDxfKkgjgWK3/P37uuToIIR4UQmwVQmw9aalFsKLJUVXFojKNhnECY/xUCDaiTE4Gvv+evZGaItd1vg59E/vi+dXPo6K6Xott4+xeW8UWISGW22IrFC7AboUghJhq7uFO4cwhpXxfSpkhpcyIi4uz/QJFk2DXLgaGY2LMH/fzY83azz9bbvntywghMPfyuThefBxvb3m77kHjRCGDjbIenc5yy2uFwgU4ElQuq/WoATACQIoLZTkBoHa/3nbnnlM0MUpKGAd4+20+Vqww39q/Nhs2MJ5qjehotpk5e9Z1snqSQSmDMDx9OGavm42iyiLTgeBguoFs7f6LithX3FfQ6TglKTu7waxpRdPE7hiClPKV2v8vhHgZwG8ulGULgI5CiFRQEdwK4HYXXl/hAdasAT77jBXDxp5CW7YA//kPcNddlptFlpXZTrMXgi6lpjywavbQ2ejzfh+8vOFlzBgyw3TgyiuBbdu4yJr7IM6epXXQp4/nhLVERQWwdClHm1ZUmAZ8DxjAQR2uSI09c4a7hD/+4JepUyde38LEOoVrcCSoXJ8QcBfvEqSUeiHEP0ElowXwbymlmhLehNi0CfjgAybK1M/6qaoCPvyQz5sZS4z4eG42rQ2jMralCA93ncyepneb3hjVdRRe3fQq/nnRP5EQdm5mQadOwD33UJsGBvID0Wr5wRUUUEk8+6z3O/eVlwOvvsoBF23aMMADME94zRpg+3aO0bMVILfGqlX8HAB2bxWCCujXX9nS9pZbLA+xUTQKR2IIu4UQf5x77AVwEEAjZwXWRUq5WErZSUqZLqWc5cprK9yLXs/CsdatzaeABgby2IIFPLc+l13GmQnW4gMFBRxsZcu1ZAkpOeNk3z66nryV0j9j8AxU6aswc83MugeGDAGmTAEuuICCZmXRMhg5Epgxg8O0vc3ChfzwUlPrjjb18+OEtupq4J13nA/0bN3KmQQJCcwkiIzkLsE4qWzRIuCnn1zzXhQNcMRCuLrWv/UA8qWUZn7aipbIwYMsJGvf3vI5oaF0kx84wFGatUlIoNJYt44bz4SEupZAaSkVyciRzsm3Zw/w7bdcY40DekJDeb1hw+xrnucqOsZ2xOjeo/HetvfwZL8nkRadZjqYns7Zp6NHc3ENDPSscNYoKwNWr7butomL44d85IjjlYIGA8dYxsWZ31VotVQSP//MkYAqwO5yHLG7xtRKNz1xzsUzz22SKZoUp0/bvyk8c8b0b72ea8C4cVz0ja18Fi3i2nPyJGOWpaXAE09wPXCUDRtY31BYyNcnJfG/ISG0WN5/38NdUvV6TO3yMLRCi2nLJ5s/x9+fAvqKMgBYEFJTY7mhFGAK9OxzonzoyBH+wa35BI19zptq/rGP44iFcAWA5+o9N8LMcy2C8nIugkLQ3dvS+47ZSqE3d66UwBdfMAspOZnPp6ezS8ORI2xTsXs3R2MOHOjcMLAzZ+iBaNu2rocDYHJPaioVRq9e5mMbLqWyEli2DPjtN7QtK8PjUR3x4p4FeKa0J3pc/xDzan0Zvb7hrGFzGGMfjlJUZN/1/fzoP1S4HJs/YyHEIwDGAEgTQvxR61A4gA3uEsxXOXOGjdZWrzaljQcGMtY1bJhvDHr3BkbvgMFgOd5n/LyM5x49yvhhSorpNRoNFWx8vOmcoCDnJ0Nu2MD71lcGRoRgUsyiRUDfvjbWo5oa+ruWL6fZEhDAF11yiSm4aomKCgZjDx6kTyw2Fs8hDu/hT0za/yb+t7uAwVhfHtMYHc0P0zjm1BLV1c4Fle3dVdVOYVO4FHtcRv8BcA2An8791/joI6W8w42y+RwFBYztrVjBBSspiY+ICMbajG0XWiIJCdxl5+ZaPic3l32HEs4l1qxaxTXAWsJIfDzwyy+2a7YssX277Y13ZCRw/LiNWomKCjaXe/FFKgWtlimiP/7Igc8bbOyNvv0WOHyYJsm5XUM0gvEc+uPnkONYpztM35UvV90lJzOwa60QxNij3Jn2Gunp/FytRfulNDW4UrgcmwpBSlkkpcyUUt4GoBhAAoD2ALoJIXyoSsa9SAm89x7XhfotmIOC+DvPzAS++cZrInqdu+/mJjI7u+5vurqaz0VHsxbh9Gm6hQ4etL3zDw2lJ8HZuie93naGotHtbTGOICXzaffupTkTH89FPSyMX4a4OODddzn82RylpTQpExMb7KzHoi/ayDCMj90Beeggc299FSGAW2/lH8Tczkev5x/6hhucSwULCeHshxMnLCvG3Fz+2Pbto0X1yCPAc8+xJqKoyPxrFHbjSNrpaHBIzm8AXjj33+fdI5bvkZXFbDvj7tYciYlMxbZVldtciY7mTIIrrmBs8NgxPk6eZAygXz9g1iy2qn72WXpeDh+2Pg3NuCG0x7VsjpQUZj9Zo7KSSt1iLPP4cZoaycnmBTH6tBYuNL+QHTlCbWMm0BICf0zFQKwXx7A45IRlpeIs5eW0aPbuZb/wxtK1K/DYY1x8s7Ko2U+fpiLIyQFuvpmtaZ3lhht4jyNHqEiNVFZyx6XR8O+xYAF3GrGx/Gy/+gqYOJGvUziNI0HlxwFcCGCTlHKwEOI8tKARmgcPmgoyLeHnx/XgyBG6RloikZHAbbexK6kx7hcezg308uWm9HKA68iePfxv//7mMw2Ns2GcrT0YNMgU77FkKeTnU16LgfFNm3jQ2h8/JobFWidPmgIgRqqr+dqKCpNLpVaw6QH0wstyAyZEb8eIinKHUv8sUl5Od9aKFSZ/m8EAdOnCwq6UFOev3acP3WfbtpkqidPSGEux1IzKXgID2eJ77Vr6CrOz+dkFBTFIt2oVLYnoaNNrwsL4OHsWePll7jpqH1fYjSMKoVJKWSmEgBAiUEp5QAjR2W2S+RiVlfYXR5orvKrNmTP8LRUU8LvdsyetYGd3wb5IcLCpJuHTT7lW1n+PHTpwk1lcDOzY0TDLR0puQG+6yfnPJjWV69T69XWD18br5+dzDRs0yMpFCgosR6WNCEGlUVLSUCGcPMmF0+i/MhgYPO7cGYiLgz+0mIkhuC3gOyww7MIduMG5N2ukvBx46SXuqNu2Nfk3peRzM2fSzdKxo/P3CAlhK4kBAxonqzkCAug6GjyYucJScqexbBkVqqU6iKgoKpD169lCQ+EwjiiE40KIKAA/AFgqhCgEh+W0CBISbOeqGxtWWtqc6PXA11/zew3we6/XcyNnrEfy5SQTZyguphvNjPsc4eH0DvzxB70ApaWmWiOdjq7kjAwm8jiLEMD993PjuXo1/z8wkOuKXk8l8eijNmIZERHW/VpGzKUzrVgBfPklj/n78zrl5dzNZmbyzXXsiFt0HTFPxGJK9qe4uWYiArSNyGP+3/947fpVgsYc6aIi4M03uZu2VlPgbTSaun2Rli+3/QOJi+MPTCkEp3CkuZ1x2/K8EGIlgEgAv7pFKh+kRw8u4JZ6jwFc/Nq0Md9hQErulFev5u+0dr2RsaXCvHnsXGCtn09T488/rbf679CBC/Tvv9N9ZNz8+fsD117LhyM1Dubw9wfuvRe46io22isooAuqVy8qYpvWR0YGFyNrlJRwsWrTxvRcdjbw+efMzDlzBti4kW/G+IbKy9mfR6+HJiAAc0aOxYhD0/DBtg/w6EWPOvdmKyqohGrLUZ/ISCqMPXua1rCds2etvy+ACrmgoHGBpxaM3T81wengdwBIk1JOF0IkA7gAwGZ3CedLhIQAN97IzZ65Qe8VFfzNjxtn/nuYlcWdcn23BcDzW7fmb3TVKi6CzQVb/YKE4OdpMND7cMEFVLgdOri+piMuzsnWF507M784P998VkFNDRehBx+s+8dduZKLf24uo+tJSTzPYOCOwFh1u3IlMHEiho8ai4GfrcCMNTNwzwX3ICzAidYMx4/zQ7eV0x8YyEBzU1II4eHckVn7Yuh01PZKGTiFI/GrtwH0A3Dbuf8vAfCWyyXyYYyNFnNyuPkzpk9mZdHV+eijtCTMYU/OfevWzJ6zFYNoSjhSUNajB+OV3bv7WIGfRgOMHctdQVYWtT9gCnJkZTGz5tJL675u0yb6D/fsodkXHU2TJCGBO9mgIP67VSsgPx9CCMwZOgf5Zfl4Y5OTfSOtRc/rvydvdfdzlsGDbc+MKChgk0CFUzhijPeVUvYWQuwAAClloRCiRTVsEAK45hpmxGzcyDRUjYaN2jIyrPfaOnrU9uJotHZLSppPkkTHjnzfZWWWM4V0Om5Y6ze8s4Zez1T0Zcu4+TYWDV96acOYrkuIiwOmTWP2y2+/mVKozjuPyqB794a70qoqU7DC6Cby8+Mf1/gHlpK+xtxc4MgR9Evvh+s6X4cXN7yIhzMeRmyIg7MF4uKoFGwphspK650IfZH+/VlSXlJiPke4tJSWlzsC3S0ERxRCtRBCC0ACgBAiDoCT9aNNm5gY+qMdQau1XW1rzLlvTq3e/fyYJfTee/SY1Pdk6PVc0G+/3X6roKyMMdF9+6iEw8PptVm0iI/77+fa4XIiIviHHzGCVoJWaz37qHVrCmnNfWEciykEfY7p6Zg5ZCZ6vNMDc9fNxUvDXnJMxpgYpq3t32+5fUR1Nb9kGRmOXdvbxMayw+FrrzGeEB/PnYROR3eeRsOUVTVa12kcWXrmA/geQLwQYhaAdWhBdQiNpXdv26MfjUHp5hRUBrg433UXkJdHV1tREd/rsWN0eV9/PQeG2YOU7PBw8CDjMXFxXJNDQ6lw4uKofFxd31UHjYY3tJWKOnw4A8fW2lGUlzNgIsTfmQbd4rvhrp534c0tb+J48XHH5bvpJt6zsLDhMZ2Of4R//KNpftG6dGHa7MiR/EFlZVGRDh/O5x0xMxUNsKe53edSyrsAtALwLIChAASA66WU7vzZNSv69WMhq7Eqtj5Gd/RDDzW/eJgQrCnq3Zstf3bvprXUrx+te0f6oGVlATt3UhlYKxr+/nuuHV7lootMxRbGyV+1KSnhopyQQBdUmmkuwguDXsCC3QswffV0vH/N+zZvVVPDGPGSJcDRo4mIqxyPGzPfQkpYJsKj/SG0GioDrZbmmL0a2BeJj6dCu/FGvnGttvn9aLyEkDaaaQkh9gG4HMAvAAaByuBvpJRnzLzMI2RkZMitW7d66/YOs3YtW+LExNRdHyor6ULOyADGjPGtFvi+xldfMW7QzsrwVim5CX7xRR/wHpw6xXYOOTn0bfn50U+m1/NLcPHFPKdvX2Yp1eLxXx7HW1vewt4xe9G5leUa0KoqDinbsYPep6gofganC2rQ6uR+XJO+F317VUMktbMd7FK0CIQQ26SUDXyG9sQQ3gWwHEAagG21rwnGE9LMvUjRkAEDuCZ8+y0XLOPkrqAgWvkjRihlYIuTJ23HGozN6kpLfUAhtGoF/Oc/bN505Ah93mFh9G+Fh9OP1q4d+33UY9Jlk/DRjo8wZeUU/Pfm/1q8xRdfmLea2rTTQt+6G97O7oaqZPaTUiisYVMhSCnnA5gvhHhHSvmIB2Rq1lxwAWN+x47Rjx4QwB9yUxmwU13NLMoDB+j2SU/ne7LlTncV9hQNGyvGzfVG8gpt2gCvv84K4nXr+FxlpWkm6MiRZnft8aHxGNdvHKavmY6tOVuR0bZhEPjUKV7SUt89Pz+65L7/nhlYasOhsIY9LiMhbZxkzznuoKm5jJo6e/YwYFtaypoqIeiuCAwE7rmHPYPczf79rOi21putuJgyzZ7tg67l0lKaOcZqRBuatLiqGGlvpKF3m95YcteSBseXLWOxpK0M0uxsdotOTWWH2ZISfkYdOqhZMy2RxriMVgohvgPwo5Qyu9YFAwBcCuAeACsBfOIiWRU+yMGDwCuvMP5Rf/GpqGA3U43G/WMojUXDeXnmg9E1NVxvx4zxQWUAmDpz2klEYAQmDZiEp5Y8heVHlmNo2tA6xwsL7WvtISUr5d96yzSpUkoq9iuuAK67rulYqQr3YU/a6ZUAagAsEELkCCH2CSGOAPgTrFp+XUr5iRtlVHgZKdmSJyLCfD1QcDAX5y++cH/xq7FoOCyMrT6Mc1oMBiqCrCwWD7p9PrIHeeTCR5AUkYQJyyegviEeEWG76SJAF+WPP1IBtG9PF1P79gxx/Pwz8PbbzatCXuEc9kxMq5RSvi2l7A9OShsKoLeUsr2U8v+klDvcLqXCq2Rmsl7A2ijKkBB6Q/budd19S0rYp23+fOCNN1h0duaMqWh41Ci6rLKyuOClpTF2e/PNPmodOEmQXxBeGPQCtuRswfcHvq9zrGdP05hjSxQUsHni+ec3rBY3xrC2bWODQUXLxmYMwZdRMQTPsHkz0xrt8VM3dmCWkQ0bgI8/5q7VWMhbWsqF76abGIcVgothVRWDpc3Z5aE36NHjnR6QkNj9yG74aUx+ojff5EC3pKSGitBgYO+8kBDWfViiqIjnzJzZvJSpwjyNiSEoWjj2ZqYYm3g2lh07GJNo27ZuzDUmhi6pr77i4n/FFXQheaQRXkEBB69s3sw0p6Qk4PLLWf3mgdQdP40fZg2ZhRv/eyM+2/UZ7u91/9/H7ruPC/rBg6YaFynZfLGkhJZdZxujrCIiqNDLy52fTqdo+jSjrjkKd2HM6LHWi8loaHbo0Lh7GQwcl9uqlfkEHH9/pu1/+y0zNz3CunXA+PFMG9XrKcShQ5xK9tpr5gfOu4Hrz7seFyVehGmrpqFSb3rzoaGcU/3ww4zxGF1oqal0oXXtars/lnE8bBN2GChcgFMKQQjhQLMBRVMnNpZtJ3JzLZ9z6hRdSuaGAznCkSMMDltrsxMYSDfR7t2Nu5dd7N3L5knx8YzEGnsYxcdTU+7dC3z0kUdWUiEE5g6di+PFx/H2lrfrHAsIYNrvjBkU56OPgKefZmufTp1s99EqK6MloVJQWzbOWgiLXSqFwue58066Io4dq5vVYjCYFMWDDzbe/3zmjH3X0GhMHajdhpSs6IqKMm+uGKf7bNvGqLsHGJw6GMPSh2H22tkoqiwye45WW9ciGDyYCtSahVdQwEaudnXaNRhYmfjxxxzD+eGH7OpqT7qTwqdxViGosFMLIzoamDSJu9CcHCoG46NbN47+TExs/H38/e3bbHukErmggEMvrA2nEIKFAJs2uVkYE7OHzMbpitN4ZeMrdp2fksKZMZmZDVNLpaQuS0mxs7Dw7FlGnufO5Xs+dgzYupWNo55/noELRZPF2aDyBy6VQtEkiI4GHniAmUQnTnAxSUhgILOigq704ODGWQnp6dylGptYmsOoMGwFShtNSQmFsfWGgoJsT/JyIX3a9sEtXW/BqxtfxaMXPoqEMDNjPWshBC280FDgl19Mn60xXbV3bwambQbnq6pYnZiXZ75UPC+PimHaNOV7aqI4qxAWulIIIcTNAJ4H0AXARVJKlUvqw0REmHoKbdoELF7MtUAItu0ZOZLNO+vPnbb32v37m/rzmCM/n8rAWsdTlxAUZJ+5otOZr9hzIzMGz8B3+77DrLWzMH/EfJvna7VM1x0+HPjjD7rmQkJo3ZkbE22W7duZimQpUGQcDP777/RTKZocvhJD2APgRgBrXHxdhZuoqABefZXuY52OG8bkZD7/3nssJKuqcu7ao0bxWkeP1s0kqq7mehQaCvzf/3kgX75tW6Y7lZRYP0+n4+wDD9IpthMe6PUA3t36Lo4WHrX7deHhVLjXXAMMHeqAMgA4OtTWbNfYWODXXx24KOjHsjVOUOERfCKGIKXcL6U86MprKtzLf/7DvPfUVFNGkBAMPKelsRHefy13bLZKaCjTJW+4gY3qsrPpqj55krUHU6ZwnTbitgQfjQa49lrGEiwFTPPyqL06dnSTEJaZOnAqtBotpq2a5pkb5ubatoTCwmjC2VrgS0uBpUuBceOA0aP5ePdddt5TeI0mF0MQQjwI4EEASLbkU1C4lcJC1mi1a2d+ly4Ej61ezfGYznhTQkL42pEjqQik5ObT6Oc+eZIDh1as4NoSFcU6sf79bW9iHaJ/f2qkX3+l5ouN5RssL6eiiIsDHnvMK4OwEyMSMfaisXhpw0t45pJn0D2hu3tvGBjI3by1QryaGlMrXEucPMlYw8mT/PxSUvi6HTuAjRtpIo4YoUqmvYBT32Ip5du2z6qLEGKZEGKPmcd1Dt77fSllhpQyI87r009aJsZZCNbWBT8//sYPHGjcvQICmL3Urp1JGezZw4ynRYu4IW3fnmvVwoXA5Ml0NbkMITi85umnaQlkZ7Pyq7ISuOUWYOpUr07hee7S59gRdcUk99+sb1/bwfOCAp5naTGvqeFsiJISKgJjWbRWywBUu3YsRd+1y6WiK+zDY60rpJSXe+peCvdSUWHfeVI6H0ewRH4+4xORkXW7SAcHc70+e5YZkU8/zeynmBgXbDSFAHr04KOqiotaUJBXrIL6xATH4Ln+z2HiiolYn70e/ZP7u+9mgwZxAINxCEZ9qqt5bOjQhseM7N/PFDVLAy38/Wnu/fQTO/cpK8GjeP8brWhyWKsiro0Qrk++WbWK1om5kQIlJax0XrmSXpxnnmEG5ObNLowzBAbSn+UDysDI2L5j0TqsNcYvH9+gPbZLSUzkJKQTJ1iabryXlExbys6mNWWtXH3jRtv5rdHRNPNUTYPHsWkhCCFi7LiOQUppozje6j1uAPAvAHEAFgkhdkophzt7PYV76dqVrhydznKH0aoq/u7PO89195WSi725zJjTp9khFaBVUFrK/nNFRcC//gVceSVw+20S4vgxunyMRRQdO/rU4u4MoQGhmHrZVIxZPAa/HP4FIzuOdN/NBg1i246ffmJWgVZLiyk9nalf3W3EMYqKbLelNQ7FttcUVbgMe1xGOece1mw3LQCnI7xSyu8BfG/zxEZgrLZfupTfY4D94S+/nDntyjK1n+Bgpi3+97+0/OvHEmpqWP16xx2urSbW66lo6q8n1dWshwgI4P1qaujiF4Leh/BwYNPCHAxc+xHaVf5leqEQ9P/fey+/DE2Y0b1H45WNr2DC8gm4ssOV0Ag3Krnzz+fjzBk2QQoONgXbbREbC/z5p/VzjEOxHZgsp3AN9iiE/VLKXtZOEEL49JAcvR7497+ZGRMaaspC2bsX2LKFNTR33aUGkDvCVVcxJXTpUrp9Y2L4Oy4s5AJ99dXAsGGuvaefH1339S2T3Fze0xifrKmpq4giy3Mx7M9ZyMmSSBzZHkJTa+EqKmLX0nHjWKXVRPHX+mPG4Bm4feHt+GrPV7i9++3uv6kxSOMIl1zC9DNrnD7NtuIuTRdT2IM92wgrYzUcOsdrLFzIytf27Wnt+vvzU8U2mwAAG4NJREFUkZDA51asYMaKwn40GuD22+mjN/bAMRiYJBIfz/m9Tz8NfP219S6pjiAEe/Lk59d9/vjxugqgvJy1EEa67/4SQf41yNHHo7Kq3i42MpKL2ocfNvkZkqO6jULPhJ6YsnIKdDW6ugfz8oBvvqHie/RRYNYs7oZ0OvMXcxcdO/JhqRlgZSV3Gtc5lHyocBF2jdC0dEwIcZ+tc7xNaSmwZAkzUMy5ijUaLmKLFnmwv34zQQjGD++7j+meUVEM6lZWcp319+dnP2kSFYQrGDiQllzt4mG93vS3ragwpaoCQGhpPuIL9qAsLAFCWKgvCw+npbBvn2uE9BIaocGcoXNwpPAIPtz+oenAunXAxImspfDz4x8qPx946y1g9my+d48JqQH++U/+gTIzTSZlRQWrDwsK2DbX7Y2qFOZorKPxBZdI4UaMXXn9rDjHjAHS/fs9J1dzQkoOaT96lDGFmBh+psHBDOwmJJg6JDeW+HjgySep6LOzuY6EhtKVXVREWS65xGQxhJfkQAoNDFJACCsxDY2GweYmzpUdrsRl7S/D9NXTUaYrY8Dsgw/4wSUl8Y9i9PGlpHCn/uabnm0dERnJXcKYMYzhnD3LH+Dw4bRcLr3Uc7Io6mBPltEflg4BcKQTilcwzuG1h7Iy98rSXDlyhMq0fXvzccXAQK4BP/xgPXar03Ft0GjoPrYU0+nShRvb9euZdRQcTKXfvTvXPHOjC4xZR8403GtKCCEwZ+gc9P93f7zx+xuYuMmfwVlzHwrAnfqff7JlRKdOnhM0IID9nzzcA0phHXuCygkAhgMorPe8ALDB5RK5GEe68HpkNm8zZONG/r6tJZnExHDdOXWqbh8igO6fZcsYoK6qogKPimL3goEDzWcpxsayzdC11/L8119ncWt9C6AkIhH6KgNEkAEdOlgxiA0Gy8VSTYxLki7BtZ2vxbx1c/HQ4asQ287KQm80m9av96xCUPgk9riMfgYQJqXMqvfIBLDKrdK5gPPP507TWrywupoupS5dPCdXc+L0acsbUCNC8O9QWlr3+TNnOG/lxx9pRSQlMd6j1QKff86RxbZiO0JwnvAFF9Drk5ND99Hp08Dek/E4HtMDl3UusFwkV1xMDdSMvgCzhsxCia4U86L22jfP4dQpzwim8GnsCSo/IKVcZ+GYB3LbGkdEBGtpjh0z7zqSkseuuELN9HCWyEjbLSqkpFunthUmJd3bhYXcnNfe3YeGMmC9fz+nWNoiOBh4/HF2Qr3kErqmU1OpKEZ+cTsiY/0YsKz/JTh7lgKMHm090NTE6BbfDXd1vhn/CtuH49JG0Fin4x9R0eJpPr8AK4waxZ3o1q30TRvTm8+c4U6yXz+2WlY4R9++bClhjaIi7vzj403PHT9uij2YQwi6uFesoGvIWGdgCSFYMJueXv9IawYx//1vU1GUEFQOrVtTa7iypNpHeGH4XCw4+B2mVy7F+4E3WT6xstLO+ZmK5o49QeXtUsrejT3HmwQEMPX6jz84QtDYcr1zZ46E7N5dFaU1hs6dudjn5XF9rY9eT/fNXXfV9V7s38//t+bR8Pfn6//6i73lnKZNG6ZeHj9Ok9BgYPqTcWZnMyQlOhWPpN2Ct/76CuOq89DZ38wfJz+fn00zcpcpnMceC6GLlUwjgMFln7c3/fw4O7Z3b64FthYihf1oNMDYsSz4zcykuyY0lJ9zQQFTQ2+6iZ99bSoq7FPEQjDO02iEYJAiKckFF2saTLrhNXz02veYUvYr/nv6Sv5xtFqm1Bkj/E88oXZECgD2KQR7bGkL46R8k2a6IfQqrVrRf79pE60wY0p/Rgb7RXXq1FABx8XZVxwspXJxO0t8WALGXfospq+Zjq3J0cjYcYIfaHQ0O5NeconqGaT4G5sKQUpZp1pHCDH93Ot2AtgppTzkJtkUTYywMC7+Q4eaBmtZU749etByq662XB9QUkLFUbsVhcIxxl0yDm9teQsTo7ZhyYe/mio1lYmsqIfDe2Up5VQAbwAoAnCDEMJr4zRbCidPsrXz6tWMg7jEfeJGhOACb8sSCwtjsDg723xLiaoqupxuuUVZdY0hIjACkwZMwtIjS7E8c6XtEZeKFouwd6CGEOINAE9It07gcIyMjAy5detWb4vhNoqKgM8+A7Zvr5stGRbGRXLAgKb/uzYYgG+/BRYvpkUREcH3WlzM93b33SxOUzSOSn0lOv2rE1qHtcbvo3+HaOpfHEWjEEJsk1Jm1H/ekbTTEgA/CSFulVKWCSGGA5gqpXTjzL6WS2kpMG8ed8hJSXV3yBUV7A1UWen6FtOeRqMxKbd165gVqtHwffXrpzogu4ogvyC8MOgF3P/T/fj+wPe4scuNXpXn7FlWuG/YwO9x69Z0NXbr1qzKQZocdlsIACCEuB3AkwB0AEoBzJRSrnWTbDZpzhbCwoUcSmWpm4JOxzTPV15Ri6bCPvQGPXq80wMSErsf2Q0/jXdW3j/+YKNVnY4tTfz8mPRUUsIs4CeesH9Mq8I5LFkIdntmhRBDAfwfgDIArQCM9aYyaM7odOzr06aN5XMCAuha2bTJc3IpmjZ+Gj/MGjILB04dwGe7PvOKDFlZ7DsVEcGCxPBwVpm3asXK8uxsYP58zzZfVZhwJFQ3CcAUKeUgADcB+FoIMcQtUrVwTp2iUrA1fjI8XLXsVjjG9eddj4sSL8K0VdNQqff8AJBffmFM21LVeWIiC0eNY24VnsVuhSClHGLsaSSl3A1gBICZ7hJMoVC4HiEE5g6di+PFx/H2lrc9eu+yMmDz5rrtS+ojBHvtuWqgksIxnE7mk1LmAhjqQlkU52jVitaBrYZxJSWq44DCcQanDsaw9GGYvXY2iio9Ny2ttNTU9dYaISFMplCYITcXWL4c+PlnRuTrtw9uJI2KKkkpK1wliMJEQAC7r/74o/WgskbDTByFwlFmD5mNjA8y8MrGVzB98HSP3DMwkLEBKa2nS+t0qni6AUVFbM64axf/X6s1FRhedRULelzQfkSV+/goV1xBf+rx4w0DbOXlfP7229nGX6FwlD5t++CWrrfg1Y2vIr803yP3jIoCOnZkl2FrlJYC/VUyu4myMjYK27ePkfiUFOaip6SwQePChcDXX9s/GtIKSiH4KGFhwHPPAb16MfMiM9P0qKjgHPIhKqSvaAQzBs9Apb4Ss9bO8tg9r7mGm11LPazOnGEqas+eHhPJ91m9mh16ExMbmlb+/lQMS5YAJ040+laqBMSHiYhg2+5Tp5h5UV3NmoPOnZv/bGCF++kU2wkP9HoA7259F09e/CRSo1Pdfs9u3ViI+M03zDRq1Yquz6oqduIOCQGefNJ2hl2LoaYG+O03833ljWi1dB2tXg3ccUejbqcUQhOgVauGc4gVClcwdeBUfPbHZ5i2aho+u8H9tQlC0OWdns51btcu01jnq6/mdMOYGLeL0XQoLqYPzVb1aVQUcKjxfUaVQlAoWjCJEYkYe9FYvLThJTxzyTPontDd7fcUgtlxXbrQMqiuZnGaGslgBuNkP1tI6ZIOkCqGoFD4MtXV3EavXs00QzfkYz536XPsiLpiksuvbYvAQMbLlDKwQEQE3QMlJdbPO3uWox8bibIQFApfREpg7Vpmj5SV1c3V7N0buOcel00NigmOwbP9n8WkFZOwPns9+ierFB+fQaMBRo5kymlYmPl8XZ2O349LL2387Rp9BYVC4XqWLgU++ICLQEoKG/2kpHB49e7dbIXrwqKkx/s+joTQBExYPgE+1OFeAXCqXffubARVfxhKWRkzkEaNsl4CbidKISgUvkZhIfDVV8w1Dw6ue0yjYfphTg5TDV1EaEAopg6cirXZa/Hr4V9ddl2FCwgI4NDy4cPpMszKMj30euDhh13WB9+h9tfuQgjxEoBrwLbafwG4T0p51tbrmnP7a0ULZvFiTg1KTrZ8TlUVE/pff50LhgvQ1ejQ5a0uCA8Ix/aHtkMj1H7R5ygrYw66Tsf4QocOTgVgGt3+2s0sBdBNStkDwCEAE7wsj0LhPQ4cYCtbaxibXZ065bLbBmgDMGPwDOzK34Wv93ztsusqXEhoKKv2LryQBUkujsb7hEKQUi6RUhprFzcBaOdNeRQKr+Mly/3WbreiR0IPTF45GboanVdkUHgPn1AI9bgfwC+WDgohHhRCbBVCbD158qQHxVIoPESXLrbTDKuqaCW4uGJRIzSYM3QOjhQewUfbP3LptRW+j8cUghBimRBij5nHdbXOmQRAD+BLS9eRUr4vpcyQUmbExcV5QnSFwrP068fgsc7KDj0nhx0QXRQ/qM2IDiMwIHkApq+ZjjJdmcuvr/BdPKYQpJSXSym7mXn8CABCiHsBXA3gDukLkW6FwltERbGV7bFjbG1bG4OBzycluSyzpD5CCMwZOgd5pXmY//t8t9xD4Zv4RGGaEOJKAM8CGCilLLd1vkLR7BkyhC6hb74BTp6sW5h24YXAXXdZnkPpAvon98c1na7BvPXz8FDGQ4gJVg2GWgK+knZ6GEAggNPnntokpXzY1utU2qmi2aPXM+vo7Fl2tOzYEYiN9citd+fvRs93e+KZS57BvCvmeeSeCs9gKe3UJywEKWUHb8ugUPgkfn7sGe0Fuid0x5097sT8zfMxtu9YJEYkekUOhefwxSwjhULhI7ww6AXUGGowfbVnxmwqvItSCAqFwiKp0al4OONhfLTjIxw63fh++wrfRikEhUJhlUkDJiHILwhTVk7xtigKN6MUgkJhg8pKZnpmZ3OedUsjISwBT/V7Cv/d+19sy9nmbXEUbsQnsoycRWUZKdxJaSnwyy/AsmWmrsN+fsDgwcCIESwXaCkUVRYhfX46+rTtg9/u/M3b4igaia83t1MofIriYmDOHGDRIs74TU7mIy6OXadnzQLOnPG2lJ4jMigSEwdMxJK/lmDF0RXeFkfhJpRCUCjM8OWXQF4eZ9IEBpqe9/cH2rdn5+mPWlirnzEXjkG7iHZqiE4zRikEhaIeZ84AW7ZwDo0l2rQB9u1jS6GWQpBfEF4Y9AI2n9iMHw784G1xFG5AKQSFoh5//cVOERorvw4heM7hw56Tyxe4u+fdOK/VeZi0YhL0Br3tFyiaFEohKBT1qD+21hrWGpI2O/Ly4Lf4V8wSQ7H/1H58vuI1r81tULgHn2hdoWh5VFVxMQ0OZuaOLxHjQB83F48j8E0qK4FPPgE2bQI0GtwQ4I8LY1ph2uoXcNvWKgQ98hgQGeltKRUuwMd+iormzoEDwG+/ATt30u0SEAAMHcpUTl9ZXDt2BKKjmXYaFmb+nIoKICQEOP98z8rmcWpqgHfeAXbtYjRdCAgAczESQ8VneCfvZzz5chUwcSK1u6JJo1xGCo+xeDFTOQ8eZDv/5GTuxn/9FZg2DcjK8raERKvlOIKCAm6O66PTMZh8661umU/jW+zfT+19ThkYGYJUXCHTMCvqDxQfO0zrQdHkUQpB4RH27gW++gpo1w6IjzcFbAMCqBz8/IDXXjO/AHuDjAzgoYeA06eBzEzOsj99mkorPx+45x5gwABvS+kBli6lmVRLGRiZjaE4LSrwSttMVvCpeEKTR7mMFB5h0SIgIoJ5/OaIjubCu3MncPHFHhXNIv37Az16MAV13z6ud507A337tiCXeWYm/3BmyEBb3CzPxysB2/BoXjLidbq6RRuKJodSCAq3U1JCz0NysvXzIiKAtWt9RyEAQHg4h5cNGeJtSbyEnx/HdlpgBgZjIfZjVvRuvKHVelAwhTtQLiOF26mspIvIjNehDv7+QJma6e5b9O5NX5kFOqMV7q/sgnciDiGz9LgHBVO4A6UQFG7HmKlTU2P9vIoK9gpS+BADB/IPZ6k4w2DA1PzzoNX4YdqqaZ6VTeFylEJQuJ3gYLqB8vOtn1dezvVH4UO0aweMGsXe38XFdQPHZWXA0aNoN/g6PNb3MXy+63PsKdjjPVkVjUYpBIVHGDGCG83SUvPHT5wA0tKALl08K5fCDoYPB8aOZcA4O5vDIbKyaDXcd9//t3fvMVaUdxjHvw8gLKKgiHdAhKLWC6UCpoq0KErF1FrSNtVAjVFrDBov1FpwVQqUWELSNBCbaoKpGtJKIqWNRcQKYtECCgWLogStN6wX8ApCKfDrH/OuLutezpHdnTPL80k2ObPMOe+zu+z89p2Z83th7FgmDJtI105dqV5cnXda2we+qGytomdPGD8eZs3KTkl3755dr9y2LSsSffrADTdk7wGwCiPBkCHZvbivv5790KqqsvcmpB9Y987duWXoLVQvrubpN57mrF5n5RzavgwvkGOt6qOPYMUKWLYsu2Zw1FHZO5VPOaXhW1KtGLbt3Ea/mf044bATWHr5UtTUXQSWm4YWyPEMwVpVt24wcmT2YW1Ll45duONbd3DtgmtZuHEho/qPyjuSlcnXEMys2Vx1+lX0PbQvEx+fyJ5o+P0LVplcEMys2XRs35Gp50xl7TtreXDdg3nHsTK5IJhZs7rk1EsYcOQAbltyGzt3708LRhSfC4KZNat2asedI+7klQ9eYfbq/Wzh6YJzQTCzZjfqK6MY1nsYU56cwrad7kdSFC4IZtbsJHHniDt5e+vbzFwxM+84ViIXBDNrEUN7D+WiEy5i+lPTeX/7+3nHsRK4IJhZi5l27jQ+/u/HTF82Pe8oVoKKKAiSpkp6TtIaSYskHZN3JjPbd6cdeRpjB4xl5sqZbPp4U95xrAkVURCAGRExICIGAg8Dd+QdyMyax+Thk9m9ZzdTlk7JO4o1oSIKQkR8XGuzC1DcBktmtpfjDz2eawZfw+x/zmbDlg15x7FGVERBAJA0TdIbwBgamSFIulrSs5Kefe+991ovoJl9adXDqqnqUMXtS27PO4o1otUKgqS/SVpXz8fFABFRHRG9gDnAdQ29TkTcExGDI2Lw4V5ey6wQjjzoSMafOZ65z89l1Vur8o5jDai49teSegMLIuLUpvZ1+2uz4vhox0f0m9mPQccM4tGxj+YdZ7/WUPvrijhlJKl/rc2LgRfzymJmLaNbVTduHXYri15exOJ/L847jtWjIgoC8Kt0+ug5YCRwQ96BzKz5jRsyjp5dezLx8YlU2tkJq5CCEBHfj4hT062nF0WEb1g2a4OqOlQxefhkVm5ayfwX5+cdx+qoiIJgZvuPy752GSf1OInqxdXs2rMr7zhWiwuCmbWqDu06MO3caazfvJ4H1j6QdxyrxQXBzFrd6JNGM+SYIUx6YhI7du3IO44lFXfbaTkkvQe81oJD9AA2t+Drt6SiZnfu1lfU7EXNDflnPy4ivvBGrkIXhJYm6dn67tUtgqJmd+7WV9TsRc0NlZvdp4zMzAxwQTAzs8QFoXH35B1gHxQ1u3O3vqJmL2puqNDsvoZgZmaAZwhmZpa4IJiZGeCC0KSirvcsaYakF1P2P0k6JO9MpZL0Q0nPS9ojqeJuzatL0gWSXpK0UdKEvPOUStK9kt6VtC7vLOWQ1EvSEkkvpP8nhWiGKalK0kpJa1PuyXlnqsvXEJogqWvNEp+SrgdOjohrco7VJEkjgcURsUvSdICI+HnOsUoi6avAHuBu4OaIqNhFLyS1BzYA5wNvAs8Al0bEC7kGK4GkbwJbgftLWX+kUkg6Gjg6IlZLOhhYBXyv0r/nkgR0iYitkg4AlgE3RMTynKN9xjOEJhR1veeIWBQRNZ3DlgM988xTjohYHxEv5Z2jRGcAGyPilYjYCfyRbE2PihcRTwLv552jXBHxn4hYnR5/AqwHjs03VdMiszVtHpA+Kup44oJQglLXe65gVwCP5B2ijToWeKPW9psU4ODUVkjqA3wdWJFvktJIai9pDfAu8FhEVFRuFwSab73n1tZU7rRPNbCLLHvFKCW7WWMkHQQ8BNxYZyZfsSJid0QMJJuxnyGpok7Vdcg7QCWIiPNK3HUOsACY1IJxStZUbkmXA98BRkSFXSwq43te6TYBvWpt90yfsxaUzsE/BMyJiHl55ylXRHwoaQlwAVAxF/U9Q2hCUdd7lnQBcAvw3Yj4NO88bdgzQH9Jx0vqCFwC/CXnTG1aujg7G1gfEb/OO0+pJB1ec7efpM5kNyJU1PHEdxk1QdJDwIlkd728BlxThCU+JW0EOgFb0qeWF+HuKABJo4FZwOHAh8CaiPh2vqkaJulC4DdAe+DeiJiWc6SSSPoDMJysFfM7wKSImJ1rqBJIOhv4O/Avst9LgFsjYkF+qZomaQBwH9n/k3bA3IiYkm+qvbkgmJkZ4FNGZmaWuCCYmRnggmBmZokLgpmZAS4IZmaWuCCYmRnggmBthKQ+kranPjE1n/tCW2pJnVMr852SeuzjmJ0lLU0dT5F0vaT1kspqEyLpEEnj9iVLCWN8odW1pI6SnpTkjgUGuCBY2/Jy6hNT05b6LmAUcDJwqaSTI2J72uetZhjvCmBeROxO2+OA8yNiTJmvc0h6blmUKfV3+PdkbRI+k7qzPg78qNyxrW1yQbDCSIuinJ8e/1LSrEZ2b4221GOAP6c8vwP6Ao9IuknS2LQYyhpJd9eaRcyXtCotkHJ1ep1fAf3SvjPSbKf2X/I3S/pFetwnzXruJ+uB06uhsWprpNX1/PR1mLkgWKFMAqoljSFreXxjI/u2aFvq1Leob0S8CpDagrwFnAMsJPure2iajezm84PuFRExCBgMXC/pMGACaXYTET8rYfj+wG8j4hTgwEbGKsU6YEgZ+1sb5nOHVhgR8WRqbDYeGF5zqkbSVLJmZ62pB1mfpfqMAAYBz2Rx6UzW/x6yIjA6Pe5FdnB/u8yxX6u1ylZjYzUpInan6ykHp8VmbD/mgmCFIek04GhgS83BS9JRZCtP1VV2W2pJ1wI/SZsXAqNrb0dE7esO24Gqhl4KuC8iJtZ5/eHAecCZEfGppCcaeI1d7D17r7vPtqbGKlMnYMc+PN/aCJ8yskJQto7uHLLrAFtTe2+AgcCaep5SdlvqiLgrnbYZGBFv1d2us+8HQHtJ9R3QHwd+IOmIlL27pOOAbsAHqRicBHwj7f8JcHCt578DHCHpMEmdyNa0aEhDY5UknbLaHBH/K/U51na5IFjFk3QgMA/4aUSsB6by+SJF9RaEtJ70dcCjZGvuzo2I55s52iLg7HrGfgG4DVgk6TngMbKZzUKgg6T1ZBeSl6f9twBPpRXjZqSD8xRgZXpugz3zGxlrL6nV9T+AEyW9KenK9E/nAH/9Ml+8tT1uf22FJmk22Wmd3sDDEVHSkoSSXgUGR8TmfRj7dOCmiPjxl32NvEmaB0yIiA15Z7H8eYZghRYRV0bEHrK7a7rVfmNafWremEZ23WFPY/uWMPZqYEl9t3kWQTqVNt/FwGp4hmBmZoBnCGZmlrggmJkZ4IJgZmaJC4KZmQEuCGZmlrggmJkZ4IJgZmaJC4KZmQHwfyDHwTuSEudxAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "oEQswfCGrmHw", + "outputId": "3cf61882-60e1-4baa-8e51-0c31ea80875c" + }, + "source": [ + "pred = tf.matmul(valid_x,W)+b\r\n", + "fig,ax = plt.subplots(1,2)\r\n", + "ax[0].scatter(valid_x[:,0],valid_x[:,1],c=pred[:,0]>0.5)\r\n", + "ax[1].scatter(valid_x[:,0],valid_x[:,1],c=valid_labels)" + ], + "execution_count": 27, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 27 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5xU9b3/8ddnZnZnC0WaIqKCXSxE3UiIGsSKFTXq1dxr7MRekvvLtXujN15bOpZwNZY8jDUxoqLYg8aoLIoCIoqFACIsSN02Mzuf3x+7moWdhZWZnTMz5/18PHiw8z2H832rn/l4OHPmfM3dERGR0hcJOoCIiOSHGr6ISEio4YuIhIQavohISKjhi4iERCzoAOvTv39/HzJkSNAxpERNmzZtqbsPyPe8qmvpTuur64Ju+EOGDKG2tjboGFKizGxeEPOqrqU7ra+udUlHRCQk1PBFREJCDV9EJCTU8EVEQkINH2hY3ciMV2czf87CoKOI5NSiTxYz49XZrF6+JugoUgAK+i6dfHj0FxO558oHcRxPO4N33IKbn7+avgP7BB1NZKOtWVHP1UffyOw3PsIihqed4y4+nLNvPgUzCzqeBCTUZ/hTJ0/nrsseIJlIkUq00JJKM2/WfC7e56qgo4lk5boTfsHM1z6gJdVCKpGiJdXCo794kifGPxt0NAlQqBv+PVc+SLol3WH8i0+X8NE7nwSQSCR7q75czfSXZmbcdu81D+U5jRSSUDf8RZ8u7nRb7eTpeUwikjtrVtTT2ToX9Ssb8pxGCkmoG/5mW3f+rfpB2w7MYxKR3Nl0q/6dbovGQv2WD71Q/9c//fqTIMPnV2XxMkYcsVf+A4nkQCwWY5d9dsq4bd/vfyfPaaSQhLrhjzhiL75/6ZFEYxHMjEg0QllFGddP/C8qquJBxxPZaNf99acMHLppW21DrCzKVjsP5pI7xgUdTQIU+tsyz7n1VMaeN4baydOpqK7gu2NrqO5dHXQskaz06teT+z76HW+/MIMFcz5nq2GD+dboXYhEQn2OF3qhb/gAm2+zGUede2jQMURyKhKJUHPIcGoOGR50FCkQ+t+9iEhIqOGLiISEGr6ISEio4YuIhIQavohISKjhi4iERE4avpn9wcyWmFnGJzaZ2f5mttLMprf9uiYX84p0J9W1lJpc3Yd/LzAeuH89+7zq7kfmaD6RfLgX1bWUkJyc4bv7FODLXBxLpFCorqXU5PMa/kgze9fMnjGzXTrbyczGmVmtmdXW1dXlMZ7IRlFdS9HIV8N/G9ja3YcDvwP+2tmO7j7B3WvcvWbAgM4fXyxSAFTXUlTy0vDdfZW7r2n7eRJQZmadP7RbpAiorqXY5KXhm9lAa1s52cz2bpt3WT7mFukuqmspNjm5S8fMHgT2B/qb2QLgWqAMwN3vBI4HzjWzFNAInOSdrcEmUiBU11JqctLw3f3kDWwfT+vtbSJFQ3UtpUbftBURCYmSafirvlzN3OmfUr+yPugoIjnj7iz4aBGfzZpPOp0OOo4UuaJf8SqVTPHrcyfw0gOvURaPkUqkOPr8MZx9039oOTcpav/8YCH/fdwtLJlXh0WMql5VXPGnixk+qtPb/UXWq+g74t1X/IlXHvw7yeYkDasaSTQlefKO53j8t5OCjiay0ZKJJD/Z/xoWzFlIc2OCpvpmvly0nKuO/F+WLVoedDwpUkXd8NPpNE/d+RzNjYm1xpsbmnn0F08GlEoke29NeodEY5J17/lJt6R5/v5XAskkxa+oG34q2UJinWb/ldXLVuc5jUjufLloOS2plg7jiaYkS+brVn/ZOEXd8MvjZQzabmDGbTt+e7s8pxHJnV333Qms43hljwr2GL1r/gNJSSjqhg9w4fiziFeV0/aFRyIRo6I6zjm/PDXgZCIbb+huW/OdI2uoqI5/PVZeWc7gHQcx8uiaAJNJMSv6u3T2PGh3fvm36/jTDX9h3vsL2GGvbfnBFcey9bAtg44mkpXLH7iIyfe8wtO/f45kIsUBP9iPYy48jFhZ0b9tJSBWyN8Er6mp8dra2qBjSIkys2nunvfTZdW1dKf11XXRX9IREZGuUcMXEQkJNXwRkZBQwxcRCQk1fBGRkMhJwzezP5jZEjOb2cl2M7PfmtlcM3vPzPbMxbwi3Ul1LaUmV2f49wJj1rP9MGD7tl/jgDtyNK9Id7oX1bWUkJw0fHefAny5nl3GAvd7qzeATcxs81zMLdJdVNdSavJ1DX8LYH671wvaxkSKmepaikrBfWhrZuPMrNbMauvq6oKOI5ITqmspBPlq+AuB9g+3Gdw21oG7T3D3GnevGTBgQF7CiWwk1bUUlXw1/InAD9vuavgOsNLdF3X3pJ9//AX/eLKW+XMyvgdFshVIXbe0tPDelPd5c9Lb1K9q6O7ppITk5LF7ZvYgsD/Q38wWANcCZQDuficwCTgcmAs0AKfnYt7OJJqT/PykX1E7eTqx8hipZAu77bcz//2X/0dFVXzDBxCh8Ooa4ON3P+Pyw35OU30TZkYq2cL5vzmdw886qLunlhKQk4bv7idvYLsD5+dirq64/9qHqX3uXRJNSRJNSQBmTHmfO39yH5fcMS5fMaTIFVpdt6RauOyQ61lRt2qt8dsvuYcdarZlu28NzVcUKVIF96FtLjz9fy90WPow0ZTk+fteoZAfBy2yPu+8NJNEc7LDeLI5xdMTng8gkRSbkmz4zQ3NGceTzSnS6XSe04jkRv3KBshwvpJuSbNq2Zr8B5KiU5INf/fvDcMyrAe604jtiEaj+Q8kkgO7jxpGKpnqMF5RHWffY0cEkEiKTUk2/PN+cwZVvaooi5cBECuPUdmjgotuPzvgZCIbr8+mvfmPa04gXhX/+oSmojrOtt8ayn7fV8OXDSvJxTG32mkL7n7/10y87VnmTJ3LNsOHcMwFY9h0K93/LMXt5MuOZdjIHXh6wgvUr2xg1AkjGX3yPlrnVrqkZKuk3+Z9OP1/1nuThUhRGj5qF4aP2iXoGFKESvKSjoiIdKSGLyISEmr4IiIhoYYvIhISavgiIiGhhi8iEhJq+CIiIaGGLyISEmr4IiIhoYYvIhISOWn4ZjbGzOaY2VwzuyzD9tPMrM7Mprf9OisX84p0N9W2lJKsn6VjZlHgNuBgYAEw1cwmuvv76+z6sLtfkO18Ivmi2pZSk4sz/L2Bue7+ibsngIeAsTk4rkjQVNtSUnLR8LcA5rd7vaBtbF3fN7P3zOwxM9uys4OZ2TgzqzWz2rq6uhzEE9loOatt1bUUgnx9aPskMMTddweeB+7rbEd3n+DuNe5eM2CAnl8vBa9Lta26lkKQi4a/EGh/VjO4bexr7r7M3b9aaPYuYK8czCvS3VTbUlJy0fCnAtub2VAzKwdOAia238HMNm/38mhgdg7mFeluqm0pKVnfpePuKTO7AJgMRIE/uPssM7sOqHX3icBFZnY0kAK+BE7Ldt5cSSaS/OXXT/PMXS+SSrUw+qR9OPny46jqWRl0NAlYsdf2go8Wcf+1DzPj1dn0G9SHky8/jn2O2TvoWBIgc/egM3SqpqbGa2tru+347s7lY/6Hma99QHNjAoCyeBlbbD+QO6bdrHVCS5yZTXP3mnzP2911DfD5x19w7l4/pWlNE+l063u8oirO6TeczHEXHdGtc0uw1lfXof6m7QdvzWXW63O+bvYAyeYkiz+r4/UnpgaYTCQ7f7zuUZrqm79u9gBNDc3ce9VDJJqTASaTIIW74b/5ES2pdIfxxjVNzPz7BwEkEsmNma99QLqlY21jsOiTxfkPJAUh1A1/wJb9iJVHO4zHK8sZOHTTABKJ5MZmW2e+9TOVbKHPZr3znEYKRagb/ogj9qSyRyUWsbXGo2VRDvr37wWUSiR7J19+LPGq+Fpj5RVl7DP22/Tq2zOgVBK0UDf8svIyfv3q9Wy/51DK4mWUV5QxeMdB3PLitfTqpzeFFK+9Dh7OhePPpEefaiqq45TFy9j3uBH85O7zgo4mAQr9bSibb7MZt711E8sXryCVbGHA4H5BRxLJiUNPG81B//E9lsxfSq9+PanuVRV0JAlY6Bv+V/pstknQEURyLhqLsvnQzYKOIQUi1Jd0RETCRA1fRCQk1PBFREJCDV9EJCTU8EVEQkINX0QkJHRbZh698dQ0nhj/DKtX1LPf90dw9LmHUtlDj2GW4ubp1XjDH6HpRYj0wapPxeL7BR1LMlDDz5N7rnmIv/zqKZrqWxdH+nTGP3nu3r9x29QbqVjnK/AixcLTa/Blx0DLEqC1tj0xFe9xPpEe44INJx3okk4eLF+8gkdvnfh1swdINCZYPK+O5+//W4DJRLLjDQ9DSx1fNftWjbDmd3h6VVCxpBM5afhmNsbM5pjZXDO7LMP2uJk93Lb9TTMbkot5i8X7//iQsvKOf5lqbmjmrUlvB5BIukq1vQHNLwFNHcetDJIz8h5H1i/rhm9mUeA24DBgGHCymQ1bZ7czgeXuvh3wK+CmbOdtz92ZU/sxbzw1jS+/WJ7LQ+dE7wG98HTHlcUi0Qh9N+8TQCLpikKo7cb6JqY++w7TX55JKpnK5aFzI7opYBk2tEBEtV1ocnENf29grrt/AmBmDwFjgffb7TMW+O+2nx8DxpuZeQ7WV1z6+Zdcdsj1LJ5XRyQaIdmcYuwFYxh38ymYZSrE/Bs2cgd69e9JU0PzWo2/LB7j6PMODTCZbECgtf23R17n1jNvJxJtPS+LlkW5/onL2OW7O2Z76Jyxqh/iTS+y9ll+BCJbQGznoGJJJ3JxSWcLYH671wvaxjLu4+4pYCWQ8bGUZjbOzGrNrLaurm6Dk193/K3Mn/M5TfXNNKxqJNmc5Kk7n+OVh1/fmH+WbhGJRLjlhWsZvMMgKqrjVPWqpKpnJZdOOIdthw8JOp50Lme1/U3retEni7n59Nu+ruuGVY2sXraGKw7/OY31GS6hBMTK94BeV4NVgvUAKiG2Pdb37oI54ZJ/Kbi7dNx9AjABWhd7Xt++S+Yv5ePpn3VYyq2pvpnHf/s0o0/ap/uCfkObb7MZd8/6FfPeX0DDqga223MbyuNlOTn26uVr+OCtuWwyoBfb7TFUb7QC9E3qGuC5+18hnWrJdBzefGoa+/9b4dR2pOoEvPIoSL4Pkd5YbNucHdtTc6Hlc4jtjEUzr+IlXZeLhr8Q2LLd68FtY5n2WWBmMaA3sCzbiRtWNRKNRYGOizKvWdGQ7eFzzswYssuWG97xG3jopsf5488epSxeRkuqhU236s//PnMlm26lN0cOBFbbq7+sJ5Xs2PDTLWnqVxZibVdA+Z45O56nV+LLx0FydusHwN6MV/0b1vMqndBkIReXdKYC25vZUDMrB04CJq6zz0Tg1LafjwdeysU1zi13HEQsw90vZeUx9jnm29kevuBNnTydB/7nzySaktSvbKCpvpkFHy7iqqNuDDpaqQistkccvgcVPSo6jHva2ePA3bI9fMHzFT+F5EygCXw1kICGx/CGR4KOVtSybvht1y0vACYDs4FH3H2WmV1nZke37XY30M/M5gI/Bjrc3rYxorEoP/6/c4hXlX/9wVa8spw+m2/CCf959Ab+dPF7/LdPr3VvP7SeAX7+8RfMm70goFSlI8ja3uuQ4ez+vZ2pqP7Xl/IqquMcdd6hDNp2YC6mKFieXgWJ1+j4N/dGaLgviEglIyfX8N19EjBpnbFr2v3cBJyQi7nWte+xI/jdP27gr+OfZfG8OmoOGc5hZx0YiuXcVi5dnXE8GouyZnl9ntOUpqBqOxKJcN0T/8WUR9/gpQdfpTxexpgzD6TmkOG5nqrweD2dnov6yrxGKTUF96Htxhi629Zc+vsfBR0j7/Y5Zm8+m/FPEk1rnwml0852ewzJ6VzuSWiZB9YHi2rd33yIRqOMPmmfgrr5IC8iAyGyCaQXr7MhCvH9cz6dp5dDehlEt6L1yl3p0qMVitjY88fQf3A/4pWtRWpmxKvinPfr04hX5u75POmGJ/AlI/Flx+N1o0h/eZa+Ni/dxsyw3jcAFUC0bbQcrDfW48KczePeRHrFJfiS/Vpre8kI0vX35+z4hagkzvDDqrpXFXdMu5lJ//cCbzw1jX6D+nDMhYez84jtczaHJ6bCqqtZ64s1iX/gKy7E+up6qnQPi+8H/f6MN9wLqXlQPgKr/ncs0jdnc/jKK1qf8EkCPNE6uPoXeHQQVnFQzuYpJGr4Ra6qZyXH//gojv/xUd1yfK+/m47PSklC4m28ZSEWXfd7SCK5YWXbY71/3i3H9vRqaHoOSKyzpRGvv7NkG35JXtJZvXwNX3y2hHQ6veGdZf1aPs88bmXQsjS/WULO3Vk8r46VS3U5LWvp5fzrctE6Wtb97KB0lNQZ/urla7jp1PG8/fx7RKJGVa8qLv39jxh5VE3Q0YpX+Xch9TEdbpHzFOTwG5WyftNfnsnNp41n1dLVpNNpho3ckSv+dDF9B+oBZRslOqjtC12N62yIQHnpfoenpM7wrz3mZqY99y7J5iTNDQmWf7GCn5/8K+a+82nQ0YqWVZ/R9oyU9ucGldDjAizSI6hYobLok8VcfdSN1M1fRnNjgmRzipmvfcBPD76OHHzHK5TMYtDzclo/GP5KBKwK63FxULG6Xck0/AUfLeLD2o9JJdZ+hGyiKcljv3wyoFTFz6KbYv2fgMoTILollO2BbXKLVjPKoyduf7bDo5FbUi0smbeU2W9+FFCq4hep+j7W53Yo2xuig6HiaKzf41hs66CjdZuiuqSzdOEyJt4+mbnvfMoOe23DUeeNoV/b8+Tr5i8lVh6juXHtD2E87Sz6pHSvyeWDRQdivX8WdIyS9u4rs5h014s01Tcx6sTvMuqEkW3PiYJFcxdnfK6OmVE3fxl8J99pS4fF98Xi+wYdI2+KpuF/OvOfXLLvVSSbkySbU0x/eRZ/Hf8sv3n952y982C2HT6EZHPHh6iVxWN8a/SuASQW6Zr7f/YIj9wykURjM+7w9gvvMfnel7lh0hVEo1G+dcAuTHvhXZob1j6ZSSVT7FCzTUCppRgVzSWd351/V9vz7lv/aptsTtKwqoHbLvoDAL369WTsBWPWevZINBahqmclx1x0eCCZRTakbsEyHr7przQ3tDZ7aH289/v/+JA3n2pd/vKQ00bTq19PYmX/uqskXhVn/5P2YfOhmwURW4pUUTR8d2fm3z/IMA7v/W3W16/PvukULrrtbIbuvhX9t+jLIaeN5o63b6HPpr3zGVeky6a/PJNIrOPtgU1rmnh94lSg9Qt2t9fexJHnHMKALfux1c6DGXfLKfzkrnPzHVeKXFFc0jEzyivKaW5o7rCt/SMEzIyDfziKg384Kp/xRDZada8qIhme7x6NRejZ9193QW0yoDfn/+YMzv/NGfmMJyWmKM7wAQ49bX/KK9ZeIaq8oowxZx4QUCKR7NUcOvzrR3u3Fy2LMeb00QEkklJWNA3/7JtPYbfvDSNeWU5Vr0rKK8sZPnpXzrzhB0FHE9lo5RXl/O+zV9Krf0+qelZS1auSeGU5F912FlsPy+3qaCJFcUkHoKIqzo3PXsW82QtYMOdzttp5C7bcUc9xkeK3097b8/DCCcx4dTbNDQl2HzWMqp6VQceSEpRVwzezvsDDwBDgM+BEd1+eYb8WYEbby3+6+0YvR7X1zoPZeufBG/vHRbok37UdK4uxxwGlv3ShBCvbSzqXAS+6+/bAi3S+vFuju3+r7Vfprz0opUC1LSUn24Y/Fvjqoej3AcdkeTyRQqHalpKTbcPfzN0Xtf38BdDZt0AqzKzWzN4ws/W+ccxsXNu+tXV1dVnGE9loOa1t1bUUgg1ewzezF4CBGTZd2f6Fu7uZdfbovq3dfaGZbQO8ZGYz3P3jTDu6+wRgAkBNTY0eBSjdaQczm5lhPOe1rbqWQrDBhu/unS79YmaLzWxzd19kZpsDSzo5xsK23z8xs1eAPYCMDV8kjz5094yLJai2pRRle0lnInBq28+nAk+su4OZ9TGzeNvP/YF9gPeznFeku6m2peRk2/BvBA42s4+Ag9peY2Y1ZnZX2z47A7Vm9i7wMnCju+tNIYVOtS0lJ6v78N19GXBghvFa4Ky2n18HdIOxFBXVtpSionm0goiIZEcNX0QkJNTwRURCQg1fRCQk1PBFREJCDV9EJCTU8EVEQkINX0QkJNTwRURCQg1fRCQk1PBFREJCDV9EJCTU8EVEQkINX7qdN79OetnxpBfvRXrpsXjzlKAjiWTNPUF6zXjSS/YjvfjbpFf8FG9ZHHSs9VLDl27lzVPw5edA8j3w1ZCahS+/gHTj80FHE8mKr7gY1kyA9GLwldD0JL7sODy9Ouhoncqq4ZvZCWY2y8zSZpZxqbi2/caY2Rwzm2tml2UzpxQXX3Uj0LTOaBOsuTGIOF2m2pb18dRcaP47a9d2C6RX441/DirWBmV7hj8TOA7o9O/oZhYFbgMOA4YBJ5vZsCznlWLR8mkn4/NxT+c3yzej2pbOJWeDRTNsaILE23mP01VZNXx3n+3uczaw297AXHf/xN0TwEPA2GzmlSISGZB53PpiVrhXFFXbsl7RLQHPsKEcYtvmO02X5eMdtwUwv93rBW1jGZnZODOrNbPaurq6bg8n3azH+WCV6wxWQo9zAomTY12ubdV1iSkbDtGt6LBKrMWwqn8LJFJXbLDhm9kLZjYzw69uOZNx9wnuXuPuNQMGdHJ2KEXDKk+EHpeC9QTKW3/vcT5WdWrQ0QB2yFdtq65Li5lhfe+D+CigDIhBbAes7x+x6MCg43Vqg4uYu/tBWc6xENiy3evBbWMSAmaGVZ+GV50CvgqsJ2YbLLt8+dDdO/1AtgtU2yFmkT5YnztwbwRPYZGeQUfaoHxc0pkKbG9mQ82sHDgJmJiHeaWAmEVb3yCF0+xzQbUtmFUWRbOH7G/LPNbMFgAjgafNbHLb+CAzmwTg7ingAmAyMBt4xN1nZRdbpHuptqUUZXW65e6PA49nGP8cOLzd60nApGzmEskn1baUosK9L05ERHJKDV9EJCRK6hM06bqlC5cx5bE3SDQlGXHEngzddaugI4lkzd2Z8epsZrw6mz6b9mbUiSOp7l0ddKyCoYYfQi8//HduPf12wGlJpfnjdY9y9HmH8qNbfhh0NJGNlkqmuGbsTcx4dTaJxgTlleX8/j/v56bnr2anvbcPOl5B0CWdkFmzop5fnHE7iaYEiaYkLakWEo0JnrrzOWa9vqEnCYgUrmfufon3psymqb6ZdNppqm+mYXUjPzv+VtwzPQYhfNTwQ+atZ94hEuv4n725IcGLD+g59VK8Jt/zMs0NzR3G16xo4LOZ/wwgUeFRww8bnelI6Kjmv6KGHzLfPmwPWlIdH0tcXlnOAT/YL4BEIrlx6OmjiVfFO4z36F3NEN2UAKjhh07PPj34yd3nUl5RRnlFGdFYlPLKco4852B23WenoOOJbLTDzjyA3fbdiYrqOBYxKqrjVPWs5JrH/hMzCzpeQdBdOiF0wEn7svt+OzPl0TdINCVab8vcbeugY4lkJVYW44ZnruS9Ke8z89UP6LOZbstclxp+SPXfoh/HXXJE0DFEcsrMGD5qF4aP2iXoKAVJl3REREJCDV9EJCTU8EVEQkINX0QkJNTwRURCItsVr04ws1lmljazTtcGNbPPzGyGmU03s9ps5hTJB9W2lKJsb8ucCRwH/L4L+45296VZzieSL6ptKTnZLnE4G9C32KTkqLalFOXrGr4Dz5nZNDMbt74dzWycmdWaWW1dXV2e4olstC7VtupaCsEGz/DN7AVgYIZNV7r7E12cZ193X2hmmwLPm9kH7p7xWbzuPgGYAFBTU6PH3El32sHMZmYYz3ltq66lEGyw4bv7QdlO4u4L235fYmaPA3sDevh6EXN3SH0AnoCyYZiVBR1pY3zo7p1+INsVqu3S4+k1kPoQIv2xWGk9ZbPbn6VjZtVAxN1Xt/18CHBdd88r3ceTH+LLfwS+HDAgBpvcisVHBR0tr1TbpSe95m5Y8xuwGHgSL9sF63M7FukbdLScyPa2zGPNbAEwEnjazCa3jQ8ys0ltu20GvGZm7wJvAU+7+7PZzCvBcU/gX54C6YXgDeD14Cvx5RfiLQuDjpczqu3w8aaXYc1vgSbwNUAzJN/Dl18YdLScyfYunceBxzOMfw4c3vbzJ8DwbOaRAtI8BUhk2NCCN/wZ63lRvhN1C9V2+HjDPUDjOqOp1qbfsgiLbh5ErJzSN23lm0kvA++4YhYkIa27T6SItXRSvxaD9PL8ZukmavjyzZTvDWRo+FaFxffNexyRnImPAjq5+SC2XV6jdBc1fPlGLDYUKscCle1GKyC2A8QPDCqWSNas+myI9AbK241WQM8rMSvv7I8VFa14Jd+Y9boeykfiDQ8BTVBxNFZ1ImYqJyleFu0H/Z/C6+9r/awqOhCrPgMr/3bQ0XJG71D5xswMKo/AKrVEopQWi/TFel4KPS8NOkq30CUdEZGQUMMXEQkJNXwRkZBQwxcRCQk1fBGRkFDDFxEJCTV8EZGQUMMXEQkJNXwRkZBQwxcRCQk1fBGRkMh2xatbzOwDM3vPzB43s0062W+Mmc0xs7lmdlk2c4rkg2pbSlG2Z/jPA7u6++7Ah8Dl6+5gZlHgNuAwYBhwspkNy3Jeke6m2paSk1XDd/fn3D3V9vINYHCG3fYG5rr7J+6eAB4CxmYzr0h3U21LKcrlNfwzgGcyjG8BzG/3ekHbWEZmNs7Mas2stq5OS+ZJQci6tlXXUgg22PDN7AUzm5nh19h2+1wJpIAHsg3k7hPcvcbdawYMGJDt4UTWZ4d81bbqWgrBBhdAcfeD1rfdzE4DjgQOdHfPsMtCYMt2rwe3jYkE7UN3r+lso2pbSk22d+mMAX4KHO3uDZ3sNhXY3syGWuvCkCcBE7OZV6S7qbalFGV7DX880BN43symm9mdAGY2yMwmAbR98HUBMBmYDTzi7rOynFeku6m2peRktaatu2/XyfjnwOHtXk8CJmUzl0g+qbalFGkRcyk57mlIvIY3vQSRXljlMVhsm6BjiWTN06vwxschNRdiu2CVR2GR6i7/eTV8KSnuLfiK8yHxBngDEMPr78V7XU+kSrfIS/Hy1N54wcoAAAOCSURBVKf4shPBE0AjUInXj4d+j2HRgV06hp6lI6Wl+YV2zR5a76hsglVX4+k1QSYTyYqvvAp8Fa3Nntbf08vwVTd2+Rhq+FJSvPGpds2+HYu2/o9ApAi5JyE5DVj37uAWSLzc5eOo4UtpsQrAMm0Ai+c7jUiOROi8XZd9o6OIlAyrPB6oyLAlAuUj8h1HJCfMohA/kI4fu5ZDRdc/m1LDl5Ji8RFQfRoQByrBqsGqsT530vrdKJHiZL1/BtEhrTVNBVgVlA3Dev64y8fQXTpSciI9L8UrT4DE661vjvhoLFIVdCyRrFikL/R/ChJvQss8iO0AZXtglukSZmZq+FKSLDYYYicGHUMkp8wiEB8JjNyoP69LOiIiIaGGLyISEmr4IiIhoYYvIhISavgiIiFhmRfyKQxmVgfM68Yp+gNLu/H4uVAMGaE4c27t7nlfb1B1/TXlzJ0u1XVBN/zuZma161virhAUQ0ZQzkJSLP+Mypk7Xc2oSzoiIiGhhi8iEhJhb/gTgg7QBcWQEZSzkBTLP6Ny5k6XMob6Gr6ISJiE/QxfRCQ01PBFREIi1A3fzE4ws1lmljazgrvtyszGmNkcM5trZpcFnScTM/uDmS0xs5lBZ+mMmW1pZi+b2ftt/70vDjpTdyvk2i6GuobSrO1QN3xgJnAcMCXoIOsysyhwG3AYMAw42cyGBZsqo3uBMUGH2IAU8BN3HwZ8Bzi/QP9d5lJB1nYR1TWUYG2HuuG7+2x3nxN0jk7sDcx190/cPQE8BHR9LbM8cfcpwJdB51gfd1/k7m+3/bwamA1sEWyq7lXAtV0UdQ2lWduhbvgFbgtgfrvXCyjxJpUPZjYE2AN4M9gkoaW67iZdqe2SX/HKzF4ABmbYdKW7P5HvPBIcM+sB/Bm4xN1XBZ0nW6pt+UpXa7vkG767HxR0ho20ENiy3evBbWOyEcysjNY3xAPu/peg8+RCkda26jrHvklt65JO4ZoKbG9mQ82sHDgJmBhwpqJkras83w3MdvdfBp0n5FTXOfRNazvUDd/MjjWzBbSuCPy0mU0OOtNX3D0FXABMpvWDmEfcfVawqToysweBfwA7mtkCMzsz6EwZ7AOcAhxgZtPbfh0edKjuVKi1XSx1DaVZ23q0gohISIT6DF9EJEzU8EVEQkINX0QkJNTwRURCQg1fRCQk1PBFREJCDV9EJCT+P2uLgQUAFhskAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HUjdeIefsIsg", + "outputId": "f267f505-8ba4-43ef-9ebe-df124c3c05a1" + }, + "source": [ + "tf.reduce_mean(tf.cast(((pred[0]>0.5)==valid_labels),tf.float32))" + ], + "execution_count": 28, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 28 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_95qF9lY2kHp" + }, + "source": [ + "## Используем оптимизаторы TensorFlow\r\n", + "\r\n", + "Tensorflow достаточно плотно интегрирован с библиотекой Keras, которая содержит в себе множество полезного. Например, мы можем использовать оптимизаторы, реализующие немного другие алгоритмы обучения, чем градиентный спуск.\r\n", + "\r\n", + "Также попробуем выводить точность на всех этапах обучения." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ups7nlV22ofp", + "outputId": "aa4dff06-82b9-4b2f-ca00-33970ea2b989" + }, + "source": [ + "optimizer = tf.keras.optimizers.Adam(0.01)\r\n", + "\r\n", + "learning_rate = 0.05\r\n", + "\r\n", + "W = tf.Variable(tf.random.normal(shape=(2,1)))\r\n", + "b = tf.Variable(tf.zeros(shape=(1,),dtype=tf.float32))\r\n", + "\r\n", + "@tf.function\r\n", + "def train_on_batch(x, y):\r\n", + " vars = [W, b]\r\n", + " with tf.GradientTape() as tape:\r\n", + " z = tf.sigmoid(tf.matmul(x, W) + b)\r\n", + " loss = tf.reduce_mean(tf.keras.losses.binary_crossentropy(z,y))\r\n", + " correct_prediction = tf.equal(tf.round(y), tf.round(z))\r\n", + " acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\r\n", + " grads = tape.gradient(loss, vars)\r\n", + " optimizer.apply_gradients(zip(grads,vars))\r\n", + " return loss,acc\r\n", + "\r\n", + "for epoch in range(40):\r\n", + " for step, (x, y) in enumerate(dataset):\r\n", + " loss,acc = train_on_batch(tf.reshape(x,(-1,2)), tf.reshape(y,(-1,1)))\r\n", + " print('Epoch %d: last batch loss = %.4f, acc = %.4f' % (epoch, float(loss),acc))" + ], + "execution_count": 30, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 6.1202, acc = 0.6667\n", + "Epoch 1: last batch loss = 5.9872, acc = 0.6667\n", + "Epoch 2: last batch loss = 5.8651, acc = 0.6667\n", + "Epoch 3: last batch loss = 5.7551, acc = 0.8333\n", + "Epoch 4: last batch loss = 5.6570, acc = 0.8333\n", + "Epoch 5: last batch loss = 5.5700, acc = 0.8333\n", + "Epoch 6: last batch loss = 5.4931, acc = 0.8333\n", + "Epoch 7: last batch loss = 5.4255, acc = 0.8333\n", + "Epoch 8: last batch loss = 5.3660, acc = 0.8333\n", + "Epoch 9: last batch loss = 5.3140, acc = 0.8333\n", + "Epoch 10: last batch loss = 5.2685, acc = 0.8333\n", + "Epoch 11: last batch loss = 5.2289, acc = 0.8333\n", + "Epoch 12: last batch loss = 5.1945, acc = 0.8333\n", + "Epoch 13: last batch loss = 5.1649, acc = 0.8333\n", + "Epoch 14: last batch loss = 5.1394, acc = 0.8333\n", + "Epoch 15: last batch loss = 5.1177, acc = 0.8333\n", + "Epoch 16: last batch loss = 5.0993, acc = 0.8333\n", + "Epoch 17: last batch loss = 5.0840, acc = 0.8333\n", + "Epoch 18: last batch loss = 5.0714, acc = 0.8333\n", + "Epoch 19: last batch loss = 5.0612, acc = 0.8333\n", + "Epoch 20: last batch loss = 5.0532, acc = 0.8333\n", + "Epoch 21: last batch loss = 5.0472, acc = 0.6667\n", + "Epoch 22: last batch loss = 5.0430, acc = 0.6667\n", + "Epoch 23: last batch loss = 5.0404, acc = 0.6667\n", + "Epoch 24: last batch loss = 5.0391, acc = 0.6667\n", + "Epoch 25: last batch loss = 5.0391, acc = 0.6667\n", + "Epoch 26: last batch loss = 5.0403, acc = 0.8333\n", + "Epoch 27: last batch loss = 5.0424, acc = 0.8333\n", + "Epoch 28: last batch loss = 5.0454, acc = 0.8333\n", + "Epoch 29: last batch loss = 5.0491, acc = 0.8333\n", + "Epoch 30: last batch loss = 5.0535, acc = 0.8333\n", + "Epoch 31: last batch loss = 5.0584, acc = 0.8333\n", + "Epoch 32: last batch loss = 5.0638, acc = 0.8333\n", + "Epoch 33: last batch loss = 5.0696, acc = 0.8333\n", + "Epoch 34: last batch loss = 5.0756, acc = 0.8333\n", + "Epoch 35: last batch loss = 5.0818, acc = 0.8333\n", + "Epoch 36: last batch loss = 5.0882, acc = 0.8333\n", + "Epoch 37: last batch loss = 5.0946, acc = 0.8333\n", + "Epoch 38: last batch loss = 5.1010, acc = 0.8333\n", + "Epoch 39: last batch loss = 5.1073, acc = 0.8333\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dvAiaj_JndyP" + }, + "source": [ + "**Задание 1**: Постройте графики ошибок на обучающей и тестовой выборке в процессе обучения\r\n", + "\r\n", + "**Задание 2**: Попробуйте решить задачу классификации на датасете MNIST с помощью этого кода. Подсказка: используйте `softmax_crossentropy_with_logits` или `sparse_softmax_cross_entropy_with_logits` в качестве функции ошибки. При этом в первом случае на выход сети необходимо подавать целевые значения в формате *one hot encoding*, а во втором - в виде целочисленного номера класса." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "995iCprDrgYQ" + }, + "source": [ + "## Keras\r\n", + "### Deep Learning for Humans\r\n", + "\r\n", + "* Раньше работал поверх Tensorflow, CNTK или Theano, сейчас включен в состав Tensorflow\r\n", + "* Оперирует нейросетями на уровне слоёв\r\n", + "* Включает упрощённый \"обучатель\", средства работы с типовыми данными (картинками, ...)\r\n", + "* Много готовых примеров\r\n", + "* Functional API vs. Sequential API\r\n", + "\r\n", + "Keras даёт более высокоуровневое API для реализации нейросетей, позволяя определять нейросети как комбинации слоёв и оперировать понятиями \"модель\", \"слой\", \"алгоритм обучения\".\r\n", + "\r\n", + "Книжка от создателя Keras: [Deep Learning with Python](https://www.manning.com/books/deep-learning-with-python)" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "QJWplVfy34Eo", + "outputId": "9be976f2-4f9a-495c-bddc-a7f9ec30989a" + }, + "source": [ + "inputs = tf.keras.Input(shape=(2,))\r\n", + "z = tf.keras.layers.Dense(1,kernel_initializer='glorot_uniform',activation='sigmoid')(inputs)\r\n", + "model = tf.keras.models.Model(inputs,z)\r\n", + "\r\n", + "train_x_norm = train_x-np.min(train_x) / (np.max(train_x)-np.min(train_x))\r\n", + "\r\n", + "model.compile(tf.keras.optimizers.Adam(0.1),'binary_crossentropy',['accuracy'])\r\n", + "model.summary()\r\n", + "h = model.fit(train_x_norm,train_labels,batch_size=8,epochs=15)" + ], + "execution_count": 32, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Model: \"model_1\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "input_2 (InputLayer) [(None, 2)] 0 \n", + "_________________________________________________________________\n", + "dense_1 (Dense) (None, 1) 3 \n", + "=================================================================\n", + "Total params: 3\n", + "Trainable params: 3\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n", + "Epoch 1/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 1.1443 - accuracy: 0.3283\n", + "Epoch 2/15\n", + "9/9 [==============================] - 0s 2ms/step - loss: 0.6759 - accuracy: 0.5823\n", + "Epoch 3/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4717 - accuracy: 0.7888\n", + "Epoch 4/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.3932 - accuracy: 0.9044\n", + "Epoch 5/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4254 - accuracy: 0.8513\n", + "Epoch 6/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.3866 - accuracy: 0.8534\n", + "Epoch 7/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4578 - accuracy: 0.8344\n", + "Epoch 8/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4432 - accuracy: 0.8192\n", + "Epoch 9/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4172 - accuracy: 0.8408\n", + "Epoch 10/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.3778 - accuracy: 0.8819\n", + "Epoch 11/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4008 - accuracy: 0.8800\n", + "Epoch 12/15\n", + "9/9 [==============================] - 0s 3ms/step - loss: 0.3769 - accuracy: 0.9019\n", + "Epoch 13/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.3995 - accuracy: 0.8336\n", + "Epoch 14/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.3816 - accuracy: 0.8719\n", + "Epoch 15/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4211 - accuracy: 0.8244\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "K2Kf60IrZcqs", + "outputId": "b60b868d-3562-4715-f5d5-1f9764e45f09" + }, + "source": [ + "plt.plot(h.history['accuracy'])" + ], + "execution_count": 37, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 37 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAbKElEQVR4nO3dfXRU933n8fdXT0iAAAkE2EJIGAtsbKd+EPhpm+MmddZJeiDd7Db41E+JU7a7cZJts921d3u8OW67p/vUdM+ptz3eOMa1U7Neb9LQLq3tk6Yn3fgBCRs/AJYgWAIBGgQSYiTQ43z3jxnhsRBoECPdufd+Xudw0Ny5zHyA0Ud3fvd3f2PujoiIhF9R0AFERCQ/VOgiIhGhQhcRiQgVuohIRKjQRUQioiSoJ16yZIk3NDQE9fQiIqG0a9euE+5eM9l9gRV6Q0MDLS0tQT29iEgomVnHhe7TkIuISESo0EVEIkKFLiISESp0EZGIUKGLiESECl1EJCJU6CIiERHYPHSZvv6hUX74VidXLKzglvoqquaVBR1JRAqACj1kegeGeeiZnbzT2XduW+PS+TQ1VNFUX836hmrqqiswswBTikgQVOgh0tU3yP1Pv0lHzxn+9NdvpnpeGS0dvTS39/DX7x7jhZ2HAaipnMP6rIK/9opKSoo1uiYSdSr0kGg/McB9T79J78Awz355A7evXgzArVelf0+lnLbjSZrbe9nV3kNzey873usCYG5ZMTetXERTfTVNDVXctLKK+XP0Xy8SNRbUR9A1NTW51nLJzb5jp7n/6Z2MpVI8+5UNfGLFopz+3NFTZ2np+KjgP+g6TcqhyGDdlQvOFfz6hmqWLSif4b+FiOSDme1y96ZJ71OhF7ZdHb18+ZmdzC0r4bmHN9C4rHLaj5UcHOHtQ6doyRT87sOnODsyBkBddQVN9enhmeKi/A/PLK2cQ1NDFVcsrMj7Y+fL4MgY7x3pY+/R04ym9Fm7M6Gs2LhhxSKuu3IBpQU6DOjuHO45y65DPfQMjMzIc9yxejHXXrFgWn/2YoWu990F7B/2d7Plz3exbMEcnnv4Vuqq517W41WWl/LJNTV8ck165c2RsRR7j56mub2HXR29/MP+E/zw7SP5iH5BtYsq0idwG6pZ31DFmqWVFBUFcwK3d2CYXR29NHf00NLey3udfQyPpQLJEjflpUXcWLeI9Q3V3FJfxc31VSwoLw0ky+hYin3HkrRkXgfN7T0cTw7N6HP+/heun3ahX4yO0AvU37x3jG9se5vVNfP584c3sLRy5odE3J3k0Ch5f0k4HOo5c+4HR/Y3TGV5CbfUp4d9muqr+IW6RZSXFuc5QPrvdqjnDC3tvbR0pN+hHDjeD0BpsXFD7UKaZjiDwMDQKLsPn6K5PV2ee4+dZizlmME1yxewvqHq3OvhykUz825uYGiUtw+dOvd6fOtQL2eG0+9Usw84muqrZixDeWkRc0qm9xrTkEvIvNh8mEd/8C43raziew+tZ2FFMEcuM2X8Le14sba097A/q1yvr11IU/1H31SL58+55OcYHUux99jp9EnizPN0Z/0QyX58FXhwsgt+V0cvb3X0MpBVrulyr+KW+mrWLq+keBrv5hKnB88debd09LDvWPLcD5Frly+YlQLPJxV6iPzPnx7kD3bs45Nraviz+25mblk8RsVOnckMf2QK+J3DHw1/XLVkXtYwTTUNi+eeN8++f2iUtw/1nvsBsfvwqXNHXSuqKs4V+PqGahqXzg9smEcubnQsxQddyfR5no70/2XidOYH8ZwSbq6vOvd/eWPdIirKPv6DOJVyDnT3p39AtKeH0w73nAXSR8U31WV+QDRUc9PKRYEN81wOFXoIuDv/7ZU2/uQnB/j8DVfwnS/dSFlJYZ40mg2DI2O8f6SPlsw3dUtHL6fOpE9QLZ5XRlNDFTfWVZE4PUhzew/7jn00g+faKxacG5st9BOxcnHuTmfvR+/mdrX30ppIAlBSZFxXu5D1mauld3X0squjl76z6dfJkvlzMuWfHsJZV8AnYi+FCr3ApVLOf9i+h+fe6GDz+jr+4FdvmNZbyyhLpZyDJ/ppzrx13tXRS8fJM1SUZubYZ06yao599PWdGeGtQ73nxuF3d55ieDTF6pp56XMxmeGT+kneyUWBCr2AjYyl+Nf/+x1+tPso//yTV/HoZ6+J5ItwJvQODDO/vCQSR10yfUOjYwwOp1g4N3zDJ9OhaYsFanBkjK99/y1+/MFxfucfr+Vf3rVaZX4JtCiZAMwpKZ72jJGoUaEHJDk4wlefbWFnew+/94Xruf+2+qAjiUjIqdADcLJ/iAef2ckHx5L88ZduZNONtUFHEpEIUKHPsqOnznL/02/S2XuWpx64hU9dsyzoSCISESr0WfThiQHu++6bnD47wnMP38qGVdVBRxKRCFGhz5I9R/t48Hs7cYcXttzG9bULg44kIhGjQp8Fze09fGVrM5VzSnjuq7eyumZ+0JFEJIJU6DPs71uP85vP7+LKhRU899VbqQ3BWhEiEk4q9BnUOzDMv3j+La5akl4xcck0FpkSEcmVLrGbQduaD3N2ZIzvfOlGlbmIzLicCt3M7jGzVjM7YGaPTnL/SjP7iZm9bWbvmtnn8h81XEbHUjz3ejt3rF7M2uXT/5QhEZFcTVnoZlYMPAl8FlgH3Gtm6ybs9rvAi+5+E7AZ+B/5Dho2r+5NcLRvkIfuaAg6iojERC5H6BuAA+5+0N2HgW3Apgn7ODD+eUoLgaP5ixhOW19rZ0VVBZ++VhcOicjsyKXQa4HDWbc7M9uyfRu4z8w6gR3A1/OSLqT2Hj3Nmx/28MDt9VoGV0RmTb5Oit4LbHX3FcDngOfM7LzHNrMtZtZiZi3d3d15eurC8+xr7VSUFvOlppVBRxGRGMml0I8AdVm3V2S2ZXsYeBHA3V8HyoElEx/I3Z9y9yZ3b6qpqZle4gLXMzDMX+4+wq/eXBub9ZlFpDDkUujNQKOZrTKzMtInPbdP2OcQ8GkAM7uWdKFH9xD8IrY1H2JoNKWToSIy66YsdHcfBR4BXgb2kZ7NssfMnjCzjZndvgX8hpm9A7wAPORBfRRSgEbHUjz/egd3rF7MmmWaqigisyunK0XdfQfpk53Z2x7P+novcGd+o4XP+FTFb2+8LugoIhJDulI0j57RVEURCZAKPU/2HO1j54c9PHh7g6YqikggVOh5Mj5V8dea6qbeWURkBqjQ8yA9VfEo/0RTFUUkQCr0PNjWfIjh0RQPaqqiiARIhX6Z0qsqdnDn1ZqqKCLBUqFfplf2JjjWN8hDd6wKOoqIxJwK/TJt/Vk7ddUVfOqapUFHEZGYU6Ffhj1H+9jZrqmKIlIYVOiXYXyq4j/TVEURKQAq9Gn62FTFCk1VFJHgqdCn6YWd6amKWlVRRAqFCn0aRsZSPP9GB//o6iU0aqqiiBQIFfo0vLJnfKpiQ9BRRETOUaFPw9bXPqSuuoJf0lRFESkgKvRL9P6RPprbezVVUUQKjgr9EmmqoogUKhX6JTjZP8SP3jnKF2/RVEURKTwq9EuwrflwelXF2xuCjiIich4Veo5GMqsqaqqiiBQqFXqOXtmToOu0piqKSOFSoedo62sfsrJ6rqYqikjBUqHnYHyq4gO312uqoogULBV6Dra+1s7cMk1VFJHCpkKfwon+IbZrVUURCQEV+hS27TzE8JimKopI4VOhX0R6VcVD/GKjpiqKSOHLqdDN7B4zazWzA2b26CT3f8fMdmd+tZnZqfxHnX0v7+nSVEURCY2SqXYws2LgSeBuoBNoNrPt7r53fB93/62s/b8O3DQDWWfd1p+1U794Lr+0VlMVRaTw5XKEvgE44O4H3X0Y2AZsusj+9wIv5CNckN7r7KOlo5cHbm+gSFMVRSQEcin0WuBw1u3OzLbzmFk9sAr4uwvcv8XMWsyspbu7+1KzzqqPpiquCDqKiEhO8n1SdDPwkruPTXanuz/l7k3u3lRTU5Pnp86fE/1D/NU7R/nizStYUK6piiISDrkU+hEg+4qaFZltk9lMBIZbzk1VvKM+6CgiIjnLpdCbgUYzW2VmZaRLe/vEnczsGqAKeD2/EWfXyFiK597o4Bcbl3D1Uk1VFJHwmLLQ3X0UeAR4GdgHvOjue8zsCTPbmLXrZmCbu/vMRJ0df/t+F4nTQ3z5zoago4iIXJIppy0CuPsOYMeEbY9PuP3t/MUKztbX0lMV71qjqYoiEi66UjTLe5197NJURREJKRV6Fk1VFJEwU6FnjE9V/Ke3aKqiiISTCj3jhTfTUxUf0KqKIhJSKvSMH39wnKb6Kq5eOj/oKCIi06JCB1IpZ38iyfW1C4OOIiIybSp04MipswwMj7FGa56LSIip0IG2RBKAtcs13CIi4aVCB1ozha5PJRKRMFOhA21dSa5cWK7piiISaip0oDXRz5rlOjoXkXCLfaGPjqX4+fF+1mq4RURCLvaF3tFzhuGxlMbPRST0Yl/obV2ZGS4qdBEJudgXemsiiRm6QlREQi/2hd6WSFJfPZeKsuKgo4iIXJbYF3prV1JXiIpIJMS60AdHxmg/eYa1mrIoIhEQ60I/2D3AWMp1hC4ikRDrQv9oDRcVuoiEX6wLvTWRpLTYaFg8L+goIiKXLdaF3taV5Kol8ykrifU/g4hERKybrDWR1BouIhIZsS30/qFROnvPsnaZLigSkWiIbaHv1xroIhIxMS70fkBruIhIdORU6GZ2j5m1mtkBM3v0Avv8mpntNbM9ZvYX+Y2Zf62JJOWlRdRVzw06iohIXpRMtYOZFQNPAncDnUCzmW13971Z+zQCjwF3unuvmS2dqcD50pZI0ri0kuIiCzqKiEhe5HKEvgE44O4H3X0Y2AZsmrDPbwBPunsvgLsfz2/M/NMaLiISNbkUei1wOOt2Z2ZbtjXAGjP7mZm9YWb35CvgTOgdGOZ4coi1yzXDRUSiY8ohl0t4nEbgLmAF8FMzu8HdT2XvZGZbgC0AK1euzNNTX7rxS/51hC4iUZLLEfoRoC7r9orMtmydwHZ3H3H3D4E20gX/Me7+lLs3uXtTTU3NdDNfNq3hIiJRlEuhNwONZrbKzMqAzcD2Cfv8Jemjc8xsCekhmIN5zJlXrYkkleUlLF9QHnQUEZG8mbLQ3X0UeAR4GdgHvOjue8zsCTPbmNntZeCkme0FfgL8jrufnKnQl6utq5+1yyox0wwXEYmOnMbQ3X0HsGPCtsezvnbgtzO/Cpq705pI8vlPXBF0FBGRvIrdlaLHk0P0nR3RFaIiEjmxK/TWLs1wEZFoil2hfzRlUXPQRSRaYlnoS+aXsXj+nKCjiIjkVewKvTXRr+EWEYmkWBV6KuXsT2gNFxGJplgV+pFTZzkzPKYrREUkkmJV6JrhIiJRFq9C1wwXEYmwWBV6WyJJ7aIKKstLg44iIpJ3sSr09Ida6OhcRKIpNoU+MpbiYPcAa3RCVEQiKjaF3nFygOGxlNZwEZHIik2ht3b1A5rhIiLRFZ9CTyQpMrh6qcbQRSSaYlPobV1J6hfPo7y0OOgoIiIzIj6FflwzXEQk2mJR6IMjY7SfGNAJURGJtFgU+s+7+0k5mrIoIpEWi0If/1ALHaGLSJTFotBbu/opLTYalswLOoqIyIyJRaG3JZKsrplPaXEs/roiElOxaLj0Gi4abhGRaIt8oScHRzhy6qw+1EJEIi/yhb7/uC75F5F4iHyht3VphouIxEPkC701kaSitJgVVRVBRxERmVE5FbqZ3WNmrWZ2wMweneT+h8ys28x2Z359Nf9Rp6ctkb7kv6jIgo4iIjKjSqbawcyKgSeBu4FOoNnMtrv73gm7/i93f2QGMl6W1q5+7lpbE3QMEZEZl8sR+gbggLsfdPdhYBuwaWZj5UfPwDAn+oc0fi4isZBLodcCh7Nud2a2TfRFM3vXzF4ys7rJHsjMtphZi5m1dHd3TyPupRm/5F9ruIhIHOTrpOhfAQ3u/gngVeDZyXZy96fcvcndm2pqZn4YRGu4iEic5FLoR4DsI+4VmW3nuPtJdx/K3PwucEt+4l2e1q4kC8pLWLZgTtBRRERmXC6F3gw0mtkqMysDNgPbs3cwsyuybm4E9uUv4vS1JZKsXV6JmWa4iEj0TVno7j4KPAK8TLqoX3T3PWb2hJltzOz2DTPbY2bvAN8AHpqpwLlyd63hIiKxMuW0RQB33wHsmLDt8ayvHwMey2+0y5M4PcTpwVGt4SIisRHZK0Vbx2e46AhdRGIisoU+voaLCl1E4iKyhd6aSFJTOYfqeWVBRxERmRWRLfS2RFLzz0UkViJZ6KmU05ZI0rhsftBRRERmTSQL/XDvGQZHUjpCF5FYiWShtyUyn1KkKYsiEiMRLfT0DJfGpRpyEZH4iGSht3YlqV1UQWV5adBRRERmTSQLfXwNFxGROIlcoY+Mpfh5d78uKBKR2IlcobefGGBkzFm7XOPnIhIvkSt0reEiInEVuUJv60pSZLC6RkfoIhIvkSv01kSShiXzKC8tDjqKiMisilyhtyX6dYWoiMRSpAp9cGSM9pMDGj8XkViKVKEfON6Pu06Iikg8RarQWzMfaqEpiyISR5Eq9LbjScqKi6hfPC/oKCIisy5ahd6V5KqaeZQWR+qvJSKSk0g1X1uiX2u4iEhsRabQk4MjHDl1VidERSS2IlPo4x9qoTnoIhJXESr08RkuKnQRiafIFHprV5K5ZcXULqoIOoqISCByKnQzu8fMWs3sgJk9epH9vmhmbmZN+YuYm7ZEksZllRQV2Ww/tYhIQZiy0M2sGHgS+CywDrjXzNZNsl8l8E3gzXyHzEVbIsnaZbqgSETiK5cj9A3AAXc/6O7DwDZg0yT7/R7wn4DBPObLyYn+IU70D2uGi4jEWi6FXgsczrrdmdl2jpndDNS5+//NY7ac6YSoiEgeToqaWRHwR8C3cth3i5m1mFlLd3f35T71OW1d+pQiEZFcCv0IUJd1e0Vm27hK4Hrg782sHbgN2D7ZiVF3f8rdm9y9qaamZvqpJ2hN9LOwopSllXPy9pgiImGTS6E3A41mtsrMyoDNwPbxO929z92XuHuDuzcAbwAb3b1lRhJPYn8iydpllZhphouIxNeUhe7uo8AjwMvAPuBFd99jZk+Y2caZDjgVd6c1kWSNlswVkZgryWUnd98B7Jiw7fEL7HvX5cfKXdfpQZKDo7rkX0RiL/RXirbqhKiICBCBQh+fsqhCF5G4C32ht3b1s7RyDlXzyoKOIiISqNAXelsiqQuKREQIeaGPpZz9x5MabhERIeSFfrjnDIMjKc1wEREh5IXeOn5CVEMuIiLhLvTxNVwal+qiIhGRUBd6ayLJiqoK5s3J6fooEZFIC3Wht2XWcBERkRAX+vBoioPdAxo/FxHJCG2ht58cYDTlOkIXEckIbaFrDRcRkY8LbaG3JZIUFxlX1cwLOoqISEEIbaG3diVpWDyX8tLioKOIiBSE0Ba61nAREfm4UBb62eExOnrOaPxcRCRLKAv9wPF+3NEMFxGRLKEsdK3hIiJyvlAWelsiSVlJEfXVc4OOIiJSMEJZ6K1dSVbXzKekOJTxRURmRCgbMb2Gi1ZYFBHJFrpC7zs7wrG+QY2fi4hMELpCP3A8fUJUM1xERD4udIXe2tUPaA0XEZGJQlfoS+aXcfe6ZdQuqgg6iohIQQndR/185rrlfOa65UHHEBEpODkdoZvZPWbWamYHzOzRSe7/TTN7z8x2m9n/M7N1+Y8qIiIXM2Whm1kx8CTwWWAdcO8khf0X7n6Du98I/Gfgj/KeVERELiqXI/QNwAF3P+juw8A2YFP2Du5+OuvmPMDzF1FERHKRyxh6LXA463YncOvEnczsa8BvA2XApyZ7IDPbAmwBWLly5aVmFRGRi8jbLBd3f9LdVwP/FvjdC+zzlLs3uXtTTU1Nvp5aRETIrdCPAHVZt1dktl3INuALlxNKREQuXS6F3gw0mtkqMysDNgPbs3cws8asm58H9ucvooiI5GLKMXR3HzWzR4CXgWLge+6+x8yeAFrcfTvwiJn9MjAC9AIPzmRoERE5n7kHMyHFzLqBjmn+8SXAiTzGmWlhyhumrBCuvGHKCuHKG6ascHl569190pOQgRX65TCzFndvCjpHrsKUN0xZIVx5w5QVwpU3TFlh5vKGbi0XERGZnApdRCQiwlroTwUd4BKFKW+YskK48oYpK4Qrb5iywgzlDeUYuoiInC+sR+giIjKBCl1EJCJCV+hTrc1eKMyszsx+YmZ7zWyPmX0z6Ey5MLNiM3vbzP466CwXY2aLzOwlM/vAzPaZ2e1BZ7oYM/utzOvgfTN7wczKg86Uzcy+Z2bHzez9rG3VZvaqme3P/F4VZMZxF8j6XzKvhXfN7IdmtijIjOMmy5p137fMzM1sSb6eL1SFnuPa7IViFPiWu68DbgO+VsBZs30T2Bd0iBz8d+Bv3f0a4Bco4MxmVgt8A2hy9+tJX3G9OdhU59kK3DNh26PAj929Efhx5nYh2Mr5WV8Frnf3TwBtwGOzHeoCtnJ+VsysDvgMcCifTxaqQieHtdkLhbsfc/e3Ml8nSRdObbCpLs7MVpBei+e7QWe5GDNbCHwSeBrA3Yfd/VSwqaZUAlSYWQkwFzgacJ6PcfefAj0TNm8Cns18/SwFsujeZFnd/RV3H83cfIP0IoKBu8C/K8B3gH9Dnj87ImyFPtna7AVdkgBm1gDcBLwZbJIp/THpF1kq6CBTWAV0A89khoe+a2bzgg51Ie5+BPivpI/GjgF97v5KsKlysszdj2W+7gKWBRnmEnwF+JugQ1yImW0Cjrj7O/l+7LAVeuiY2Xzg/wD/asInOxUUM/sV4Li77wo6Sw5KgJuBP3X3m4ABCmc44DyZsedNpH8QXQnMM7P7gk11aTw9v7ng5zib2b8nPdz5/aCzTMbM5gL/Dnh8Jh4/bIV+qWuzB8rMSkmX+ffd/QdB55nCncBGM2snPZT1KTN7PthIF9QJdLr7+Duel0gXfKH6ZeBDd+929xHgB8AdAWfKRcLMrgDI/H484DwXZWYPAb8C/LoX7gU2q0n/YH8n8722AnjLzJbn48HDVuhTrs1eKMzMSI/x7nP3gv/QbHd/zN1XuHsD6X/Xv3P3gjyKdPcu4LCZrc1s+jSwN8BIUzkE3GZmczOvi09TwCdxs2zno6WwHwR+FGCWizKze0gPF2509zNB57kQd3/P3Ze6e0Pme60TuDnzmr5soSr0zEmP8bXZ9wEvuvueYFNd0J3A/aSPdHdnfn0u6FAR8nXg+2b2LnAj8B8DznNBmXcSLwFvAe+R/r4rqEvVzewF4HVgrZl1mtnDwB8Cd5vZftLvMv4wyIzjLpD1T4BK4NXM99qfBRoy4wJZZ+75CvediYiIXIpQHaGLiMiFqdBFRCJChS4iEhEqdBGRiFChi4hEhApdRCQiVOgiIhHx/wHO1VhtMoQ9YwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iJruFXmb_dur" + }, + "source": [ + "Выше мы использовали **функциональный** способ задания модели, когда мы сначала описываем входную переменную, затем - происходящие с ней преобразования, и потом определяем объект `Model`.\r\n", + "\r\n", + "Мы можем также задавать модель как последовательности слоёв с помощью `Sequential`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "iWc_kSr8_YXt", + "outputId": "345dbe65-629d-468f-ed75-1d412c966340" + }, + "source": [ + "model = tf.keras.models.Sequential()\r\n", + "model.add(tf.keras.layers.Dense(5,activation='sigmoid',input_shape=(2,)))\r\n", + "model.add(tf.keras.layers.Dense(1,activation='sigmoid'))\r\n", + "\r\n", + "test_x_norm = test_x-np.min(train_x) / (np.max(train_x)-np.min(train_x))\r\n", + "# это не ошибка, мы нормируем тестовые данные так же, как нормировали обучающие!\r\n", + "# (поэтому min и max считаем для обучающих данных)\r\n", + "\r\n", + "model.compile(tf.keras.optimizers.Adam(0.1),'binary_crossentropy',['accuracy'])\r\n", + "model.summary()\r\n", + "model.fit(train_x_norm,train_labels,validation_data=(test_x_norm,test_labels),batch_size=8,epochs=15)" + ], + "execution_count": 45, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Model: \"sequential_6\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "dense_14 (Dense) (None, 5) 15 \n", + "_________________________________________________________________\n", + "dense_15 (Dense) (None, 1) 6 \n", + "=================================================================\n", + "Total params: 21\n", + "Trainable params: 21\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n", + "Epoch 1/15\n", + "9/9 [==============================] - 1s 19ms/step - loss: 0.6962 - accuracy: 0.5596 - val_loss: 0.5533 - val_accuracy: 0.7333\n", + "Epoch 2/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.5057 - accuracy: 0.8035 - val_loss: 0.4924 - val_accuracy: 0.8000\n", + "Epoch 3/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.4173 - accuracy: 0.8726 - val_loss: 0.5273 - val_accuracy: 0.6667\n", + "Epoch 4/15\n", + "9/9 [==============================] - 0s 4ms/step - loss: 0.4405 - accuracy: 0.8145 - val_loss: 0.5641 - val_accuracy: 0.6667\n", + "Epoch 5/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3523 - accuracy: 0.9039 - val_loss: 0.5498 - val_accuracy: 0.7333\n", + "Epoch 6/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3239 - accuracy: 0.9012 - val_loss: 0.5666 - val_accuracy: 0.6667\n", + "Epoch 7/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.4043 - accuracy: 0.8568 - val_loss: 0.5883 - val_accuracy: 0.6667\n", + "Epoch 8/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3351 - accuracy: 0.8903 - val_loss: 0.5906 - val_accuracy: 0.6667\n", + "Epoch 9/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3624 - accuracy: 0.8761 - val_loss: 0.6198 - val_accuracy: 0.6667\n", + "Epoch 10/15\n", + "9/9 [==============================] - 0s 4ms/step - loss: 0.4015 - accuracy: 0.8622 - val_loss: 0.6087 - val_accuracy: 0.6667\n", + "Epoch 11/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3387 - accuracy: 0.8846 - val_loss: 0.6604 - val_accuracy: 0.6667\n", + "Epoch 12/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3579 - accuracy: 0.8694 - val_loss: 0.6177 - val_accuracy: 0.7333\n", + "Epoch 13/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3304 - accuracy: 0.8830 - val_loss: 0.6274 - val_accuracy: 0.6667\n", + "Epoch 14/15\n", + "9/9 [==============================] - 0s 4ms/step - loss: 0.3193 - accuracy: 0.8807 - val_loss: 0.6472 - val_accuracy: 0.6667\n", + "Epoch 15/15\n", + "9/9 [==============================] - 0s 4ms/step - loss: 0.3552 - accuracy: 0.8546 - val_loss: 0.6596 - val_accuracy: 0.6667\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 45 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BmHNhUU8bqEX" + }, + "source": [ + "## Функции ошибки при классификации\r\n", + "\r\n", + "При использовании Keras для классификации важно правильно указать функцию ошибки и передаточную функцию на последнем слое. Основные правила:\r\n", + "* Если у сети один выход, то передаточная функция - сигмоида, если несколько - softmax\r\n", + "* Если на ожидаемый выход подается в виде one-hot-encoding, то функция ошибки - cross entropy loss (categorical cross-entropy), если номер класса - sparse categorical cross-entropy, для бинарной классификации с одним выходом - binary cross-entropy (она же log loss)\r\n", + "\r\n", + "В целом бинарную классификацию можно рассматривать как частный случай мультиклассовой, подавая на выход one hot encoded вектор.\r\n", + "\r\n", + "| Классификация | Формат входных данных | Передат.функция | Функция ошибки |\r\n", + "|---------------|-----------------------|-----------------|----------|\r\n", + "| Бинарная | Вероятность 1-го класса | sigmoid | binary crossentropy |\r\n", + "| Бинарная | One-hot encoding (2 выхода) | softmax | categorical crossentropy |\r\n", + "| Мультикласс | One-hot encoding | softmax | categorical crossentropy |\r\n", + "| Мультикласс | Номер класса | softmax | sparse categorical crossentropy" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yX6hqiafwHl9" + }, + "source": [ + "## Выводы\n", + "\n", + "* Tensorflow позволяет более гибко определять структуру графа вычислений, описывать свои функции и конфигурации.\n", + "* Есть более удобные средства для работы с данными (`td.Data`), со слоями (`tf.layers`)\n", + "* Для массового использования нейросетей Google рекомендует **Keras**, который позволяет собирать нейросети как конструктор\n", + "* При этом возможно реализовать свой слой для Keras, и потом использовать его в своих моделях.\n", + "* Для типовых задач имеет смысл использовать Keras\n", + "* Также стоит посмотреть на PyTorch, это \"восходящая звезда\"\n", + "\n", + "Хороший Notebook про Keras и Tensorflow 2.0 от создателя Keras - [тут](https://t.co/k694J95PI8)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gZ-kWx84bMDH" + }, + "source": [ + "**Задание 3**: \r\n", + "Используйте Keras для обучения классификатора на сети MNIST. При этом:\r\n", + "* Обратите внимание, что в keras заложены типовые датасеты, включая MNIST. Для обращения к нему достаточно пары строчек кода (см, например, [тут](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/mnist))\r\n", + "* Попробуйте несколько конфигураций сети с несколькими полносвязными слоями, передаточными функциями, и разным количеством нейронов\r\n", + "\r\n", + "Какой точности вам удалось достичь?\r\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "MG64NKzawHl-" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/3-NeuralNetworks/05-Frameworks/IntroPyTorch.ipynb b/3-NeuralNetworks/05-Frameworks/IntroPyTorch.ipynb new file mode 100644 index 00000000..b37041ac --- /dev/null +++ b/3-NeuralNetworks/05-Frameworks/IntroPyTorch.ipynb @@ -0,0 +1,1726 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "celltoolbar": "Slideshow", + "kernelspec": { + "name": "python3", + "display_name": "Python 3", + "language": "python" + }, + "language_info": { + "mimetype": "text/x-python", + "nbconvert_exporter": "python", + "name": "python", + "file_extension": ".py", + "version": "3.7.4-final", + "pygments_lexer": "ipython3", + "codemirror_mode": { + "version": 3, + "name": "ipython" + } + }, + "livereveal": { + "start_slideshow_at": "selected" + }, + "colab": { + "name": "IntroPyTorch.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "En2vX4FuwHlu" + }, + "source": [ + "# Введение в нейронные сети\n", + "\n", + "## Эпизод 2b: Многослойный персептрон на PyTorch\n", + "\n", + "Дмитрий Сошников | dmitri@soshnikov.com\n", + "\n", + "http://github.com/shwars/NeuroWorkshop\n", + "-> Notebooks -> IntroPyTorch.ipynb" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fVjlmWBwwHl2" + }, + "source": [ + "## Нейросетевые фреймворки\n", + "\n", + "Мы видели, что для обучения нейросетей нужно:\n", + "* Быстро умножать матрицы (тензоры)\n", + "* Считать производные для вычисления градиента для метода обратного распространения ошибки\n", + "\n", + "Что позволяют делать нейросетевые фреймворки:\n", + "* Оперировать с тензорами, как на CPU, так и на GPU\n", + "* Автоматически вычислять производные (они вручную прописаны для всех элементарных функций)\n", + "\n", + "Опционально:\n", + "* Конструктор для нейросетей (описание сети как набора слоёв)\n", + "* Простые функции для обучения (`fit`, как в Scikit Learn)\n", + "* Набор алгоритмов оптимизации\n", + "* Набор абстракций для работы с данными" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8cACQoFMwHl3" + }, + "source": [ + "## Основные фреймворки\n", + "\n", + "* Tensorflow 1.0 - первый, получивший широкое распространение (Google). Позволял определять статический computation graph, и затем в явном виде выполнять вычисления\n", + "* PyTorch - Facebook\n", + "* Keras - надстройка над Tensorflow/PyTorch для унификации (Francois Chollet)\n", + "* Tensorflow 2.0 + Keras - динамический вычислительный граф, код получается похожим на обычные вычисления в numpy\n", + "\n", + "Мы рассмотрим PyTorch. Для начала рекомендуется установить PyTorch [по инструкции на сайте](https://pytorch.org/get-started/locally/). Либо можно выполнять код в [Google Colab](https://colab.research.google.com/), в котором PyTorch уже установлен." + ] + }, + { + "cell_type": "code", + "metadata": { + "tags": [], + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "id": "xwqVx9-bwHl3", + "outputId": "38564a63-0567-4406-ee1a-1d3618f27351" + }, + "source": [ + "import torch\n", + "torch.__version__" + ], + "execution_count": 168, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'1.8.0+cu101'" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 168 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6tp2xGV7wHl4" + }, + "source": [ + "## Основные понятия в PyTorch\n", + "\n", + "**Тензор** - это многомерный массив произвольной размерности. Удобно использовать при обучении нейросетей, например:\n", + "* 400x400 - чёрно-белая картинка\n", + "* 400x400x3 - цветная картинка\n", + "* 16x400x400x3 - minibatch из 16 картинок, используемый для одного шага обучения\n", + "* 25x400x400x3 - секунда видео\n", + "* 8x25x400x400x3 - minibatch из 8 1-секундных видео" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qG2bsaR7wHl4" + }, + "source": [ + "### Простые тензоры" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ybpnk08HwHl4", + "outputId": "54e2c89b-b373-4389-b285-49b0510be931" + }, + "source": [ + "a = torch.tensor([[1,2],[3,4]])\n", + "print(a)\n", + "a = torch.randn(size=(10,3))\n", + "print(a)" + ], + "execution_count": 169, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[1, 2],\n", + " [3, 4]])\n", + "tensor([[ 0.1288, -0.8088, -0.2092],\n", + " [ 0.0789, -1.4706, -0.8880],\n", + " [-0.0601, 0.6659, -3.2120],\n", + " [ 0.8023, -0.4045, -1.0164],\n", + " [-1.5578, -1.3014, -0.8372],\n", + " [ 1.3626, -1.0825, -2.7018],\n", + " [ 0.2258, -0.7675, -0.4102],\n", + " [-1.9663, 0.0386, -2.6183],\n", + " [ 0.7666, -1.3057, -0.4486],\n", + " [-0.9006, 0.7092, 0.8018]])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AXFMsV3r09Ux" + }, + "source": [ + "С тензорами можно производить обычные вычисления, которые производятся поэлементно (как в numpy). При этом тензоры автоматически дополняются до нужной размерности. Можно извлечь numpy-массив из тензора при помощи `.numpy()`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "e5Nu5Xgj1DnQ", + "outputId": "c1fbcd86-dde6-40b6-8edf-7a37f9d60901" + }, + "source": [ + "print(a-a[0])\n", + "print(torch.exp(a)[0].numpy())" + ], + "execution_count": 170, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[ 0.0000, 0.0000, 0.0000],\n", + " [-0.0499, -0.6618, -0.6788],\n", + " [-0.1889, 1.4747, -3.0028],\n", + " [ 0.6734, 0.4044, -0.8072],\n", + " [-1.6867, -0.4926, -0.6280],\n", + " [ 1.2338, -0.2737, -2.4927],\n", + " [ 0.0970, 0.0413, -0.2010],\n", + " [-2.0951, 0.8475, -2.4091],\n", + " [ 0.6377, -0.4969, -0.2395],\n", + " [-1.0295, 1.5180, 1.0110]])\n", + "[1.1375165 0.44538715 0.81125516]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uQ5zN6cVyrG7" + }, + "source": [ + "## In-place и out-of-place операции\n", + "\n", + "Обычно операции с тензорами возвращают новые тензоры. Однако для большинства операций сущетвуют аналогичные варианты, которые модифицируют исходный тензор:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Mjkbcw3-ACKS", + "outputId": "ca021008-9ab6-4b09-c5a5-bbe854cd1493" + }, + "source": [ + "u = torch.tensor(5)\n", + "print(\"Result when adding out-of-place:\",u.add(torch.tensor(3)))\n", + "u.add_(torch.tensor(3))\n", + "print(\"Result after adding in-place:\", u)" + ], + "execution_count": 171, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Result when adding out-of-place: tensor(8)\n", + "Result after adding in-place: tensor(8)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DLPUcVsXACKT" + }, + "source": [ + "Например, вот там можно \"наивно\" посчитать сумму строк тензора `a`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7pu0UZ-_yqfB", + "outputId": "bd2e8c6a-39e1-4f29-990b-9591e866936c" + }, + "source": [ + "s = torch.zeros_like(a[0])\n", + "for i in a:\n", + " s.add_(i)\n", + "\n", + "print(s)" + ], + "execution_count": 172, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([ -1.1197, -5.7273, -11.5398])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rIh1EHcezlNo" + }, + "source": [ + "Умный способ:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aQIdWZ1kzn6P", + "outputId": "89000bb4-f45e-493b-a7b0-39fa4e7d92c1" + }, + "source": [ + "torch.sum(a,axis=0)" + ], + "execution_count": 176, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "tensor([ -1.1197, -5.7273, -11.5398])" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 176 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5UzUmEZhACKT" + }, + "source": [ + "Подробнее про тензоры в PyTorch смотрите [в официальном руководстве](https://pytorch.org/tutorials/beginner/basics/tensorqs_tutorial.html)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U-auwezDwHl6" + }, + "source": [ + "## Вычисляем производные\n", + "\n", + "Для обратного распространения ошибки, нам нужно уметь вычислять градиенты. Мы можем пометить любой тензор в PyTorch атрибутом `requires_grad`, и впоследствии автоматически будут вычисляться все градиенты при операциях с этим тензором. Для вычисления производной необходимо вызвать метод `backward()`:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "m8vFOXr7wHl6", + "outputId": "7054c2b1-0b61-4938-937d-813f75f0b195" + }, + "source": [ + "a = torch.randn(size=(2, 2), requires_grad=True)\n", + "b = torch.randn(size=(2, 2))\n", + "\n", + "c = torch.mean(torch.sqrt(torch.square(a) + torch.square(b))) # Do some math using `a`\n", + "c.backward() # call backward() to compute all gradients\n", + "# What's the gradient of `c` with respect to `a`?\n", + "print(a.grad)" + ], + "execution_count": 179, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[0.0227, 0.1661],\n", + " [0.1980, 0.2253]])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nPj3rtrtACKU" + }, + "source": [ + "На самом деле, PyTorch может таким образом вычислять \"накапливаемые\" градиенты. Если при вызове `backward` указать `retain_graph=True`, то граф вычислений будет сохраняться, и градиенты - накапливаться. Чтобы начать их вычислять заново, нужно в явном виде обнулить поле `grad`: " + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "z_VIw8MoACKU", + "outputId": "36a28b11-6919-47ab-c3f9-c7f1d8500423" + }, + "source": [ + "c = torch.mean(torch.sqrt(torch.square(a) + torch.square(b)))\n", + "c.backward(retain_graph=True)\n", + "c.backward(retain_graph=True)\n", + "print(a.grad)\n", + "a.grad.zero_()\n", + "c.backward()\n", + "print(a.grad)" + ], + "execution_count": 180, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[0.0681, 0.4983],\n", + " [0.5939, 0.6758]])\n", + "tensor([[0.0227, 0.1661],\n", + " [0.1980, 0.2253]])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HM9sUkVgCiG9" + }, + "source": [ + "Для вычисления градиентов PyTorch создаёт и поддерживает **граф вычислений**. Для каждого тензора, который вычисляется с использованием тензоров с установленным флагом `requires_grad`, устанавливается специальная функция `grad_fn`, представляющая собой функцию для вычисления производной по правилу дифференциирования сложной функции:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PcxHb-7jC7Vv", + "outputId": "3b3fa138-6d09-4636-8a71-f4a4051c7827" + }, + "source": [ + "print(c)" + ], + "execution_count": 181, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor(1.0358, grad_fn=)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rvLfNiblACKV" + }, + "source": [ + "На самом деле, PyTorch может вычислять градиенты только для скалярных функций. Если речь идет о вычислении производной тензора по тензору, то PyTorch позволяет нам вычислять произведение якобиана на вектор.\n", + "\n", + "Например, пусть есть векторная функция $\\vec{y}=f(\\vec{x})$, где\n", + "$\\vec{x}=\\langle x_1,\\dots,x_n\\rangle$ и\n", + "$\\vec{y}=\\langle y_1,\\dots,y_m\\rangle$, тогда градиент $\\vec{y}$ по $\\vec{x}$ задаётся **якобианом**:\n", + "\n", + "$$\n", + "\\begin{align}J=\\left(\\begin{array}{ccc}\n", + " \\frac{\\partial y_{1}}{\\partial x_{1}} & \\cdots & \\frac{\\partial y_{1}}{\\partial x_{n}}\\\\\n", + " \\vdots & \\ddots & \\vdots\\\\\n", + " \\frac{\\partial y_{m}}{\\partial x_{1}} & \\cdots & \\frac{\\partial y_{m}}{\\partial x_{n}}\n", + "\\end{array}\\right)\\end{align}\n", + "$$\n", + "\n", + "Вместо вычисления якобиана, PyTorch вычисляет произведение $v^T\\cdot J$ на некоторый вектор\n", + "$v=(v_1 \\dots v_m)$. Для этого необходимо вызвать ``backward``, передав `v` в качестве аргумента. Размер `v` должен совпадать с размером исходного тензора, по которому мы вычисляем производную.\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "VUNYiQCOACKV", + "outputId": "e3127c21-fce6-420d-f347-ec40cc827e7e" + }, + "source": [ + "c = torch.sqrt(torch.square(a) + torch.square(b))\n", + "c.backward(torch.eye(2))\n", + "print(a.grad)" + ], + "execution_count": 182, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[0.1135, 0.1661],\n", + " [0.1980, 1.1263]])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dGHlkVlvACKV" + }, + "source": [ + "Подробнее про вычисление градиентов в PyTorch читайте [в официальном руководстве](https://pytorch.org/tutorials/beginner/basics/autogradqs_tutorial.html)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FnVvj4LkD15r" + }, + "source": [ + "# Пример 0: Оптимизация функции методом градиентного спуска\r\n", + "\r\n", + "Попробуем использовать автоматическое дифференциирования для оптимизации простой функции двух переменных $f(x_1,x_2)=(x_1-3)^2+(x_2+2)^2$. Пусть тензор `x` представляет собой текущие координато точки. Мы начнем с некоторой начальной точки $x^{(0)}=(0,0)$, и будет вычислять следующую точку по формуле:\r\n", + "$$\r\n", + "x^{(n+1)} = x^{(n)} - \\eta\\nabla f\r\n", + "$$\r\n", + "Здесь $\\eta$ - т.н. **learning rage** (назоём его `lr` в коде), а $\\nabla f = (\\frac{\\partial f}{\\partial x_1},\\frac{\\partial f}{\\partial x_2})$ - градиент функции $f$.\r\n", + "\r\n", + "Для начала определим стартовое значение `x` и функцию `f`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nDw5mV9KEeOa" + }, + "source": [ + "x = torch.zeros(2,requires_grad=True)\r\n", + "f = lambda x : (x-torch.tensor([3,-2])).pow(2).sum()\r\n", + "lr = 0.1" + ], + "execution_count": 184, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Wt815LWdEj77" + }, + "source": [ + "Теперь проделаем 15 итераций градиентного спуска. На каждой итерации, мы будем обновлять координаты `x` и печатать их, чтобы убедиться, что мы достигаем минимума (3,-2):" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "KfwMf555EyWJ", + "outputId": "67e2199c-61ff-4ad1-9c48-b4a646bf8bbd" + }, + "source": [ + "for i in range(15):\r\n", + " y = f(x)\r\n", + " y.backward()\r\n", + " gr = x.grad\r\n", + " x.data.add_(-lr*gr)\r\n", + " x.grad.zero_()\r\n", + " print(\"Step {}: x[0]={}, x[1]={}\".format(i,x[0],x[1]))" + ], + "execution_count": 186, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Step 0: x[0]=1.2000000476837158, x[1]=-0.800000011920929\n", + "Step 1: x[0]=1.5600000619888306, x[1]=-1.0399999618530273\n", + "Step 2: x[0]=1.8480000495910645, x[1]=-1.2319999933242798\n", + "Step 3: x[0]=2.078400135040283, x[1]=-1.385599970817566\n", + "Step 4: x[0]=2.2627201080322266, x[1]=-1.5084799528121948\n", + "Step 5: x[0]=2.4101760387420654, x[1]=-1.6067839860916138\n", + "Step 6: x[0]=2.5281407833099365, x[1]=-1.685427188873291\n", + "Step 7: x[0]=2.6225125789642334, x[1]=-1.7483417987823486\n", + "Step 8: x[0]=2.698009967803955, x[1]=-1.798673391342163\n", + "Step 9: x[0]=2.7584080696105957, x[1]=-1.8389387130737305\n", + "Step 10: x[0]=2.8067264556884766, x[1]=-1.8711509704589844\n", + "Step 11: x[0]=2.845381259918213, x[1]=-1.8969208002090454\n", + "Step 12: x[0]=2.876305103302002, x[1]=-1.9175366163253784\n", + "Step 13: x[0]=2.9010441303253174, x[1]=-1.9340293407440186\n", + "Step 14: x[0]=2.920835256576538, x[1]=-1.947223424911499\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8sfjBMBu59B5" + }, + "source": [ + "## Пример 1: Линейная регрессия\r\n", + "\r\n", + "Попробуем с помощью полученных знаний решить классическую задачу линейной регрессии. Для этого сгенерируем небольшой синтетический датасет:" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "j723455WwHl7" + }, + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.datasets import make_classification, make_regression\n", + "from sklearn.model_selection import train_test_split\n", + "import random" + ], + "execution_count": 187, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "WJNK_J6v6I-Z", + "outputId": "09e6386e-a6d4-4b81-c8d2-153f0acf9696" + }, + "source": [ + "np.random.seed(13) # pick the seed for reproducability - change it to explore the effects of random variations\n", + "\n", + "train_x = np.linspace(0, 3, 120)\n", + "train_labels = 2 * train_x + 0.9 + np.random.randn(*train_x.shape) * 0.5\n", + "\n", + "plt.scatter(train_x,train_labels)" + ], + "execution_count": 188, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 188 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAb2ElEQVR4nO3df4xlZXkH8O+zwwUuP8pVmVoYXZemzZIKZZedEJo1RtZYbFHcgFEb0xbbZpM2bYVQ2tE/ivYfJjFWbW1qJpQWW6pLXdhQUJFkN6FuFDvDLgKyNChuZbBllB0QZsrODE//uPcsd86c97zvuec957zvvd9PsnF25+yZ9+yV5773eZ/3eUVVQURE4drU9ACIiCgfAzURUeAYqImIAsdATUQUOAZqIqLAnVLFTc8991zdsmVLFbcmIhpKc3NzP1HV8azvVRKot2zZgtnZ2SpuTUQ0lETkmOl7TH0QEQWOgZqIKHAM1EREgWOgJiIKHAM1EVHgKqn6ICIaJfsPz+NT9z+JZxeXcX6njZuu3Ird2ye83Z+BmoiohP2H5/Gxux7F8soaAGB+cRkfu+tRAPAWrJn6ICIq4VP3P3kySCeWV9bwqfuf9PYzGKiJiEp4dnG50J8PgoGaiKiE8zvtQn8+CAZqIiJ0c807pw/ggqn7sHP6APYfnnf6ezdduRXt1ti6P2u3xnDTlVu9jY2LiUQ08sosCCbfZ9UHEVGF8hYEXQLu7u0TXgNzGlMfRDTy6lgQLMMaqEVkq4gc6fv1oohcX8fgiIjqUMeCYBnWQK2qT6rqNlXdBmAHgCUAd1c+MiKimtSxIFhG0Rz1OwF8X1WNDa6JiGJTZEGwf7v4Oe0WRIDFpZVKFhEToqruF4vcBuBhVf18xvf2ANgDAJs3b95x7BhjORENl3R1SFq7NYZbrrl4oGAtInOqOpn1PefFRBE5FcDVAP4t6/uqOqOqk6o6OT6eeewXEVHw8uqps6pD+vneOp4okvr4DXRn0//rfRRERAGw1VO7VIFUUSlSpDzvtwB8yfsIiIgCYWuw5FIFUkWliFOgFpEzAbwLwF3eR0BEFAhbPXVWdUi/qipFnFIfqvoygDd4/+lERAE5v9PGfEawTmbJ6eqQuqo+uIWciKjnpiu3bqjqSM+Sq94unoWBmoiop44GS4NgoCYi6tPEjNmGTZmIiALHQE1EFDgGaiKiwDFHTUQjob+ZUiiLhK4YqIloKOQF4kGP2kruOb+4jDERrKliooEgz0BNRIWEODPNC8QAcOOdj2At1SnUdtRW+p7J3y9ynqIvhdqcupqcnNTZ2Vnv9yWiZmW1+SzT2tOXndMHMncUdtotvLL6am7Hu4lOO/NNx3TP/r93aGpX+cH3eGlzSkRka1rUFFOPjsXlldwgLejOkBWvzZSTtqa2Lnh1nqfIQE1EzkI9BHaQjnUCIJ1PKNIpr87zFBmoiciZKTid024Zm+3XwXTm4evOaGVePyayIUgnXDrl1X2eIgM1ETnLCl6tTYKXT6waUwhVSJ/CAgC3XHMxJjptCLr541uuuRg3v/etmQH80x+4BBOWk8d3b584eU+gG9zRd29WfRBRkLKaFi2dWMXxpZV119kqKsowVXjccs3FxsU9U5VKiJ3ysrDqg4hKuWDqvsw0ggB4evoq7z/PVI0xSBVGSKWGeVUfnFETUSm2Zvu++VzQDGXGbMMcNRGVYlrIq2qxzfQGUGcVRt0YqImolP5Ft/6FvKpmqnW/MYSAqQ8iKq3OFEKop7BUySlQi0gHwK0ALkK3Rvz3VPVbVQ6MiMgkltyyL64z6s8B+Lqqvl9ETgVwRoVjIiLKFFKVRp2sgVpEzgHwdgDXAYCqngBwotphERGtN2ir0mHgsph4AYAFAP8oIodF5FYROTN9kYjsEZFZEZldWFjwPlAialZ6N2Dd28RDbQhVB5dAfQqASwH8vapuB/AygKn0Rao6o6qTqjo5Pj7ueZhE1KRkNlvnNvH+n53XcrTphlB1cMlRPwPgGVV9qPf7ryAjUBPR8DLNZj9xz+O5p6qUzSdn9b9OG+b66YQ1UKvq/4jIj0Rkq6o+CeCdAL5X/dCIKBR5/Z4Xl7t9PtKnqvjIJ2e9QfQb9vrphGvVx58AuKNX8fEDAB+pbkhEFBrTNvG0/pyxKZ9cJFDnpTWaOLuwKU6BWlWPAMhsFkJEw++mK7daUxCJvOBaNJ9seoPwfQxW6LiFnIissraJm5ryn99pO/XjcKkiGcXt4lm4hZyInKR3A5oOuk2CaN73XGqik8XI5ZU1jIlgTXWk0h39GKiJRliZygyXnhum7+XVRO/ePrEhkK+pngz0oxakAR4cQDSyTDPiOo6ZMh02AHTTKqaFy6zc9LBsK887OIA5aqIR1eROP1MOW4Dc6pL0YmSTG3HqxEBNNKJ8npRSVNYioQDGWXYiHeBHZVs5AzXRkDNVV5hmtQpU3ssjXUXSabesQTqr2qPJN5s6cTGRaIjlVVfk1Ub77kxnyiP3LxzmMVV71H1eY1MYqImGUBIYs4JYkhpIFuVs15UN1LZSvLxt4rbFzaw3m3ZrDFdcOI6d0weiX2BMMPVB5EHTLUDTY0kW2EyS1MDu7RM4NLULYrmuDFseOe9n2CpQsjbiXLtjAvvm5odqgZEzaqKSQmtob2tkBGxMDfhMIaTTHLb2pHnbxF3+/dIbcXZOH/DSZyQknFETldRE5UHeDN42C85alPO1VTurXM40W0/eBHxvE3dZYAzpE5ALzqiJSqq78sA2g8+bxY6JrHsTSWaYvk72znrTUmSX3i2dWMX+w/PeTxW3fToI7ROQCwZqopLqrjywbb/OWmBrbRJAgJW1brjMCk4+TvY2vTkpuiV4Se9qADi+tLJuDL6CpGmBMZmh2/79QsTUB1FJdXd4s83gsxbYzjr9lJNBOmFKz5RJC5jenCY6bZx52sZ5YRUpoqzn71+UjLH2mjNqopJ8f3S3cZnBp2eoF0zdl3kv05bsQdMCebPZG/YecRqDD3kz9BhrrzmjJvIgKXN7evoqHJraVelH6EFm8C79oYHyC6N5s1nXMVQtxh7XnFETRWaQGbwtb5vwkRYwzWZdx1C1uj8B+cBATRShootvrsGpyrRASAHS5+JlHdiPmihQTfRZbrJH9ajL60fNGTVRgJqq9Q1p1kuvcQrUIvJDAD8DsAZg1RT1iciPJmt9Y0sLjIIiM+orVPUnlY2EiE6KsdaXqsPUB1EA0vnozhktHF9a2XBdyLW+VB3XOmoF8A0RmRORPVkXiMgeEZkVkdmFhQV/IyQaclmNjF76v1W0xta3Mwq91peq4zqjfpuqzovIzwN4QESOquqD/Reo6gyAGaBb9eF5nERDIz17XjqxuiEfvfKqotNu4czTTuGiHrkFalWd7/3vcyJyN4DLADyY/7eIKC2rmsPkheUVHLn51+saGgXMmvoQkTNF5OzkawC/DuCxqgdGNIxcmvonmI+mhMuM+o0A7haR5Pp/VdWvVzoqoiHlWrXBfDT1swZqVf0BgEtqGAvR0DNt0WY+mvKwPI+i1MT2ah9MjYk+cfVbjePvf9Zz2i2IAItLK1E9N5XDQE3RqXN7te0NoegbRtEt2uln7T8hJeu5Y30Do3xsykTR2Tl9wHhq9aGpXd5+jq1BUR0NjEzP2i95bjZUilteUyYeHEDRqWt7ta2Jfh2nj7s8U3JNE6ehUz0YqCk6dZ0UYntDqOMNw+WZkmvYH2R4MVBTdOo6Ssn2hlDHG0bWs/brf+5Qjroi/xioKTq2U6Z9sb0h1PGGkX7WTruF153RynzuKsZT5kRy8oeLiUQ5fFd9ND3eovfi4mR98hYTGaiJIlfVm0Vd1TXUxaO4iCJRNOhWWVPOxclwMEdNlNJUXjarL/XH7no09+dXWZLHxclwMFBTVKoOooMES9f72sY9SNCtctZbV3UN2TFQUzSqCqL9qpihuo57kKBb5ay3ruoasmOOmqJRx8ncVcxQXcdt6qyXF3RNTZ58zXp5InkYGKgpGnXtBCwaLG1M45tfXMbO6QMnFw6vuHAc++bmCwXdok2eKE4M1BSNKoJoWhUzVNO4Ba8dxTW/uIx9c/O4dscEDh5dKBR0OesdfgzUFI2qP+YD+TPUQeuVs8YtANI7GJZX1nDw6EItNcqhbdShfNzwQlFpKsBk7dIDgNed0cLN781u+p/X8N/UulQAPD19VRWPsG5c3HEYHm54oaFh+5hfVSA3HUp7fGklc4NJVsP/dmsMn/ngNuzePmHc9VdHjXIdi7LkF8vzyIsQmvdUWb6Xt2CZVb5nK/NrskaZOw7j4xyoRWRMRA6LyL1VDojiU0d9swtTcLzxzkdKj8U2000HOVswbLJGmTsO41Mk9fFRAE8A+LmKxkKRCuWjtCk4rqmW7n+RtSDYrz/I7T88j00iWMtY/+m/rqlqjToWZckvpxm1iLwJwFUAbq12OBSjUD5K580Iy+4uTGbAnXZrw/f6g1zy6SIrSIcSDLnjMD6uM+rPAvhzAGebLhCRPQD2AMDmzZvLj4yiUUd9swvbrLfsG0cyA85bsDQtOo6JBBUMWXsdF2ugFpH3AHhOVedE5B2m61R1BsAM0C3P8zZCCl4oH6WTwHPjnY9Y0w5lf44pyJneDF5VZWCkgbmkPnYCuFpEfgjgywB2ici/VDoqikpIH6V3b5/Apz9wSWMVFVyooyoU2vDSm1H/maq+J+86bnihpoW0MYabScgFN7zQyGkqB8smSVQFbiGnxjTZb4K9Lig0nFFTcHye9efrnMHZY88X7lxHVAfOqKkRpl4XQHcx0jVIFskJJwE9ryFS/38NzC1TnfJm1Oz1QY3Iq2kusgXd5eis/Yfnse2T38D1e48YgzSQ3XbUxyGxRGUxUFMjbOVqrkHStisymXEvLq8UH2TO/YnqxEBNznx2yMvqHpfmEiRtdcumnYJpUvD+RHVioCYnvjvk9W+SMXEJkrZ2oS7BfqLTxocv39zYJhkiG1Z9kJMqOuT1984YdAu6rW457zSV9GLh5Ftez5I9ChIDNTmpskNe2U0ieZtbTI2aso7QYqMiChUD9RCoY/NG1R3yqgqS3ClIw4CBOnI+N47kCaVD3iA4U6bYcTExci51xD6E1CGPaNRwRh25Ok9XqWNm2p/GOafdgkj3pO+x3tFWRXYtEg0LBurIhXK6io1LHj2dxunfpJIcBFBVaocoZEx9RM5WRxwC1xps180p3NpNo4aBOnIx5I5NefTr9x5Zt8OxSLqGW7tplDD1MQRCr2pwacAE5G9OSQsttUNUJQZq8i6dj+6c0cLxJXNTpCSVYTtFPBFaaoeoagzU5FVWXXdrk6A1JlhZM/c+f3ZxecPmFFZ9EHUxUJNXWfnolVcVnXYLZ552ijG1kaQy6k7j8EguigEXE8krUz76heUVHJrahc9+cFswVSq+OwISVcUaqEXkdBH5jog8IiKPi8gn6xgYvcZnH+iq2fpDh1SlUteuTqKyXFIfrwDYpaoviUgLwDdF5Guq+u2Kx0aor5eHLy49QUKpUqlzVydRGdZArd3Tb1/q/bbV++X/RFzK5LMPdB352Ji61cWyq5PI6RRyERkDMAfglwD8nar+RcY1ewDsAYDNmzfvOHbsmOehjqYLpu7LfFcUAE9PX+V8n6zm/Mmp21mVFKOwyFbkBHOiquWdQu5U9aGqawC2iUgHwN0icpGqPpa6ZgbADABMTk5yxu2Jr1lf1sw8eZHS6RRbuiWrcdLi0kp0AT2m2T+NtkLleaq6KCIHAbwbwGO264dVnbNNX32gbXnX/nSKbZHN1Dgp9Px5llDy5UR5XKo+xnszaYhIG8C7ABytemChqruky1eVhMsMPAnmeYtstsZJrJog8s9lRn0egNt7eepNAO5U1XurHVa4qjjk1cbHrM9le3YSzE3plk0iTr04WDVB5JdL1cd3AWyvYSxRiLGkK0nVLK+sndyKnSwkJvrTKaagnvX3siQBfxQWJInqwJ2JBdk2dISmP1UDdINtuzWGD1++2ZhOSdItYyIb7qfoVouYJAGfu/6I/GGvj4JCOuTVZcZqStUcPLqAQ1O7jPfevX0CN+w9kvm9pKQvr+pj5/SB2lNERMOKgbqgUEq6XHcslknVmHLVE512bpAv+3OJaD0G6gGEUNLluqhZpg67zKcH7voj8oc56opV1VDJdcZa5kzFMqWBMZzlSBQLzqgrVGVDJdcZa9lUzaCfHkJJERENA6deH0VNTk7q7Oys9/vGZuf0gcxgOiaCT3/gklJBi30qiIZL6V4fNBhTemJNtfTMmjNWotHBQF2hvFO1fZSq+V7U5AYVojBxMbFCWQtq/UIqVeMGFaJwMVBXKG+HHxBWqRqPpSIKF1MfFUtSB3XvZiyaxuAGFaJwMVDXwGXhz2d+eJCyQG5QIQoXA3VN8hb+fNdbD9KKNaQeJkS0HnPUAfCZH95/eN5YaZKXxvB1QAER+ccZdQB85YeTmbmJLY0RQg8TItqIM+oA+OpxnXdMFtMYRPHijLokH4uApvzwFReOY+f0gdKVGwCYxiCKGAN1Cb4WAbOqQq64cBz75ua9VG5MdNoM0kQRG9pAXXamW+b0FNet4Xk/Y5ATUli5QTScrIFaRN4M4IsA3ojuKUwzqvq5qgdWRtmZbh2np9h+xiD3HqRRE/t7EIXPZUa9CuBGVX1YRM4GMCciD6jq9yoe28DKznTrOD3F9jMGvXeRyo0q+2UTkT/Wqg9V/bGqPtz7+mcAngAQ9H/FZcvd6jg9xfYz6jghhf09iOJQqDxPRLYA2A7goYzv7RGRWRGZXVhY8DO6AZUtd3P9+2U2idh+Rh0bUNjfgygOzouJInIWgH0ArlfVF9PfV9UZADNA94QXbyN0kM6zpismgGKz0SKLcoNuEnH5GVVvQGF/D6I4OM2oRaSFbpC+Q1XvqnZIxWT1Ud43N49rd0wMPBsdZDZb9BDbELZs8wBaojhYz0wUEQFwO4DnVfV6l5vWeWai6VzCiU4bh6Z2lb6/S1VEzOcXsuqDKAx5Zya6BOq3AfgPAI8CeLX3xx9X1a+a/k6VgTodWEwNiATA09NXFb5ff6DKCsCCbo3iRN+1Vb9ZENHwK3W4rap+E9341LiscrIkcKa55Flt5WlZVRHJz+q/lotyRFSlqJoymQJn+l3ENc9qK0+zBdrkWl9NlYiIskQVqE2BM0lFCIBOu4XTW5tww94j1kU920zYJdA+u7jMRTkiqlRUgdoUOJNc8Gc+uA2vrL6K40srTidp22bCtlPEk2tDqOAgouEVVVMmW+1x0a3jtvv1987Iyoenr2VgJqIqRBWobU2Hii7quTQx6g/AIZSyhTAGIqqXtTxvEHXWUfcb9jK5mOu1iShfXnleVDlqm1gW9YruYkywiRLRaIoq9WEzSD/mupVpLcp6baLRFHWgNuVrBw3MvvK/efcp0yubTZSIRlO0gdp303tf9yt7cktekOdRW0SjKdocte98ra/72e6TV7ud1Qmwvw6c9dpEoym4GbVr+sF3vtbX/VxObjHNil3SIqzXJho9QQVqU9pg9tjzOHh0YV3w9p2v9XU/233yFjxv2Hsk855cLCQabUEFatOM8o5v//eGrnXX7pgodYpLmq/8b5mTW7hYSERZgspR5zVd6re8soaDRxe85mt95X/L3CeWOnAiqldQOxNNOwuzuB4MEBtuEScaTaUODqhTVtqgzMEAMeJiIRGlBZX6yEobfPjyzUwHENFIC2pGDWTPKCff8nqmA4hoZAUXqLMwHUBEo8waqEXkNgDvAfCcql5U/ZCqw4U6IoqRy4z6nwB8HsAXqx3KRj4Dq+/eIEREdbEuJqrqgwCer2Es69j6XhTFXs5EFKugqj76+Q6s7OVMRLHyFqhFZI+IzIrI7MLCQun7+Q6sthPHiYhC5S1Qq+qMqk6q6uT4+Hjp+/kOrNyeTUSxCjb14TOwJouSyytrGBMBAHTaLZze2oQb9h4pdG4hEVHdrIFaRL4E4FsAtorIMyLy+9UPy1+TpP5FSQBYU0Vrk+DlE6s4vrTiZaGSiKhKQTVlqkKRRk8TnTYOTe2qeERERBvlNWUKNvXhS5HFR1aAEFGIhj5QF1l8ZAUIEYVo6AN11qJka5OgNSbr/owVIEQUqiiaMpVhOqMw68+4lZyIQjT0i4lERDGI4oQXdrYjIsoWRKBmZzsiIrMgFhPZ2Y6IyCyIQM3OdkREZkEEana2IyIyCyJQs7MdEZFZEIuJplpnLiQSEQUSqAGeNE5EZBJE6oOIiMwYqImIAsdATUQUOAZqIqLAMVATEQWuku55IrIA4NiAf/1cAD/xOJwmDcuzDMtzAHyWEA3LcwDlnuUtqjqe9Y1KAnUZIjJravUXm2F5lmF5DoDPEqJheQ6gumdh6oOIKHAM1EREgQsxUM80PQCPhuVZhuU5AD5LiIblOYCKniW4HDUREa0X4oyaiIj6MFATEQWusUAtIu8WkSdF5CkRmcr4/mkisrf3/YdEZEv9o7RzeI7rRGRBRI70fv1BE+O0EZHbROQ5EXnM8H0Rkb/pPed3ReTSusfoyuFZ3iEiL/S9Jn9Z9xhdicibReSgiHxPRB4XkY9mXBP8a+P4HFG8LiJyuoh8R0Qe6T3LJzOu8Ru/VLX2XwDGAHwfwC8COBXAIwB+JXXNHwH4Qu/rDwHY28RYPTzHdQA+3/RYHZ7l7QAuBfCY4fu/CeBrAATA5QAeanrMJZ7lHQDubXqcjs9yHoBLe1+fDeC/Mv4/Fvxr4/gcUbwuvX/ns3pftwA8BODy1DVe41dTM+rLADylqj9Q1RMAvgzgfalr3gfg9t7XXwHwThGRGsfowuU5oqCqDwJ4PueS9wH4onZ9G0BHRM6rZ3TFODxLNFT1x6r6cO/rnwF4AkC6cXvwr43jc0Sh9+/8Uu+3rd6vdFWG1/jVVKCeAPCjvt8/g40v2slrVHUVwAsA3lDL6Ny5PAcAXNv7SPoVEXlzPUPzzvVZY/FrvY+uXxORtzY9GBe9j8/b0Z3B9Yvqtcl5DiCS10VExkTkCIDnADygqsbXxEf84mJi9f4dwBZV/VUAD+C1d1lqzsPo9lW4BMDfAtjf8HisROQsAPsAXK+qLzY9nkFZniOa10VV11R1G4A3AbhMRC6q8uc1FajnAfTPLN/U+7PMa0TkFADnAPhpLaNzZ30OVf2pqr7S++2tAHbUNDbfXF6zKKjqi8lHV1X9KoCWiJzb8LCMRKSFbnC7Q1XvyrgkitfG9hyxvS4AoKqLAA4CeHfqW17jV1OB+j8B/LKIXCAip6KbbL8ndc09AH639/X7ARzQXmY+INbnSOUKr0Y3NxejewD8Tq/C4HIAL6jqj5se1CBE5BeSfKGIXIbufwehTQIAdCs6APwDgCdU9a8NlwX/2rg8Ryyvi4iMi0in93UbwLsAHE1d5jV+NXK4raquisgfA7gf3cqJ21T1cRH5KwCzqnoPui/qP4vIU+guDH2oibHmcXyOPxWRqwGsovsc1zU24Bwi8iV0V93PFZFnANyM7iIJVPULAL6KbnXBUwCWAHykmZHaOTzL+wH8oYisAlgG8KEAJwGJnQB+G8CjvZwoAHwcwGYgqtfG5TlieV3OA3C7iIyh+2Zyp6reW2X84hZyIqLAcTGRiChwDNRERIFjoCYiChwDNRFR4BioiYgCx0BNRBQ4BmoiosD9P3fFq1WlCzEwAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ng4rZmGc6oxk" + }, + "source": [ + "Линейная регрессия вычисляется как $f_{W,b}(x) = Wx+b$, где $W, b$ - параметры модели, которые необходимо найти. Функция ошибки на наборе данных $\\{x_i,y_u\\}_{i=1}^N$ может быть определена как среднеевадратичное отклонение\r\n", + "$$\r\n", + "\\mathcal{L}(W,b) = {1\\over N}\\sum_{i=1}^N (f_{W,b}(x_i)-y_i)^2\r\n", + "$$\r\n", + "\r\n", + "Опишем модель и функцию ошибки:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "QxhI4GlB6aiH" + }, + "source": [ + "input_dim = 1\r\n", + "output_dim = 1\r\n", + "learning_rate = 0.1\r\n", + "\r\n", + "# This is our weight matrix\r\n", + "w = torch.tensor([100.0],requires_grad=True,dtype=torch.float32)\r\n", + "# This is our bias vector\r\n", + "b = torch.zeros(size=(output_dim,),requires_grad=True)\r\n", + "\r\n", + "def f(x):\r\n", + " return torch.matmul(x,w) + b\r\n", + "\r\n", + "def compute_loss(labels, predictions):\r\n", + " return torch.mean(torch.square(labels - predictions))" + ], + "execution_count": 189, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JUxwj3367gD2" + }, + "source": [ + "Обучать модель будем на сериях примеров - minibatches. Для обучения используем градиентный спуск, подстраивая парметры в соответствии с формулой:\r\n", + "$$\r\n", + "\\begin{array}{l}\r\n", + "W^{(n+1)}=W^{(n)}-\\eta\\frac{\\partial\\mathcal{L}}{\\partial W} \\\\\r\n", + "b^{(n+1)}=b^{(n)}-\\eta\\frac{\\partial\\mathcal{L}}{\\partial b} \\\\\r\n", + "\\end{array}\r\n", + "$$" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "-991PErM7fJU" + }, + "source": [ + "def train_on_batch(x, y):\r\n", + " predictions = f(x)\r\n", + " loss = compute_loss(y, predictions)\r\n", + " loss.backward()\r\n", + " w.data.sub_(learning_rate * w.grad)\r\n", + " b.data.sub_(learning_rate * b.grad)\r\n", + " w.grad.zero_()\r\n", + " b.grad.zero_()\r\n", + " return loss" + ], + "execution_count": 190, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "idr2VEWb9rr0" + }, + "source": [ + "Теперь приступаем к обучению: делаем несколько проходов по всему датасету (эпох), разбиваем его на minibatches, и вызываем функцию обучения:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nOuu0qpx-wAp" + }, + "source": [ + "# Shuffle the data.\r\n", + "indices = np.random.permutation(len(train_x))\r\n", + "features = torch.tensor(train_x[indices],dtype=torch.float32)\r\n", + "labels = torch.tensor(train_labels[indices],dtype=torch.float32)" + ], + "execution_count": 191, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3zdIf6c_85Ht", + "outputId": "6520288c-da59-4a9f-c37e-cd99779c3073" + }, + "source": [ + "batch_size = 4\r\n", + "for epoch in range(10):\r\n", + " for i in range(0,len(features),batch_size):\r\n", + " loss = train_on_batch(features[i:i+batch_size].view(-1,1),labels[i:i+batch_size])\r\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 192, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 94.5247\n", + "Epoch 1: last batch loss = 9.3428\n", + "Epoch 2: last batch loss = 1.4166\n", + "Epoch 3: last batch loss = 0.5224\n", + "Epoch 4: last batch loss = 0.3807\n", + "Epoch 5: last batch loss = 0.3495\n", + "Epoch 6: last batch loss = 0.3413\n", + "Epoch 7: last batch loss = 0.3390\n", + "Epoch 8: last batch loss = 0.3384\n", + "Epoch 9: last batch loss = 0.3382\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "US6q0nCBD-LL", + "outputId": "c804b779-3231-4f6f-c854-032d211b2853" + }, + "source": [ + "w,b" + ], + "execution_count": 193, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(tensor([1.8617], requires_grad=True), tensor([1.0711], requires_grad=True))" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 193 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "_e6xRMZFDnyI", + "outputId": "79e6c360-265a-401d-ce39-8f211917a13d" + }, + "source": [ + "plt.scatter(train_x,train_labels)\r\n", + "x = np.array([min(train_x),max(train_x)])\r\n", + "with torch.no_grad():\r\n", + " y = w.numpy()*x+b.numpy()\r\n", + "plt.plot(x,y)" + ], + "execution_count": 194, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 194 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhU1Z3/8ffp7uqFrRulQWhoQFZFZOsxGowKiBhN3KNJRo3JTJiJGbcYlGQWZ/L8ZgTcMyYTiZplYhKNInFiZFFQXFFWUeluEGRp9qWbpffq8/uju9qmum7Vra5b1beaz+t5eNJ03br3XMp876lzvud7jLUWERHxr4zOboCIiESnQC0i4nMK1CIiPqdALSLicwrUIiI+l5WMk/bp08cOGTIkGacWEemSVq9efcBaWxjptaQE6iFDhrBq1apknFpEpEsyxmxzek1DHyIiPqdALSLicwrUIiI+p0AtIuJzCtQiIj6XlKwPEZGTycK1FTywuIxdlTUMKMhj1oxRXDWhyLPzK1CLiCRg4doKfrRgAzUNQQAqKmv40YINAJ4Faw19iIgk4IHFZa1BOqSmIcgDi8s8u4YCtYhIAnZV1sT1+45QoBYRScCAgry4ft8RCtQiIjSPNU+es4yhs19m8pxlLFxb4ep9s2aMIi+QecLv8gKZzJoxyrO2aTJRRE56iUwIhl5X1oeISBJFmxB0E3CvmlDkaWAOp6EPETnppWJCMBExA7UxZpQxZl2bP0eMMXemonEiIqmQignBRMQM1NbaMmvteGvteGASUA28mPSWiYikSComBBMR7xj1NOBTa61jgWsRkXQTz4Rg2+Xi+XkBjIHK6oakTCKGGGut+4ONeRpYY619PMJrM4GZAMXFxZO2bVMsF5GuJTw7JFxeIJP7rxnboWBtjFltrS2J9JrryURjTDZwBfCnSK9ba+dba0ustSWFhRG3/RIR8b1o+dSRskPa8nrpeEg8Qx9fprk3vdfzVoiI+ECsfGo3WSDJyBSJJz3vG8AfPG+BiIhPxCqw5CYLJBmZIq4CtTGmOzAdWOB5C0REfCJWPnWk7JC2kpUp4mrow1p7HDjV86uLiPjIgII8KiIE61AvOTw7JFVZH1pCLiLSYtaMUe2yOsJ7ycleLh6JArWISItUFFjqCAVqEZE2OqPHHIuKMomIeOCjiiqeWZmchX7qUYuIJOCzA8d5cEkZf/lwN3175nDNhIHkZTtnhnSEArWISAfsO1rLf7+2mT+8v51AZga3TR3Ody843fMgDQrUInKSaFtMKZFJwqO1DcxfsYWn3tpKfWMTXz9nELdPG0HfnrlJaHUzBWoR6RKiBeKObrUVOmdFZQ0ZQBOQYaDJwlfO7s8PLxnFkD7dk31rCtQiEh+veqZet8kpEAPc/dx6gmGVQmNttRV+zqaW3zdZyMnK4OIz+qUkSEOcZU7dKikpsatWrfL8vCLSuSKV+UyktKdXJs9ZFnFFYUFegLrGpqgV74oK8iI+dL54/2vsqqqN+r63Z09NvPEtopU5VY9aRFxLdBPYZHGq0VFZ0xD1fQZaA3zbXnjxqd2iBulo10wGBWoRcc2vm8A61eiIxgDh4wk1DUFmL/iQ2oam1rHoaNdMFS14ERHXnIJTfl7Asdh+Kjjtedi7WyDi8ZnGtAvSIbUNTdw9fST3Xz3WsVJeqvdTVKAWEdciBcRAhuF4fSMVlTVYPh9CSGawDt+FBeD+a8ZSVJCHoXn8+P5rxnLfV8dEDOAPXT+OIoeHzmm9crlt2ghuOKe49ZzQHNxpc+5UDvVo6ENEXItUtKi6vpHD1SeOBSdz3Nopw+P+a8Y6Tu6FZ6nMGHMar27c2264JC+Qyewvj279u1/qfijrQ0QSMnT2yxGHEQywdc7lnl/PKcPDTRZGY7CJP63eyaOvlrP3SB1jBvRi/9E69h+t6/RUQ2V9iEjSxCq277WOTGhaa1n88R7mLS5jy/7jTCwu4L+/MZFzhp6SlDZ6TYFaRBLipti+l+J9MLz76UHmLipl3Y5KRvTtwfybJjH9zH6YljHndKBALSIJSXWxfbcPhk92HWHuolLeKN9P//xc5l13NtdOHEhmRvoE6BAFahFJWCon3WI9GHYcquahJWX8ef0ueuUG+PFlo7n5vCHkRtmU1u9cBWpjTAHwJHAWzTni37HWvpvMhomIOIn0YDhwrI7Hl23mmZXbyMww/OOFw/jHC4eRnxc5lzqduO1RPwYsstZeZ4zJBrolsU0iIhFFKgh18Zn9ePLNLfxyxRZqG5u4vmQQd148gn69kld2NNViBmpjTD5wAXALgLW2HqhPbrNERE4UKX961p/WkxPI5FhdI18+6zR+OGMUwwp7dHJLveemRz0U2A/8yhgzDlgN3GGtPd72IGPMTGAmQHFxsdftFJFO1tnlTSMVhGposphgEwu/P5nxgwpS1pZUc7OEPAuYCPyPtXYCcByYHX6QtXa+tbbEWltSWFjocTNFpDOFerOpXCbe9tpOi1wA6hubunSQBnc96p3ATmvtypa/P0+EQC0iXZdTedN/f+njqLuqJNoDj1T/OpxTzY6uJGagttbuMcbsMMaMstaWAdOAT5LfNBHxi2j1nkM1n8N3VenI1lfh/uuvG6MG6VRXsessbrM+bgOeacn42AJ8O3lNEhG/cVvvOVSMKfRzpNfcBOq9R2p59NVN7Dta53hMkU+2AUsFV4HaWrsOiFgsRES6vkirAZ1Eq7kRa4OBqpoGnnjjU55+eyvBJkv37EyO17e/ptfbYPmdViaKSExuy5vC5zU3YtXjaDuG3T8/l5Ihp/BG+X6O1DZw5bgB/GD6KNZsP5zSOiJ+pUAtIq6ErwZ02ug2FESjvRb+3l1Vtby0fhejT+vJ77/7BcYMyG8N5DUNQTKNIWjtSTXc0ZYCtchJLJHMDDfFmJxem7eoNOIwytHaxtYg3TaQB61tDfQnW5AGbRwgctJy6hEne5up97ce4vonnEsFFUWZuIw0Nt3ZC3G8oo0DRKQdp9zoZG2hVbrnCA8sKuO10n2OO3wbIo9th4RPRjptywXxpQH6nQK1yEmqIzuldMTOw9U8vLScF9dW0CMni3svHc2p3bO576WPT3hQGHDcGTwkfHOAVD9sOosCtUgX5zQ04JQbbWnelzDRIYRDx+v52fLN/O+728DAzC+dzvcuGkZBt2wAsrMyWtuVnxdoXTjjJFK2R6oeNp1NgVqkC4s2NBAtNzqRIYTq+kaeenMr81ds4Xh9I9dNGsgZ/XvxZMvv2j4srppQ1NrGaJyyPVK9X2NnUaAW6YJCvehIQSw0NBCalIt1nNtA3RBs4o8f7OCnr21i/9E6LjmzH7NmjOLjXUeijiNHGr4IiTW56bQt15TRhUyesyztJxhDFKhFPOCnzAM3hYxCQwOhXu3Q2S9HHB92M4TQ1GR5ecNuHlpSxmcHqzlnyCn84sZJTBrcG4BbfvVB1HHkaNeIlYESKUVwyuhCXlhd0aUmGBWoRRLkt8yDaD3UkPChgY4OIby16QBzF5WyoaKK0af15OlbSqiqbuD2P6xtDZxOWRyhAO10TFFBnqt/v/CFOJPnLOtyE4xu6lGLSBTRMg+SJVSjeejsl5k8Z9kJdaFj9YIjTcrNmjGKvLDNX6Mt1d6ws4obn1zJjU+t5NDxeh6+fhwv3/4ljtQ08uMXPzqhbrXTnt+hh0C8147FzQRjtH8/P1KPWiRBqc48iNWDj9aLzTTmhIdIqIfpZpUhwGcHjvPgkjL+8uFuencL8K9fOZMbzy0mJyuz9f3hDy1L5NS76vpGFq6tcH1tt2J9O/DbNyA3FKhFEpTqzINYucORJtgCGQYMNASbw2Wk4BRpZ++QfUdr+elrm/jj+zsIZGZw29ThfPeC0+mVe+IO304PJwsUhKXgHa5uOKENXgVJpwnGUA89HXOvFahFEhQrMHgtVg/ebaU7p+DUdmL0tPxcxhbl8+amAzQEm/jGOcXcNm04fXtG3uE72ngz0C5XOhkBMlYPPR1zrxWoRRLk9Vf3WNz04MN7qENnvxzxXLGWZO+uqmV3VS0TBhXwyA3jGdKne9S2RXto3fXsOldt8EK0Hno65l5rMlHEA1dNKOLt2VPZOudy3p49NalfoTsy+eYUhMJ/71TVbt/RuphBGpr/He6/ZixFBXkYmnvSoRQ7t21INq8nL1NBPWqRNNORHnys4RlrLcvL9rGrqjbi++Pp9Tr1ZlM9ROQk1d+AvKBALZKG4p18ixacVm87zNxXSnn/s0NkZhiCEcraedHr9VOA9HLyMhVUj1rEp5K92nHT3qPMW1zG0k/2UtgzhzumjSAvkMm/LPwo5TWqRfWoRdJOMnN9d1XW8Oir5Ty/eifds7P44SUj+c75Q+mW3RwOMjOML3q98jlXgdoY8xlwFAgCjU5RX0S8kYxc38rqen7++qf8+p3PwMJ3Jg/l1inDOaV79gnHpduwwMkgnh71FGvtgaS1RERaeZnrW1Mf5FfvbOV/Xv+UY3WNXDNhIHdNH8HA3t0SbaakiIY+RHwgfDy6oFug3QIViG9SrzHYxHOrdvLYa+XsPVLHtNF9mXXpKEaf1svLpksKuA3UFlhijLHAE9ba+eEHGGNmAjMBiouLvWuhSBcXaTw6kGEIZJrWJd/gPpXNWsuij/bwwJIytuw/zqTBvXn8mxP5myGnJO0eJLncBurzrbUVxpi+wFJjTKm1dkXbA1qC93xozvrwuJ0iXUZ477m6vrHdeHRDk6UgL0D3nKy4JvXe+fQAcxeVsX5HJSP69uCXN5dw8Rl9Mcaphp2kA1eB2lpb0fK/+4wxLwLnACuiv0tEwkXqPTupqmlg3X2XuDrvx7uqmLeojDfK9zMgP5d5153NtRMHkpmhAN0VxAzUxpjuQIa19mjLz5cAP0l6y0S6IDdF/UPcjEdvP1jNQ0vL+PO6XeTnBfjny87gpvMGkxu2RFrSm5sedT/gxZavTlnA7621i5LaKpEuym3WRqzx6APH6nh82WaeWbmNzAzDrRcN4x8uHEZ+XsDxPZK+YgZqa+0WYFwK2iLS5TlVbnM7Hn2srpFfrtjCk29uobaxietLBnHnxSPo1yty2VHpGpSeJ2nJT5vJxsOpMNG/XzHGsf0L11Ywb1Epu6pqT9gppW/PHL4w9BQF6ZOAArWknVRupRTrgRDvAyPewkQL1uzk3hc+bE3Ta5tOte9oXbv7TtcHmESnokySdibPWea4i8jbs6d6dp3wBwKcWKAo1uuJsNbyRvl+vvvbVSfkUkcSuu9ktkeSL1pRJm0cIGknVVspxdpdPFm7j6/bUck3fvket/zqg5hBGj6/787YDV1SQ4Fa0k6qdgqJ9UDw+oHx6f5jfO93q7nqZ2+zed8x/uOKMQzIjz3+HLrvdNwLUNxRoJa0k6qtlGI9ELx6YOypquVHCz7kkkdWsKJ8P3ddPJLXZ03hW18cwj2Xjm53r221vW+/bHUl3lOglrQTbV8+L8V6ICT6wKiqaWDuolIuenA5z6/eyU3nDuaNe6Zwx8Uj6JHTPM8ffq8FeQF6dwtEvO9kPMAWrq1g8pxlDJ39MpPnLGPh2ooOn0s6TpOJIlF4nfUBUNsQ5DfvfMbPX/+UI7UNXDW+iB9MH8mgUxIvO+pl1ocmJ1Mr2mSiArVIijQGm1iwpoJHXi1nd1UtF40q5J4ZozlzQGJlR5OVkpeq7Bpppq24RDqRtZYln+zlgcVlbN53jHGDCnj4+vGcN+zUdsfGG3STvWVXPL+X5FGgFgnjZQ/1/a2HmPPKRtZsr+T0wu784saJzBhzWsSyox0JusnYsivEabm7JidTT5OJklaSPbkVCpYVlTVYPg+W8V6ndM8RvvPrD7j+iXepqKzhhpJB1NYH+d7v1nD+3OURz9eRPOhk9npTlV0jsalHLWkjFUvHE+2h7jxczcNLy3lxbQU9c7K499LRnNo9m/te+jhmuzsSdJPZ6413ubskjwK1pI1kfs0P6WgP9dDxeh5ftpnfvbcNY2DmBadz64XDye8WYPKcZa7a3ZGg61Tkyater3Yk9wcFakkbqZjcijdYVtc38tSbW3lixRaq6xv52qRB3Dl9BP3zPz/eqX0VlTVMnrOstbc6ZXQhL6yuiCvoqtd7clCglrSRiskttz3UhmATf3x/O4+9tpkDx+q45Mx+3HPpKIb37em63YbPt+KqqKzhhdUVXDupiOWl++MKuur1dn0K1JI2kv01H6L3UNvWhc7MMASbLOcMPYUnbprEpMG942p327rSITUNQZaX7k9JjrLKoaYXBWpJG6n6mh+ph7pwbQX3PP8h9cEmAIJNzWG2fM8Rdhyqjhio2wbD/LwAuYEMKqsbHHvYkJoc5VTW8xZvKFBLWon1NT8ZPcUPd1Zy7wufB+m2KmsaIwa58GBYWdNAXiCTR24Yz1UTihxX/aUiRzkVk7LiLeVRiyf8ULzHqxzokK0HjvP936/hisffpq6xfZAOiZTrHCsnujNzlLXiMP247lEbYzKBVUCFtfYryWuSpBu/fJV2Co53P7c+rrbsO1rLT1/bxB/f30F2Vga3Tx3On1bvZHdVreN7woNcrGDYmdkaWnGYfuIZ+rgD2AgkVkFGuhy/fJV2Co5Ba109OI7UNjD/jS089dZWGoJNfPMLxdw2dQSFPXM4vbBHuwnBttoGuYVrK8gwhmCEgmdtj+usbI1UTMqKt1wFamPMQOBy4D+BHyS1RZJ2/PJVOtokXbQHR21DkN+9t42fLd/M4eoGvjpuAHdPH8mQPt1bjwm9799f+pjKmoYT3t82yIW+XUQK0n4Jhsq9Tj9ue9SPAvcA7ZNEWxhjZgIzAYqLixNvmaQNv3yVjtRTbCv8wRFssry4toJHlpZTUVnDl0b04d5LR3NWUX7E94d6wNEmLCN9uwDINMZXdZyVe51eYgZqY8xXgH3W2tXGmIucjrPWzgfmQ3M9as9aKL7nl6/SocBz93Prow47WGtZVrqPeYvKKNt7lLMH5jPvurOZPLyP6+s4BTmnbxFN1iowSoe56VFPBq4wxlwG5AK9jDG/s9bemNymSbrw01fp0DWdHhyrtx1iziulfPDZYYac2o2ffXMil42NXHa0I/zy7UK6lrh2eGnpUf8wVtaHdniRzhY+PHHjucWs3lbJqxv3UtgzhzumjeCGvxlEINPbDFVtXyUdpR1e5KQTGp7YVVnDI0vLeWBxGd2zs5g1YxTfnjyEbtnJ+U/fT98upOvQnonSaZJZb6Kyup6fv/4pv37nM7Bw83mD+f6U4fTunp30a4t0hHrU4jteLpJpG3RPy89lYnFvVmzaz7G6Rq6dOJC7po+kKCzPOdK1V207FHflOpFUUI9aOoVTrQto3uXabZCMNCYMMGZALx6+fjyjTut5wrEPLC5zvG54RTuNLUsqqUctvhNtMUw8vet5i0oj5i1XVje0BumFaysiLlQJF6nsqAoViR+oKJN0iljparE2dQV459MD7HKovxF6EIR63LGCtBMVKhI/UI9aXPNyAi7WKkJwDpIfVVQxb3EZK8r3kxmjpobTSsFwkQr5tz2PSGdSoBZXvK6Q1zaNzWnMODxIbj9YzYNLynhp/S4KugX458vOoKBbgH/788eOqyLd9IiLOrhfoUiqKFCLK8mokNe2dka0Jej7j9bx+LJN/P797WRmGL4/ZRgzLxhGfl4AgEBmhmNPP1qhpvDJwpLBpyhlT3xJgVpcSWaFPKdFIhef2Y+Hl5bz5JtbqGts4oa/GcQd00bQr1duu/c7BVSnIZbe3QLc99UxJ7xPhYrErxSou4BULN5Idg2LtkGyrjHI71du58J5yzl4vJ7Lx/bn7ktGcnphjw6dF7RSUNKbAnWaS9XuKqmokNfUZHlp/S4eXFLGzsM1fHHYqdx76WjGDSpI6LzqKUu6U6BOc6naXSWZPVNrLa+X72feojI27j7Cmf178dvvjOVLI/p4VtVOJJ0pUKe5VO6ukoye6drth5nzSikrtx6i+JRu3HTuYF7buJdvPf0++XkBjIHD1Q2taXjxrFoU6SoUqNNcutQ/Dh9Hv/m8wazdXsmij/fQp0c2P7lyDN0Cmfxrm1S7totUQrnSnbVxrkhn0srENDdrxijyApkn/M5v+b+hcfSKyhoszcH2/ldKWV62j7suHskbs6Zw83lDeOTVTa4Wp7hZtSjSlShQp7mrJhRx/zVjKSrIw9C8eMNvhYScVgfWNTbx3KodLP1kLxDfcI2WdsvJREMfXYCfsxpqG4KOC07gxKGMaItTwvltaEckmRSoxXML11Ywb1Epu6pqyXCRtBEaynBT/wP8N7QjkmwK1OKpF9fs5N4XNlAfbAKgyUKmgYwMQ0PQufb5rsqadimAyvoQaaZALZ5ZueXgCUE6JGihZ3YW3XOyYhZgSvUwjrbkknSgQC0J27j7CPMWlbK8bL/jMVU1Day775KYBZhSKVWrOkUSFTNQG2NygRVATsvxz1tr70t2w+Rzfu317ThUzSNLy3lxXQU9c7KY/eXR/PadzyIW82/bYwZ/1N5I1apOkUS56VHXAVOttceMMQHgLWPMK9ba95LcNsGfvb6Dx+p4fPlmnnlvO8bAzAtO59YLh5PfLcBpvXJj9pj9kqWSylWdIomIGaht8+63x1r+Gmj54/2OuBKRl72+RHvmx+saeeqtrcxfsYXq+kauLxnEHRePoH/+56lyfuoxx5IuqzpFXO1CbozJBFYDw4GfWWvvjXDMTGAmQHFx8aRt27Z53NST09DZL0d8Khpg65zLXZ8n0thwaPupSJkUbYN6//xczht2Km+UH+DAsTpmjOnHrBmjGN63Z/sLpRGn8XK/LRiSk0PCu5Bba4PAeGNMAfCiMeYsa+1HYcfMB+YDlJSUqMftEa96fZF65qEPKXw4JTyA7aqq5YU1FQwr7M78myex/WA133r6gxNS6CqrG3zde44knXr/cnKLK+vDWltpjFkOXAp8FOv4riqVk3te1YGONe7adjjFacl3TX2Q7QerT2hP28JJfhg/j5dfxstFoolZ68MYU9jSk8YYkwdMB0qT3TC/ilRg6EcLNrBwbUVSrudVLQ83PfBdlTV8uLPSMdd5d1VtzF29VTBJxHsxx6iNMWcDvwEyaQ7sz1lrfxLtPSUlJXbVqlWeNdJPJs9ZFjGQFRXk8fbsqZ3QIncijceGywtkUtMQJMM0rygMF1odGEu84+cikuAYtbX2Q2CC561KU+mY0hUaqqlpCLYG29BEYltBa7l92ggG5OfyH//3Sbug7vS+cKHeu1/zv0XSjVYmxindUrrCe9JBa8kLZPLVcf3564Y9HKtrBOD84X145IbxFPbMASA3kMndz61v14O2EDVYh8bP/Zj/LZKuVI86Tn4q1L9wbQWT5yxj6OyXmTxnWcRxcqc87D+t3smxukauGDeAN2ZdxO/+/gutQRqag2mTwzBHKKXPAAV5AXp3C7QbP4+W/y0i8VGPOk5+Sely22N1GpKxFv5y2/mcVZTveA2nbw9uxuPTcYhIxK8UqDvADyldblcs9s/PjVh7o6ggL2qQhsRSA9NtiEjEzzT0kWRuhic6wk2PddVnh8gJG6YB98E2kdRAPw0RiaQ79aiTKJkTatF6rOV7jzJvURmvbtxL3545fG3SQN7efIDdVbVxD9V09NuDX4aIRLoCV7U+4tWV86jj4ZRznWkMD10/LqGgFSkvOicrg3EDC1i17RDds7P4x4uG8e3JQ+iWreexiN8lXOtDOsZpeCJobcI967Y91orKGnrkZFHbEGTdzkr+7vyh3HrRcHp3z+5Yw0XEVxSokyjartpeFKi/ZEw/Kipr+MXrn3K8vpFrJg7krukjKerghJ0WqIj4kwJ1EsXaVbujqWoNwSaeW7WDx17dxL6jdVx8Rj/uuXQUI/t1vOyoFqiI+JcCdRKFAlykFX4Qf6qatZa/btjDg0vK2HrgOCWDe/Pzv51IyZBTEm6rtqUS8S8F6iQLBblES5W+s/kAcxeVsn5nFSP79eDJm0uYdkZfjDERj493GEMLVET8S4E6BdykqjkF1o8qqpi7qJQ3Nx1gQH4uD35tHFdPKCIzI3KADp0r3mEMLVAR8S+l5/mAU6rdmAG9WLO9koJuAf5pynBuPHcwuREWsITrSClWbUsl0rmUnudzkcaH6xqbWLu9ku9PGcY/XDiMXrkBV+dauLbCMdMk2jCGFqiI+JcCtQ84Fk4CZs0Y7fo8oV6xk1jDGH6oYSIi7anWRyerawzSKy9ybznefOho22SpzoZI+lKPOkEdXSTS1GT58/oKHlpSTlVNQ7vtr/ICmUwZXcjkOcsSztwANNYsksYUqBPQkewKay2vl+1n7qJSSvccZcyAXvzX1WM5eKyOB5eUtwblKaMLeWF1hSeZG0UFeQrSImmsywbqRJdDu3l/vItE1mw/zNxXSlm59RCDT+3GzecN5tVP9vKtp99vd43Jc5bFvQAlkfrRIuJfMQO1MWYQ8FugH83zW/OttY8lu2GJSHQ5dKK7p4T/fvO+YzywuJTFH++lT49sfnLlGLoFMvnXP3/seI2OLEDpSOaG6nuI+J+bHnUjcLe1do0xpiew2hiz1Fr7SZLb1mGJLod2+/5Yi0T2VNXy6KvlPLdqB92ys/jB9JH83flD6Z6TFbPH3NEFKPFkbqi+h0h6iJn1Ya3dba1d0/LzUWAj4Ov/Fye6HNrt+512Mfn+lGHc/8pGLnxgOS+s2cm3vjiEN2ZdxO3TRtA9J8vVNVKxQ4o2oBVJD3GNURtjhgATgJURXpsJzAQoLi72oGkdl+hyaLfvDx9q6J+fy6TBvZnzSilH6xq5enwRd00fyaBTusV9jVQsQFF9D5H04DpQG2N6AC8Ad1prj4S/bq2dD8yH5iXknrXQhfBx1vCMCYivNxrPpNxVE4r4ytn9eX71Th59dRP/9+Fupowq5J5LR3NG/14JXSPZC1BU30MkPbha8GKMCdAcpJ+x1i5IbpPiExpnraiswdI8zvrC6gqunVTUoU1Zwf2mrtZaFn20hxmPrmD2gg0cPF4HQPneY5TtOerJNZJJG9CKpIeYRZlMcx3N3wCHrLV3ujlpKosydaQAUTycsiLe23KQuYtKWbu9kr49c6isbjgHR4kAAAn7SURBVKA+2NT6vnQpaKSsDxF/iFaUyU2gPh94E9gAhCLRj621f3V6TzIDdXhgcSpAZICtcy6P+3xtA1WkinJt5ecF+PFlo3ns1U3sqqpt97pXDwsR6foSqp5nrX2L5rjX6SKlkxmak7vDuRlnjZWeFq12BkBdQ5CcrEx2RwjSoEk5EfFGWhVlihQ4Le2fIm7HWWOlp8UKtLWNTTywuMzxoaBJORHxQloF6mjlQEOTcgV5AXIDGdz17Domz1nGwrUVcZ9vV2UNx+sa6ZEbOylmV2WNJuVEJKnSKlA79VBDY8GP3DCeusYmDlc3tGaA/GjBBsdg7XS+XnkBLnxgOUdrG4my41XrOfyQwSEiXVdaBepYPdd4V9pFOp8BqmoaGFbYgwW3fpGHrx/fWhc62hDLVROKeHv2VLbOuZy3Z09VkBYRz6RV9bxYq/XiXWl31YQirLX8v5c3cvB4PQD983P5z2vGctHIQowxTCzufUIWSGensvmhDSKSWmkVqCH6ar14V9qt31HJn1bv5ODxegb2zuPuS0Zy5bgiMhzGOzp7qyoVURI5OaVdoI7G7dLvLfuP8dCScl7esJtTumdz31fP5JtfKCYnK/YO317oaK840aqAIpKeulSgjjU0su9ILY++tolnP9hBTlYGt08bwXe/NJSeLnf49kIivWIVURI5OaV1oHbqmYYHvCO1DTzxxqc89dZWgk2WG79QzD9NHUFhzxxX5/OqXZBYr1hFlEROTmkbqN30TGsbgvzvu9v42eubqaxu4IpxA7j7kpEMPrV7h87nRbti9YqjBXlttSVyckrbQB2tZ/rVcQNYsGYnjywtZ1dVLReMLOSeGaM4qyi/Q+eLJ1DHOk+0XnGsIJ+KGtUi4j++C9Ruhx+ceqYVlTV8+bEVlO89xriB+Tz4tXF8cXifmNf1avzXzc4tTr1iNw+Lzs48EZHU81WgdupRrtp2iOWl+08I3tEq5zUGLT//24l8+azTaK7SGptX47+J7Nxy17PrIp5Tk4UiJzdfBWqnHuUz721vrZAXCt7XTipqt4sLwPUlA/nPq8cSyIxv0aVX47+J7NyiyUIRicRXS8ijFV1qq6YhyNJP9jK2zZhzr9ws5l17NvOuGxd3kAbvdlxJ5Dwq7iQikcTcOKAjOrpxgNNuLU6yszK45YtD+N6Fw+jdPTvu6/mRloiLnJwS2jgglSINGzhtDNAtO5OlP7iwtWBSV6HJQhEJ56uhj0jDBt84ZxCBsNobOVkZ/NfVY7tckBYRicRXPWr4vEdpreXlDbt5aEk5DU2W7MwM6oNNFGk4QEROMr4L1ABvbz7A3EWlfLizipH9evDkzSVMO6Ov61Q7EZGuJGagNsY8DXwF2GetPSuZjfmoooq5i0p5c9MBigryePBr47h6QhGZsbZZcUkTdSKSjtz0qH8NPA78NpkNqapp4Gu/eJecQAb/cvkZ3HjuYBZ9tIcL5i33JLCqlrOIpKuYgdpau8IYMyTZDcnPC/DETZMYX1xAr9yA54FVtZxFJF35KuvjgpGF9GqpDR3v/oexqJaziKQrzwK1MWamMWaVMWbV/v37Ez6f14HVaRm2lmeLiN95FqittfOttSXW2pLCwsKEz+d1YNXybBFJV74a+mjLy8AayvaoaQiS2ZLiV5AXIDeQwV3PrmPynGUsXFvhSbtFRLwWM1AbY/4AvAuMMsbsNMb8XfKb5V2RpNCkZKiGSNBaAhmG4/WNHK5uwPL5RKWCtYj4ka+KMiVDPIWeigryeHv21CS3SESkvWhFmXw79OGVeCYflQEiIn7U5QN1PJOPygARET/q8oE60qRkIMMQyDxxWboyQETEr3xZlMlLTnsURvqdViiKiB91+clEEZF0kBY7vKiynYhIZL4I1KpsJyLizBeTiV4XYBIR6Up8EahV2U5ExJkvArUq24mIOPNFoFZlOxERZ76YTHTKddZEooiITwI1NAdrBWYRkfZ8MfQhIiLOFKhFRHxOgVpExOcUqEVEfE6BWkTE55JSPc8Ysx/Y1sG39wEOeNicztRV7qWr3AfoXvyoq9wHJHYvg621hZFeSEqgToQxZpVTqb9001XupavcB+he/Kir3Ack71409CEi4nMK1CIiPufHQD2/sxvgoa5yL13lPkD34kdd5T4gSffiuzFqERE5kR971CIi0oYCtYiIz3VaoDbGXGqMKTPGbDbGzI7weo4x5tmW11caY4akvpWxubiPW4wx+40x61r+/H1ntDMWY8zTxph9xpiPHF43xpifttznh8aYialuo1su7uUiY0xVm8/k31LdRreMMYOMMcuNMZ8YYz42xtwR4RjffzYu7yMtPhdjTK4x5n1jzPqWe/mPCMd4G7+stSn/A2QCnwKnA9nAeuDMsGNuBX7R8vPXgWc7o60e3MctwOOd3VYX93IBMBH4yOH1y4BXAAOcC6zs7DYncC8XAX/p7Ha6vJf+wMSWn3sC5RH+G/P9Z+PyPtLic2n5d+7R8nMAWAmcG3aMp/Grs3rU5wCbrbVbrLX1wB+BK8OOuRL4TcvPzwPTjDEmhW10w819pAVr7QrgUJRDrgR+a5u9BxQYY/qnpnXxcXEvacNau9tau6bl56PARiC8cLvvPxuX95EWWv6dj7X8NdDyJzwrw9P41VmBugjY0ebvO2n/obUeY61tBKqAU1PSOvfc3AfAtS1fSZ83xgxKTdM85/Ze08V5LV9dXzHGjOnsxrjR8vV5As09uLbS6rOJch+QJp+LMSbTGLMO2AcstdY6fiZexC9NJibf/wFDrLVnA0v5/CkrnWcNzXUVxgH/DSzs5PbEZIzpAbwA3GmtPdLZ7emoGPeRNp+LtTZorR0PDATOMcaclczrdVagrgDa9iwHtvwu4jHGmCwgHziYkta5F/M+rLUHrbV1LX99EpiUorZ5zc1nlhastUdCX12ttX8FAsaYPp3cLEfGmADNwe0Za+2CCIekxWcT6z7S7XMBsNZWAsuBS8Ne8jR+dVag/gAYYYwZaozJpnmw/aWwY14CvtXy83XAMtsyMu8jMe8jbKzwCprH5tLRS8DNLRkG5wJV1trdnd2ojjDGnBYaLzTGnEPz/w/81gkAmjM6gKeAjdbahx0O8/1n4+Y+0uVzMcYUGmMKWn7OA6YDpWGHeRq/OmVzW2ttozHmn4DFNGdOPG2t/dgY8xNglbX2JZo/1P81xmymeWLo653R1mhc3sftxpgrgEaa7+OWTmtwFMaYP9A8697HGLMTuI/mSRKstb8A/kpzdsFmoBr4due0NDYX93Id8D1jTCNQA3zdh52AkMnATcCGljFRgB8DxZBWn42b+0iXz6U/8BtjTCbND5PnrLV/SWb80hJyERGf02SiiIjPKVCLiPicArWIiM8pUIuI+JwCtYiIzylQi4j4nAK1iIjP/X8W5zmilIONHQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0giuwC9GHzi8" + }, + "source": [ + "## Вычислиения на GPU\r\n", + "\r\n", + "Для проведения вычислений на GPU PyTorch поддерживает перемещение тензоров на GPU и автоматическое построение вычислетельного графа там. Традиционный способ вычислений состоит в том, что вначале мы определяем доступное вычислительное устройство `device` (CPU или GPU), и затем перемещаем туда все необходимые тензоры по мере необходимости с помощью вызова `.to(device)`. Мы также можем заранее создавать тензоры на нужно устройстве, указывая параметр `device=...`. Такой код работает без изменений как на GPU, так и на CPU: " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "HK7HPLz3Hyrl", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "7e14cccb-d376-4e59-be66-4ab3f5c3f6f4" + }, + "source": [ + "device = 'cuda' if torch.cuda.is_available() else 'cpu'\r\n", + "\r\n", + "print('Doing computations on '+device)\r\n", + "\r\n", + "### Changes here: indicate device\r\n", + "w = torch.tensor([100.0],requires_grad=True,dtype=torch.float32,device=device)\r\n", + "b = torch.zeros(size=(output_dim,),requires_grad=True,device=device)\r\n", + "\r\n", + "def f(x):\r\n", + " return torch.matmul(x,w) + b\r\n", + "\r\n", + "def compute_loss(labels, predictions):\r\n", + " return torch.mean(torch.square(labels - predictions))\r\n", + "\r\n", + "def train_on_batch(x, y):\r\n", + " predictions = f(x)\r\n", + " loss = compute_loss(y, predictions)\r\n", + " loss.backward()\r\n", + " w.data.sub_(learning_rate * w.grad)\r\n", + " b.data.sub_(learning_rate * b.grad)\r\n", + " w.grad.zero_()\r\n", + " b.grad.zero_()\r\n", + " return loss\r\n", + "\r\n", + "batch_size = 4\r\n", + "for epoch in range(10):\r\n", + " for i in range(0,len(features),batch_size):\r\n", + " ### Changes here: move data to required device\r\n", + " loss = train_on_batch(features[i:i+batch_size].view(-1,1).to(device),labels[i:i+batch_size].to(device))\r\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 195, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Doing computations on cuda\n", + "Epoch 0: last batch loss = 94.5247\n", + "Epoch 1: last batch loss = 9.3428\n", + "Epoch 2: last batch loss = 1.4166\n", + "Epoch 3: last batch loss = 0.5224\n", + "Epoch 4: last batch loss = 0.3807\n", + "Epoch 5: last batch loss = 0.3495\n", + "Epoch 6: last batch loss = 0.3413\n", + "Epoch 7: last batch loss = 0.3390\n", + "Epoch 8: last batch loss = 0.3384\n", + "Epoch 9: last batch loss = 0.3382\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A10prCPowHl7" + }, + "source": [ + "## Пример 2: Задача классификации\n", + "\n", + "Рассмотрим пример двухмерной задачи классификации на 2 класса. Примером такой задачи может быть классификация опухоли на 2 типа - доброкачественная и злокачественная, в зависимости от её размера и возраста.\n", + "\n", + "Сгенерируем тестовые данные случайным образом:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "id": "j0OTPkGpwHl7" + }, + "source": [ + "np.random.seed(0) # pick the seed for reproducability - change it to explore the effects of random variations\n", + "\n", + "n = 100\n", + "X, Y = make_classification(n_samples = n, n_features=2,\n", + " n_redundant=0, n_informative=2, flip_y=0.2,class_sep=1)\n", + "X = X.astype(np.float32)\n", + "Y = Y.astype(np.int32)\n", + "\n", + "split = [ 70*n//100, (15+70)*n//100 ]\n", + "train_x, valid_x, test_x = np.split(X, split)\n", + "train_labels, valid_labels, test_labels = np.split(Y, split)" + ], + "execution_count": 196, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "id": "c-_BjSHPwHl8" + }, + "source": [ + "def plot_dataset(features, labels, W=None, b=None):\n", + " # prepare the plot\n", + " fig, ax = plt.subplots(1, 1)\n", + " ax.set_xlabel('$x_i[0]$ -- (feature 1)')\n", + " ax.set_ylabel('$x_i[1]$ -- (feature 2)')\n", + " colors = ['r' if l else 'b' for l in labels]\n", + " ax.scatter(features[:, 0], features[:, 1], marker='o', c=colors, s=100, alpha = 0.5)\n", + " if W is not None:\n", + " min_x = min(features[:,0])\n", + " max_x = max(features[:,1])\n", + " min_y = min(features[:,1])*(1-.1)\n", + " max_y = max(features[:,1])*(1+.1)\n", + " cx = np.array([min_x,max_x],dtype=np.float32)\n", + " cy = (0.5-W[0]*cx-b)/W[1]\n", + " ax.plot(cx,cy,'g')\n", + " ax.set_ylim(min_y,max_y)\n", + " fig.show()" + ], + "execution_count": 197, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 283 + }, + "id": "tq0vFchQwHl8", + "outputId": "919f1922-f789-4779-cbdc-4f9e742c358b" + }, + "source": [ + "plot_dataset(train_x, train_labels)" + ], + "execution_count": 198, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd5iU5dX/v2dmtrINdpfOsoBKV9AFsQRFNIgx1thiNJZojDH6mkSNKSbRn0l8TaKmGDUxRo2vFbEiYgsIEaWIIFUEgaXsLmzvOzP374/vPM6WqbvT93yua67dnXn2mTPzzJxz36eKMQaKoiiKYou3AIqiKEpioAZBURRFAaAGQVEURfGgBkFRFEUBoAZBURRF8aAGQVEURQEAOOItQF8oKioypaWl8RZDURQlqVizZs1BY0xx9/uT2iCUlpZi9erV8RZDURQlqRCRXb7uV5eRoiiKAkANgqIoqYgxQHs70NERb0mSiqR2GSmK0gecTmDjRuDNN4GdOwG7HZg+HZgzBxgzBhCJt4Th09YGrFoFLFoE7NvH+w47DDjjDOCoo/gaFb+oQVCiijFAYyN/5uQANt2TJgatrcCDDwKffALk5gKFhYDbDXz4IfD++8BZZwHnnptcRqGpCbj/fmDrVqCoCBg9mvdXVgL33QccfzxwzTWAQ9WeP/SdUaJCRwfwwQdcqFVUUK/k5wPz5wNf+QqQlRVvCfs5TzwBbNgAlJZ2VfrDh3PnsHAhlers2XETMWwefxz4/POeu5tBg4CBA4GVK4Fhw4BzzomfjAlOwqzXRGSUiLwnIptEZKOI3BRvmZTe0d4O/PnPwN//zt9LSnhzOIB//xv43e+AhoZ4S9mPqayktR41yvcOwOEAhg4FXnoJcLliL19vqKqiq2jkSN+vSYTGbvFi7o4UnySMQQDgBPAjY8wkALMAfF9EJsVZJqUXLFwIrF8PjB0L5OV5v58DBvC+8nLgscfiK2O/5uOP+TOQ/27AAKCmhrGFZGDDBvolA72mjAyuUD77LHZyJRkJ4zIyxuwHsN/ze4OIbAYwAsCmuAqmhEVTE/D228CIEf7dzyNHUidVVABDhsRWPgVAbS2Qlhb8OJuNFzRW1NRwlb9vH5CZCUydCkyYEFoguKkp9ICx7hD8kjAGoTMiUgpgOoAP4yuJEi7bt9MFHUjfiHAxt3EjMHgwvRJ2e3LFL5OavLzQ0jGNiU2wx+Wie+r11/mcmZn8EC1ezA/ID35A91YgCgpCc29Z2Q2KTxLOIIhIDoAFAP7HGFPv4/FrAVwLACUlJTGWTglGW1toit3lApYtA155hQtWhwM49ljg1FMZE1SiyLRpwLPPUjn6u1gtLcw+Gjs2+vIsWAC8+iqzgrpnAB06xKDTHXcE3k5OncodjbW68IX1mg4/PHKypxiJFEOAiKSBxuApY8yLvo4xxjxijCkzxpQVF/doxaHEmbw86plANDQAa9YAH30EpKcz0WXoUHoLfv1r4N13YyJq/2XYMODooxnM8XWx3G66bc48M/opmlVVwBtv8EPg67kKC7nKeP11733G0O3TeUdQUADMnQvs2uX7NTmdfE3nnqtppwFImHdGRATAowA2G2P+GG95lN5x2GFML21uBrKzez7ucgHLl/P3adNoEAB+R4cPZ8zvX//i7xMmxEzs/sfVVzNnf9s2pmValvzgQRaOfPWrVLDR5oMPuLIP5P8fOhRYsQI47TQGn5YsYcxABJg5k/ePGwdceCFXGx98wA9fQQFf06FDdJGdcw5w8snRf01JjJhgy7kYISInAngfwAYAbs/dPzXGLPL3P2VlZUab2yUe778PPPIIU027xxLKy4H33gPKyoCJE33/f1UVPRU/+lH0Ze3XtLcDq1dzhV5eTgU7dSqNwaRJsQnqPPAAA0+FhYGP27yZsQW7nXGFzEyuLioruYO47DIaMLebRu6tt1iTIAIceSSrr0eP1kCVBxFZY4wp635/wuwQjDHLAejVSgFOPBGorwdeeIGLv4ED+T2srfWmo44f7///CwsZcG5ooMtXiRLp6azePf54KlKR2CtMh4PPHQinE/j0U+4GOscN7Xa6v9rbWZQ2ciQ/WBMm6PaylySMQVBSBxHga1+jm3r5choBY2goLAUfKF3cZuM5rBigEgPi1VNk+nQGjwLFA3ftotHwl2mUns7MoUWLAq80lKCoQVCixrBhwAUX8Gbxl78AmzbRZe0Pt5sGxFcMQkkxpk/nhW5s9J0Oagw/MIcfHthoFRVx5eHvPEpIJFSWkZL6nHQSA86BqKwEjjlGv9f9gqws4PrrGfg9eLBrhlBbG/DFF/Q5jhsX+DzWtrKtLaripjpqEJSYMnEid/779/t+vKWFGYVnnBFbuZQ4MmUK8POf84OxaxewZw+wezeDTmefzQ9De3vgc7hcNAi6rewT6jJSYorDAdx8M/CHP3DxV1DAtjkdHVwg2mzADTdocVrCcegQW0s4HOxLEkrri3AYNw647Tb2M6mu5vOUlLD/0Cef0B0UKM5QWQnMmqVtdPuIGgQl5gwaBPziF8C6dUwpr6hgFuGZZzLwPHhwvCVUvmTXLuDFF6mQbTZvcOf004F58yJvGIYM6VmRPHmyd1s5bFjP/2lq4opi3rzIytIPSZg6hN6gdQiKEkW2bgXuvZdKv7jYG9RtaaFynjaNfYYibRR8UV3NbWV5eddt5aFD3E18//uciKaERMLXISiKkkC0tXGoRV5ez5SwrCz69NatA955h7uFaDNoEPsZffIJi84qK7mtPO881lEMGhT6uZqaaNDcbm5HCwqiJ3eSoQZBiQvxqoNSQmTdOirOoiLfj4uwpcQbb7AjYSz6A2VksDht5sze/X9DA/Dyy8DSpV37IM2cyR5H2otdDYISO+rr2ZLmrbfoAcjMZMxgzhzGKZUE4uOPgwdos7OZCXDgAKuEE5n6enZN3b+fjbIsN5fLxU6Ln34K/PSnfKwfo2mnSkzYu5eB5Oee42Jy9Gi2qFi6lPevXBlvCZUutLeHNnBGJDnGbD7zDLMXRo/uGvOw27kaMQZ46KHgrXpTHDUIStRpbWU80OXi9zE7m3okLY3fxcGDgYcfZhqqkiCUlASfluZyUYEOHBgbmXpLbS1XHIFW/0VFrH9IlpGhUUINghJ11q6li8ifOzozk+1oliyJrVxh4XJxbu8//gHcdx/w5JPsppmqK8rjjuNrC7T6r6hg7n+gPiSJgLXSCLTjsYJZ27ZFXZxERmMIStRZtowzEgIxeDDw4YfAt7/N2GFCUVHBNs3WrN/0dPbXeecd5sh/73up12djyBDOGVi8mNu67sq0poZK9Otfj4984RCqS8tmC220aAqjBkGJOrW1wZW83c4FaUtLghmE+nrgnnvo9yot7fqYMezT/6c/AbfemnqTuC66iEryzTep/LOyqFxbWxkAuv1234ViiUZhobdjYqC0Nrc7OV5PFEmxT7CSiAwcyKByoDYzTqdX5yQUy5ZxNTx6dM/HRFhBu3UrdwxHHhl7+aKJ3Q5cfDHTSleu5EXMyGCH0smTk8cAjh7N61Rb6z/e0drK1zZ1amxlSzCS5IoqycycOcCDD/qvHWpsZNq7zcaF9ogRHNo1ZUqcdY7bzcBGoF4aInQXvfVW6hkEi6Ii9hVJVkSAb30L+O1vmcnQ3b3X1kZj953vJNj2NPZoUFmJOkcdRZ1aWdnzsV276JHYvZuT1LKzmexx333A//4vjUXcaG1lMVOwbUtuLuMLSuIyfjzwwx/SJ7lrF2snKioYcD54ELjySmD27HhLGXd0h6BEnYwMfhfvvZffv8JCKv59+zh/ecAA7iKGDvUeP2gQR+0+9BBnK8elotnanrjdgYezuFwMNCuJzZFHAn/8I9tfbNrE6zp2LAd8p1pSQC9Rg6DEhKFDgTvv5LTEt97i4mzzZg7CmjKFRqEzlnv+009pROLSDjs9ncLt2BG49fKhQ+zb3wusrFVt4REj+tr+IsVRg6DEjAEDgJNP5q2qivGCkhL/ytAqXvvgg14ahMZGYONGZgoNGABMmhR+I7PTT2eW0cCBvgMaLS3cPZxwQsindDpZ0vDGG9wFiXBw0Lx5jNXGa7yxoqhBUOJCYyOTWIKtjDMz6eINC6eTPfyXLOHvFjYbZ3hefHHowcNJk7j6f+kl+rry8ym0MRSssZF1CP6q7rrR1gb87W8s1svP97YA2rmTLrXZs4ErrkieBB4ltdCPnRIXMjO9nQ8CGYX29jALYd1u4LHHGJwoKemqWV0u4L336OK58cbQtK4IWyyXlACvvMLot83G57GMxRFHhCzes88yo2rMmK6vu7iY9mbZMv4899wwXrOiRAg1CEpcGDqUrWXq6wNXMbe3A8ceG8aJt24Fli9nEVl334vdzpz0devYzXPGjNDOKcJjy8qYKtXWxiBkOD34wTT4//yHsRFfRtBm445h8WJg/nwaTUWJJeqtVOKCCHDWWYwldPbqdMbqqjx+fBgnfvttpjD5c8SLMB6waFHYMkOELR1KSsI2BoA3sSVQS530dBrBLVvCF09R+ooaBCVuzJxJb8zu3Vx4u928v6mJmUW5ucBNN4UZZN26NXj3zYICOu1j3La5oSH0bKKWlujKoii+SCiXkYj8E8CZACqNMVPiLY8SXUToK584kcVpn3zC+wsKgG9+kxk3tbVAXR3dSyG1tQhV48YhzzMvL/TmqIHafChKtEgogwDgXwD+AuCJOMuhxAgr5XLiRC7YnU4agZdf5kwTK6HH4QBOOYUdFALWEE2ezBiBVeXmi+pqFkCEMgAmgkyezKd0Ov3Hs9vamAAVlptMUSJEQrmMjDHLAFTHWw4lPtjtTAC66y4WsI0YwQBsSQmzOhcvZklAQ0OAk8ydS3+LP3eQMdxyzJ8fldcQiLw8GrU9e3zvFNxuoLwc+NrXNKCsxIeEMghK/8YY5ugbQ2PQeQGfns7Eob17geefD3CSww5jH/8vvuByuzMdHYwdzJoVt0Z0F17I2MnOnd6AutPJyu1du2jPzjgjLqIpSsK5jIIiItcCuBYASkpK4iyNEkk+/5wr5ECXdfhwYMUK4Bvf8FOfIMIAxMCBwKuvdjUKaWlMbTr77Ji7izqLcN11LEB7803gs88o8qRJtGMTJmgbCyV+JJ1BMMY8AuARACgrK0vR+YX9EyvVMpBCdDi4g/jiiwCLfJuNfpdTT+VJGxsZpR0/PiGitXY72+7389b7SgKSdAZBSV06OkJfuPurXehCRgZ7byuKEhIJFUMQkacBfABgvIiUi8jV8ZZJiR3DhwdX9MYw+FpYGBuZFKU/kVA7BGPMJfGWQYkM9fXeRqO5uUy5DNSiAqALKCPDm3oJAHZXO4bsX4fDPn8TefXlaOlw4OCYGShxnwygNMqvQlH6FwllEJTkp6ODWUDvvMPMT6uOwGYDvlpWjfOG/Rfp2zfz4IkTgeOP/7INRFYW57o/9hjTTXPQiGM/uB+F1Z+hNbMANWmD0druwpyMFZBfvceD58/XKKyiRAg1CErEcLmAv/8d+PBDKvQvi6+MQen2tzD8vmewbSAw6dhctqPYvJltqi+6iEOURTBnDt1Gzz1rcOT6h5HZuBPl2aVwtgnS0oCZJ9pRMGQELc/TT3M2Z1lZPF+2oqQMahCUiLFpE7ByZc/WziW738f0DU+idsgobKtPR2E7MGwYmBra3g78+9/cHsyeDRHahuNH7EbDDzdg/9DRyLYJiotZfPylkUlLY7XaSy8BxxyjuwRFiQAJFVRWkpslSxgv6Kybba4OTN74PBpyh8M40pGZySlhX5Kezmjyc89x1e8hZ/MqDBtpx9HHCKZPZ9fTHu0e8vI4mFkH3CtKRFCDoESMbdt6NhotOrgF6e0NcKaxM11WFlsJdWndkJXFWoHNm733VVcH798gwuBEU1PfhW9sZN+M1ta+nyuWVFez7HnfPm+7WEXpJeoyUiKGzdazR09WSzXS25swuGID0tsb4LKlQWwjANcQwNGt6KC21vt7fn7P1hPdMYaBi74Um23eDLz+Ov1dIrwdfzwHHI8Y0fvzRpvPPwcWLmQql/XGFxUBX/865zvrYGalF6hBUCLGpEnUq0OG8G+HsxVHbH0FQw98jLaMPLhtaXB2uDHJsRfydiZ7CnUeep+W5v195kxOoQ80Y7OhgU/WW8W9eDHwf/9HP5c1xszpZCDkv/8FfvhDvqhEY+1a4M9/piEcNcqr/BsagEceAXbsAC67TI2CEjb6iVHC4uBB4LXX2ITu0Uepm9rb+dhpp9F743YDMAbT1/wduQ170Z6eg/b0XHQ4stAiA5A9vICKfsWKTv8AtqS2KC1lYx9/8QGnk93hzjmndwHlrVuZpTRqFAcaW+dwOBjTyM8H/vQndkZNJOrqgIceoszFxV2Vfm4uI/rvvAOsWRM/GZWkRXcISki4XMALL3BRLcLFqcvFWfb5+cAPfsBWQaecArz7LjA1eweG71+D6sLxyGyrx4D6/ahHPvLzgQEDAEgWq9a2b2fZ8YwZdHlYiADf+x7wxz/SR15YSIXndtMQtLRw3FpYA5c78eabfBGddyUADdWhQ3yOigrm0V53XZAhDDHkww9pDP25yWw274jQsjLNvlLCQg1CClBVxWZvbjfT8ktLI68Hnn+eOmb06J79hurqgP/9X+AXv6CnorAQqP7jMtS1pKMWghr7UTjc2YiitFoMyM6B9bFz2jNg1m1Ey/zzkP3Ny3t+GPPzgdtvB1avpvto924+eVkZ+0QfdljvXmhbG/Dxx9wddH8hq1YxwGyzMevpySfZIO+882JfBFdfT3k++ojbsJISYMMGP21eO1FQwF7aDQ3Bj1WUTqhBSGIOHmQKvzV6UoRGYeRI4PLLgSOOiNzzLF7s2xgA1NstLSwJ+MEPGNfsWF2OA5tysP4LoL4lA58POxGNTTswuPxzHNrTBIcDaHc50Jxegservov2X+Vi/nzq+S6L9sxM4MQTebMi1n1Vyu3t3gwli8ZGYPly3m/FNTo6+JxDhtC95HZzZFssWLeOfrn2dr7BdjvwwQeMbwwezPiLw0GlX1/P/8nJoQGwguMhdQBUFC9qEJKUQ4eA3/yGeqxzXNEYJuv89rfALbdEJia6ciX1S6BOpIMHc9FdU0OPRYdkYOd2F9w2FpSJpKO1aAK2NR2OAztb4GwDikZmYnTaXhSNyYVUb8MX/+9DvP9wDWafPRCOE2b13AFEanWelcX6h/Z2/gS4C3C5uq6oOzqojNPSuDp/8UUaps6B8GiwfTvwwAN0oQ0Y4L0/N5dWeetW73tRVcWfVo+QgQN50dPSeLwSHk4nd2GLF3t3pEcfDcyZE52td4KhQeUk5bnnuDgcPrzrQleEOmHgQODhhyOzSNyzJ3hmp83GW7VnAOp/mmfC1ljfpVDN7QbK99vhzMyBLS8HTZWNaEgvQtlHf8Npq3+Dsub34dz8GfY9swy4+25avGgEdR0ObkUOHODfra0cxdY9TtDeDowdy9/T0qhwP/oo8vJ058UX+YZ3NgYWpaWUf+1aoLKSBquggD/z8xmkf/ttDlvoHh9RAtPcDPzhD8D99zOZobCQC4SVK4Ff/5oDl3zNPk0h1CAkITU1dC0HmiOfl0ddunFj358vPT20mie32+vFePXADKTnZSK9vfHLx5uavAPm7eJGfkcV9h1MQ379btQVlKIxbziksBAba0bAPWo0AyN/+EN0isXmzKE7qqaGiqC7C6m+ngp28GDvfdnZDHBHk4MHWRvROcDemUGDuJOxZm/62kFlZHAUW4orr4jzz39yp1haSmPgcHgr6UeOZCBt5cp4SxlVwjYIIjJAROIzf1ABwMUsEHyYjMPRrU1EL5k+nTGCQLS0cEE7fDiTc9ocA7Dq2BuQ2VKDnMYDEOP+Mlab7azHsNad2JAxA1vaxqAxz1tHkJbGhXlDo7C+YNcuroYjTVERcOut/H3vXhqd9nZardpaWtTjjuv6JlsWL5rU1fFN8ueaaGykIcvP57H19TRojY2UOy2Nk+JqayNz8fsL+/YxVbekxPd773BwcbBwYUpXhAc1CCJiE5FvisjrIlIJYAuA/SKySUTuFZHDoi+m0plAtVqdsYLMfWXqVK/+8SfP/v0s7rW8FCJA1eDJWHrSHagYPAV59eUY2FSOYR274LSl45Vh38VHOBbt6b7TOb+Ue+BA+nOjQWkp8LvfATfc4A3GFhXREMye3bN1RktL9OdepqcHXtnX11PO4mLGCsaO5Wp2+HDKPXeuNw6ya1d0ZU0l1q71BuP9kZvLmM3u3bGTK8aEstx5D8DbAG4H8Kkxxg0AIjIIwBwA94jIQmPMv6MnptKZoUOpMN3uwMWoHR3UeX0lLQ248UbgnnuoEwcP9j5vSwuNwZFHskupJZ+V5FJXMBqrZt6AjNY6lG+qx9YdaXAVDoaBIG/3Cgwb2rUPkaULv9TFVgO7aJGVRffRLbfQR+zvDWts5BYo2iM5hw9nTKCpyXcMwVoNuFwMund2afk6VgmN6mpvgkEgItU7K0EJxWV0qjHmLmPMessYAIAxptoYs8AYcz6AZ6MnotKd4mIq4IoK/8c0N9Pl7XcQfZiMGwfccQcnn5WX87Z7N5/nwgtpMKzvU04OF9id9XhbZj6yx4/CQcdQuIwNHU6Bw+7C+OzyLs/T1ESDkpXlucPp9BkcdbkivHM/4wyutnft6tJ1FcbQr3/oEHD99d5RbtHCbmfe7oEDvl9gTg6tcHa2/ziDRSL3Yko08vO7Xnd/GNPpw5l6BN0hGGM6RGQCgBEAPjTGfBklFJHTjTGLjTEhvJNKJLnoIuCuu7iDLSrqutNtaqKx+MEPIqu/Ro7kOWtquKCyujz4SmY56yxm7+3dy9kHNhsXvOPGMWbqcABnT6jFoOZ9aEwfDoD1Ym43O1Z8SVUVm7V5Hl+1ijVqe/fynFOncmcyaVIfMwKzsoAf/5g+4v/8hxYHoECHHQZcfDF/xoLZs9m8bulSXlzLndXeTpdRXh6LTPxtD+vr6UYaPz428qYCRx/Nax/IH9vUxN3b6NGxlS2GiAmyrRSRGwF8H8BmANMA3GSMednz2FpjzNFRl9IPZWVlZvXq1fF6+rhTXs7OCpar2EpFz89nxXC8B4nV1ABPPMEaKwur84TdDkzK2olzN/waNVkj0OZOQ0YGe9oVFnoO7uig5v/1r9EwaDTuu496ctAg6kRr8d7YyPjFJZdEqJ9bUxO3P04nhRk2LPb55243M1oWLfJaP4eD7q0jjmDRWnZ2z37j9fV843/8Y27nlNAwhm1SNm/myqc7Lhe/aFddBZx8cszFizQissYY00NDhGIQNgA4zhjTKCKlAF4A8KQx5gER+dgYMz0aAodCfzcIAD/HO3eywaXLRd01aVL0k2HCobLSK19hIXvYHTzIwltZ/AYmfPw08scMRNG4AjjSxFtdV10NXHopzLzTcd99TKHt3m0C4Hm/+AK48kr2UkoprPfC5WJQ09ry7djB1cCBA11XtYMGUWmpMQifhgb4XHVUVdE3evrp3CmmQHFaXwzCRmPM5E5/54BGYROAU4wx0yItbKioQUgBjOEW4qWXuCq32bg6Li0Fzj4bmDYN5eXAz3/Onbq/72JzMz0qv/99YhnDqOJ2M7V0927+PmIE/W3B8pEV/7S1eXtn7d3LD9y0aWzlO2FCShgDwL9BCOWrUyEi04wx6wDAs1M4E8A/AUQ5B09JeURY6DBtGrcSVjR88OAvv3zr1gVOzQf4LwcPcqcQK1d/3LHZ6D6KVNMqhTuwE07gze0OnoqaYoRiEC4H0KUBgjHGCeByEXk4KlIp/Q8R72SdbtTVhdaFQSR4AZ2ihEw/HDAUSpZReYDHVkRWHEXpycCBoWUEut2+U/eTjpoaNrBra6Mfe+LE4POlFSUC9Bdvq5LEHH00m/kFyghsbGTAOhKFeHGjuZkjPf/73641CJmZnAw3b16/cl8osUcNgpKQGMPEGrudhWqzZnFYmK9WM04n6y6uuy6Jd/ltbcxw2b6daY+dA8NtbcBTT9HqnX++GgUlaoRsEEREAFwKYKwx5k4RKQEw1BgTsX7AInI6gAcA2AH8wxjzu0idW0kO6uuZjvrGG8y2dDhYmzB7NvXhhg3MvszP5yL64EFmF11wAXD88fGWvg+sWEE30ZgxPRV+RgZTrF57jZbRV568okSAcHYIDwJwAzgFwJ0AGgAsADAjEoJ4Oqj+FcBpAMoBrBKRV4wxmyJxfiXxOXCAozhratieo7SUu4Q1a6gvL7mE2X9vvslsIrud/dxOOSVwSmrC43azAK1TZlUPHA7eli4FLr00tvIp/YZwDMKxxpijReRjADDG1IhICN2gQmYmgO3GmB0AICLPADgbrHdQUpyODhaKtrV17Qxgt7PYrr2d7vWf/MTbtTplaGxkEV5JSeDjCgpYSasoUSIcj2uHZxVvAEBEisEdQ6QYAWBPp7/LPfd1QUSuFZHVIrK6yhofqCQcLhdX+jU13rZAgfj0U5YhFBf7fjw9nQk3r70WWTkVRfESzg7hTwAWAhgsIncD+AaAn0dFqgAYYx4B8AjASuVYP78SmNZW4P336QGx5ifk5wPz5/seMWCxfHnwlNHCQg60qqvjOVOGnBy2Smhs7DnGszO1tZx3kAg0NbGid8MGWvxx4+i/+7IRVQRoaGDDLrebNSrBursqfSYkg+AJKC8DsAbAXAAC4BxjTCT3r3sBdO5UM9Jzn5IkNDXR7fPZZ/z+Wh6Qpibg3/9mltAPf+hb8dfXB+/MahWNtrSkmEGw2YCvfQ147DG+Ob7iCE4nFe9XvhJ7+bqzZg3wyCP07+XkUP5PPgEWLGB67Fln9S2gU1fHc63oVObkdnMWxYUXssWuEhVCchkZNjxaZIzZYoz5qzHmLxE2BgCwCsDhIjLGE5u4GMArEX4OJYo8/TQb7Y0d21XpDxjA+3buZPakLwYNCl5l7HYzHbUvxWetrRxt0NDQ+3NEheOPZ6+c3bt7+tja2thp88wz459htHEj8Oc/0yKXlnLVPmgQrf+IEcALLzBFrLfU1gK/+Q23jMOGsZuhdduyBbjzTmDPnuDnUXpFOC6jtSIywxizKhqCGGOcInIDgDfBtNN/GmMiMAuADNAAACAASURBVCJeiQU1NaynCjSTZcQIppRecEHPrs2zZtHN1NLCZJqiop7upaoqtjzKzQ1fvv37mZ20YgX1rTEcF/C1rwFTpiRAhlJGBnDzzbSqy5d7C9NEOKvhW9/yjqSLF8ZQvvx8No/qjsNBw7BwIXDSSb2z3M8+y1zi7gF2m40FKYcOAQ89xGEgSVt0kriElWUE4FIR2QWgCXQbGWNMhGZyAcaYRQAWRep8SuzYupU6LFCjTbudOmXrVhoAgP+zZAl1yJ49fCw7m3qwpITKOi2NhqKlhQo8XLZtYxdUY+jKcjj4+969wL33starr16OsLDmjhrDKLo1Azkri62rzzuPb0R7O63fxInRn9QWCrt3800LlA2VlsaUsY8/Bk48Mbzz19TQrxhoVVFYyN3S9u3a1C8KhGMQ5kVNiiTDGCqvigqv4go02rY/0N4e/rHG0FX8yiv0hMybxx1EfT31ys6d1BFjx9KYfO97jF2GQ1MT8MADdHVbehfgdSss5GJ3wQKed8qUEE/qcvHEDofvlbI/GhuZJvXee4wJWMyaRd+7lWJVUAAce2zo540VNTXB284CvHgHDoR/fmvSU7D23SK08moQIk7IBsEYsyuagiQLO3dyCtjOnd4gpzGcXfytb/lPm0x1OivbQIh4jy0vB15/na5oSwfMnk1d8vnn3krkuXPZlsJPM9SArFrFFkH+rovDQXkWLQrBINTVsTBsyRKe1BhO+znjDAY8AynKhgZW3e3Zw6CoNYDa5eKqeP164Kc/pd88UbG2VsFwuXrXjM9qNx0Mmy20XGYlbMJpXXGHr/uNMXdGTpzEZvt24He/486+c2Ws2816obvvBn72s/5pFCZO5GK5tdW/Lmht5Xs3cSL/XrqUi8nOC0KHg7sFK3ba0MDFeG/f0xUrgmckFRby+gXM+qys5MWvqfGmQBpD188f/kD//qWX+vdrP/cc3S3du+/Z7XyxlZX0jf/qVwkQ0PDDmDGUt6PDfz9yY/iFsC5yOBQXewM8gd4Dp5Nb8i1b+OHIzuYQjFB6pCsBCcdl1NTp90wAZ4JzlvsFLhfw8MNUGAUFXR+z2bjo27eP1bQ33RQfGeNJRgZd348/TmPZfWqZ00mPwDHHMLvS7WbsNFhqeW4uXdfNzYFT9P3R0hJcT4jwGvp1e7nd9Du1tHQto7b8TgMHctcwejS3ON2pqwsecS8u5htkpWklIgMGMFj89tv+28pWVfF96M1rGDmSRufgQf/1DM3NDCw/8QSzryyys5mF9dWvarC5D4TjMvpD579F5PdgRlC/YOtWftYDtVceOpTTvQ4e7J81NHPncpX98sv8TlqGs7aWC2mnkxXJVvLJxo3UtdOn++7pFgmGDgU2bQqc8NLRwYWv32O2bvW9urew2bhife01BlK7K6SdO7nqDeQbt/yPW7dGxiBYw7bffZd1A04nUzdPP52pWum97Dpz/vlsJPXZZ3RvWdvBjg76+nJyGOzpzcUUod/1N7+hFe/uh2xpoWEdMICPZWV5H2ttZU5zZSVw2WWJu8tKcPrS/jobLB7rF+zYEXzhYcXb9uwJbBA6Orjbra/n9/KII1Kj0EqEsdFjj+Xqf5OnC9XgwXytY8d2/Q6PG8dYwbp11JWdF98WDQ18L8OJ3XZmzhwW1AbyQlRUACefHCCR56OPgitQayuzb1/PWoHqan4oNm3iqjYjg2/GqFFd3xC7PbzovD+MoXFasIByFxfzw3noEPDXvzLu8T//07stV1YW8KMfAW+9xTzeqiqvMTvpJKaB9aVaedw44JZbgAcfpOFJT+e529poEHJygLKynlvQzEyuKt5+m49PmtR7Gfox4cQQNsDTxwisEygGcFc0hEpEgrk1Lawgs79zLF3K2p2mJu85RViAevHFXfVDsjJsGGsNAO4AbruNmVjdX9vo0TS0AwZw59B9DIAx1DdXXtl7L8CECdR/O3bw/N2vYV0dn/O00wKcpLExtBW1zdbVjQFwJf3oo1RuhYV8UYcOccuUnc1WrVaAxOmMTFD5ww8Zs+juuxs4kNu2HTtYaXzzzb1bSWdlMU/39NO5Ine7+doiNa5uwgTGZTZu5I7J5eLu7KOPuJLqbgwsbDYajLfeUoPQS8LZIZzZ6XcngArPbOV+wahRwRMbrKEuQ4f6fvy11/g9HT686w7C5QKWLaOO+NGPEiPlPFJ8/jldaL5W/3l5VNZbt/K9q6z06kOXi1lIEyb0bc6B3Q7ceCPwpz9RN2dlUW91dHDXkpPDBWnADKYhQ5hXHwjr4neumquupmIrLqby//xzCmSz8fjmZuD55+n7HjKEF/7IPpb1uN3ASy9xW+ZLcYrww7x+Pd/gUaN6HhMq6enRq5xOS6Nra9o0731PPBF891FYyP5Koa7glC6Es+663hizy3Pb66ksvidqkiUYkyfzu97U5P+YqioqMF+tVg4c4A7e10rZbuf9W7bQ1ZJKHDrk/3spwoXckUfSU7JjBxfSX3xBl/1XvsJFbF8NZG6ut2321Kk0CMOGsQbsnnuYoBKQY4/l6j1QymV1Na1b54KU5cv5whoaqPxtNiq69HS+KGtF/e67XA1ffHHfZyeXl9MHFqicW4QfutWr+/ZcscbtDr5VFOk6flQJi3B2CKcBuK3bffN93JeSpKVRgdx/Pxd83d2v1dVcIH7zm77/f/lyrz7whQh1yaJF9HunSqJEsNR1ESpky202YQL15YQJPbO5fFFdzZvDwSQef++v3U7j0ytPwsiR9EuvXu17Ek9LC7cb3/9+1/vffZd9fpYu5cXNy+M2sKOj6wVuaOAH6qSTeiFcNyzDE4yMDL5xycS4cTR4gXYJdXXRy1DoBwQ1CCLyPQDXAxgrIus7PZQL4L/REiwRmT6dK9bHHuMq1mbzLkaGD2cnT39V/Z9+2rN/T3dycph5WF8fmjJMBqyEGWs+si+sBJwzzgjdA7F7N9tdfPKJ9zpkZ9Ot/dWv9j6JxiciwNVXU5GvW8cnys31Dn2w2ZhZM2FC1/+rr6el6ujgbiAtjT+t4grAuyNIT+fFHzOmb7JmZ4e2Qm5vD/6BTDTmzWM73UGDfCt8Y3g9Lrss9rKlCKHsEP4PwBsAfgvgJ53ubzDGJNkSo+9Mm0a38KZNTCgRodIbNy4yq/pUW9gMGsTODB995N9dXVFBXRooTb8z27ax6DctjQbEet9bWxmj2bKFcYOIGoWsLBaYbNvGlf+uXVS+c+b4nwOQl0ffV+eLarMxpcxKK+vooGGx2Rhs6atBGDmSW9iGBv9uI2PoAisr69tzxZopU7gq++QTrrw6f+HcbmZyTZ3KqnGlVwQ1CMaYOgB1AC4RkYEADgcL0yAiMMYsi66IiYfDQb93OPG/CROYERfItdvURB3Sm26eicwll1B/7tpF372lqDs66EEpKAC+853QjGFbG7sv5+X1TFO3Mg/Xr+d7fcYZEX4hNhsvZPedgD9OOYVpnoF8Zs3NPF+wOoUQaGgAVq2yocl+DiYu/xuyJmRhyHBHz9jy3r1Urn0JKMcDh4M7saee8gbb7HZvtsesWcDll2vFch8IJ+30OwBuAmsP1gGYBeADAKdER7TUYvZsYPFiLsz8Zc1VVLAup496IeHIywNuv53xkXff9fZ1E2H+/5lncicRCuvXU/H5qxETodF54w2mksZVN5xwAvDMM1y5+sp6aWvjxR4xgtH3QFWPATCGmZbPPUfdmJV5HMpzKzB12UvYkZGJ8ScWY/BQO/3rVrfA7343ObejmZl0351zDt13dXX8gB11VP/sGRNhwgkq3wRgBoCVxpg5IjIBwG+iI1bqMXw48PWvMyNw5MiumTNuN2NlY8cmxkCsaJCbC1x0EdPXKyu9nZ/DTV3/+OPgRWpZWcz4OnAgzovgwkLgl79kj6OKCv7tcNAiNjfTGBx3HEu5Z8wI3Sp24513mJFZUmLtvgQVg89BxxFTMHzzO3D/Zw0ypriQP2UUM5mmT0/+3ObCwsQZJ5pChGMQWo0xrSICEckwxmwRkfFRkywFOfdcKrOXXmJMz/IkiFAfXH554hemNTQwHvDppzRkhx/OOoFQdZnVGLC3dE/Q8YfN1rXDdNwYP54Nnm65hZFwh4OrXCs/uaaGRSn+0tOC0NzMncGoUd1iJiKoLjwc1ScejkOHgKVFBr+4Q5JyU6DEjnAMQrmIFAB4CcBbIlIDQFtih4HNxmHzc+Yw7byujgu18eOTo/fRihXAv/5FRZuTQ0O2YQPw4otscXPGGdH3QowaxdY8gXC5aKx6ueCOPJMnMzVtwQL247YmBdXV0Zd47rm97l3yySc0koEW/IMGATu/EOzdy6detYq7tJwcNhuMVEKEkvyE09zuXM+vvxKR9wDkA1gcFalSnMxMfhGTibVr2e11+PCutVODBlEhPfMMldKpp0ZXjuOOY7ppoDTWigruuBKqP9SQIcD113NHYE1WGj68zxkEe/f6j0lZWLVaDz/McIbNxmvY3s64VmkpcMMNybEoUaJLyOsCId8SkTuMMUvBwPK0YP+nJD9uN0fpFhf7LqS10j+ff56pn5GkqYktJ7Zt44K6uJh1Bl984buVSG0tFeBZZ0VWjogxcCDdRePHRySdLCMjtJYqW7dyNzF6tHfC38iRNAYVFazYrq/vszhKkhOOy+hBAG4wq+hOAA0AFoCBZiWFCdSPyCIjg8bg008jk95eX8/RmkuXUuFZTQNnzfLOP37zTe8MepeLzz9wIJvphVrTkOyMHx+8Dq2ykklMJ5/s26U3ZAgN7LJlzPhS+i/hGIRjjTFHi8jHAGCMqRGRSJb+KAlKTU1ox9ntzO7pK3V1HE5WUcEUUit11OWi/3vDBk6mO/VUYOVKuk0yMlg0OHlylFJNnU5WI65fz3TRUaPol4pzte9hh9H4BZrBsW4ddwOB2iQNGUIDO39+6qU9K6ETjkHoEBE7PC2wRaQY3DEoKY7DEVqw2O2OTHXwM8/QsHRvA2Kl7FdW0h9+xx0xWtHu3An85S/s/ZOeTkGWL6egZ57JnPg4RWWtrhm/+Q0r54cM8Sr09nYW/tlswYt3rVTd5ubUK4xUQiccg/AnAAsBDBaRuwF8A8DPoyKVklCMG0eDEKwfkTF0YfSFmhqu+gP1NCoupo7etavXtVyhs28ftyuZmT19Zk4nI9wA54fGiZEjaRwXLuQOyuqubbez/U9xcXC3knX9dHfQvwmlud2TxpjLABQBuBXAXAAC4BxjTL+Zqdyfyc9nrcGKFf6b9x04wLnqffXd79zJn8GmTQIMNEfdILz0En/6cg05HDQSr71GB30c81yHDuVO4ZJLuNK35nxnZfH28suBB6RZBcwh1cE0NzPt7N13GcXPy+PrnzEjckNylLgQyj73GBEZDuAqABUAngYb3lWISKJkeitR5uKL6TbfubNrJlF7O1fqubnANdf0vQ4hWMaMRUwKz2pr2fI60PQcK+dz5cooCxMaBQUsFhw3zqvcTziB18VfBpjbTYMwf34I12/vXuDnPwf+8Q+vC62ujsV3P/0pPwxK0hKKQXgIwDsAJgBYA2C152b9rvQDBgxg9s5557Faec8ettuoqmIg97jjqDu3bOnbfBIrMBqoHxzA5/A3mS5iWNN9gsUHsrISWhEWFQFXXEHvV01N1/e2qYlGfs4c4Oijg5yosRH4/e+5CigtpfXJzOQWcvRonvjee2lIlaQklG6nfwLwJxH5mzHme9EQQkQuAPArABMBzDTGqKFJQLKzgbPP5kqyqor1AS++yFTTjRt5jNWj6NpruVINl9JSZhbV1fmfCdHSQlmmTOn1SwmNUB3qbnfCd9j8ylf4fr74oneWhzHU5VdcEeJQpo8+orL3l388aBDbc6xYAXztaxF+BUosCCWGIIb4NQbWMX2Q41MA5wF4uA/nUGJEejoXi48/TiXT3Y9fW8s47O23hzCeshsi7PhqzTvo7pJua2PmzDXXRHjegS+ssuzW1sA5my0tXWf/JihTp9KI7t/PXV5GBt2AIQeSlywJHicpLmbr1VAMwr59dLXt2cP3t6yMQkb9wir+CCXL6D0RWQDgZWPMbutOTw3CiQC+DeA9AP/qrRBWcFq081ZS4HYDjz5KY+CrPURBAY/517+Au+4KP64waRIn0z38MPPrMzK8PnCHg6NMZ85kRs3771O5DRzICZSTJwdv5RAy6elM01mwgFbP1wupr2dQNZzhGHHE6pjRK2pqus6M9kVmJgtIAs0/7ugAnnySF89mo9V3Omkc8vM53cgatafElFC+OqeDAeWnRWQMgFpwQI4dwBIA9xtjPo6eiF0RkWsBXAsAJf5SXpSosn07XUaBKpcHDqT3YOfO3n23jzoKuO8+FlVt3kz3xtixXETW1TGuWVXFzJmMDOqgtWuZgnnzzYHH7obFvHksSNu8mZrU6iLndtNatbcDt97aP1a1Awbw9QZKRerooFHwZwyMYa/upUtpZLsfV1vLPhq/+hV9h0pMCSWG0Aq2rXhQRNLA9NMWY0xYkSMReRuArzDgz4wxL4d6HmPMIwAeAYCysrK+uKmUXrJ/f/Cgr7WY3r+/94u9jAzg2GN5s6ivpzupo6Orq8oqptq3j7rkqqt4n3dGQC/JyKCFWbSIrpDWVioxl4v+l298o2/9vJOJk05i/mqghVhlZeA5BXv3cmfgyxgA3F42NzOV95pr+iyyEh5hba6NMR0A9vfmiYwxUe6DqcSKcFxAkfYCrljBHYIvHVxVxdqE8nLuTAYP5mJ2/nwu9Hsd983IYIvqM87gyZ1ObkH6W3vQE04AXn+dASRfRQ3NzVwpnHyy/3OsWEGfXqAI9pAhdB9dckng4gkl4kTK25o0VFdz1QrQAxDnVjRJiTWFzNdUSAur8jWSE8uMYb8dX5MS9+5lEkxmJnV1YyPdS62tHCCzYwdw/QVVcHz0X7b9dLmYBjV7NoUMxXJlZDDBv79SVAT84AfAAw/QKg8eTCvrdNJn53IB3/9+YFfPnj3Bi9esKHdtrRqEGJMQBkFEzgXwZwDFAF4XkXXGmHmRfI6KCrZnXrvW+903hsWVF1zQ/xZ7faG0lCv0Q4f8v29VVdS3gVpQhIvLRR3RPauprY1Dc3JyqJ9cLubXAzQQY0oNml99F1VvPEVdVVDAD8HSpcDbb7NL3iWXaN+GUJg6FbjzTs7tXLaMxsBu5+7h1FODrwBC6dcNMEYTsewAJVR69Y6LyFBjzIFICWGMWQj2SYoKBw4Ad99NxTFypHe36nLRQGzdyu6ZOqM7NEQ45/zuu+kyLi72Gtn2dhantbQAJ57IwHJJSWRcR3a7dyRxZ11RXt61FKB7WcCIfR/h6L3/wua8URg6M90rS24uPwRvvknf0vnn913IRMUYrs63bOG2afBgZkYFG1Dti+HDgcsu49jP1lYq+VCV94wZtN6BVmCNjdzmBctoUiJOb03wIgDB6hoTAmOYIul09ky3s7pn7t/PnPof/zg+MiYjJSXM9HnySfrtAb6PO3dSzx5xBNPWlyxhLcK11/b9+y3CiugPP+x6Lffv7zpCsrmZzw8AYtyYvOl5tBcMQUNL+pdFbV9it3O7s2gRJ++kYqvPgweBRx7hhbIqr10uRtvPO48Blt5YbLs9/N5F06ZxK2el63bHGG7nr7pK53rGgd6+40lTMFBezoraQMpo6FBW2x6I2J6nfzBqFIvP7r6b33Oru+bXv86up5Zrqbwc+O1vGb/pK3PnchfS3u69z+326rOODv5uJcIMrNmB7OZDaE+nL9pndpTDQQX5ySd9FzDRqK3lm797Ny/G6NG8cKWlXKU/9RTw6quxkyczk3UG9fX8wnXuc9LUxGDP8ccztqPEnN4ahL9HVIoosnNn4OAnwMdEvJ02ldAR4Yp7/Xpg+vSe+f8iNLh1dUxQCURzM/XWnj1dFX5nSkuBb3+bRqaykvpk4EC6AxsbqVPKyrwL14zWOhgRuFxccPodRu9wRGa6T6KxeDEt8dChPb8E6ek0EAsXchcRK444AvjlLxmPKC/nBd+9m0b5yiu5ndR4TlzorcvoxYhKEUWs8Yuh0JembP0Zq9FnoLTOYcMYgzz//J5u6+pqpp2//773GmRkAKedxh1H9+PnzKHL6PXXOT0tLY3G4PDDeeucOeaypwPGoLGRriu/rm6Xq3f+9ESmtRV4773AWT9WCugHH3BrFytGjWJGUl0ddzEOB42WGoK4kvIxhECdiztjjMawestnnwV3JaeleYt7O9c1VVZy2ldDA/WBZVRaWzmKYP16xna6n3/8eN4aG7mzWLiQKe7d3dLVgw5Dc3sasgZ0YOxYPxbLypGNere8GHPoEINnwSrzcnJ4EeNBfr7v/idKXEj5GML48VwxNjb6P6aujoajP6eY9wW7PXjlskXn3ZoxwEMPUfmPGtV1h5GZCYwZw67Szz/v/3w5OTTkV18NnH466xF276Z7urwc2L43C+UTT8VXxpQjK9OPkPv2sQlSr5v8JCjhbI21j5iCfhBDsNuByy/3zovtTmMje3ZdfrkmNfSWyZO9ef/+aGujG6jzLmzXLsYQA+3ihg/3NrALhMPBLMh77wUuugiYNYvupp/+FLjwqXOQNesoBok6n6ilhb2gi4oiM90n0SguZjqtv8k4Fo2NSdOcT4kuvXIZGWMejLQg0WT6dCY2PPooXRTWDrq9nSvMm2+mUlN6x4wZwP/9H/Wrv75n+/dzlkLnoO6WLd6Avj8cDu4kduwIPigeYFB7Xo+SxnRW2P73v0wv3e1p2jtgAHsRnXxyalbEpqVx2/T889xu+cJqITtzZmxlUxKSflMKeMwxdBGvX0/lAjAAOWVK/2hUGU0GDAC+8x3gr39lu/zOLmGnkx6Z0aN7Kuq2ttB3ZX0el5mWxuZsX/kKUx6NYc1BqlfDzp3L6sudO1l00/n11tczqHPddalZf6GETYp/G7qSkcHV7IwZ8ZYk9Zg5k37/p5+mK8ha9YuwYvmii3om8QwZEryLgRXvjdj8epvN/yi2VCQri1H5F15gmpeVdmcML8DNN3MLrSgIbWJaKF9Fd7jtsJXQsDoObN7MFXVxMV0niZgheeSRTC3fuZOppA4HW1/7Kki1jk9Pp+vO3y6tvp5Zk937FylhkJ3NINm553J73NHBTIsxYzRwpnQhlB3CPs8tUMTNDkCn1USYgwc5Neyzz7p2HEhLY8eB009PvDioCI1AKDMQsrPpwn/ySaaidq9jaGmhYfnxjxPvdSYlubmhBWJiSGOjN5anrtv4E4pB2GyMCbinFJGYTUzrL1gdB5qa6H/vrBDb2xnEbWsDzjknfjJGgtNO4+t50VPqOGAAd0WNjVQQN9zAXYeSOhjDWN7rr3OxY7MxG/Ckk/h50Hqg+CEmSAK5iGR6pqb16ZhoUFZWZlavXh3rp40JzzzDJpz+hnE5ncy5v/fe1GjdXV3Niuft26kcpkzp2oJCSQ2MofF/+WUmHwwaxMVORwd72jkcwG23+U+KUiKDiKwxxpR1vz/UEZr+TnqlMeaxeBiDVCbUjgMizKQ866zYyRYtBg3iQDIltfn4Y1agl5Z27VKRlsbW9LW1wP33c6xyZmbcxOy39DWi9OuISKF0IdSOA7m58es4oCjhYgx7Vg0c6L9lUUEBOwesWxdb2RQSSpbRen8PAQixU5ASDqEmfhijSSJK8lBbywy0kiDpJ7m53PnOmhUbuZKKxkbOtWhvp8/t8MMjWksTypmGAJgHoKbb/QLgvxGTRPmSoiJm4ASq/AXYhUE7DijJQns7FzDBMsbS0oK3Qul3tLay4nzp0q61JHl5TNU78cSIpOKFYhBeA5BjjOmxiROR//RZAqUHaWms6tWOA0oqYRVDu1yBu1w3NQETJ8ZGpqSgvR144AH2eulebd7cDPz971wdRiAIF9ThYIy52hiz3M9j3+yzBIpPTj2Vu8Fdu3q2baivZzuIK6/UjgNK8pCdTTdQRYX/Y4xhOrUOTOvEihXAxo30tXV3D2Vns1Xw888HfmNDRD3QCUpmJvDDH3IYzIED7Me2ezcNRHo6Ow4cf3y8pVSU8Jg/n922fXWvtaryJ07kYkgB36zXX2dxhj+XUFoafXErVvT56UIJKq81xgQchhPKMUr4WB0HzjsP+Pxz7hQKCrTjgJK8jBzJyvMHHmA2XUEBF71NTfR+TJzIQWr6+fZQX88inWCR+IICjg8877w+PV0oMYSJATKNAAaXdeRRFMnJSbiOA0osaWqiYkhLY3/vJO/jMWECCypXrWI2UWsrW53MmcNxyzpFsxOhTp4CIjIDOBSDMCGEY4L0rFQUJWz27wfeeINa0xh+4YcNA848k874JF5G5+TQAMyZE29JEpy8PAYKm5sDd7Ssq2Npfx8JpVJ5V+e/ReROz/+tA7DOGLOtz1IoitKVHTtYrut2c9i0NSmovh7429+YcXLFFUltFJQQsNvZxfKZZ/ynHLpc9CdHIBIf9qfJGHMHgAcA1AE4V0SSZpymoiQFbW3s35CV1TXNUITFSGPGsLfJBx/EV04lNpx0EmMI5eU93UIdHRwDO39+RGaCh2wQROQBETovjTEVxpg3jTH3GGOu6asQInKviGwRkfUislBE+tEEE0XpxiefcCfgb5CPzcbBGK++Gp6PWUlOsrOBW29lFeru3TQA1s+qKuDCC4ELLohZYZpFA4BXRORiY0yTiMwDcIcx5oQ+SwG8BeB2Y4xTRO4BcDuA2yJwXkVJPj78MPiM59xcKoWqKu0X3R/IzeVg+IoKTstqbWVHyClTIjotK2SDYIz5uYh8E8B/RKQdQCOAn0RCCGPMkk5/rgTwjUicV1GSEqsMPRDWxKT29tjIpCQGQ4bwFiXCcRnNBXANgCYARQBuNMa8HwWZrgLwRgA5rhWR1SKyuqqqKgpPryhxZsQINjELhDWMOl8zvpXIEU5Q+WcAfmGMORlcwT8rIqeE+s8i8raIfOrjdnanY34GWx6JlAAADjtJREFUwAngKX/nMcY8YowpM8aUFRcXhyG+oiQJJ5zArJFA8YGKCuDYY7V3iRJRwnEZndLp9w0iMh/AAgAhNVAwxpwa6HERuQLAmQDmmmBj3BQllSkpYdfCDz/kJJnuwcL6ehoLnSikRJheN9I2xuz3uJH6jIicDuBWACcZY5ojcU5FSVpEgKuvptJftYoVyjk53DU0NHCu6C23sA+EokSQPk1WMMa0REiOvwDIAPCWJ7N1pTHmugidW1GSj4wM4Prr2c3w/fc5QDsjgzuHo48OPChDUXpJ5Ebt9AFjzGHxlkFREg4RuoxKS+MtidJP0Lp3RVEUBYAaBEVRFMVDQriMFCVROXCALvyNGxnjnTCBPcSGD0/6LtSK0gM1CIriA2M4qGrBAir+gQN5/zvvAG++yYzPb3xDm40qqYUaBEXxwbJlwLPPsiQgLc17f24ui4RffZWZoFoKoKQSur5RlG50dAAvvEC3UGdjYGG3c675yy8DLZFKvFaUBEANgqJ0Y9s2thIKlOqfns6xBRs3xk4uRYk26jJSlG7U1YV2nAhQUxNdWRKGtjZg3ToGUaqr6S876SRgxozgrbqVpEENghJzqqqArVupYwoKgMmTgczMeEvlJT09tOOM6ScFw5WVwO9/z5/5+bxYdXXA448z6v6jH/kf76gkFWoQlJjR0AA88QSwejUnAYrwlp4OnHsuMG9eYqRyHnEEs4ecTv9jCSz5J0yIrWwxp6WFxqChoWvFdFYWjUNtLXDvvcBddwGFhXETU4kMGkNQYkJzM/XK2rUMyI4ZQ/0yejQHPz31FLBwYbylJHl5rDUoL/ffgXrvXqCsDCgqiq1sMWftWu4M/A1lKSjgQJ9ly2IrlxIV1CAoMWHpUo6AHTWqZ+5+RgYNwyuvsBAsEbjoIq7+d+7sOqumuRnYsYOv49vfjp98MeOdd/zPdrYYMgR46y2d75wCqEFQoo7TCSxeDAwd6v8Yh4PpnO9HYwZfL8jMBG6+GbjiCv69ezdv7e3At74F3HZbP4mlVlUFn9mbkUHXko7zTHo0hqBEnbo6uqCtal9/5OcDW7bERqZQyMgATjkFOPlkzqQBWJhmt8dVrNiSk0NFHyjS7nTyTfFVtKEkFbpDUKJOIgSK+4LNRq9JQUE/MwYAgymHDgU+prISOP547eORAugVVKJOfj5vTU2Bj6utZQqqkkAcdxy3Sg0Nvh9vbWVp96kBJ+QqSYIaBCXq2O3A/PmcC+8v7tjRwcdOPDG2silBKCgAbrqJPrPycl4ogA2d9u1jFsB3vsOmT0rSozEEJSbMns3xwNu3cxRwZ9dLSwuwfz8zewYPjp+Mih8mTgTuvBN4912mizmd9AMeeyxw2mlalJZCiEniVLGysjKzevXqeIuhhEhzM/DMM8CKFSzsssjJYSvp2bOTP96Q8jiddBNlZGgQOYkRkTXGmLLu9+sOQYkZ2dnAVVcB55/PBnLt7YwtjB+vuiVpcDj6Sb5t/0QNghJz8vPZE01RlMRCg8qKoigKADUIiqIoigc1CIqiKAoANQiKoiiKBzUIiqIoCoAEMQgicpeIrBeRdSKyRESGx1smRVGU/kZCGAQA9xpjjjTGTAPwGoA74i2QoihKfyMhDIIxpr7TnwMAJG/5tKIoSpKSMIVpInI3gMsB1AGYE+C4awFcCwAl2lBLURQlYsSsl5GIvA3A18ysnxljXu503O0AMo0xvwx2Tu1lpCiKEj5x72VkjAm1YfpTABYBCGoQFEVRlMiREDEEETm8059nA0igQYqKoij9g0SJIfxORMYDcAPYBeC6OMujKIrS70gIg2CMOT/eMiiKovR3EsJlpCiKosQfNQiKoigKADUIiqIoigc1CIqiKAoANQiKoiiKBzUIiqIoCgA1CIqiKIoHNQiKoigKADUIiqIoioeEqFRWoo/TCezYATQ1AVlZwLhxQFpavKVSFCWRUIOQ4hgDLF0KvPQSUFcHiPC+7GzgzDOBr34VsNvjLaWiKImAGoQUxhjgxRdpDIYNA0aP9j7W2go8/TRw4ADw7W8DNnUeKkq/R9VACrNzJ/DqqzQE2dldH8vMBEpLgffeAzZujIt4iqIkGGoQUpj33gPS0wGHn32gzQbk5gJvvhlbuRRFSUzUIKQw69YBhYWBjykq4g7B7Y6NTIqiJC5qEFIYtzu02IAxvCmK0r9Rg5DCjBsH1NYGPqa+Hhg1SjONFEVRg5DSnHYa0NDgf/VvDHDoEDB/fmzlUhQlMVGDkMJMmgSUlQG7dvWMERgD7NkDjB/PYxRFUbQOIYWx24Hvfhd46ing/fd5n8PBqmVjaAiuvJKZSIqiKGoQUpyMDOCqq4Czz2bWUU0NkJcHHHUUMGRIvKVTFCWRUIPQTygsBObOjbcUiqIkMhpDUBRFUQCoQVAURVE8qEFQFEVRAABikrhEVUSqAOyK4lMUATgYxfNHk2SVXeWOPckqe7LKDcRf9tHGmOLudya1QYg2IrLaGJOUWfrJKrvKHXuSVfZklRtIXNnVZaQoiqIAUIOgKIqieFCDEJhH4i1AH0hW2VXu2JOssier3ECCyq4xBEVRFAWA7hAURVEUD2oQFEVRFABqEIIiIneJyHoRWSciS0RkeLxlCgURuVdEtnhkXygiBfGWKVRE5AIR2SgibhFJuNS87ojI6SKyVUS2i8hP4i1PqIjIP0WkUkQ+jbcs4SAio0TkPRHZ5Pmc3BRvmUJBRDJF5CMR+cQj96/jLVN3NIYQBBHJM8bUe36/EcAkY8x1cRYrKCLyVQDvGmOcInIPABhjbouzWCEhIhMBuAE8DODHxpjVcRbJLyJiB7ANwGkAygGsAnCJMWZTXAULARGZDaARwBPGmCnxlidURGQYgGHGmLUikgtgDYBzEv09FxEBMMAY0ygiaQCWA7jJGLMyzqJ9ie4QgmAZAw8DACSFBTXGLDHGOD1/rgQwMp7yhIMxZrMxZmu85QiRmQC2G2N2GGPaATwD4Ow4yxQSxphlAKrjLUe4GGP2G2PWen5vALAZwIj4ShUcQxo9f6Z5bgmlT9QghICI3C0iewBcCuCOeMvTC64C8Ea8hUhRRgDY0+nvciSBckoVRKQUwHQAH8ZXktAQEbuIrANQCeAtY0xCya0GAYCIvC0in/q4nQ0AxpifGWNGAXgKwA3xldZLMLk9x/wMgBOUPWEIRXZFCYSI5ABYAOB/uu3kExZjjMsYMw3csc8UkYRy1emAHADGmFNDPPQpAIsA/DKK4oRMMLlF5AoAZwKYaxIsWBTGe57o7AUwqtPfIz33KVHE44NfAOApY8yL8ZYnXIwxtSLyHoDTASRMUF93CEEQkcM7/Xk2gC3xkiUcROR0ALcCOMsY0xxveVKYVQAOF5ExIpIO4GIAr8RZppTGE5x9FMBmY8wf4y1PqIhIsZXtJyJZYCJCQukTzTIKgogsADAezHrZBeA6Y0zCrwBFZDuADACHPHetTIbsKAAQkXMB/BlAMYBaAOuMMfPiK5V/ROQMAPcDsAP4pzHm7jiLFBIi8jSAk8FWzBUAfmmMeTSuQoWAiJwI4H0AG8DvJQD81BizKH5SBUdEjgTwOPg5sQF4zhhzZ3yl6ooaBEVRFAWAuowURVEUD2oQFEVRFABqEBRFURQPahAURVEUAGoQFEVRFA9qEBRFURQAahCUFEFESkWkxdMnxrqvR1tqEcnytDJvF5GiPj5nlogs9XQ8hYjcKCKbRSSsNiEiUiAi1/dFlhCeo0eraxFJF5FlIqIdCxQAahCU1OJzT58Yqy31XwHMBzAJwCUiMskY0+I5Zl8Enu8qAC8aY1yev68HcJox5tIwz1Pg+d+wEBLqd/hfYJuEL/F0Z30HwEXhPreSmqhBUJIGz1CU0zy//z8R+XOAw2PRlvpSAC975HkIwFgAb4jIzSLyLc8wlHUi8nCnXcRLIrLGMyDlWs95fgdgnOfYez27nc4r+R+LyK88v5d6dj1PgD1wRvl7rs4EaHX9kud1KIoaBCWp+CWAn4nIpWDL4/8JcGxU21J7+haNNcZ8AQCetiD7AMwBsBhcdZ/g2Y244FW6VxljjgFQBuBGESkE8BN4djfGmFtCePrDATxojJkMIDvAc4XCpwBmhHG8ksKo71BJGowxyzyNzX4I4GTLVSMid4HNzmJJEdhnyRdzARwDYBXFRRbY/x6gETjX8/soULkfCPO5d3WashXouYJijHF54im5nmEzSj9GDYKSNIjIVADDAByylJeIDAUnT3Un7LbUIvJ9ANd4/jwDwLmd/zbGdI47tADI9HcqAI8bY27vdv6TAZwK4DhjTLOI/MfPOZzounvvfkxTsOcKkwwArX34fyVFUJeRkhQI5+g+BcYBGj3tvQFgGoB1Pv4l7LbUxpi/etw204wx+7r/3e3YGgB2EfGl0N8B8A0RGeyRfZCIjAaQD6DGYwwmAJjlOb4BQG6n/68AMFhECkUkA5xp4Q9/zxUSHpfVQWNMR6j/o6QuahCUhEdEsgG8COBHxpjNAO6Cd0iRT4PgmSd9A4A3wZm7zxljNkZYtCUATvTx3JsA/BzAEhFZD+AtcGezGIBDRDaDgeSVnuMPAVjhmRh3r0c53wngI8//+u2ZH+C5uuBpdf0BgPEiUi4iV3semgPg9d68eCX10PbXSlIjIo+Cbp0SAK8ZY0IaSSgiXwAoM8Yc7MNzHw3gZmPMZb09R7wRkRcB/MQYsy3esijxR3cISlJjjLnaGOMGs2vyOxem+cIqTAPjDu5Ax4bw3GsBvOcrzTMZ8LjSXlJjoFjoDkFRFEUBoDsERVEUxYMaBEVRFAWAGgRFURTFgxoERVEUBYAaBEVRFMWDGgRFURQFgBoERVEUxYMaBEVRFAUA8P8BBYGhJSJXix0AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SjPlpf2-wHl8" + }, + "source": [ + "## Обучение простейшего одноуровневого персептрона вручную\r\n", + "\r\n", + "Используем возможности tensorflow по вычислению градиента для обучения одноуровневого персептрона.\r\n", + "\r\n", + "Для начала, задаём архитектуру сети, в которой будет 2 входа и один выход. Соответственно, матрица весов $W$ будет иметь размерность $2\\times1$, а вектор сдвига $b$ -- $1$.\r\n", + "\r\n", + "Для удобства сгруппируем все параметры в отдельный класс:\r\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "J1KaixW-cMWJ" + }, + "source": [ + "class Network():\r\n", + " def __init__(self):\r\n", + " self.W = torch.randn(size=(2,1),requires_grad=True)\r\n", + " self.b = torch.zeros(size=(1,),requires_grad=True)\r\n", + "\r\n", + " def forward(self,x):\r\n", + " return torch.matmul(x,self.W)+self.b\r\n", + "\r\n", + " def zero_grad(self):\r\n", + " self.W.data.zero_()\r\n", + " self.b.data.zero_()\r\n", + "\r\n", + " def update(self,lr=0.1):\r\n", + " self.W.data.sub_(lr*self.W.grad)\r\n", + " self.b.data.sub_(lr*self.b)\r\n", + "\r\n", + "net = Network()" + ], + "execution_count": 199, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rQ7W6TOacIAI" + }, + "source": [ + "> Обратите внимание, что мы используем `W.data.zero_()` вместо `W.zero_()`. Это делается потому, что к тензору, состояние которого отслеживается через механизм *Autograd*, нельзя обращаться напрямую.\r\n", + "\r\n", + "Функция обучение будет такая же, как в прошлом примере, но функция ошибки будет представлять собой логистическую функцию ошибки. Для этого нам нужно получить на выходе сети значение **вероятности** класса 1, т.е. необходимо привести выход сети $z$ к диапазону [0,1] с помощью передаточной функции `sigmoid`: $p=\\sigma(z)$.\r\n", + "Далее, если для примера с номером класса $y_i\\in\\{0,1\\}$ был получен выход сети $p_i$, то ошибка вычисляется как $\\mathcal{L_i}=-(y_i\\log p_i + (1-y_i)log(1-p_i))$. \r\n", + "\r\n", + "В PyTorch оба эти этапа (применение сигмоиды и взятие логистической функции ошибки) делается одним вызовом `binary_cross_entropy_with_logits`. Поскольку мы делаем обучение по минибатчам, то необходимо усреднить ошибку по всем компонентам минибатча - это функция `binary_cross_entropy_with_logits` делает сама, возвращая одно число. \r\n", + "\r\n", + "> Следует отметить, что вызов `binary_crossentropy_with_logits` эквивалентен последовательному применению `sigmoid` и `binary_crossentropy`" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "kdDxWeCqwHl8" + }, + "source": [ + "def train_on_batch(net, x, y):\n", + " z = net.forward(x).flatten()\n", + " loss = torch.nn.functional.binary_cross_entropy_with_logits(input=z,target=y)\n", + " net.zero_grad()\n", + " loss.backward()\n", + " net.update()\n", + " return loss" + ], + "execution_count": 200, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zAAgw0h6KzUd" + }, + "source": [ + "Для чтения данных воспользуемся встроенными функциями PyTorch по организации датасетов. Концепция датасетов основана на двух понятиях:\r\n", + "* **Dataset** - собественно источник данных, может быть **Iterable** и **Map-style**\r\n", + "* **Dataloader** отвечает за загрузку данных и разбиение на батчи.\r\n", + "\r\n", + "В нашем случае мы определяем датасет на основе тензора, и далее разбиваем входные данные на минибатчи по 16 элементов. Каждый минибатч включает в себя два тензора, входные данные (размером 16x2) и выходные (вектор длины 16 целого типа - номер класса)." + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PfyqjVb2wHl8", + "outputId": "b3a685a9-304c-4e7e-adf9-2858cc47c3a5" + }, + "source": [ + "# Create a tf.data.Dataset object for easy batched iteration\n", + "dataset = torch.utils.data.TensorDataset(torch.tensor(train_x),torch.tensor(train_labels,dtype=torch.float32))\n", + "dataloader = torch.utils.data.DataLoader(dataset,batch_size=16)\n", + "\n", + "list(dataloader)[0]" + ], + "execution_count": 201, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[tensor([[ 1.3383, -0.9861],\n", + " [ 0.5128, 0.4330],\n", + " [-0.4474, -0.2681],\n", + " [-0.9866, -0.2869],\n", + " [-1.0694, 0.4172],\n", + " [-0.8694, -0.8607],\n", + " [-0.9894, 0.3734],\n", + " [-0.9951, 1.2320],\n", + " [-1.3667, -1.0529],\n", + " [ 0.5239, -1.9724],\n", + " [-1.2474, 0.7092],\n", + " [ 1.8675, -0.2057],\n", + " [-1.1899, 0.4243],\n", + " [ 0.7644, -0.6796],\n", + " [-2.0532, 0.4223],\n", + " [-0.5973, -1.2011]]),\n", + " tensor([1., 1., 0., 0., 0., 0., 1., 0., 0., 1., 0., 1., 0., 0., 0., 0.])]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 201 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xrwgkbQjhkEp" + }, + "source": [ + "Теперь мы можем пройтись по всему датасету и организовать процесс обучения:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "QGchp9D6gVJa", + "outputId": "b4c4751d-cb56-4104-d5b5-f1ae9d3d858d" + }, + "source": [ + "for epoch in range(15):\r\n", + " for (x, y) in dataloader:\r\n", + " loss = train_on_batch(net,x,y)\r\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 202, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 0.6447\n", + "Epoch 1: last batch loss = 0.6046\n", + "Epoch 2: last batch loss = 0.5817\n", + "Epoch 3: last batch loss = 0.5680\n", + "Epoch 4: last batch loss = 0.5596\n", + "Epoch 5: last batch loss = 0.5542\n", + "Epoch 6: last batch loss = 0.5507\n", + "Epoch 7: last batch loss = 0.5485\n", + "Epoch 8: last batch loss = 0.5470\n", + "Epoch 9: last batch loss = 0.5459\n", + "Epoch 10: last batch loss = 0.5453\n", + "Epoch 11: last batch loss = 0.5448\n", + "Epoch 12: last batch loss = 0.5445\n", + "Epoch 13: last batch loss = 0.5443\n", + "Epoch 14: last batch loss = 0.5442\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5QaDiCQUkFOT", + "outputId": "45b4a66b-1222-40f4-c758-d58f1c7daf8c" + }, + "source": [ + "print(net.W,net.b)" + ], + "execution_count": 203, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[1.0563],\n", + " [0.4450]], requires_grad=True) tensor([0.], requires_grad=True)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s4_Atvn5K4K9" + }, + "source": [ + "Для демонстрации того, как сработало обучение, построим граничную прямую $W\\times x + b = 0.5$" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 283 + }, + "id": "PgRTHttLwHl9", + "outputId": "d9abf92f-cb70-4c56-ccd0-5e027239da58" + }, + "source": [ + "plot_dataset(train_x,train_labels,net.W.detach().numpy(),net.b.detach().numpy())" + ], + "execution_count": 204, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd3iUVdrG7zOT3nsIISGh92YA6QgIglhZP+xlLWtvKEp1V0AF17oWLKyu3V0bKCKCUqVIly4ISYBUEtLLZDLn++NmnJSpybQk53ddcxFm3nnfZ9p5ztOFlBIKhUKhUGg8LYBCoVAovAOlEBQKhUIBQCkEhUKhUJxHKQSFQqFQAFAKQaFQKBTnUQpBoVAoFAAAH08L0BxiYmJkSkqKp8VQtCCOFx5Hma4MfeP7Qiu0nhZHofAIu3btOiuljG14f4tWCCkpKdi5c6enxVC0IHZm7cTgdwbj/8b9H2aNmuVpcRQKjyCEyDB3f4tWCAqFo6S1T8OUrlPwwtYX8MDQBxDiF+JpkTzL2bPA1q3AiROAVgv07QukpQHBwZ6WrGlICWRkABs2AEePAhoN0L8/MGoU0K6dp6XzekRLrlROS0uTykJQOMq209swbNkwLJmwBI+PeNzT4ngGgwFYvhxYsQIQAggJ4X3l5YCfH/C3vwGDBnlaSseorQU++ghYtw7w9QUiIqggzp3ja7vmGmDyZL7eNo4QYpeUMq3R/UohKFxFdTWwdy9w+DB/l506ec/mc+KHE7Evdx9OPnQSQb5BnhbH/Xz7LfDf/wIdOwI+DRwFFRVAXh7wxBNAz56eka8pfPkllVxKCi2DutTUAJmZVHQjR3pEPG/CkkJQWUYKl7BvH/DII8CbbwLbtgG//gq89x7w8MO05j29D5k/Zj7yyvPw9q63PSuIJygt5cKZnNxYGQBAUBAQHg58/rnnPyh7KS0Fvv+er6mhMgBoMbRrR6VRW+t++VoIXqMQhBBJQoh1QohDQoiDQoiHPC2TomkcOgS89BItgZQU/g7j4/l3TAywbBnwyy+elXFk8khclHIRFv+yGJU1lZ4Vxt3s2cNF0dfX8jEREUB6OnD6tNvEaha//cbXZE7BGQkOBoqKgOPH3SdXC8NrFAIAPYAZUspeAC4EcJ8QopeHZVI4iJTAxx9zPTHnGvL3BxISgE8+AXQ698tXl/lj5iOnLAfL9izzrCDuJjvbujIA6GfXaOh/bwkUFjIobgshgJIS18vTQvEahSClzJZS7j7/dymAwwASPSuVwlHS04GsLHocLBEYCFRWAgcOuE0ss4zpOAajkkfhuc3PoVpf7Vlh3ElAgP1uE2s7bmeTlQX87380L19/Hdi+nYEoewgKsu81ScldicIsXqMQ6iKESAEwEMB2M4/dJYTYKYTYmZ+f727RFDY4e5abMFuJHEIAOTnukcmyDALzRs/DmdIzeG/ve54Vxp307s3F01p8oKaGysAdhZ81NcC//w3Mng2sXg2cPEm/49KlwGOPAceO2T5Hr/POBFuvydcX6NLFOXK3QrxOIQghQgB8CeBhKWUj205K+baUMk1KmRYb26jQTuFhtFr74pB6PfDHH8AzzwAPPcSElhUrgIIC18tYlwmdJuDCDhfi2c3PQlfrYR+Wu+jcmcHXs2fNPy4lcOYMMG4cd96u5qOPgPXrKVOHDkBUFBAba8qAWrLEdiwjIQEYOJBym0NKnmPCBPe8phaKVykEIYQvqAw+llJ+5Wl5FI5j3FBas94rKpiFtH49rYSgIFNa/JNPArt3u0NSIoTA/NHzkVmciQ/3fei+C3sSIYB776X2PnWK2tlIZSV36N26AVde6XpZsrKAjRvNp4oC9D1qNNwt2OK226gYTp5kPQVARVBSwsK7AQPc85paMF5ThyCEEAD+A6BQSvmwPc9RdQjeyZtvAjt3AklJjR8zGIBVq6gwLrussWupooIb1/nz3eOtAAApJYa8OwQFFQU4ev9R+GptBFxbC2fPMlVz40YunFIyE2DSJO6kAwJcL8MXX/ALYe7LYqS2ljv/l16iwvr1V+4kgoNZPNe9u0mZVFQAmzfznMXFfE0JCcCUKcDQobaD6W0Ery9ME0KMBLAJwH4AhvN3z5ZSfm/pOUoheCfFxcCzz7K2KSHB9BusrWWR2sGD/H2Ghpp/fk4Ouw3cc4/7ZF5xdAWu+OwKvH/F+7hlwC3uu7A3UF7OLB2NBoiLc++i+fLL3L1HRVk/LiODRXKHDtGy8fenZVNdTTfTAw/Ub01hMABlZaYqbFWdXA+vVwhNQSkE76WkBPjmG2DTJv42heC/lZX8LVvb/dfW0pPw6qvuq2qWUmLgWwNRUVOBw/cdhlajOqG6haVLWUMQF2f5GCmBH38EEhMZPG7oWsrLY7uNp54CIiNdK28rQVUqK9xKWBhw883cAD75JDBzJvDii9zkRUdbf64xndzoBnYHQgjMHzMfxwqP4fODn7vvwm2dCy6gm8caWVncYfTsaT7OEBfHx3/+2TUytiGUQlC4lOBgoGtXxigjIugmspVaLiWtCXe4sOtyZY8r0SeuDxZuXIhag2pv4Bb69+fuobjY/ONS0seYmmq98Cw+Hli7lqmliiajFILCrYwcSdeuNQoK6BkIC3OPTEY0QoN5o+fh8NnD+PLwl+69eFvFz495x+XltATqpqeVlTG+EBHBHYU1/P2501BVyM1CKQSFW+nXjy4jSynwNTXsU3bppe6Vy8i0ntPQI6YHFmxcAIM02H6Covl07kz/f1oalcKpU+xMCjCV9MILbRe3GLOk3FlZ3QpR757Crfj7A48+CixezN98XJypk0JeHjd5N9xgKjx1N1qNFnNHzcWNX9+I5UeW46qeV3lGkLZGYiJw113AddexAZ2PD78cWi13CZ99Zr0fSkkJs43cbVa2MpSFoHA7iYnA008DV19Nr0BGBtPMBwwA5s4FJk70rHzT+0xH16iueHrj02jJWXgtktBQ1iQkJJhiBkOH8u9KC11ppaTJeemlKr20mSgLQeERIiKAqVP5G66uZuq7Pc0q3YGPxgdzRs3BrctvxXe/f4fLul/maZE8h7F4ZO1atn7w8+MCPWIEe5m7g4gIDrZ57TVaAFFRpoW/ooLdW0eNAoYMcY88rRhVh6BQmKGmtgY9Xu+BqMAo/HrHrxBtcedZXs5F+NAhFneFhFBBGIvY7riD/n13ceQIB9wcP87dg8FABXHppey75C07ihaApToEZSEoFGbw1fpi1shZuPPbO/HD8R8wuetkT4vkXqQE3n6bg+pTUuq7YkJC6L55800WgnXv7h6ZevRgR9S8PKap+vkxbqACyU5DxRAUCgvc3P9mJIcnt81YQkYGOxAmJZn3ywcG0t+/fLl75RKCNQfdulFROaoMpGTzuw0bgHXr6A4zqGwyI0q1KtyOlKZ2++YKT70FP60fZo2chXtW3oOfTv6ECZ0meFok97F1Kz8ga66y6GguqAUFtsvPvYFTpzi/NT2dX0Lja4uJYXpr794eFc8b8OKfo6K1ce4cuxg/8ABjhHfdxbkoGRmelswytw24DYmhifjHhn+0LSshL49WgDWMYzZLS90jU3M4cwZYtAjIz+echdRUWhgpKbQQ/vlPz4/w8wKUQlC4hcxMYN484Ouv6YLu2JHNKbdtY03Spk2eltA8/j7+eGLEE9icuRkbMjZ4Whz3ERpquw2EscdISxhJ+fHHVGCxsY2tHmPm0rvv1p8N0QZRCkHhciorgRde4GayY0dTjyIfH6B9e6acL1vGCWreyB2D7kC7kHZYsHGBp0VxH0OGAFVV1o8pKeGHV7fttDeSnc1MKWsdVUNDWRB3+LD75PJClEJQuJxdu7h2WGp57+9PJbF6tXvlspdA30DMHD4TP5/8GZu3fEbXQkaGfbNCWyo9e1Jb5+WZf7y2lsVg5qYceRvZ2dyN2JJTq/Vu/6UbUApB4XI2brTdUSA2llPWbG1KPYLBgL8VdUacIRALVszg5K6//50pkK21DkarZdO5gAAuksYqYWNVcEYG8//dWYfgalqzgrcTpRAULqekxLab2VhTZKs1ttsxGID330fQJ//DY/oh+DEwC9uSBAfCV1dzis+PP3paStcQH88Az7RprAjOyOCtY0fg8ceB6dO93zoA2CvFGO+whsHAYHMbRqWdKlxOdDSDykFBlo/R62nV20pscTv79gHr1wOpqbhHk4TF8lcswEasFNfT7AkIAD79FOjbl/701kZ4OHuMTJlCK8HHp2UEkesSHw/06QMcO2Y53lFczC9qjx7ulc3LUBaCwuWMHWs7MzE3l+1xvK7odNUqLooaDULgh0cxDN+LY9iFLD7u50dNtqGVZyBpNJx21NKUgZEbbuCXKze3vmtISuZDFxczD7qNt79QCkHhcvr358YsN9f846dPMwnkxx+Bv/4VeOIJbsptTVZ0OdXVwO+/14uG348hiJABWICNpuOio4E9ezwgoMJu2rUD5syhuyszk8VpRhdYSAgwa5b7WnB4Md62H1O0Qvz8gBkzWPuTns7mlcHBXG/37eNvsl8/oEsXbkTLyoD332fW0eOPW85OcjkGA33kdfzkYfDHI7gQT4n12CtzMADtKLQa3ej9JCRwwPeZM1QKUtKd1KlTy4iFuAFlISjcQmwsE3PuvJP90IqLmdGo19NF3acPrXUhmBKeksKOCP/6lwdbzQQEUNjy8np3P4ihCJP+WGi0EoqLKbCiZZCYCAwbBgwfzmltShn8iVIICrcRGMg4wVNPcaFPTTXNWDdH+/bsQ3bsmHvl/BMhgMmTG+XiRyAAD2IIvhSHcUDm0rc1oQ31OVK0WpRCUHiE3Fxa7damIgJ0N23f3owLSQnodE3PMR8xgi2Wz5ypd46HcSFCpB8WVfzAWcAO+p9rahh2+Ppr4JtvgP3723zXBIUXoGIICo9QWWlyEVnD358dBRwmP5+ZP2vXstrN15fpThddRNPDXoKCGMhYupQDWjQawM8P0Tod7g/vjsURB/DUtDHo4UDb1t272dSvrIyJL1Ky8DciArj77jaf+ajwIEohKDxCcDAXwbpdiM1RVdWEzsq//87mSXo9+9f4+3NLvm4db/fdBwwcaP/5IiKY+pSZyW19aSkQHY1Hez+GVz8ejEXbn8eHV31o16n27gVeeYViNZxAWVICLFnChJeuXR14vQqFk1AuI4VHiI1lcse5c5aPMc5NcKg7QlERW0sEB3O4izFv3teXrp+oKOD119nfxhGEYMrilVcCN90ETJmC2I69cE/aPfhk/yc4VmA70KHXM3sqNpbiNSQsjAH1//xHdVFQeAalEBQeQQjgiiuoEHQ688ecOUP3SadODpx4yxbms1qKVBvLpdevd0Rcizw2/DH4af3w7OZnbR575AgTkkJCLB8TEcG6jPR0p4inUDiEUggKj9GvH3Drrdysnz7NdVyvp5I4eZIb+nvvdTArcP162z6m+Hge54RteLuQdvjbBX/DB/s+wMlzJ60em5Nj+5LGmTOWivgUCleiFILCo4wfDyxYwHhvWRljwdHRdPPPnm27S2ojSkuZmmQNX19qn9rapopdj5kjZsJH42PTStBo7NNBUnr3aFFF68WrgspCiH8DmAogT0rZx9PyKNxDhw7AjTfy1hDjAmq3lRAVZcoqskR1NV1HTupb0z60Pe4YdAfe3vU25oyag44RHc0el5LC12EtkC4lb8nJThFNoXAIb9uHvA/gEk8LofAsNTUcM7BgAXD77axufuUV9juyucMePx4oLLR+TF4ecPHFTq1QfWLEEwCAxb8stnhMairj0mfPWj5PTg4bp3r7EDJF68SrFIKUciMAG79mRWumshJ4+WWOGTh7ltZDQgJw9CiweDHw4Yc2WlkMGcJqt4IC848XF9N6GD3aqXInhSfhtgG3YdmeZThTcsbsMUIAd9xB+XNy6r8Og4FB9IAAJjEpFJ7AqxSCQvHhh7QEUlPZRkijoWcnLo6767VrgTVrrJwgJAR47DE+KT2dyf01NQxQZGTQXfTYY00obrDNrFGzYJAGLPllicVjkpKAuXPZyC8z03Q7dYqWwdy51kf/KhSuREgvS3gWQqQA+M5SDEEIcReAuwAgOTn5gow2PgO1NZGfD8ycyUXTUlC1spKtg1580XqYAGVlwI4dwE8/MW0pLIxVykOH2u6X0QxuX347PjnwCU48eAIJodYH5uTm0lIQgv3WXKCjFAqzCCF2SSnTGt7vVUFle5BSvg3gbQBIS0vzLm2maBb79vFfaxk2gYEMARw/zjnwFgkJoQK46CKnymiLWaNm4f197+OfW/6JFya9YPXY+HjeFApvQbmMFF5DSYl9E9OE8ILhORboEtUFN/S9AUt3LUVeeZ7tJygUXoRXKQQhxKcAtgLoLoQ4LYS43dMyKdxHeLj9c2bMtX7wFuaMmoPKmkq8uPVFT4uiUDiEVykEKeV1UsoEKaWvlLKDlHKZp2VSuI/+/bn7t5ZFVFlJZdC5s/vkcpTuMd1xbZ9r8dqvr+FshZUcU4XCy/AqhaBoXZw7x5YUtsoCjMTEcPyAcbqhkaCKs+h29FtcsO11dNvwNm7svgO+hmrXCO0k5oyag4qaCry87WVPi6JQ2E2LCyorvJ8jR4AVK4DDh5n9WVvLds5XXAH0TigEsrJ4YPv2jQYm33gjYwl79wIRobUYnv0luv6xClXVAhUyCKOSDOiyYwtwJAR44AGvHYzeO643pvWahle3v4oZw2YgMjDS0yIpFDbxurRTR0hLS5M7d+70tBiKOmzYACxbxizP6GhTqwbdmXx0/+1/uDhyJxI7nK8QNhiACy4Apk9nT+jz6PXAgQPAiSX/Q8KOFSgMTUFcghadOlF/CAFqjZISYN48Fih4Ifty9mHAWwPw9zF/x1Njn/K0OArFn7SatFOF93LmDPv9JyaaxhAAQEh5LkbtXwSNrhI7czoguKcGERGgQti3jwNt5sz5MwfTxwcYkFyIAQGrgGtTzPccCgtjQOGrr4BHHnHHy3OY/u3648oeV+Ll7S/j4QsfRniA6+ofFApnoGIICqexfj3X7rrKAFJi4N73oK3VoSIiERofDU4au0RrNNQeOh3w3nv1AwfGQcrWGtDFxgK//Wa5TYUXMG/0PBRVFeG1X1/ztCgKhU2UQlA4ja1b63l+AAChpVmIPnsE5cHc/QcHM9Bcj/h4Bh6MsQWAvRwCAqxfUKPhzd6otQcYlDAIU7tNxYvbXkRpdamnxVEorKIUgsJp6HSNC8siitL5x/nOokKYZin/ibHraN0xYf7+9s0rkNK+ajZbVFQwLara+dlL80bPQ2FlId7Y8YbTz43CQr5vWVk2uv4pFLZRMQSF02jXjvNp6rYKEtIAH30Vos7+Dj9dGaqlL/xD20EgBkCD9tN1F7QBA4B166xfsKqKvSwSE5su9NGjwKpVjGUIQRfVqFFsj51gvReRvQxJHIJJnSfhha0v4P4h9yPYzwlVdX/8AXzzDaPvWi3fu9hYYOpU5u6qCTuKJqC+NQqnMWlSfe+NprYGSZmb0T5rJ2LPHkZoWTbCC9PRt/gX4OefqT2MCFHf39S7N1OKzp0zfzEpOXtz0iTbE9IssWYNsGgRg9pJSZxKEx8PbNwIzJ9PN5aTmD9mPvIr8vHWrreaf7K9e4GFC6kUkpLYIzw5melZb78NfPSRU8aDKtoeSiEoHEJKZhPt38821XV7Cl1wAdemrCwe2H/vfxCbfwgVQbHQ+wSgHMGoDQpFUPsI7u43b2amUEkJc1S7dTOdzMeHdQZVVTxhXfdRZSWHLvfpA1zSxHlKx45x4ezQgf2mjTtqHx/eFxbGqTwlJU07fwOGJw3H+NTxWPLLElTWVDb9RMXFwBtvsIqvrtwAEBrKvuFr1gC7djVfaEWbQykEhd0cP84N9dy5wEsvAS+8ADz0EPDxx1QMAQEcNZCYCJQezETs75uRE5iKE2EDYKiohL9Gh+Tk84lDwcEMOhw8yCyhW25p7OZITQWeegoYNIha6PRpBpsrKoD/+z9evKnWwQ8/cIymuedLyfuLixkpdxLzRs9Dbnku3tn9TtNPsn07LQFLzZw0GlpWK1cqK0HhMA4XpgkhggFUSSmdM6G8GajCNPdx4ABnEAQH1ykOA5vRZWVx7X78cbr0DQYg+/kPUfLdRhT4JyIwEEgNyEZE+h5oanSmhb+mhge/9x4nnVmjuJj+KK2Wvn2rwxBsUF0N3H1348ELRvPn+HGgqIgLb0AAMHs2MG4cWDzRPMa8PwbHC4/jjwf/QICPjSwqgJZRfj5liYykJi4osD7TQUr2/3jlFVo6CkUDmlyYJoTQALgWwA0ABgOoBuAvhDgLYCWAt6SUx50sr8JOioqAbdvoVtbrOYlr1Ch2hXDWyOCqKuDNN6kIQkLqP+bry0LhEycYm7366vPlBbp0JA4ORbmWa9OerAQYtHFI8M1FUug5BAVKFBgiUVlSg9+PpqBzFBvWWZQ5PNx5g210Ol6ooTLYu5euqMBAXkuvp9L67jvGFZ58stnDjuePno8JH07Ae3vewz2D77F8oF7Pa65cyTiKUdbcXLq0rL0Xxtem1zdLVkXbw54so3UA1gKYBeCAlNIAAEKIKAAXAVgshPhaSvmR68RUmOOXX7i5rq3l+qDRcMTk6tWcNX/dddbruuxl3z6gvLxxjUFdEhKAH38ELr30fGGary/OZNZi5x98PDAQEFot0qvbY/ex9tDpgKhIibjqU/hulRalP3HDfu+9TkvusUxgIF1COp3JZZSeTmUQGVnf/AkPZ8A2L4+DnhcsaNabOi51HIYnDcezm5/F7YNuh5/WjMtKr6cG/vVXBrmTk033Hz3KYPLEidTO2dm0GKSkxm7fnn/7+jbW3gr7qa3l1D2tlmaxs3ZXXo49CmGClLJRl3opZSGALwF8KYRohv2uaAq//Qa89RZ//3Xrt8LC+F1evZoL8zXXNP9a+/dzDbWGvz/X1+xsICUFOBmdhlN7PkZwQmS9MoGSEv7OhABQVgqZGIf41CD0zdmG2LUbsP/7IoROiUDI1LHsh22rOK0p+PhwktqPP1ILGQzMNAoJqf/D1+mATp34d1wcZzL//ruNUW3WEUJg/uj5uOTjS/Cfvf/BnRfc2fign36iMujUqb48Pj5Mx127ljdjrYbxDc7I4BcjPh647bamx1faMqWlbMi1ejVjVQYDFfKUKUBamnN2WF6MzaCylLJGCNFDCDFeCFFvyyGEuMR4jKsEVDRGSuDzz7khNLdearV046xaRdd7c9Hr7UtrN84ykBL47MQQaAN9ESBNaUh6Pd3hAQFAgL+Eb1E+jkYOw7h1czF411IkGM7AUFOL9C1ngKVL2bguO7v5L8Ac48ZxF11UxEWgqqr+AlpaSu1ad+K9ry/ghJjVxM4TMbj9YDyz+RnU1Db46ej1wPff00wytyuNiuItI4MWTEQEFVlICP+Wkv47FTtwnIICWoBffMGEA2MqcnEx8NprwLvv2lcs2YKx+TMXQjwIYDmABwAcEEJcUefhZ1wlmMIymZkM5FpzI/v4cHHevbv510tNtT2y0lh9HBPDwfFHs8Owf9jfEFyeh8AKujSMGZwBshLtq0/iQNBgFBzJh09NFYojUlAVGAW/iCD8URiF6nYdedHnn6dJ4WxiY4GZMyl4ZiatAZ2O1ywq4gI7bFj9HaGvL31nzUQIgflj5iO9KB0f7/+4/oNZWVRGQUGWTyAlU0zPnaPJVVnJ96ioiKbcRRcB335r//g5Bd/TN97g4p+SYjKJhaCi7dSJPtoff/SomK7GnrTTOwFcIKW8EsBYAPOEEA+df6xtONa8jOJifk9tuTX9/Oj6bi5Dh/Ja1taXnBzgwgu5MS0t5Tqak3gBNo+ajdLQBIQXZyL43Cm0r82Ev6zC2rjr8XPQVJRWalERbApOGC2Rigpw0T53ju4TV9CpE7B4MVNe/f158chIvpAxYxr7yaqq6lsMzeDSrpdiYLuBWLRpEfSGOsFfvd76B3vuHN+cxEQOmejUiTK3b08FNn48XUZlZUzpVdjHyZOMzVhKGhCCVtv337dqRWtPDEEjpSwDAClluhBiLIAvhBAdoRSCR7A341Kvt77RtJeICGYPff45LeiG1z97lsrnivO2o7+/KQW+ILobNo+ajeCyXBzfVYycAl+URSahVvgg9PA2+ISb93P/uSZGR9OfO25c81+IOYKCgMsvZ15tdjavZw4paU1ceKFTLmu0Eq76/Cp8duAz3NjvRj4QEUHTzmAw76czmmp6PZVTjx6WLuCc3UBbYe9e7mKsKePAQPo8MzKYztcKscdCyBVCDDD+57xymAogBkBfVwmmsExqKhdlnc7yMVJyTenTxznXvPRS4PrrmfVo7KV26hT/Dg0FZs36c5wBOnTgulbX01MeEg99p27I0KSiVvhQwJoadA6v37par+fv8s8EmaAg97S3njaNpk2lmSpiKbkIXHghd+JO4vLul6NvXF8s3LgQtYbzvumoKKBfPy485jAuWFLyjbaEMdNIYR9lZfa9X0K4pAGit2CPQrgZQE7dO6SUeinlzQBGu0QqhVUCAtjC58wZy8WoubnM609Jcc41hQAmTwZefhm4/XZu2C+9lIpg0aL6a5NWy013bm79fnXt2jG2odMBZeUC8QElSPar99VCWRnl/jMzSadzTaZRQ7p3B+67j8VvGRn0zZeX02rIyGBfjttuc+olNUKDeaPn4WjBUXxx6AvTA1ddxdddaqZddng47+/Y0XJaqfFL0bmzU+Vt1cTGWt9hAaZdVisO2Nt0GUkpG3avr/vYL84VR2EvU6dyd75nD3fmxrWhupr+/Kgo4J57nJ8+HRLCwjdbjB7NWO3atVzDoqK4AevThy2MwsOBYQOrEJKVh9LwDjAYuAZHRDSwxvPzqf3qUFREN5VWyw17vYE8zWHwYPrlt24FduzgApGWBowdS6FckIs+rdc09IrthQUbF+Ca3tdAIzTU4o8/zrqHggK+6VotFZRez1iBtRedl0d5k5KcLm+rZfBg4L//teyqAxi869DBumXWwlEzlVswNTXApk2McxUWcr3y9WXn5gkTnFfY21SkZFHb99+zlxzAzVXPnlRmVSezcfWeuSjzj4HeN2cw7loAACAASURBVBAdO/KxP7M/Kyu5IC5cCLRrh+xsdnzescO0Nhtf75QptmslvJVP93+K67+6Hl9c8wWm9ZpmeqCykhp/504qp44d2do6MBB49llqxfbtTeZUbS0tmuBgttsw+vAU9vH++2y5npLSWPlXV9MknzGD9TEtHEutK5RCaAUYDEw+MRioBLyxHqmqyhTk1mioLE6fBio2/IrYL95EaJQffBPjuBOureUuV6ejGyctDZmZXANra7nOGbNBq6sZz+jcmY31nBFEdze1hlr0eqMXAnwCsOdve2gl2KK4GFi+nO0tjL9hKWk9XHUV838VjqHTAf/+Ny3EgADuXow/LiGAm29m9lkroNkKQQghwH5GnaSUTwshkgG0k1K6KCfQNkohtBLS05nfvX07FzUhGMCdOBHo2BEGA9sIVVZaTgJKT6dn6brr3Cm48/hg3we45Ztb8M30b3BFjytsP8FIWRmtAimpKT1tFrZ0pGT66fr1TEX18WG33REjWpWSdYZCeBOAAcA4KWVPIUQkgB+llIOdK6r9KIXQyqiu5qofGFjPR374MMsFrAXIa2oYbnj55ZZpJegNenR/rTsiAyKx484dEG2kd47CM1hSCI7MQxgqpbwPQBUASCnPAfBC54SixeLvz6hyg4DpwYO2xyb7+tIllZHhQvlciI/GB3NGzcGu7F1YdXyVp8VRtFEcUQg1QggtAAkAQohY0GJQKFyKTmdfTzEhWnarmZv63YSO4R3x9Ian0ZJje4qWiyMK4VUAXwOIE0IsArAZqpeRwg0kJtqfIm4pxtAS8NX6Yvao2dh+ZjvWnFjjaXEUbRC7FML5gPJGADMBPAsgG8CVUsr/uVA2hQIAY3parfUWMgUFLCFw+SwFF3NLypXo4BeDpz+4HfLRR4DnnjPVRCgULsYuhSBpv34vpTwipXxdSvmalPKwi2VTKACwNcbVV7PQzZxSKC1li5/p090vm1M5dAj+c+bjyezO+EVzGusDcphT+/rrwD/+4Z4WHoo2jSMuo91CCJdmFAkhLhFCHBVCHBdCPOnKaym8G73e1NkZYNuM6dNNnSTy8019laqrWdjbojs1ZGVxXnJwMG6PmoAEGYKn/baxxDslhUVoL73UqjttKjyPPd1OjQwFcIMQIgNAOdjpVEop+zlDkPMB69cBXAzgNIAdQogVUspDzji/omVQUAD8/DOHhul0jA307k2FcOmlwMiR7IZtTBHv25eFo05rX+Ep1qz5c85BAIAnMAIPi9XYKDMwGh3pC0tPZ1fWgQM9La2ileKIQphk+5BmMQTAcSnlCQAQQnwG4AoASiG0ETIzgSVL6P5p144V11Jy8V+8mAW4V17JerVWhU7HHiR1evHfiQvwjNyMBdiINbiJd4aEsLWCUggKF2G3y0hKmWHu5kRZEgGcqvP/0+fvq4cQ4i4hxE4hxM58Sy2CFS2O6moWlWk0nLlgbL8hBBtRJicDX3/N3kitDuPs3jrFFkHwxeMYjrXiBLYafxZBQZbbYisUTsBuhSCEmG/u5krhzCGlfFtKmSalTIuNjbX9BEWLYN8+toyJijL/uI8Pa9a++85yy+8Wi3GikKF+Wc/dSEOMDMICbOQdOp3lltcKhRNwxGVUd5hsADgkx5mZRmcA1O3X2+H8fYoWRmkpMyWPHOH/e/Rgd+HQUMvP2bKFTTqtERnJNjNFRfy71RAYSDfQ4cP1OpSGwA8zMAyzxE/YIc9gcLGeg3y8BZ2OkX0pWQBi6wNUeD12KwQp5Qt1/y+E+CeA1U6UZQeArkKIVFARXAvgeieeX+EGNm4EPviAFcPGnkI7dgCffALcdJPlZpHl5ba7tApBl1KrHFh1ySXArl1cZOu8EfdhMJ6XW7Cg5iesCLmUg3o8TWUlg+CrV/Nv44DvUaM4qMMZ1YGFhdwl/PYbv0zduvH8TpxYp2iMIxZCQ4LAXbxTkFLqhRD3g0pGC+DfUko1JbwFsW0b8M47nB/SMOunuhp4913eb24scVwcR3JaG0ZlbEthzdJosXTrBtxyC7Wpvz/fEK0WodXAw1U9MT9iN/bcMgkDPd25r6ICePFFDrhISGCAB2Ce8MaNwO7dHKNnaVi9Paxfz/cBYPdWIaiAfviBLW3/7/8sD7FRNAtHYgj7hRC/nb8dBHAUwCvOFEZK+b2UspuUsrOUcpEzz61wLXo98OmnXAfMpYD6+/OxTz/lsQ0ZPZozE6zFB/Ly6HpqqmdCSs44OXSIrievS+kfNw6YNw8YMICCZmQARUV4YNhDCPcLw8IT73laQuCrr/jmpabWH23q48MJbTU1wJtvNj3Qs3MnZxLExzOTIDycuwTjpLKVK4EVK5zzWhSNcMRCmFrnbz2AXCmlmZ+2oi1y9CgLyTp2tHxMcDCTZI4c4SjNusTHU2ls3syNZ3x8fUugrIyKZMqUpsl34ADwxRdcY40DeoKDeb6JE+1rnucWOnfm7NM77uDi6u+PCK0WD637A09vfBr7c/ejb3xfz8hWXg5s2GDdbRMbyzf5xAnHKwUNBo6xjI01v6vQaqkkvvuOIwFVgN3pOGJ33Vsn3fTMeRfPYpdJpmhRFBTYvyksLDT9rddzDZgxg4t+cTFdTytXcu3Jz2d9QlkZ8PDDXA8cZcsW1jecO8fnJyXx36AgWixvv+3mLql6PQdfZ2XRLDKHry8FPK+pHrrwIYT6hWLhpoVuFLQBJ0/yjfL1tXyMMdBzqAnlQydO8AO35hM09jlvlfnHnscRC+FiAE80uG+ymfvaBBUVXASFoLvXG8dWuhNb8wrMHSsl8NFHrExOTub9nTuzS8OJE2xTsX8/R2OOGdO0YWCFhfRAtG9f38MBMLknNZUKY+BA87ENp1JVBaxdy2Bs+fmkPV9f4KKLGFSOiLD41KjAKNw/5H48t/k5/H3M39EztqeLhTWDXt941rA5tNqmRf6Li+07v48P/YcKp2PzZyyEuAfAvQA6CSF+q/NQKIAtrhLMWyks5ND4DRtMaeP+/ox1TZzYcge9Nxejd8BgsBzvM75fxmNPnmT8MCXF9ByNhgo2Ls50TEBA0ydDbtnC6zZUBkaEYFLMypXA0KE21qPaWvq7fvqJZoufH580fLgpuGqJykoGY48epU/MmImj01FB7NjBYKyVMY2PDnsUr25/FYs2LcJHV39k/XquIDKSb6ZxzKklamqaFlS2d1dVN4VN4VTscRl9AuAyACvO/2u8XSClvMGFsnkdeXnAggXc0cbF0fWQlMSY11dfmdoutEXi47nLzs62fEx2NvsOGVPt16/nGmAtYSQuDli1qlHNlt3s3m114w2Ayub0adZPWKSyks3lliyhUtBquZgvX86Bz1ts7I2++AI4fpwmSd1dg58fAy/l5fRdWfG7xQTF4N7B9+LTA5/i94LfrV/PFSQnM7BbVGT5mJoaWj1Naa/RubPtPudSmhpcKZyOTYUgpSyWUqZLKa8DUAIgHkBHAH2EEKNdLaC3ICXw1ltcF5KT67tRAwL4O09PB/7XhidE3HwzN5EN21TX1PC+yEjWIhQU0C109KjtnX9wMD0J5eXWj7OEXm87Q9Ho9rYYR5CS+bQHD9KciYvjoh4Swi9DbCywdCkLy8xRVkaTMjHR8s46Ph74/Xfm3lphxrAZ8Nf645lNHphNJQRw7bX8QMztfPR6ftBXXdW0VLCgIGD8eGZYWVKM2dn8sR06RIvqnnuAJ56glVVc7Pg1FfVwJO30DnBIzmoA/zj/799dI5b3kZHBbLs6haSNSExkKrbVnWYrJjISmDMHuPhixgZPneItP58xgGHDgEWL2Kp65kx6Xo4ftz77xbghbOrM+ZQUZj9Zo6qKSt1iLPP0aZoaycnmBTH6tL76yvxCduIEtY21QIuxuMuSUjlPfEg87k67Gx/99hH+KPzD6rEAuHAfOUJllptr+3hb9O4NPPAAF9+MDGr2ggIqgqws4Jpr2Jq2qVx1Fa9x4gQVqZGqKu64NBp+Hp9+yp1GdDTf288+A2bP5vMUTcaRoPJDAAYD2CalvEgI0QNtaITm0aOm36wlfHy4Hpw4QddIWyQ8HLjuOnYlNcb9QkO5gf7pJ1N6OcB15MAB/jtihPlMw5ISKtqm1h6MHWuK91iyFHJzKa/F9XrbNj5o7cOPimKxVn6+KQBipKaGz62sNLlUzAWbtFrLWUd1eHz443hjxxt4dvOzePfyd80fVFFBd9bPP5v8bQYD0LMnC7tSUmxexyIXXED32a5dpkriTp0YS7HUjMpe/P2Bhx5i99dVq6hohKDSnTiRfsagoPq9S0JCeCsqAv75T+46WlVvE/fhiEKoklJWCSEghPCXUh4RQnR3mWReRlWV/cWR5gqv6lJYyN9SXh6/2/370wpu6i7YGwkMNNUk/Oc/XCsbvsYuXbjJLCkB9uxpnOUjJTegf/lL09+b1FSuU7/8Uj94bTx/bi7XsLFjrZwkL89yVNqIEFQapaWNFUJ+PhdOo//KYGDwuHv3+sHo2trGzzVDQmgC7hx0J5buWop5o+ehY0SD4o+KCuD557mjbt/e5N+UkvctXEg3S9euNq9lkaAgtpIYNarp57CEnx9dRxddxFxhKbnTWLuWCtVSHUREBBXIL7+whYbCYRypQzgthIgA8A2ANUKI5QCc2f7aq4mPt52rbmxYaWlzotcDH3/MnPtPPqF76bvvgKef5m/07Fnny+1pSkr4Os25z0ND6R2oraUXoK6HQKfj2pWWxkSepiIE8Ne/sgj41CmuF7m5vF5GBpNhnnzSRiwjLMy+mcbm0pl+/pkfusHAhbm2lgGR48fp9z52jMfpdFQodpqWT4x8AhqhwXObn2v84Lff8s3r2LF+sMuYIx0aCrz2mheWajdAo6FLKCaGr+Onn6xmYQGggl271j3ytUIcaW531fk//y6EWAcgHMAPLpHKC+nXjxuXBr3H6lFSwozC1NTGj0nJnfKGDfyd1q2MNbZUWLyYnQus9fNpaRw71qjVfz26dKGXYPt2uo+Mmz9fX+Dyy3lzpMbBHL6+wK23cuLajh3c8AcHMxGmc2c7rI+0NC5G1igt5WKVkGC6LzMT+PBDZuYUFgJbt/LFGF9QRQX78+j1/FJdd53d6ZQdwjrgrwP+imV7lmH2qNlICj/fKLiykkqorhwNCQ9vmdPXioqsvy6ACjkvr3mBpzaMI0FlIYS4UQgxX0q5AcBeAANcJ5p3ERTEQe+nTpnfWFVW8jd/3XXmv4cZGdwpp6Q0bpMgBHeq+fl0kbYmbG1ChWBMYfBgxiPvuw945BEOy5k2zXpRrKPExrJVxa238lpduti5ZnTvzvxiS0HZ2louQpddVt8ntW4dF//sbH5xkpL44RsMvLCvLxeudevol3cwGPvkyCchIbHklyWmO0+f5ptuK6ff35+B5pZEaKhtS02no7ZXyqBJOOIyegPAMADXnf9/KTgDuc1gbLSYlcXNnzF9MiODrs777qMlYQ57cu7btaMXwVYMoiXhSEFZv35cF/v29bICP40GePBB7goyMqj9AVOQIyODi/nIkfWft20b/YcHDtDsi4ykSRIfz51sQAD/jolpUgZQx4iOuKX/LXhn9zvILj1fAGItet7wNXm7y6ghF11ke2JcXh79g4om4YgxPlRKOUgIsQcApJTnhBBtqmGDENwEjhhB6/+PP/i76tOHXgVrvbZOnrS9OBqt3dLS1pMk0bUrX3d5ueVMIZ2OG9aGDe+sodczFX3tWm6+jUXDI0faFZd1nNhY4KmnmP2yerUphapHDyqDvn0b70qrq7no6vUmN5GPDz9c4wcsJX2N2dlNagg3a+QsvL/3fTy/5Xm8OOlFymkw2FYMVVXWOxF6IyNGsKS8tNR8jnBZGS0wVwS62wiOKIQaIYQWgAQAIUQsgCbWj7ZsoqLoj3YEo6fAGsac+9bU6t3Hh1lCb71Fj0lDT4ZezwX9+uvttwrKyxkTPXSISjg0lF6blSt5++tfuXY4nbAwfvCTJ9NK0GqtZx+1a0chrbkvjGMxhaDP0UGF0DmqM27sdyOW7lyKJ0Y8gfioeAamDx+23D6ipoZfsrQ0h67lcaKj2eHwpZcYT4iL407COLlNo2HKqhqt22QcWXpeBfA1gDghxCIAm9GG6hCay6BB1iv+AVNQujUFlQEuzjfdxAafmZmsaSopoSI4fZo1AJdcYt+5pGSHh6NHGY+JjeWaHBxMhRMbS+Vjo76reWg0vKCtVNRJkxg4ttYGtqLCFMxoYg/u2aNmo7q2Gi9sPT/U8C9/4TXPnWt8sE7HD2HatJb5RevZkyl5U6bwB5WRQUU6aRLvd8TMVDTCnuZ2H0opbwIQA2AmgPEABIArpZSu/Nm1KoYNYyGrsSq2IUZ39N/+1vriYUKwpmjQILb82b+f1tKwYbTuHemDlpEB7N1LZWCtaPjrr7l2eJQhQ0zFFsbJX3UpLeWiHB9PF1SnTk26TLfobpje61q8tv0NBO+dibz0RMRWPYmr019HSkg6QiN9IbQaKgOtluaYvRrYG4mLo0K7+mqahlpt6/vReAh7XEYXCCHaA/grgA8AfGp8QAgRJaUstPhMxZ9ERwO33caWOFFR9deHqiq6kIcM4SLZWomJMaWSNpVt25icY2/RsEe9B4GBLDI5fJiZCKGh9KHp9byFh7MaLz+f/9rqwmeB6mogOWMOKvWfYkXJS7gkYhF04al43XcxYvIP47Kwgxg6sAYiqYPtYFdLwlgMqHAa9rybSwH8BKATgF117hdgPKFp25o2yKhRXBO++IJWu3FyV0AArfzJk71ocpeXkp9vO9ZgbFZXVuYF7uSYGFYhzpzJoLG/PxfkpCR+GXJyWKdw3XW2z2WBjz4Ccvb3Qq+Ua3Ag6F+YgBkIRBQSOmihb9cHb2T2QXUy+0kpFNawqRCklK8CeFUI8aaU8h43yNSqGTCAMb9Tp+hH9/Oj+6OlDNipqWEW5ZEjdPt07szXZMud7izsKRo2Voyb643kERISWFjx7becEQrQLDTOBJ0ypcm79rNnecrkZCAQc3FI/Bfb5Cu4CP8AwA10u3Z0oY0cqTYcCuvYE0MQklhUBsZjnCta68VYjNXSOHCAAduyMpPbZs0aLry33MKeQa5myBDWcVmjtJSLoK2iVrcSHc2KuL/8hWaOsRqxmZp0717+q9EA8eiLnvJqbMcrGIZHEAC6oIKCTFPoUlPZNaO0lJ9bly5q1ozChD0uo3VCiC8BLJdSZhrvPF+DMBLALQDWAXjfJRIqvIKjR4EXXqB/vmH6emUlu5lqNK4fQ2ksGs7JMR+Mrq3lenvvvV4aZzR25nQS587Vd6OPwlwcFl9hu/wXxmDen/dLyUr51183TaqUkor94ouBK65oOVaqwnXYk3Z6CYBaAJ8KIbKEEIeEECcAHAOrll+WUr7vQhkVHkZKtuQJCzNfDxQYyMX5o49cX/xqLBoOCWE7HuOcFoOBiiAjg8WDLp+P7CWEhdVvupiAgegmL8M2vIRqmAZznDrFbti+vlToycn8NyaGDRbfeKN1VcgrmoY9E9OqpJRvSClHgJPSxgMYJKXsKKW8U0q5x+VSKjxKejrrBawlwQQF0ZXkzPY4paXs0/bqq8Arr7DorLDQVDQ8fTozbDIyuOB16sTY7TXXeKl14AL69zeNOTYyGvNQJc5hx/nOMnl5bJ7Yq1fjanFjDGvXLjYYVLRtHMrZklLWALAyNVfRGjG6vG0tshoN02cHOKHl4ZYtwHvvcddqLOTdt48ZWn/5C+OwkyezHqm6msHStujyaNeOmaS7d9OVJgSQiMHoIidjK15AmuF+7N8fguRky+EKIWgpfP8940BtRZkqGqOSeBU2sTczxWBwThbLnj2MSbRvX38Ri4qiS+qzz7j4X3wxlZBbGuHl5XHwyq+/Ms0pKQmYMIHVbx5O3bntNsYFjh411biMkvPwnnY41pxbivYRj6G7jVFWYWFMha6oaPp0OkXLpxV1zVG4CuO0RWu9mIwuiy5dmnctg4HjcmNizO9ofX2Ztv/FF3ZNm3QOmzdzis6339Jk8fUFfv+dU8leesn8wHk3EhzMOdV3380YT0YGIE8NQ1fNBKQnPo9uvSts9scyWoAqV7Bt0ySFIIRwoNmAoqUTHc22E9lWnIVnzzJIaW44kCOcOEEXlbU2O/7+dBPt39+8a9nFwYNsnhQXx0issYdRXBw15cGDwLJlHl9J/fzo7lmwgOIsWwYsu3k+CqvzkJXwjs0+WuXljBGpFNS2TVMthO+dKoXC67nxRroiTp2qn9ViMJgUxV13Nd//XFho3zk0GlMHapchJSu6IiLMmyvGgpJduxh19xK0Wr4/ozqOwtiUsfi5ejEqdFVWLby8PDZytavTrsHAysT33uNQ+3ffZVdXWzNmFV5PU2MIKuzUxoiMBObMYYO+LVtM9xsMtB6uucaxJnWWMA4Rs4VbKpHz8jj0wloVobGfzrZtjCt4GfNHz8e4D8ZBXLAM6TvuQ3Jy/boF4/jWlBQ7CwuLipj2ZWzDERDASrfNm/n6H36YJqWiRdJUhfCOU6VQtAgiI4Hbb+fif+YMF5P4eAYyKyvpSg8MbJ6V0Lkzd6nGJpbmMCoMW4HSZlNaSmFsvaCAANuTvDzE2JSxGJE0AuuLnsOSqXdg7Wr/P99bY7rqoEEMTNsMzldXszoxJ8cUWKpLTg6wZAlzgpXvqUXSVIXwlTOFEEJcA+DvAHoCGCKl3OnM8yucS1iYqafQtm1MV8zJ4bqZkMCU0KFDmzYPOSyM8xOM/XnMkZtLZdChQ/Neh00CAuwzV3Q68xV7XoAQAvPHzMekjyahtPN/8PLLd+G33+iaCwri+ID4eDtPtns3U5EsBYratWPRyvbtHHepaHF4SwzhAICrAWx08nkVLqKyEnjxRbqPdTpuGJOTef9bb7GQrLq6aeeePp3nOnmyfiZRTQ3Xo+Bg4M473ZAv3749051KS60fp9OxyZKXcnGnizE0cSie2fQMAoJqMGIEq7nHj3dAGQAcHWprtmt0NPDDD44JqNfbHieocAtNVQhO/SlKKQ9LKY8685wK1/LJJ8x7T001ZQQJwcBzp05shPff/zbt3MHBrDi+6ip2hM3MZDA7P5+1B/PmcZ024rIEH42Gwxvy8iwHTHNyqL26dnWREM3HaCVkFGfgw98+bPqJsrNtW0IhITThbC3wZWXsjDhjBnDHHbwtXcp4hMJjtLgYghDiLgB3AUByS2wZ2go4d441Wh06mN+lC8HHNmzgeMymeFOCgvjcKVOoCKTk5tPo587P57z7n3/m2hIRwTqxESNsb2IdYsQIaqQffqDmi47mC6yooKKIjQUeeMDrB2FP7jIZFyRcgEWbFuHm/jfDR9OEn76/P3fz1grxamttTzDKz2eswTjBKCWFz9uzB9i6lSbi5MmqZNoDNOlbLKV8w9HnCCHWCiEOmLld4eC135ZSpkkp02I9Pv2kbWKchWBtXfDx4W/8yJHmXcvPD0hMpIIxKoMDB5jxtHIlN6QdO3Kt+uorYO5cupqchhAcXvPYY7QEMjNZ+VVVBfzf/wHz53vBFB7bCCEwb/Q8nDh3Ap/s/6RpJxk61HbwPC+Px1lazGtrORuitJSKwFgWrdUyANWhA0vR9+1rmoyKZuG21hVSygnuupbCtVRW2neclE2PI1giN5fxifDw+l2kAwO5XhcVAc89x/U7Koq3Zm80hQD69eOtupqLWkCA11sFDbm8++XoH98fizYtwg19b4BW42DLjbFjgbVr+R6Yy/mtqeFj48dbPsfhw6Y8V3P4+tLcW7GCnfuUleBWWtY3WuEVWKsirosQzk++Wb+e1om5kQKlpUyPX7eOXpzHH2cG5K+/OjHO4O9Pf1YLUwaAyUr4veB3/PdgEwI8iYmchHTmDEvTjW+qlExbysykNWWtXH3rVtv5rZGRNPMKChyXUdEs7JmYFmXHeQxSShvF8VavcRWAfwGIBbBSCLFXSjmpqedTuJbevenK0eksdxitrubvvkcP511XSi725jJjCgpMBXNRUYwrJCWx6du//gVccglw/XUS4vSp881+zhdRdO3aIhf3pnJVz6vQO7Y3FmxcgOl9pkMjHHztY8eybceKFcwq0GppMXXuzNSvvn2tP7+42HZbWuNQbHtNUYXTsMdllHX+Zs120wJocoRXSvk1gK+b+nx7MFbbr1nD7zHA/vATJjCnXVmm9hMYyLTF//6Xln/DWEJtLTs53HCDc6uJ9XoqmobrSU0N6yH8/Hi92lq6+IWg9yE0FNj2VRbGbFqGDlV/mJ4oBP3/t97KL0MbQCM0mDt6Lq778jp8eehLXNP7GsdP0qsXb4WFbIIUGGgKttsiOho4dsz6Mcah2E6cLKewD3sUwmEp5UBrBwghvHpIjl4P/PvfzIwJDjZloRw8COzYwRqam27yeBfjFsWllzIldM0aun2jovg7PneOC/TUqcDEic69po8PXfcNLZPsbF7TGJ+sra2viMIrsjHx2CJkZUgkTukIoamzcBUXs2vpjBms0moDXNPrGvx9/d+xcNNCTOs1zXErwYgxSOMIw4cz/cwaBQVsK+7UdDGFPdjzTRjmpGM8xldfsfK1Y0dau76+vMXH876ff2bGisJ+NBrg+uvpozf2wDEYmCQSF8f5vY89Bnz+ufUuqY4gBDBuHAPLdTl9ur4CqKhgLYSRvvs/RoBvLbL0caiqbrCLDQ/novbuu617hmRODvC//wEzZkD7wIOYe64vfsv9DSsOfuleObp25c1SM8CqKu40rnAo+VDhJOwaoWnpMSHEbbaO8TRlZcCPPzIDxZyrWKPhIrZypRv767cShGD88LbbmO4ZEcGgblUV11lfX773c+ZQQTiDMWNoydUtHtbrTZ9tZaUpVRUAgstyEZd3AOUh8RDCQn1ZaCgthUOHnCOkt7F5MzB7NmspfHyAiAhcmx+PLjWhePqLByFt9cZ2JhoNcP/9/IDSQR12+gAAGq5JREFU000mZWUlqw/z8tg21+WNqhTmaG407R9OkcKFGLvy+lhxjhkDpIcPu0+u1oSUHNJ+8iRjClFRfE8DAxnYjY83dUhuLnFxwCOPUNFnZnIdCQ6mK7u4mLIMH26yGEJLsyCFBgYpIISVmIZGw2Bza+PoUeCdd/jGJSXxQ/H1hU9UDOb4jMMekYOV/3rAva0jwsO5S7j3XsZwior4A5w0CVi0CBg50n2yKOphT5bRb5YeAuBIJxSPUFZmf8pheblrZWmtnDhBZdqxo/m4or8/14BvvrEeu9XpuDZoNHQfW4rp9OwJPPMMY0Lr1nGNq61lgktSkvnRBcaso6Y03GvRrFjB4KyZN+UG9MU/5AY8XbkKlx47BuHOXbmfH/s/eXEPqLaIPUHleACTAJxrcL8AsKXx4d6FI1143TKbtxWydSt/39aSTKKimFxy9mz9PkQA3T9r1zJAXV1NBR4Rwe4FY8aYz1KMjmabocsv5/Evv8zi1oYWQGlYIvTVBogAA7p0sWIQGwyWi6VaKufO0Syz0OLFF1rMxkjc5f8dfvzpbUzq/oKbBVR4G/a4jL4DECKlzGhwSwew3qXSOYFevbjTtBYvrKmhS6lnT/fJ1ZooKDC/K6+LEPwcysrq319YCCxcCCxfTisiKYnrl1YLfPghRxbbiu0IwXnCAwbQ65OVRfdRQQFwMD8Op6P6YXT3PMtFciUl1ECt7QtQVmZznsMtGICk2hD8o+BLSDVQuc1jT1D5dinlZguPXe98kZxLWBhraU6dMu86kpKPXXyxmunRVMLDbbeokJJunbpWmJR0b587x8153d19cDAD1ocPc4qlLQIDgYceYifU4cPpmk5NpaKY8tH1CI/2YcCy4ZegqIgC3HGH9UBTSyQwkK/XykLvBy1mlQ3EVkMGfj75sxuFU3gjrewXYJ7p07kT3bmTvmljenNhIXeSw4ax1bKiaQwdypYS1igu5s4/Ls503+nTptiDOYRgMsrPP9M1ZKwzsIQQLJjt3LnhI+0YxPz3v01FUUJwoWzXjlrDmSXV3kJ0NLXi2bNWc/r/ei4Fi9odxdMbn8b4Tlb6EClaPfYElXdLKQc19xhP4ucH3Hcf8NtvwKpVppbr3btzJGTfvqoorTl0787FPifH/FxlvZ7um5tuqu+9OHyY/7cWe/D15fP/+IO95ZpMQgJTL0+fpkloMDD9yTizszUiBEvKX3iBgWVzEfXcXPgndMDMUbPw0I+PYEP6BoxJGeN+WRVegT0WQk8rmUYAg8vhTpLHZfj4cHbsoEFcC2wtRAr70WiABx9kwW96Ot01wcF8n/PymBr6l7/wva9LZaV9ilgIxnmajRAMUiQlOeFkLYT+/YFrr2VRWkAAPxytlil1xgj/ww/jzshQPPPLc1iwcYFSCG0YexSCPba0hXFS3klr3RB6kpgY+u+3baMVZkzpT0tjv6hu3Ror4NhY+4qDpWScQtEEhGCfke7dmcplbP0aGcnOpMOHAyEhCAQwc8RMzPhxBn7J/AUjkkd4WnKFBxCOZhYIIZ4GFcleAHullL+7QjB7SEtLkzt37vTU5RUWkNI0WMua8i0rAx5+mJ4bS/UBpaU8z3PPKUXuFAwGU6VmAw1dritH6iupGJQwCD/c6OBcZEWLQgixS0qZ1vB+h39iUsr5AF4BUAzgKiGEx8ZpthXy89naecMGxkGc4j5xIUJwgbe1gIeEMFicmWm+pUR1NV1O//d/Shk4DY3G4ojLYL9gzBg2A6v/WI1fz/zqAeEUnsZuC0EI8QqAh6UXJSu3dguhuBj44ANg9+76mYMhIVwkR41q+XEQgwH44gvg++9pCYSF8bWWlPC13Xwzi9MU7qG0uhQpr6RgeNJwfHvdt54WR+EiLFkIjqSdlgJYIYS4VkpZLoSYBGC+lFI5G11AWRmweDF3yElJ9XfIlZXsDVRV5fwW0+5GozEpt82bmRWq0fB1DRumOiC7m1D/UDx64aOYu24udmfvxqAE1yQPFhWxwn3LFn6P27Xj5M0+fVpfOUhLwqEYghDiegCPANABKAOwUEq5yUWy2aQ1WwhffcU2NJa6Keh0TPN84QW1aCqcS3FVMVJeScHYlLH4errz51b99hvw+uv8DkdFUQGUlzNe1Lkz40r2jmlVNI1mxxCEEOMB3AmgHEAMgAc9qQxaMzod+/okJFg+xs+PrpVt29wnl6JtEB4QjoeHPoxvjnyDfTn7nHrujAz2nQoLY0FiaCgLqmNiWEOXmQm8+qp7m68qTDgSqpsDYJ6UciyAvwD4XAgxziVStXHOnqVSsDV+MjRUtexWuIYHhz6IUL9QLNy00KnnXbWKMW1LVeeJiSwcNY65VbgXuxWClHKcsaeRlHI/gMkAnPttUSgUXkFkYCQeHPogvjz0JQ7mHXTKOcvLWQZRt31JQ4Rg/ZyzBiopHKPJyXxSymwAqvGJC4iJoXVgq2FcaWnra9Cp8B4eufARBPkGYdGmRU45X1mZqeutNYKCmEyhMEN2NvDTT8B33zEi37B9cDNpVjxfSlnpLEEUJvz82H11+XLrQWWNhpk4CoUriA6Kxv1D7seSX5Zg/pj56BHTvAaA/v6MDUhpPV1ap2NqtaIOxcVszrjvfExHqzUVGF56KQt6nNCQTZX7eCkXX0x/6unTjQNsFRW8//rr2cZfoXAVjw57FAE+AXhm0zPNPldEBNC1K7sMW6OsDBihktlNlJezUdihQ4zEp6QwFz0lhWX+X30FfP65/aMhraAUgpcSEgI88QQwcCAzL9LTTbfKSs4hH6dC+goXExcch3vS7sEn+z/B8cLjzT7fZZdxs2uph1VhIVNR+/dv9qVaDxs2sENvYmJj08rXl4rhxx+BM2eafSmHexl5E625DqEuZ88y86KmhjUH3bu3wdnACo+RXZqNTq92wvV9rseyK5Y161xSAitXsvlqcDDjZRoN42W5uYwfPPFE22pIa5XaWuDRRxlptzbj9/Rp4KKLgBtusOu0zqhUVniImJjGc4gVCneREJqAuwbdhTd2voG5o+ciNTK1yecyNl/t3BlYvZoucSEYX5g6ldMNo6KcJ3uLp6SEPjRb1acREcDvze8zqhSCQqGwycwRM7F011I8t/k5vHXZW806lxDMjuvZk5ZBTQ03v2pIlRmMk/1sIaVTOkCqGIJC4c3U1HAbvWED0ww9lI+ZGJaI2wfejvf2vofM4kynndffn/EypQwsEBZG90BpqfXjioo4+rGZKAtBofBGpAQ2bWL2SHl5/VzNQYOAW25x+9SgJ0c+iXd3v4slvyzBa1Nec+u12ywaDTBlClNOQ0LM5+vqdPx+jBzZ/Ms1+wwKhcL5rFkDvPMOF4GUFDb6SUnh8Or9+9kK18lFSbZIDk/GrQNuxTu738GZkuZntCjsZPhw7v4zMhoPQykvZwbS9OnWS8DtRCkEhcLbOHcO+Owzpto0zCzRaJh+mJXFVEM3M2vkLNQaavH8lufdfu02i58fh5ZPmkSXYUaG6abXA3ff7bQ++F7hMhJCPA/gMrCt9h8AbpNSFnlWKoXCQ2zdyn/9/Cwfk5DAGclTp1o/zsmkRqbi5v43461db+HJkU+iXUg7t127TePvzxnYl1/OHHSdjvGFLl2cGoDxFgthDYA+Usp+AH4HMMvD8igUnuPIEbaytYax2dXZs+6RqQ6zR82GrlaHf275p9uv3eYJDmbV3uDBLEhycjTeKxSClPJHKaWxdnEbgA6elEeh8DheXDDaJaoLru97Pd7c+Sbyy/M9LY7CiXiFQmjAXwGssvSgEOIuIcROIcTO/Hz1ZVS0Qnr2tJ1mWF1NK8FDFYtzRs1BZU0lXtz6okeur3ANblMIQoi1QogDZm5X1DlmDgA9gI8tnUdK+baUMk1KmRYbG+sO0RUK9zJsGIPHOp3lY7Ky2AHRjfGDuvSI6YHpfabjtR2voaCiwCMyKJyP2xSClHKClLKPmdtyABBC3ApgKoAbZEtusKRQNJeICLayPXWKrW3rYjDw/qQkp2WWNJU5o+agTFeGl7e97FE5FM7DW7KMLgEwE8AYKWWFreMVilbPuHF0Cf3vf0B+fv3CtMGDgZtusjyH0k30ieuDaT2n4dVfX8WM4TMQEaB6sbd0vKLbqRDiOAB/AEbbc5uU8m5bz2sr3U4VbRi9nllHRUUchtK1KxAd7Wmp/mRvzl4MfGsg/jH2H5g/Zr6nxVHYiaVup16hEJqKUggKhee58rMrsSFjAzIezkCYf5inxVHYgSWF4I1ZRgqFogUxb/Q8FFUV4bVfVX+jlo5SCAqFollc0P4CTOk6BS9ufRFlOvf2V1I4F6UQFApFs5k3eh4KKgvw5o43PS2KohkohaBQ2KCqipmemZmcZ61ozIUdLsTEzhPx/JbnUVGjEgVbKl6RdqpQeCNlZcCqVewhZ+w67OPD0bWTJ7NcQGFi/uj5GPneSLy18y08MuwRT4ujaALKQlAozFBSAjz7LAfCR0VxDEFyMhAby67TixYBhYWeltK7GJE8AuNSx2HJliWorFGmVEtEKQSFwgwffwzk5HAmjb+/6X5fX6BjR6C4GFi2zGPieS3zRs9DTlkO3t39rqdFUTQBpRAUigYUFgI7dnAOjSUSEoBDh9hSSGFiTMcxGJU8Cot/WYxqfbWnxVE4iFIICkUD/viDnSI0Vn4dQvCY48fdJ1dLQAiB+WPm40zpGby39z1Pi6NwEKUQFIoGNBxbaw1rDUlbHTk5wHffAe+/D3z9NUc4mul0MD51PIZ1GIZnNz8LXW1beoNaPirLSOERqqu5mAYGMnPHm4iKsv9YD40jcC9VVVQC27bRbPLzo9ZcvpyzG+6+GwgP//NwIQTmjZ6HKZ9MwQf7PsAdg+7wnOwKh/Cyn6KitXPkCLB6NbB3L90ufn7A+PFM5fSWxbVrVyAykmmnISHmj6msBIKCgF693Cub26mtBd58E9i3j9F0Y8dVgNbBsWPAP/8JzJ5N7X6eS7pcgrT2aXhm0zO4pf8t8NX6ekB4haMol5HCbXz/PVM5jx5lO//kZO7Gf/gBeOopeiC8Aa2W4wjy8rg5bohOx2Dytdd6bD6N+zh8mNq7oTIA+P8OHVixt21bg4cE5o+ej5NFJ/HJ/k/cKLCiOSiFoHALBw8Cn33G9SMuzhSw9fOjcvDxAV56yfwC7AnS0oC//Q0oKADS0znLvqCASis3F7jlFmDUKE9L6QbWrKGZ1FAZ1CU2lhV8DeIJU7tNxYB2A7Bo0yLoDXoLT1Z4E8plpHALK1cCYWHM4zdHZCQX3r17gQsvdKtoFhkxAujXjymohw5xveveHRg6tJ7LvHWTns4PzhohITxOp6tXtGG0Eq7+79X4/MDnuKHfDS4VVdF8lEJQuJzSUnoekpOtHxcWBmza5D0KAQBCQzm8bNw4T0viIXx8OLbTGsZpblpto4eu6HEF+sT1wcJNC3Ftn2uh1TQ+RuE9KJeRwuVUVdFFZM3rANB6KC93j0wKOxk0iL4ya5w9y+i6mXQxjdBg3uh5OHL2CL48/KWLhFQ4C6UQFC7HmKlTW2v9uMpKuqMVXsSYMfzgLBVnGAw0AS+5xOIppvWchp4xPbFg4wIYpA1rQ+FRlEJQuJzAQLqBcnOtH1dRwfVH4UV06ABMn85MopKS+oHj8nLg5ElgwgSgTx+Lp9BqtJg7ei4O5B3AN0e+cYPQiqaiFILCLUyezI1mmYWBWmfOAJ06sc5J4WVMmgQ8+CADxpmZHA6RkUGr4bbbgBtvtOkPnN57OrpGdcWCjQvQkue4t3ZES/5w0tLS5M6dOz0thsJODh4E/vUvVilHRdHlXF5OJZGSAjzySBvK3mmJSEmFUF4OBASwNsFMINkS/9n7H9y6/FasuHYFLut+mQsFVdhCCLFLSpnW6H6lEBTupLgY2L4d2LyZMYN27Vip3Lu35ZRUReugprYGPV7vgajAKPx6x68QtrIMFC7DkkJQaacKtxIeDkycyJuibeGr9cXskbNxx7d34IfjP2By18meFknRABVDUCgUbuOm/v/f3v3HVlXecRx/fwaUFoqiCMqQ4VyMm3OGiTqZZNEozpltzrhfhpnMHyOIRFCZq2MZE9S4kCwmxmUu0UwNbiHC1Dh0oGMgy1DRoKL114hMZCoyyWRqHPDdH+epXmvbeyv2Pue2n1fSpOf23Pt82rTn2+ecc7/POUzYdwJXrr7S1xJKyAXBzOqmaVATbVPaeOjlh7h/0/2541gnLghmVlfnTjyXcSPGsWDNAs8SSsYFwczqaujgobRNaWPtP9eyevPq3HGsgguCmdXdBUdfwNjWsSxYvSB3FKvggmBmddc8uJnLT7icVS+u4sHND+aOY4kLgpllMX3SdMYMH8PCNQtzR7HEBcHMshg2ZBhzJ89l5aaVrNuyrvoTrM+VoiBIWijpCUkbJK2Q9Mncmcys71147IWMahnlWUJJlKIgAIsi4qiImAjcA/w8dyAz63utTa1cNvkylj+/nPVb3YYmt1IUhIj4T8XmcMA3J5sNEBcddxH7Ne/nWUIJlKIgAEi6WtJLwDR6mCFImi5pvaT127Ztq19AM+sT+wzdhznHz+HuZ+9mwysbcscZ0OrW7VTS/cBBXXxpXkTcVbHfFUBzRMyv9prudmrWP+x4ZwcTrpvA1EOncsd378gdp9/rrttp3WYIEXFKRBzZxcddnXZdDJxVr1xmlt/I5pHM/tJslrYvZeNrG3PHGbBKccpI0mEVm2cAz+TKYmZ5zDl+Dq1NrVy15qrcUQasUhQE4FpJGyU9AZwKzM4dyMzqa/+W/Zl17CyWPLWE9m3tueMMSKUoCBFxVjp9dFREfCMiXs6dyczq79LJl9IypIVr1l6TO8qAVIqCYGYGMHr4aGYeM5Pbn7yd57c/nzvOgOOCYGalMvfLc2ka1ORZQgYuCGZWKge2HsiMSTO47fHb2PTGptxxBpS6vQ+hL0jaBmzuwyEOAF7vw9fvS42a3bnrr1GzN2puyJ99QkSM7vxgQxeEviZpfVdv3mgEjZrdueuvUbM3am4ob3afMjIzM8AFwczMEheEnv02d4C90KjZnbv+GjV7o+aGkmb3NQQzMwM8QzAzs8QFwczMABeEqhp1vWdJiyQ9k7L/UdLI3JlqJek7kp6StEdS6W7N60zSaZKelfSCpLbceWol6WZJr0lqqH7TksZLWiXp6fR70hDNMCU1S3pY0uMp95W5M3XmawhVSNqnY4lPSRcDR0TEjMyxqpJ0KvCXiNgl6ZcAEfGTzLFqIulzwB7gRmBuRJR2FSRJg4DngKnAFuAR4OyIeDprsBpI+gqwE7g1Io7MnadWksYCYyPiMUkjgEeBb5X9Zy5JwPCI2ClpCLAWmB0R6zJHe49nCFU06nrPEbEiInalzXXAwTnz9EZEtEfEs7lz1Og44IWI2BQR7wJ/oFjTo/QiYg3w79w5eisi/hURj6XP3wTagXF5U1UXhZ1pc0j6KNXxxAWhBrWu91xi5wH35g7RT40DXqrY3kIDHJz6C0mHAF8EHsqbpDaSBknaALwGrIyIUuV2QaBY7zkt0NP54wyAiJgXEeMplveclTft+6rlTvvMA3ZRZC+NWrKb9URSK7AUmNNpJl9aEbE7IiZSzNiPk1SqU3WDcwcog4g4pcZdFwPLgfl9GKdm1XJL+iHwdeDkKNnFol78zMvuZWB8xfbB6THrQ+kc/FJgcUQsy52ntyJih6RVwGlAaS7qe4ZQRaOu9yzpNOBy4JsR8VbuPP3YI8Bhkj4tqQn4PnB35kz9Wro4exPQHhG/yp2nVpJGd9ztJ6mF4kaEUh1PfJdRFZKWAodT3PWyGZjRCEt8SnoBGApsTw+ta4S7owAknQlcD4wGdgAbIuKreVN1T9LpwHXAIODmiLg6c6SaSPo9cCJFK+ZXgfkRcVPWUDWQNAV4EHiS4u8S4KcRsTxfquokHQXcQvF78glgSUQsyJvqg1wQzMwM8CkjMzNLXBDMzAxwQTAzs8QFwczMABcEMzNLXBDMzAxwQbB+QtIhkt5OfWI6HvtQW2pJLamV+buSDtjLMVskrU4dT5F0saR2Sb1qEyJppKSZe5OlhjE+1OpaUpOkNZLcscAAFwTrX/6R+sR0tKW+AfgacARwtqQjIuLttM/Wj2G884BlEbE7bc8EpkbEtF6+zsj03F5Roda/4d9RtEl4T+rO+gDwvd6Obf2TC4I1jLQoytT0+VWSru9h93q0pZ4G3JXy/AY4FLhX0iWSfpAWQ9kg6caKWcSdkh5NC6RMT69zLfCZtO+iNNup/E9+rqRfpM8PSbOeWyl64IzvbqxKPbS6vjN9H2YuCNZQ5gPzJE2jaHk8p4d9+7QtdepbdGhEvAiQ2oJsBU4C7qP4r/uENBvZzfsH3fMiYhJwDHCxpFFAG2l2ExE/rmH4w4BfR8TngWE9jFWLjcCxvdjf+jGfO7SGERFrUmOzS4ETO07VSFpI0eysng6g6LPUlZOBScAjRVxaKPrfQ1EEzkyfj6c4uL/Sy7E3V6yy1dNYVUXE7nQ9ZURabMYGMBcEaxiSvgCMBbZ3HLwkHUSx8lRnvW5LLeki4Edp83TgzMrtiKi87vA20NzdSwG3RMQVnV7/ROAUYHJEvCXpr928xi4+OHvvvM9/q43VS0OBd/bi+dZP+JSRNQQV6+guprgOsDO19waYCGzo4im9bksdETek0zYTI2Jr5+1O+74BDJLU1QH9AeDbksak7PtLmgDsC7yRisFngePT/m8CIyqe/yowRtIoSUMp1rToTndj1SSdsno9Iv5X63Os/3JBsNKTNAxYBlwWEe3AQt5fpKjLgpDWk54F/Jlizd0lEfHUxxxtBTCli7GfBn4GrJD0BLCSYmZzHzBYUjvFheR1af/twN/SinGL0sF5AfBwem63PfN7GOsDUqvrvwOHS9oi6fz0pZOAP32Ub976H7e/toYm6SaK0zqfAu6JiJqWJJT0InBMRLy+F2MfDVwSEed81NfITdIyoC0insudxfLzDMEaWkScHxF7KO6u2bfyjWld6XhjGsV1hz097VvD2I8Bq7q6zbMRpFNpd7oYWAfPEMzMDPAMwczMEhcEMzMDXBDMzCxxQTAzM8AFwczMEhcEMzMDXBDMzCxxQTAzMwD+D0I+ZtV3h/8cAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1W4TZfXOmIlS" + }, + "source": [ + "Посчитаем точность на тестовом датасете:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HUjdeIefsIsg", + "outputId": "a1a363d4-a307-4769-9ccf-fe8a857b62af" + }, + "source": [ + "pred = torch.sigmoid(net.forward(torch.tensor(valid_x)))\r\n", + "torch.mean(((pred.view(-1)>0.5)==(torch.tensor(valid_labels)>0.5)).type(torch.float32))" + ], + "execution_count": 205, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "tensor(0.8667)" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 205 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_95qF9lY2kHp" + }, + "source": [ + "## Нейросети и оптимизаторы\r\n", + "\r\n", + "В PyTorch реализованы специальные модули `torch.nn.Module` для реализации нейросетей в виде набора слоёв. При этом есть два способа описания нейросетей:\r\n", + "* **Sequential**, в виде перечисления набора слоёв\r\n", + "* В виде **класса**, унаследованного от `Module`\r\n", + "\r\n", + "Первый способ проще позволяет описывать стандартные сети, второй - более гибкий, позволяет описывать произвольные сложные конфигурации вычислений.\r\n", + "\r\n", + "Внутри модулей можно использовать стандартные слои, среди которых:\r\n", + "* `Linear` - линейный слой, аналогичный нашей нейросети, приведенной выше\r\n", + "* `Softmax`, `Sigmoid`, `ReLU` - слои для передаточных функций\r\n", + "\r\n", + "> Обратите внимание, что большинство передаточных функций и функций потерь в PyTorch доступны в двух вариантах: **функциональный** (внутри `torch.nn.functional`) и **в виде класса** (внутри `torch.nn`). Для передаточных функций удобно использовать функции из `torch.nn.functional`, не создавая отдельного слоя.\r\n", + "\r\n", + "Для однослойной сети мы можем просто взять экземпляр слоя `Linear`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "D77pXPR6oFRs", + "outputId": "efa49e5c-72d4-4781-89d4-4ab6597d2b0e" + }, + "source": [ + "net = torch.nn.Linear(2,1) # 2 inputs, 1 output\r\n", + "\r\n", + "print(list(net.parameters()))" + ], + "execution_count": 206, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[Parameter containing:\n", + "tensor([[-0.1892, -0.5979]], requires_grad=True), Parameter containing:\n", + "tensor([0.1538], requires_grad=True)]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0tbe0Et_oiNo" + }, + "source": [ + "Мы видим, что у сети есть метод `parameters()`, возвращающий все параметры, которые необходимо подстраивать.\r\n", + "\r\n", + "Методы обучения, такие, как градиентный спуск, реализованы в виде отдельных объектов, которым передается список параметров:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "B4AxyrFMozh0" + }, + "source": [ + "optim = torch.optim.SGD(net.parameters(),lr=0.05)" + ], + "execution_count": 207, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6eB8v58eo9pp" + }, + "source": [ + "С использованием оптимизатора, наш процесс обучения будет выглядеть так:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ups7nlV22ofp", + "outputId": "503d8ae9-35f3-4ecb-e2ff-4da2ec2914eb" + }, + "source": [ + "val_x = torch.tensor(valid_x)\r\n", + "val_lab = torch.tensor(valid_labels)\r\n", + "\r\n", + "for ep in range(10):\r\n", + " for (x,y) in dataloader:\r\n", + " z = net(x).flatten()\r\n", + " loss = torch.nn.functional.binary_cross_entropy_with_logits(z,y)\r\n", + " optim.zero_grad()\r\n", + " loss.backward()\r\n", + " optim.step()\r\n", + " acc = ((torch.sigmoid(net(val_x).flatten())>0.5).float()==val_lab).float().mean()\r\n", + " print(f\"Epoch {ep}: last batch loss = {loss}, val acc = {acc}\")" + ], + "execution_count": 208, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 1.1270943880081177, val acc = 0.6666666865348816\n", + "Epoch 1: last batch loss = 1.0163036584854126, val acc = 0.7333333492279053\n", + "Epoch 2: last batch loss = 0.9239333271980286, val acc = 0.800000011920929\n", + "Epoch 3: last batch loss = 0.8493780493736267, val acc = 0.9333333373069763\n", + "Epoch 4: last batch loss = 0.7907424569129944, val acc = 0.8666666746139526\n", + "Epoch 5: last batch loss = 0.7453570365905762, val acc = 0.8666666746139526\n", + "Epoch 6: last batch loss = 0.7104158401489258, val acc = 0.8666666746139526\n", + "Epoch 7: last batch loss = 0.6834256649017334, val acc = 0.8666666746139526\n", + "Epoch 8: last batch loss = 0.6623864769935608, val acc = 0.8666666746139526\n", + "Epoch 9: last batch loss = 0.645784318447113, val acc = 0.8666666746139526\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vRLXEQ4Qrcvx" + }, + "source": [ + "> Обратите внимание, что для вызова (применения) нейросети к входному аргументу мы используем синтаксис `net(x)`, вместо `net.forward(x)`, поскольку `nn.Module` реализует метод `__call__()`\r\n", + "\r\n", + "С учётом описанного, можем описать универсальную обучающую функцию:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5c6WsBhlrlIs", + "outputId": "54de8404-4170-4a15-abba-039d06d5e946" + }, + "source": [ + "def train(net, dataloader, val_x, val_lab, epochs=10, lr=0.05):\r\n", + " optim = torch.optim.Adam(net.parameters(),lr=lr)\r\n", + " for ep in range(epochs):\r\n", + " for (x,y) in dataloader:\r\n", + " z = net(x).flatten()\r\n", + " loss = torch.nn.functional.binary_cross_entropy_with_logits(z,y)\r\n", + " optim.zero_grad()\r\n", + " loss.backward()\r\n", + " optim.step()\r\n", + " acc = ((torch.sigmoid(net(val_x).flatten())>0.5).float()==val_lab).float().mean()\r\n", + " print(f\"Epoch {ep}: last batch loss = {loss}, val acc = {acc}\")\r\n", + "\r\n", + "net = torch.nn.Linear(2,1)\r\n", + "\r\n", + "train(net,dataloader,val_x,val_lab,lr=0.03)" + ], + "execution_count": 214, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 0.6797666549682617, val acc = 0.06666667014360428\n", + "Epoch 1: last batch loss = 0.6262387037277222, val acc = 0.06666667014360428\n", + "Epoch 2: last batch loss = 0.5835054516792297, val acc = 0.20000000298023224\n", + "Epoch 3: last batch loss = 0.5517064332962036, val acc = 0.2666666805744171\n", + "Epoch 4: last batch loss = 0.5291333794593811, val acc = 0.2666666805744171\n", + "Epoch 5: last batch loss = 0.5140607357025146, val acc = 0.3333333432674408\n", + "Epoch 6: last batch loss = 0.5050028562545776, val acc = 0.4000000059604645\n", + "Epoch 7: last batch loss = 0.5006228089332581, val acc = 0.6000000238418579\n", + "Epoch 8: last batch loss = 0.4996645152568817, val acc = 0.6000000238418579\n", + "Epoch 9: last batch loss = 0.5009827017784119, val acc = 0.7333333492279053\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KzuIDqJ8sFYm" + }, + "source": [ + "## Описание сети в виде набора слоёв\r\n", + "\r\n", + "Простейшую многослойную нейросеть можно описать в виде набора последовательно-применяемых слоёв. При этом такая сеть будет обладать всеми характеристиками вышеописанных - она автоматически соберёт в методе `parameters` параметры всех промежуточных слоёв" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "tBtytmEAsq-O", + "outputId": "06ad840b-c2b7-409e-e01e-a9170548151d" + }, + "source": [ + "net = torch.nn.Sequential(torch.nn.Linear(2,5),torch.nn.Sigmoid(),torch.nn.Linear(5,1))\r\n", + "print(net)" + ], + "execution_count": 215, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Sequential(\n", + " (0): Linear(in_features=2, out_features=5, bias=True)\n", + " (1): Sigmoid()\n", + " (2): Linear(in_features=5, out_features=1, bias=True)\n", + ")\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5r5RbLB1s6YB" + }, + "source": [ + "Обучать такую сеть мы можем с помощью описанного ранее метода `train`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ogXKdcfIs_ND", + "outputId": "957ccd8d-0076-4e9b-89f1-edc1de75f18e" + }, + "source": [ + "train(net,dataloader,val_x,val_lab)" + ], + "execution_count": 216, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 0.6622872352600098, val acc = 0.6666666865348816\n", + "Epoch 1: last batch loss = 0.6727332472801208, val acc = 0.800000011920929\n", + "Epoch 2: last batch loss = 0.6326467990875244, val acc = 0.800000011920929\n", + "Epoch 3: last batch loss = 0.5876082181930542, val acc = 0.7333333492279053\n", + "Epoch 4: last batch loss = 0.5610458254814148, val acc = 0.7333333492279053\n", + "Epoch 5: last batch loss = 0.5546905398368835, val acc = 0.7333333492279053\n", + "Epoch 6: last batch loss = 0.5626044273376465, val acc = 0.7333333492279053\n", + "Epoch 7: last batch loss = 0.5754281878471375, val acc = 0.800000011920929\n", + "Epoch 8: last batch loss = 0.5850474238395691, val acc = 0.800000011920929\n", + "Epoch 9: last batch loss = 0.5874522924423218, val acc = 0.800000011920929\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jY4R1XEGtEzJ" + }, + "source": [ + "## Описание нейросети в виде класса\r\n", + "\r\n", + "Это более гибкий способ описания нейросетей, поскольку он позволяет выполнять произвольные вычисления." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "SlsJmGu0tMsZ", + "outputId": "240d5c89-096c-4392-99cd-1ade5ff3e3e1" + }, + "source": [ + "class MyNet(torch.nn.Module):\r\n", + " def __init__(self,hidden_size=10,func=torch.nn.Sigmoid()):\r\n", + " super().__init__()\r\n", + " self.fc1 = torch.nn.Linear(2,hidden_size)\r\n", + " self.func = func\r\n", + " self.fc2 = torch.nn.Linear(hidden_size,1)\r\n", + "\r\n", + " def forward(self,x):\r\n", + " x = self.fc1(x)\r\n", + " x = self.func(x)\r\n", + " x = self.fc2(x)\r\n", + " return x\r\n", + " \r\n", + "net = MyNet(func=torch.nn.ReLU())\r\n", + "print(net)" + ], + "execution_count": 230, + "outputs": [ + { + "output_type": "stream", + "text": [ + "MyNet(\n", + " (fc1): Linear(in_features=2, out_features=10, bias=True)\n", + " (func): ReLU()\n", + " (fc2): Linear(in_features=10, out_features=1, bias=True)\n", + ")\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HwdapRxft-7M", + "outputId": "6eb900cf-4902-4a04-c62b-497b68455406" + }, + "source": [ + "train(net,dataloader,val_x,val_lab,lr=0.005)" + ], + "execution_count": 231, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 0.6930208206176758, val acc = 0.8666666746139526\n", + "Epoch 1: last batch loss = 0.6779811382293701, val acc = 0.9333333373069763\n", + "Epoch 2: last batch loss = 0.6627538800239563, val acc = 0.9333333373069763\n", + "Epoch 3: last batch loss = 0.6472628712654114, val acc = 0.9333333373069763\n", + "Epoch 4: last batch loss = 0.6321150660514832, val acc = 0.9333333373069763\n", + "Epoch 5: last batch loss = 0.618022620677948, val acc = 0.9333333373069763\n", + "Epoch 6: last batch loss = 0.6052160859107971, val acc = 0.9333333373069763\n", + "Epoch 7: last batch loss = 0.5925578474998474, val acc = 0.9333333373069763\n", + "Epoch 8: last batch loss = 0.5803073048591614, val acc = 0.9333333373069763\n", + "Epoch 9: last batch loss = 0.569009006023407, val acc = 0.9333333373069763\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dvAiaj_JndyP" + }, + "source": [ + "**Задание 1**: Постройте графики ошибок на обучающей и тестовой выборке в процессе обучения\r\n", + "\r\n", + "**Задание 2**: Попробуйте решить задачу классификации на датасете MNIST с помощью этого кода. Подсказка: используйте `crossentropy_with_logits` в качестве функции ошибки. При этом в первом случае на выход сети необходимо подавать целевые значения в формате *one hot encoding*, а во втором - в виде целочисленного номера класса." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gZ-kWx84bMDH" + }, + "source": [ + "**Задание 3**: \r\n", + "Используйте PyTorch для обучения классификатора на сети MNIST. При этом:\r\n", + "* Обратите внимание, что в keras заложены типовые датасеты, включая MNIST. Для обращения к нему достаточно пары строчек кода с использованием библиотеки `torchvision` (см, например, [тут](https://pytorch.org/vision/0.8/datasets.html))\r\n", + "* Попробуйте несколько конфигураций сети с несколькими полносвязными слоями, передаточными функциями, и разным количеством нейронов\r\n", + "\r\n", + "Какой точности вам удалось достичь?\r\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "MG64NKzawHl-" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/3-NeuralNetworks/README.md b/3-NeuralNetworks/README.md new file mode 100644 index 00000000..cb710a41 --- /dev/null +++ b/3-NeuralNetworks/README.md @@ -0,0 +1,44 @@ +# Introduction to Neural Networks + +As we have discussed in the introduction, one of the ways to achieve intelligence is to train a **computer model** or an **artificial brain**. Since the middle of 20th century, researchers tried different mathematical models, until in recent years this direction proved to by hugely successful. Such mathematical models of the brain are called **Neural Networks** (sometimes *Artificial Neural Networks*, ANNs, in order to indicate that we are talking about models, not real networks of neurons). + +## Machine Learning + +Neural Networks are a part of larger discipline called **Machine Learning**, whose goal is to use data to train computer models that are able to solve problems. Machine Learning constitutes a large part of Artificial Intelligence, however, we do not cover classical ML in this curricula. + +> Visit our separate **[Machine Learning for Beginners](http://github.com/microsoft/ml-for-beginners)** curriculum to learn more about Machine Learning. + +In Machine Learning, we assume that we have some dataset of examples **X**, and corresponding output values **Y**. Examples are often N-dimensional vectors that consist of **features**, and outputs are called **labels**. + +We will consider two most common machine learning problems: +* **Classification**, where we need to classify an input object into two or more classes. +* **Regression**, where we need to predict a numerical number for each of the input samples. + +> When representing inputs and outputs as tensors, input dataset is a matrix of size M×N, where M is number of samples, N is the number of features. Output labels Y is the vector of size M. + +In this curricula, we will only focus on neural network models. + +## A Model of a Neuron + +From biology we know that our brain consists of neural cells, each of them having multiple "inputs" (axons), and an output (dendrite). Axons and dendrites can conduct electrical signals, and connections between axons and dendrites can exhibit different degrees of conductivity (controlled by neuromediators). + +![Model of a Neuron](images/synapse-wikipedia.JPG) | ![Model of a Neuron](images/artneuron.png) +----|---- +Real Neuron | Artificial Neuron + +Thus, simples mathematical model of a neuron contains several inputs X1, ..., XN and an output Y, and a series of weights W1, ..., WN. An output is calculated as + +Y = f\left(\sum_{i=1}^N X_iW_i\right) + +where f is some non-linear **activation function**. + +> Early models of neuron were described in classical paper [A logical calculus of the ideas immanent in nervous activity](http://www.springerlink.com/content/61446605110620kg/fulltext.pdf) by Warren McCullock and Walter Pitts in 1943. Donald Hebb in his book "[The Organization of Behavior: A Neuropsychological Theory](https://books.google.com/books?id=VNetYrB8EBoC)" proposed the way those networks can be trained. + + +## In this Section + +In this section we will learn about: +* [Perceptron](03-Perceptron/README.md), one of the earliest neural network models for two-class classification +* [Modern multi-layered networks](04-OwnFramework/README.md) and [how to build our own framework](04-OwnFramework/OwnFramework.ipynb) +* [Neural Network Frameworks](05-Frameworks/README.md), such as [PyTorch](05-Frameworks/IntroPyTorch.ipynb) and [Keras/Tensorflow](05-Frameworks/IntroKerasTF.ipynb) + diff --git a/3-NeuralNetworks/images/artneuron.png b/3-NeuralNetworks/images/artneuron.png new file mode 100644 index 0000000000000000000000000000000000000000..29ce411177c224ee9de5513ee342d94fafa68cf2 GIT binary patch literal 8166 zcmaKRc{o&W__vZTg=j&ROo^moQnqLa*_W})Iu#;IiZsj^zV@WZ64}ONcn z4vwP$2gi|R9!|EUTO~q-Jsk7YHq++ds7>N!KICHSM*^)3bU3PU5^HSb_%lO?Kn@Pk z(f{rv=Y9-su#NT>rZ#$qhlk(3b>T-wqGMl?$rKS0k-@=1bpuDXe`jaM*x124T=3yQ`nDc164>eM-XtTAV8wrmeQd-RXVaQ}~ zv(x@>S43e&>)@D{wG4>=m7Xkrj68~QsmFlxk(^@>z?3b)o@$scI@01k)NfwKy`ARr zDevdsI(|T|K=Ph-^W@m#Py$`zf8zo%NBOXcA47{z`}i@E!% zcT>j2geP`abe@P9K5_S%_LTb$YAglM8F?3&8ET3zchfw{6X|3>Vl}8^@xqTDZ6*Be znwNNh<_VtIx(u|Huufq%RvHt>aRwQC>Vib8o`aPH&+F#sz~l#tv0{$ITdsRZ^~ygX8>I2!fcN~P&iC*m^GF*AJD zDeu=>v`>0neL?-cgO?>yUOD6NPuSiru}by#G|vw-!0 z<@do+qnDxf3lVl$Zt4_nx#JjE$Qim$?5);rykEe=A%N+S;8(PsM>;xF{@-uhJ<>UU zZMg15NV?Aq^HbQZF1g(Z5`0#=5tV=&9eGVm5&L4!9BH}plQDT>)4O&CyZt`lTT=^Nh~f##}{4-jn*MH zqtIFgORCK~#vNK4=1(0`OtN%I!}pEXgnT&`?8d^8k^szRWLU;G}8_TS~@se|=v%Fui0<7cgBOkV*P`WPyeV{2X^;UeRyo({;!_-_coMO4PZ z>OcTQqEXA_*$ZUk@zE#=>hM;$raTDemJLMf8W=U4YZMF9yc<{*ouaMua z%HP~eZ{2?nJ^O7-GZ(b{Wu7D$O`DiT;CLf0v?nFRzmnkAsq{IrCCNeke~$yf!J5MU z_Z1c7AHSI9dc`v?{67dkB(ei)5op4-_zNO{pa8*bnhcFK*D1T)+3t9cI^#-9RF!MgkKC4=oJ zn{zCofc?TrgMCHf#lAjdXFMTsP^-rt#uGt1P={?)F)wIB9lMdRS*kXUb_7J-CUw#1 zYPfw8QsvD&)>CHvD^7r9J-&dt@f;nTMC&+Z|1^h6akQ)6ItYC_n7flpd;Qh4T|m0! z;_z37V5;~}iFR6HU~1lc=nBeQm79t`zMEA4(H|>2y~vUiWwA{AH@8Wsmqobu9e&I2 zmgZ1u6Z{S*N!%Z1*a|j~ipM|g57;8r(w4eJ)|{vlkIsFdO~{e@G>xDEAW2IyiP3*; z#c5;0Ka%#4P)leCrs5dWT5D7L$CSP>zI-%5wrg2&ES2v1ai2N9rFmoRha@%W56NxK zX=7WG8a1%y>G`OTgA^wNQI;oBsi=AEuF~dqm?yP}*ARb`Z6c#WkFSXier24;g27Ud zOCN$evuVrEMJZH)f7pQ{RASIa&z2KZ+IGk1ce_df4@9VoCPq*_UX5UXh0Bm~2y+G* z1fg83R%Z?{76N{zPIiH?N#HC0c?OVyMr%eJ^I+17eIvRW!W?D%E&Q3f_=SqC+01A} zYo28OMqMun*9CEtb~{F#ie5MKn#_9Pi%!ftQGZF#{G}2at4i}@sO{h^*fs6q>s<28 zbrg-EV=kw0SDSY>c(&GbBcYG@UknJ_Y>f;IS#v0eQEjhrk3mFYwWi-Vv>(tmd@@LQ znNt@~MUak=h#yTZAv@~XaNg{27Y&qw_pL^-`+evq+9iT))2K^XE-@KVA0I5e_ShS& z!{z`J{7f6=p(Q}tXVls*M~&2k@dmHlXyCalk>7UDl7uhXP0*qjEwp4{Iv-`n87bpV zU$M}N#!QV>rLXY~W}`Fjk|ij|_gvo%ckK&4+1Zz$*XFXD>9Kg2BT(k>+AX%d5G^x7 zXBG?HF6cPE2fMOjB4S{cj&}=hb(Dgv^tSUx{xAWLq)yt3?*V5SjUvYrLk#QCwa36^ zC37TsOQPF)lE^S4^@_U#-uS(MDP;e%$=Z(M-Zib$Ib?+sWCwq%oa(UQw&;>;Plj(* zX=(AxgX$_*v`>Z=t3AmZSuKD+VH!RQcsM#6qEsMo7TOj!P|x1sTntv2&9Y{6Zc4r8 zmNWWoT+F!FD^{4aTCEag;_j4_NZ)oZetJnb2-aBPnP&yf#5Mf;^tJjgNBe4PG*bbG zaNLV>PjD%$eJSd>6Lwq$rO>^H9U3=&7Qm9=2@D;^jfs73YM75hPIuR^;iAy zana>nP(k>5U62Db{V|R^@KO9|>H;4lGbPvoroK*Tx;fT!u8+NYs*AK6-Xw+zbZ0L9 zSai+@UOY$F25qJcsF?Svr96G(Y6b(lgn5{uMU|ONFWQGcp1uO%wRNjB3oy|sT?f7kP307veOd7GYa& zNO@WBeQ1w+F`59e-E#`Mk6VT(zZ-{sZOl2^xmZx6W>mQyr5>_>x#5>wjByDdy=Q$j zFY!67?cl_B-=RwyyHgu7an>hpviXqLN?)N}OlTOm<=LKklZTUI-~h!q|4ow$VEs~# z<1T%oaaZN6(yUQ=m#VB%m27SEiOF_oP&>=?QLa}jo4L($&*jWU-$Aj+lb6eC`v}8E zv}I1n>_;CW17T&M@o`ScaOSrozJB7L(%sRP4YUzVqe|^j%V@nhaD66a%tse(n5V8e z>{<=vq&)}%pINHe&0G|>n(6t5OYwwgd_Qsl8%hBwJzvI}KpE*Jn%amK`J5r8=aa<2 zB@fkYx@YI(P}hj0cvRb)e0{!A?nfx>^p?^Km%S|GH4zWn1*WzI2qkcPM%~G{=|Id= zdx;{~?E()k3lo%ensp{k?_5d@_yispg}I+L04E)t3UV)z5ca@-xD`fW4L?ttKbun3 zX-EkWTHk@_qFBypW{?ogSKMdeUq@jc2fm`~G(O;0InPo6CD^7p+)`{V%~fHXK4j{$ z{Ia-ZfopVjGv3WvWBi(Pkhy!Q6%6gk8XumzMQ)J^GTNRC@JkeRp~?M zp0f4zJewx_vSz5>hmFfjxJiQ;K1Bqv6_VMmcE`F7kX68y7^NSoj8jF)K5ggAJmaL_ zT+^AHI-c94?8y5+w>d8F=CIn^EZMzKB!Y&7dA%0DO#cD+bscp_;vgCr@l} zzGziR3PWYcS>nVWVf}GhQCJEVGz+>Hc~(tQl8d zH65(umSk~IC$w`cVukxh&d(Op4EqS%3eKvryp31CHkz)v6fi?Qo2dsyq6ev7G z7^jQ%UDKfas>1vWa68n5*X~RN*@2zuz4RAsO0=eie_6!=@#&$`bNcV2OM4;Fn~F6r z6XQv?0gaYA&7%n!2g>WkCy0okZ;Ob~f-m$NRsJR`2;WeH2nW3A@rRE7HEsVi1*Pvi zJDLa#V2qsCp6xk&B33Ft6}{is{&X1b!GAYt_&lHh6;))m(4BYCGz9Xxbkv)EgVxt{&O-}JwC8SPU6|CWN+;J zq>fiSJZoNx_RGTjmx24?fcvp zr&kCofnh{2uMvZ1LW5_XSWl?iy8I)`Q8-dJ5pXuSZvfo6%B2cE4ty$xGcHKwj(R^l zyfk7F-Zmc{`UVY&#ng5R3axAL10EI1JG@6MoRadgtM@N;TfnZqXL(Z1(-yl6uKZh9 zKJ;id)#%UR?Co`arIE;VF?pKpZEGVyEfK|JzBqN1Y<6zF`G?Psb+9RLk|2I#ZhXg8 zE4-}reU;Td(EJmnt*vk>h$a>8&H7GFNQ6v+h%5=c(UXvb#{KqpoBZrw5^Y_GenLVWjL)`pmk(dN?u&bIPC1QH^4i;_LX|qZ&Ge+V$A>hx zC%Yxlzc*y!t<&FbwO;w7C*&8jer61g~2=y_qsp4t-TJ$O1qXyWhYPQ6k1 z9u8*#*KR}oNxcCWU^~{f-~JV5zSrXj2P}7%R2TlKw{_Q@UMdYZJcA)msAT_!0JYyQI$Ui z=BYNU1k1?l&auGXuDi|PyforocIaB;KP)?T6jUZ6G?8QdPk9 z%oGLF-YWR_E|xEO7}xy%^m>%4R*PN*?e(47d{<(>{>+7yjg9%EvN<-SYrJwg3^rQk4=Xj@K866rqJ+)Uk{rV(1Suf}^))M7{XMetYgeKNUhX+u3KW9rj2G{iZEo$z%Us_5#!$&(o zD(P@umye}Gdt<#sC)`iIXLMnQP&X1o;pL@6ZHD;=bb64`ME9ROi=~fW<EO8A5fleF2I~6sTr|Dj`da`?thIur z(Rq9M7)JcWQ%fx<`3I%ORE2y`<@B4|Ux!K-oCx+YTtLY^mA=g={|}|<$H+rti|BmU z#r|b;+SlTh%e)qc>a%yT%gMv@S_uZK z^D|+B_l|4z-68zi;FK>Qz3E@xkf6yi^D})VsuM%XU>ltgh+k6K&0E=)vY$;g-%5}Y zZES-uX(gal3Jp(s@bmx@6aXH;(FU| zqq9r4o%Lz|)hb?ntDurhORux=f9hV;NoUWuWb#`F^3cNEp^6iBJNa!B`H5vfWMwAe zo{AV|cJcuq@XEKr4l0m$#ixXAYsF~!0+%Zc*kAamScgp8Kbe1yhpDdZx)I&tom?{};4Pa@infEnq!JI+#1 zCLWLVU#e3yL1kJC5&O>LdqDdepox`PAt~`4LZ=^)@>nhRIh=)>%R7j?P^W0Yns2(5 zX^nPl+Gw^ zeh)XmwE{Az1w{g{Eq;VtJWMHiDwx+yU;Wv3*WK*{BHa#5wXN8a!!~ zp#X0~OBj&h zW`J_aUF;3OVKtWZ-RI2M`I-)-^g}ig2H%U+{!LdmNZ@+eKo?~HF-VoI)Q~PF0;}Q$ z5XK$k!=7n!v7JBst!oEgoI)1k)9;d505QRkC<=p@ zYv5-{t_MtAu}zCN%RV#5$T8*cDG&y-*ZtPzX6V9Nr;gxn@#mzNnv#Fu~LH1XuV4#{QHl+zi-s&HIKsk&}Lj%wfz?$oZ$Q2<}^bh zWaW}MCbXj+Fq^tc`vkRd4`-5SiLaqvQE zO3qf0vdkX^wnfC6PsoJaZsaH5#`sHUhKssd3djG04GXy%c?$|IQs|PURpjYQbRL35 z11|TqYdrWk9%pT1!wIbYxq8UF-k^`tFS)*R8CY!MRDYwgHo|b(ua?fv@A%$~maopD z169($W}>6$Ufz4E<)E4`y)vE3`qlN1s)iH!2`-h-Fp=}EHU zouZ8O6a|IvtISp`C)Wb2sfHf# zilkOurM$tZUbF%n9W#?BSSEt$2v>?P?P;92^fU~Q<$`A;)O`eMdsNKiYzzY=2J~2(9CxUqhAp^IIQS!$0!r-g3Dy z8!}qqAO5_+HJ?5&XA@iIFS@Tjv={r#eCO=wPS49uga zD`x7MnTt^T6RD$hUgtWrUKx3e03aIiDv7A%fLX~k<}1p(rV81R?${O;Dbza=v~R0+Hm&^*t%4@F`7A9dEeZDv)G>;-->K{m6R-QaxZiqeRWvB1Mv zl|)K42~9OE(rj6L6P-$)GW)1V2(;Jo0a`KkrIåoKg1e4c{n7Jh(L~NcC;vodu z!HB{L?00ezXy2S-G& ziS=;5PHjk>JrbPc8ih#PBxjV%5Q_LLV9S#1!Mf8}9*QxDjj}RlsW;xMF?dpgehSf( zAX_?OYMRheCxI)uMA?v|z?Ck7|8L^uMLy_%5XRW%#xy-Pr0M{uxK1civhNfk z>Ozct0OSrop=j3@Mi3?xeYb)wr6nNT)vkCDsCd;?;EEaX7i<3?vSn7qY>Orra0Ny5 zpEhP^z!E|HfJM?zYt|17|BGGPYh@h@SF`{&LWp3$pWVrZw3dUD;t*v9Yo(;2YrJGj z{qj)2d<2dg{ymFONJfqV*ry=`+*l%S)uZ$h$zsIj*Fy@d_qt?Ev4!>2X^}zU0B7cx>_4!Q_s^;dCM+MpTqNgaX8g5ddeA+}DZ~}BGI6mQ17DUygH5(p zo;X&$Xhev3^JS5rXAraRG_p)7tYZ1C6{GPLZ=kj(`{@yD$N?O{Wwz|Dg}Ocp*UUA~ zNJoYlpTAXO2lJl#9ZeC3;M6PnI{)ZF$J#T#W!wF2j}Hj}D&35Y-ge!1qn%`o>xCWA zR0bn=!`kB09f&WN?OEI!{~>?Oq*ot=+PM;ISs5uKE1P#rQ9YAM$Ol+z}WTTGD-3{y(Ru%*!jzW2^kVPj5as z^y>QoxPE*P#q20`?@^WDCrw_xy_1UNUz|nn_Bw6TmS1FNZNjxDKBRu;TU;WR-Ym0_ z@fdafxPnPHGQpcTGO=s zrbZoC?#Bx+y<=32Mc&Q8tRqW^t_M=*yNTDP(8sH_reb`%ZoC4kRJLBjtF%2>_E@Vf zcKazEo;-+2>hmsK>eAL!^vj^c4>bLScJ#WjrNlYqsX?GYRERgJ+TTgFlsamw;G_%0j|ujGJZ{o zV$f**^3*v9k-F&B^xeO(zc*2_cdjQ-!qy#?wHJ6Xp;=f@g7vldTqAJ-_6$uwS+V^& zO}^Mu^%{)-SFN|aXjRn7pDrdjvKwm462Daoo8#tN=EA`fW0O0l${gFpO32uZoW18i we8!ZY^x0Lv!uSgmJ<|~S-_Ef)<}QYEHaGBYx~KPh{}*JWXMV3*=RxHE0gBoeMgRZ+ literal 0 HcmV?d00001 diff --git a/3-NeuralNetworks/images/synapse-wikipedia.jpg b/3-NeuralNetworks/images/synapse-wikipedia.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0ab40f436ac6b380df08a3552ac7e64cd5c858e5 GIT binary patch literal 48364 zcmeFZ2UJwcwl>-%5lN!tEFd6JvSby>l7$AEAQGFl$vL%vh~y-oNR%9!oTKC<85(So zaJB?tE%Q#vu1s(W);^H*FONamE;uV02mk; zfE@H6;Ck*hO4bc*0RX6|05|{uz)b))hByEd9V0>CCeF5cm@T{)c=bVtjmJ{348e55)u?i3vUg{8btN@C*P2fB{Yb2Y@?(5ugRI z1vmn%05<3}2s-|gm|V}{Gg~=1*@^M;+BouYCUX!Df?FzL z>J%Z&&-6|#7uyX+e|>fOhkz95x)iBE2h6!Z)W;O?RL%% zuYarC%#_#swK>$>#>o+l1|KgI@825zKUF6hqCYDAOZd?;qVtH!IGCF_nLk4Zsh=H< zmV}#MCd*?I{dAAEpz+-UT?n! zOZ_vw{+p!eAO03WSAo_)JUl!J-hZt9U*Y&I1+9MQ=l)Pu^plMLFI>MW`Hw*U1=laQ z{v!nbqs+gw>la-A5d!~F=3mPEkLDP+$cS$fkWi9QQ&EypQqa(|-J_wqPe(z?#KUx-{Q)NzCp9Cl051nW8wV%H z&q6S8Z{ECxcZ(DspOk}^l9uB?zOKIlNN!>jVfJBRFaa=0FtA84uG;_%Xa^+@+NJr^ ztNHf}0}~4y=LYUgyj%F_3>CKlm>5`CnAli2IM`@^CWbfqcK|jC&YgRFPj1}RFu`TA zC*}8zO1sJYw4|9#b7+@E;FW_P-Ys$pN-FC6tZeKLI0S`+MIMQY$vl&llUGnw(t4q- z1Ju>iH#IZ2u(SeOJ32YLxVpJ}_`eN!7x+FXI6CG-Y+U@uPYGYrGcvQXb8_=a%gQS% ztEy{iTUy)NJ370%dxl3w#}MNa-;s0k3yVw3E30ekd;156N5?0pXXiiV!T@0XO{_m8 z`zN_b&~jm7V`E|C{*(&?(-j@CNU(A4@!h!dL<85v{w@>0?@iLDQE4U3c+3KtyJW8% zhHjCw2+rN#`zhLQlKp#v`Tdt9`$MolP_Z26_>4oZ z0n#L}*;gS=r+X==nyURm{g2_=%YrmzT2F0Po-#)0XgNPV$!?g=0o<7GdPXCJFJi4_^0O1F$!YL@M?O zw*wY>rOV~9>7B$HDr#;wJWCIiV!;YIEJSy6X?-(JK z!gGLI4{5Mhc4=)}4K<*R?5trw;I^6wFF~l@%7u01#7$frhZ~|uF@nXT4hV2=dI=ub zw!`q<$WVrGY-}4k=Xz`SfRiqD-|i+Gnaa_=?Y0Y9xljI{xQg_ApnnBu5~L!nr&q{( zpHJqtpZd4fQL-P7qd%Mvg|f3wtDcP=Zd_c6o%C0ZOfS^J9*p;zQJMH*JfiEaxaAx>?@;KW}-K4aC0U=NX!Z!81$7>@^WXWFqumO?Rr`Lv<- z!&yFCTgYob{8>21PS5e>HK6O5Te@A0CwFdOYeVv>K;~Rf?FY&aIFCeUFld9*QZHq8 zvx6#blne=dEVTJ>cZt;P`jmAKDiP!i)e6rh-w@MR*}4~r`!I~3#lT!pP5!Jy zC_XFw&hX5^dycfCy6|kPsn4H+n$?9bQ#~pLwhS#k%{Nf!R*dopP7a813(E(u)n^-X ze=rYZw8R<_T1h5tC?9E*K5fC@$bl}^-iFsI%{?!8nv_SZHSk0ig}=udlWK*MDnZz$ zmDP!yaK-Hx>#=lkU>50!(&3nY8xuVL-jpibhz_JJvH?yRX3uapOGqoZ}F5;_|*yT_NSAJ-tjHC@Gt9 zDJ48E8uq`PJNF>z%F3TJNrm}GAQBrUX2*9)on1uxGP2h#WKvg0Ii>2BtEjKQx`HmkDRbMrIatr-c$go_la5PUh1kmoC&G`%vreIhD1a1q7@>L&1 z@Q4|QJ&!xj_S@lM(M=81(P9a=%i#}XIi|c3#G@zcS~dS@b7$q0t|MpxM(h@%0ik11 zKiJlpw(+X;Q|JyWc}MfX%&RpUF5%YsR0IkC*!b;SLJqQYyXzUp4aYhgyv~=3KgO~4 zh)~3YZe^`aB!bJJd~%(#I!_mZ(vGhmUdrP(Nws{MBjj*vBsl$SJSuVq0ys)8D>F1Md?NsSzD15Z4Qy&9W5XOZ>M1EAd&R9_ibv(LIV-KIU4 zx!*ms28cfzYP!7U8(}@R=iM+}7_nPOsK7rh?}*t`bn*eU;lC}WJvmdAVC;@JZQPB* z%{`6iK+*2T$Tc#Ky(*8^_z(dRkJ_ks^5~?4W4b8F3DgJJa6!h_tp#TyH+;w*ZzLW{ zbtW9>@}kEEk_J?>r`<+wZ!p0&!NW7b0d2v=62VF*?$24DVcI>4Xld}d_==+Mi&=jv zN~8W(1-xGnVtXb7(Yiqjc7NTV+u7LQGyj51p z8m`d0P#Uj}cT5(M>6QYlIo*M~xj-6_+Hk)Q(~da_RYh%xZw2VNdgB2YT5>Pt-z~#1jo}|( zwL%^lWny7M=$s>NPJxqYd{aeEz)SFQVU-|8FM>FXcr4~7MD6O_f|$XUNWQ+)@Cj`A zVZL^YLEfaXaaU};1=6-nQfEMaRFjTp2FEHYomTYw2$sD>vS?wIGU9OBx@`s=50<#4 zU_oV=;oZ@K6~XAuT>^EsR;5r&U#cC zEgmFnum%_}IDbsUqZTLpT8v_Z_h~;Tq*Po#)V!3`oLyP(3TJfymad&P8Sfv85ud8n z^B$A9UKLTq)>#M(qvyGru-R}9Z!P_blol;(51-;nm!&75MgM-1%dg)3sE=C-is(5bpRmX(%g{eHKe-Dh;~%*UQHsTr3sxf(NLs9ru}V~rV1k!!znb=0h5 z>|KD-+2FpN_l(>Jv%ZO8k1LqH9$^xzgZ#+0%aB$WV9Mm4now2c;8C(NqYu9x-obu$0+LA8QSDTErs9u>r zQkpJoyl}k+@XCw?eeM1M6u$=GodlpJkBzt7yrw4|D=ICoE}@r>p!oc|x(3`Gb0k6j zo7k08j&DQiy<>#&l`(p~%sFx#4ZBTtp{&(&scgbyD?26A<+kJEcHVN%yHqC8b z1Jtzg8WgpZX2<6vfYdX|&I6kf<*!aVm){3J>^XneLA;_+yPb$^LuyXhH26siEBNst z>m5}2rtA|Iy!4p$L8q=AxAiM(D)!rUG5n$)T{OK@xU?Sd<(}E^1n$udM+xV0emp=5 zbzvQJUE+wm^Gn|X?Oy|`iq1e9a`~~cM8`1RQ*T1GQ(Ygj+bpxlwTg)x`WK%|C4n}N zS90eoiiG<3Ta^vvwvX92r!b>e?j3Y>h@r@qXGkjp8VNTFhwi!XN4L9+Ybc@QLg1>( zycnmdI3TNl)*mxdUrR$?i_BnD<0h^kYuS9Ww?yxG-?weKYWmhe`KkIq37LgjF&%V* zky+JV7LmvJ#B}*PvLfU<3n?*WwNpA9sVc#7-WnunA6nL5c{c}Ygw-%l*&7%K&fMMe zkuF=`x1L=spkMoVyRRy;dI2l^!!-bR-Ge8k;4}xq-;A&5$VzZZ|H|fd&VW|eLZj@1 zAGz(7F&=C-wRJ96xU|ovV=9j?d1sF#Z*S;Zp=a6(l}%}r$-05*{e|q6WWI4BWu?71 zU-*o7yvk^_UKfj8R~KllngB(%a?Tx)fmR0_EyLP`mLAaj@xjdm75_JQ>QdWJt^w2~ z*&8C%u=j_a;&n+3Z6!VPoSyGV^yk03QFSr!h%Ll4FKQf7daem0ZKj2d;(R<(&%eRR zw=7XG@5Y1Lv3kQaD_3RPJZrvI0fI9Vy)OETC!GuWapUSnAbEQ*hYEqegF@iFKt%mV znh-T=w+}{ce%yEDU!5fn-M@Hd@Z*8wS~#w8OC9e!L_YM#;l4$69G-rZzeIaEU$b@4G&oWt*f(i157o2f^p=|>0@|ugdPaiu{CM_bg&J1D7{@&BG_g26~uJ1EEx=;G29}l509T_n2w{F2fsm z2rimAhcb02*N~=9NtRMkDKz7hMDulU7NXiJQI}00uK~CIDD{nKh@^0*Y6-cAcTxj9 zZ{_xrJ6{Nn&K`G)<~Uv9$`4ya8*(a@lt!yH9wWM`V7cl7hPKD|t4@|KJ5M#Pf}mGt zFg_G%n}g$Of;J+PJk2>vug{Z6qV5y-P{wnoR+1=19^dqV9HC33N@#p&o&M5RWS1OY z*hD{HyK~b--0W=p0mGd9?{)jtIJ1D;eOrPimzg1H&-%EOxl8Sz)9lb{T9ROaJ8WBAVR z7^Yv`nY`FXnO?GEMb+-2!aRNA$7^N7+L5N_ZIxa7JYpvAyK;99C+T!XC#~ajPAV7Q z@foz0vdYV#dsgiaxnZwBFdyS@umE!WkGJyPlaX@ddJn zY{_E1H2S#eXs82E_fBuFs%1@J4*XW^seomd`?06cPK-z#2>uJX>J2>oT%zMmzg<%N z@Prb#)XFB;m}eVVdpHz>GT$p`@#wyGQEx4(aEedRNy+KPP6!C77@3s*gr#jr|3y=*u%c6OPV9I%{sN2LqXCn@Xcg(#<_$z12uux4YBkUY9~VEL^-xIF zHa%jrs7?77-Fc9Oc&{(n59Z=pXz=XXVpPt(F`i_KG2ubJ?m^rW;z^^dct3g=K}qj2 z%_&Q%b?@kvShAY_8tivt@Q2E#Cm6PLEqxhwTID62qV_Iuqhf_1@N%O)e6TUp5O%D0m^HyQcD zeC3~1Wl82%dUz)%QAMbg`;Ad97LZ_LUjYg~|1p^UEK4*KV!#!yc0sH-&4=nO{kCwI zp)7kx{Bp|ibTljzymJgaCWjvzc&9wf-M7D*kuvhn&UF4$n&10bYbC|rp|iyQ1xRLC ze`iZ2^E*xG_CZ*!?LgtCMeV@9b_pB%X2-B+vikxd4QPA|R_3%3LdnIZ&O`sQL*v~* zZ#duAjR?36`1tl(tzpAktdob&nAPRgZ-}0c|HY^V<+G`1tp+U_TwbXf(ebKPm3ltn zf_|xp5&xm^g4xnP77JHUIR51^r`fS^pH&0*Hx5&VUD{W*e|GHe$Z)u4PPrI{>r1k! zzOyn9RI3PAAT8i2&f)V6cd-pn(o5XRS z2k>`1{?vPWxaNb5lWRcW{gDvxr0ssmfIFW%z1oLlo#{rSp(^^1hk1wusZZJ8%!^0h z+!@R`bH3_PN0_xm|AHB((1ZJCy``d_eVl5eRMe;rwG7qvC?D8=99Mieq#BR^OTpB->O>gX7kaz9^1(-3!_<{XaYUbE z?a5^*weiKmHQ*_!|Zvy{^B#WA!@E9L7|^cJ<@(~^a!MJ4Tv=F zymGW%Qr+YVZe*xWzED0WO!4~M)el(Mg^RWCW?%Zkw(?<0Nci)o|3z{~#5->pTmi$e zs!TXeg3!&RKpoPx-sT=i#4R|`?)k~D8A?Uue}^C?E3-3I%a8`CZquIrSK#xcu*%$w z=?UX!Wgvr=J!Kjv^OsK;qKe3_;tSbq@i*dZ1?tJiCjuxJS%Hb(R#?;QUxESucYy<6 z`9gf}K}nt8qRhQuHP*)$w9Z=LZ#`ZdElANOue5jsTmyo}W>i>LfHmb~OzZbBwU)l& zmOPT`sAV70eo--={dv<|(<{!hn0qUKrXgXfWn95e{*8i2UUDcfhyF;&4&h27zvr)8 z!_%m=THIwo8&(%Iu=Ij?4^`utB~@c4h&}QZ%v@Yu%e1ZA;oc>#{hkWzM*jLRjM)4% z1j&h}xggl5HBul)GB2dhwe|Me8jbFT%ZO?pQwpC;syP-eR*Zjh{v9{JH3?_uFYo6r z1jT@p+LcL5CtJ;;9ei6U9(_Xzvt=+x$RUqFXyyJT)b`tIP9c`eUUHND9K)&7QLncc z;eDu)&N~dY@^KlSG)XVLu&(HD&TXTa5J8 zt;r5{(?mnR<<_NLEFMbojcQ+EwHdLFSBN9R3Hj+n)JE@D#C|g4@5zC^!FppKuBPch zpy}rFCDSMV5Yc)Zyaj?cIMpDBj9hb0TiadqC6J2`-WV^f_S8K>)zT%A zbz99aBTV>WH7cjSxwJ2I9trRCC;=%vM~K+WO#!6NRY<`|XW zxqvAJRib6n@`oFKI1Iq#I*^J7q*$y(K25{GCT*UVM0w$J5z(qJ61ELzuA!j#yhpEy zeLH)3fD(&t-#Q>Z*vmX0TDuwXhyy&g4&peD$hA!n=m+ z;A8j|oS9FcvteuCgW~>VC~c@^bpbbtm&#S<0*nXBCgpzMD$6#zA5Y_B^V!?I4>*>T zB2cD$R2=(k3JTvk&AJB2j50^v50y|Y3`~E}CAf}$=_hiFSKl3I*&-{{?3jm)&^3Oh zvjA@peAC!Dc=#C~$7^{WUi#L{#{5y7@4|kA>yJevn zag|N|Z5`zXTHe~IgQ_ppeRtdF*~TmSiH54vVX>VZF{D>KR zRVTKRB!X)Jp?YA{EK90m{+rQsti1+Y29`ISuSy@@GbD3tWJDXVIZB@7^N*K?;xVz)0iW__M8BQN4vVSB>=kziuIi%M~{22wKX2t z6BJn!i1NQ=qS28@=F?V%m4f z{3Fa*c2B2-D5Fm*`WUmnXSFLc)HCR&oMS9@XjC{yK$ZWIA+MYmmk;jX5q?f{Frl@^ zjp6L=`uDx{#dLB+3Pr##wLUc(b&ZufBZUztjQHYUeVFlBIJ5TdaZy(9T#~$Og}+9M zT)2Wo#nZSJ^;ObRJ2E>t`>z=zf&;gwPv=iSW91W@snx?O zD;;pnH5jnDh;0ORbpA-`Ixy5G<0(&cH$H;*s4D@D5pfcJyf_)EtS9U&KTRXMa?d`et+PUgcq!D4F&T3pRA@o+jzj9bH-SG^hL0SV)pmh-IA4a+jq{M-qK%7RKOL7@unUUqf$Di=5#0q(UqYYBP?y_Htn9ZO0! z(kub>3hqRFU}m~}L+===;(DlEZ!mqRKigz@j&j|06 zq_^M9=1LF8Oh6pJpqIApzWfr*4*18A8tO+ZBTI_W@1}m{X#CV8KVqFTz0mggH9&UH zd6O+dfJ>U;^ZB?Uh&z;0LzeXGKO3Zf=l*rd`s6C41!g{bhItx;YQee&JSlSd3-cm~ zu|ujM3;6E_GP-X?gay&Ecb~6j6jCE=#J8@p~{> zG+r|jjtb<&G?fm#1{jRXu!NARG4JMNMhpU_=bXSx^^GwFMWLvyED#x8|4%j>_E)Tv z$WaGPc=u$JW@RIBOwVB&hIFQG$)f|A?s-yHAlJE>A}|nUGjcCQ^i?(NBv2nAPNnQH zO{(!iak=pkLTH+Eo-C{*jAaC*2Zp=4zASVZpp4|4^RU4vC9VLUHW^;1hRlV@JZp^Q z-l+(A(nxjdEUH9Qr_-j5L8;^s?t)}7Gvlh}BE$K1MY~)2^Sa%tAVDb84F5UVaUIm% z%2U6)X(eRU)rKLW!K2(1uhxFO;%IEzCx2zpLEA=Z*iiaH+Y)8@WkZU$Y*^_2$g8rZ z(VZT5I5fL?Sps~u+>IeWI)WQ^n6+E6U7hc^q-$m{DG0I*ddDB28%i(l|^AihCQjvvbRHpw>qcQ}OtdEB6z#MQ2+Z!vxU4v;7>7z--EzLaCF*qzP`L)cr7Y8}XBaDn$~QohIm>R_ z*}>Q(IV^&Kw^S7tk|84Mdwoga4UXyUS?lp9mFwYq-zS5lbf~72T7AEYcCv+zGtaA0 zuIV59e$fZ>a<_HoeE7LBY|3&DV$1K<$K4;__>N<)nKEMu$rY3&BEhcR>Wz=_Eat zzxM9P)VC@iNe5e1sKved8#>c|8T;vt-eO~h_-)j#zB?7u;_Y`DK8&nu89m`_`Buf( zcoX?pl$o+qzpK8iE@V!%BuC9$sN6MSa8&cgQ+jrSE(c)MEft7rDCsqJD2HGj=R95h+(_Te#KNrUSNt^I$2zFmb4GQ$^*^P z=<5yO6zE&zy2H5e?T&RSn=^fhWmt7Nk|+8h90=3^@U$%3coHXx|bdyF@ zN}G;R9XzutqJq=VSU0{^r-QOS2+B3u>%-U0|KXKO%$syXSr}z^&i;nQ`AJ0`DpRz= z)m!jV(^c%_rE_NQ9OTMQ)};izm^oGu^*WiAvL}{>U|)PbHDsRr;|U_N+dQ`!;>j31 zsh%(nt^ek;N|<;s5#@hacw;OcMb_DdP3+ORCEnaLkldpoT((H0K()$Qve)x=*XLc% z!~C3yH@GJC+Vzs`eR7j^-fPmMDPR%uk?+!9$HpYUknHss@Nun8Zr&6q2NE!{)#gVK z`lc zjn?K4LNtZlu}KLVe<7+J?O+&lDP1I!Y5e}&t_~ue#Hm-&U^Z(D5sG7}Mem%7dwDm- zYk}^Z+I`O@so>NU^r0{}TPo!)eCj?nP~9uTuRaow);}TS1>n+n&_N(fY%#({r}#+i z49^Q~4sn@6;v$Gyj~de#d(McILbs=rY842c%|dQ{J-{<} z9^d!SZZ7`blh3Iu5@SV#Zl%Z9fX*4llKQey9Rx*~3D{INeC#Qoc$bX`{gV|5OR~=} zZ`%yT-JKVs8T(u>E1cq7G_~<3a|{+nM@zbC+(tT(oGOdP>&JUn0lFm15oN_J^oi66 z^$6w@4@qyfb8c^mP!rZOo2|Udl#SER47;UHi8KMlV<2T9(pkHlb;8kavaHU=ZMurg z6SSA~E-2^wL_@{aR{JJ5cd6bp7XCN#SW*2%-Vcd|mUfl1ntvP)+X~Izy#_!s7|dbB z={P*6lO^)eJ~A5v^he)8-@W}iQ6%L~&JZZMRZG|0cZHh|bpd!=kvMl-HAB(ECn*Lt zR!$31=^RjtMH&}rRhQI?RNkKY><$mjVd$}dzZPz{^bn~ z+j-?V^=`^{SJdN17BuW{&>SB!jdr8y%*q(uy+Chi>7r;$Ilj$sP6+U9|F5pA{PS~; z2{R#%IM;yVmB1GGc_Ewsm+90^81hhPXy`E&lC=t2sC)Hg(m{+a=IH|Q-SE7Ya!4Yg z9Fp&OQ0V4{;gf#%t19`Hyy?#gk^8PbdP0;Mg49r{cBk0~26bwiObp%#`>jfK1wgbN zuGVA98P@s>sZWb(SBooFSR)Mv!(6D=%vQXMiK%OV7NX=O+P!cu1r9HoJ~_M`5E?-^ zSVTRMC4E8uR};Rl0zx-Igh~gSWXoXNpBY@6*%2@LqjGhi=o(M}RMa(ME^&D_%I&~f zwHUQ3jv~`=G3z#@Dc6?dRaa4Jw@{@?FG|ag&5cBp7QlZRzDti)4UWG>^)~I}?6Fs^lDwVOcQ_SEh()rojsxv-pc8YW#doM$Db)uEc^ePGbr zg?su0RsOknxF#(Vyp}alz}b7`awOys%*l*qkq^+IYK>NeiO5$8tdAcfGTi6U zFdTyCczpOyWm)*ZF*6$i#AlXS6TQMdVQ;bza^%~=nI^}uz!?E2Hk$r~JmTfUP0nP0 zU6C0^mzoRKC<-KNKR!mJRLX+d9^GY>ekTMPCDh%4YoHSsk&go3{-;pl&6n#0qVC$-ES4b*NGjOB0oj%697?H>mNOyueGri1=7ii8vM= zP%@Nny9Vrrt|3UQqa@p>O2SZj00L(gL}E5e3CEr21TaqnKDAB*-DZkqjT+M;3~zPz z3T8cCpkm#^t)`-)(?;O=(fMri+@9&3R}wt`lYVQ&C;-|6&omOD=|Ye zD%IHULJ9 z#AsIn`=fUEpt1Q;FA7(vW8u1nb@KC86v-9OH&6!mKv%PAfAeT`tHii`7UePmMO{1h zW_X$LoZ+4uuWzAM{Y}@4eS_XKp$p5HMb3hx2e+{_VFvabVb5RrIkeYXN=DVh$My#4XWNRC{ zXue*G%{3E7?ZF!$=oPio$pm|Xi|vQ_8Z2y7X^|Cmw)b{ip!jA z-SK(;icNE<@Q9aRSz%L|dl6`58a?G3P3LXDmN@=E`u%GO={<->HKMsH*pO#he^IVK zvT|8m_l+HK(S^`w@2nZOBKf_-{B)4=F;oGBFJ#6({*+$NFWGJyxA1|t5pDh_Jw3>X zPRymynF~X?7t(u{Gs(QF%}`hmjvC8euOyFGy)$#ZKho>>)MuwJTF#Rkv%uQuQQ1hv z+M(Owv?}WS=CDq1&y?ba*O#r!v;+hzbaDa)O~GEk%+uBu?zt-r^O~`u=k_7eWwjN# z1o#4rCQldOHdfscAySO}_&h0A0_bxfumCnW5>?F*oyO$2vP)Q(QbuyntnJ46S!}(~ zeDAbOXTnw3Qrwrg62Vu@f>tHS962_3!j6S5iIvcK_%)!Vd)Ur~yj2^@Z44gM8T@S0 zk8jhzRkPqSr6Z;g*wH9FL~qNiKDM=RajTlSuZ;i?H~U77?){OMyB=VXgqy2w!?**HQn;n z@px4gMR9p$xB2bJ9t>I{`A4FMUnLM@VM7hY?-nQfUYkEgskCfG34DL2N@#a{Z2S`K zi+>BHqtl<1O?X)P$VC&AfE|{+LSn@A zIo}T*icf+r#CPydYghD&!NN1`IYuF|R(lBHTXwA@X@xN=<8WscEv=FHaH~SW?>eG! z+E{kxJ*7Vqa~V{VII@-F=UlJEte;zY9}2D(>cHM7NH)9$j>x)kXEjE5Jis=o?|kg= z^+vbm3t!)8Pwg<~4i}HYVyE2KLfrg^Z5F}YPiK1t7}tT=D`?&)8K7*`dK}eI==1`} z>A8p5F|)qSDai{QV=i^1>uV1yJpE#F5`X`l!cORXR@2^02zo1v(U%=TL_#{WDw^`- zj!PV%$RN=aa5tI#m`rfQ?zgf&0L3%ih6l7pSwF?=rtLpmVaD5MX!#LK1SYOMI4Hfi!aYZ)PB?&+kX!< zHM{}Om(6UBJ6kr{mlMD8&KFS{Hm|Qn4ULsTbu$WPF>S?d_&CYsXmw)Xv!_R5w(0eGsdv}1rxUs zFZa@vs7N@TaE=@%*+|6H%qj-s>jNRK3$T${}t?58SD2Kdyu)9~r9JdN`#)$>Wz% zRp_@k-DgT^>O6z##f7FV83p5e#I=PtuV5{;gmQaaPG^>mVfmdkTe?2fvK1p{wKdKW zTl{vq3zC8BDwf>gr`xtevSgofv^_T{o$+4=!?vZBZ2vDuW)4 zlHnnFqYv75ev)(3$yI5~y@2hyC&A`UoZ6DR1=JAKeoCI?Q z?lFXKm%}J!Ol@tk6UkIKBuDwC3@p2Y7BLn(*{mcIJ+(cPTouY1L7WYm zJ^0xj6ZflST?U>%5Y5qu`A&)i#job@@Is;Vg0(g8Zp2IJcQb09Qau_l04N^kYzldc zXak|V>J<$niK3OC%#hwh!}ju=6`K?w^3^ZFQ33z_GWPquBR%JMp&;j^NW)0u_}p34 z@cD~?OgEZS-5eBA%@4HWYGeqA$KJDLpdwrl!4hqYzdpk}oOKGe>CMZWvRH=DxcZS+ z4eVgoVA#pOwiJRde7BVlnP;y^Vr6v~3XvQB4n%n-o7jGfG72%c8hBS|d#Oc~nX^7% zZ(=nMQF-(@ucWt%=6ff1Xf(#Eii65-tvz3;JrY)B|Mg5zEoYYlCqMi#FD7KSuo$woUOOPryLrbR}OFOPDJ`h`I# zLYIRBfmkZ>x|2&NN}j(Ko*n9Q&kATn&+Q1#$k;bDy7^t5LM5M)g&=WXzQS<}2Bq-x z@=jM~?R&ygxUEw6fN+V=KsxP4gGX@tD`pz>Y3Y=t3u;f{=}E*jATvVLv#XwOK~mZf zA=8;Ty;_3go{%&5d)!<+agi2?UZ-Vy=Uk@(Qy(o7>=WoIpL(AVrSk67w;PaiADGIL zV!fJ^n`G5zU2$~KI6Iyy_c>$;Jq9t+{=a|V13ErGjr!!`<-xiI9ti-TdJ}>*1kk)2g&k9>xUvbPXu4FsKi8ct72+cg30Z zCAhmTA@gM+bDL_dT(yui(dGEcQeA~wjU9T~VvAUR_2^2rfa_^1s?oGUvOz`s$q;k# z8?{HX&WybRj#U%dvUgcc);v0@<~g+-v&uD?N`TJ+U9EYM4o~vK9OpTSlb3O7LY==d z45#f$rl0z(p2o;CJg>9@}4YB(`M$ zZ|)Q>zusG&5v&Lv(DCmJfAFjieD93Ke61RG8#j?&Qr3yDBeJ8UE%<%ZVa znCut-IKf2}a~a`_phx-c$c z^0j-j7f{JOz$S@&vY{Q=p8V$8xj~jrSg570G_6_=T1r-HVwP=(bx;;Jbl6V*MfUPu;rXXfq%5 zb5ZMOV61AatW9=ZBG|=k0j_^P-46FB}iF`WbLc?PbKEF z%}5L?!u}VY3D)DIM4gJ|d#qJ+Di$(xA8kh33QrioF&es`)I;TjUrRdyfy(d_R@rX|6`-14-cE z*x4mTszIM4%P~#ZbdRBGqn9sBtkeeCbomvZ{}lCB9Mj!e>z=iI+pHvG+jiz;!6h2; z=?x#``nl^Rk3Ebqd4AzV{pU8QL^5M=51(0;5(fEaA03UDgS#geD`ck4)WZIGULtWP zh7}db%X8UKrTICGC9eC68FMz3s3yYr zsAj#d$k{*?-ID^FYDJqY0`&35xTw6w!I{(?tSO+n;Dw~X0@?c!s7W~}jLBH&I2%mKkk*!HUs-kPv&9*yy ziOyJw7D4kLl8%jzldEX0kir|;KYBlC$lB5U;YR!2o%Y8!*~jy(av!#kr*Wdu%Mgp2 z;HS88b0C~2{jLNnlxr`(EERuNG)!>P5pc^_iO?8>T37}Ne$)ij#bfoTe2uK>SAJ

myGEbCb@Ax{N+9lt_)@6>nh?iTU+T8Ty@#|=TNjs|i7uJ$dJoccnye~D=zz&^S zn|#*K(5!V2_~^|dG4S!c8jHyElp+?31E4#vHZVxAK}})sh|;ElJ+*dp{vDsSR~El{ zU_86+kFI-x#qC0JHvPzl&P+v%&gi#|e7E7==VQhyRmG#upsq+o2{GrU1QV?hdlro3 zR?*{vB(1iR_3`A}^+UWCAW)xddaU**{8z#~VizQX=UQQSJ^e^Nr3~%<`QYUWURpif z^_U60HDoeeGKT?->aRQ?|IHU65Z=mKDHvunU@*eeG-7@LUFyffzWt;boBj#5DH&1X z0=-2!X972J3CaGYjGcp~uA1xhQf*yKuOBHdbEN8^STct-I~IBx2?&`}JIC+52KZ!L z1Lm#%IRTRO^)MZ4_>E}Fscs`FnyuBtPS({>tp0lxEva{}3IZpdp*OOqI8D&U_HC3& zrRTI~Ad~qCGF+|!N4EegcRP$2#)SMG^Q;wVNzf-Z-*>D?M_z8Ep{og+doluyKwY6} zVB5G16hf9+`sCJ|_YXM?h?Ra$W~Tpkx^zLkJt#y-fq^saG|~0fuSm-(MUcfOoTSfDR(YYh38s*$go4qm*@l~)X$<~bv67dM>;V} zy%Sa}m=9vy;Siw23}uFr_sfn>z@E;?g69(UL;X zmJ$IOM_0z?WVM0IguMSuWcaKE8imuu*?S+H6r)##|f+wn{JEwkhU447l#Qr z6UU+SEqg+cdw#?_k9T$q>G-WQ=P80UKG^#`$TktOk8{pGF;S z>3DNEcDTx^!O$zJh0&FKzt1wF|NDGWXxYlJ&*%0T)ue^qr4YQ9^S^Cq_{Yr+=r{)D zmZtX&$a0c!ul7|Mic!TqUkTwuKD5MM*)>&lA-(gEw#wvdQKs47d@(o|LNhYir^f9- zSIth+hci2tQ7t)Vo-xC|>C>BB+)fQr_;ga(Rh3nbqhu*t^1{bn6-(Tk&$lIYv!GM! zt5j5qt9~cy@0}9!W+o7;kyEKvfA~lz%BexA|0w-pe#SYnXdL_qeqYrtCNysHQ-^)D zrkcYR*^{F!Ch^#+80`|h1g%?b<2Nb0_XPIoO7F?~ca4>9xM357{M-_+qASCD8S z8m35AU|N7V-m(TEjFQWfv*6M7AQ z6)yYY!`Yx8p2f$z0Y?1UgVY8Eu5y`V$+f8PlPDxB{MEPsyVV>8yJ0gGHjl#-+R+na zYt3{e@QJ(bi&w0h@C=%Ac*4?EG)xv$N4pjo3wp&8F7G4sRQxImE5sd-sp;MIS~v zgs=ta>STblFiNk=L+ zYFkuH#`Qf0{2fwJIRFoY%ElV?5xnq|NzrM#enWM_!*jD3E(?N(_cK@-K0IK^UNGvD zb?zjepR58Eao5+!A^2kH%^`tm>I2grb$aU_&@v zn*Vv*1GJY91?x$fuL)k;`e*5nIoxRUJa_<=D{=ryTYpg$37+KFdpZUx=-x%ieU|1QE zpNe}r0a|cW%a&NlX%?=>Sz_-?=JUh*m9o%RX_ z{hAb=!`f!c(Aw#ri6b`0QDd^-9ba3uLZK&tP{qII&+`atSsWWenj|Qf|Q?6i+hdoFz?q0yv{(jo7g-3Z8Oqqzm#ton_O1+rcJF zmyM*TK~pZCEw3WWVVvG9O2gEE`TML%#8Q%7C{y&BxjaEmNkbQ6x+X2+DEYT>a9(QZ zQd*p}q5|``Zxu_^wNWF#)O0N%EG3M8%^lZ+c>e$x$Rk^K&ZJL`V5$H2;JgX*+yx?f z%e?%1J;qkCQPj}v({X@d>)(JonzOM0Y_>3A*-{_H{{diRyZiWZ4^k@}YIZ!aqpfkb zn3t#BQTrOJFh8r>KZKOQnrxm-Dsi~yIGjcC_}M9??OBV2rM^GmV(1yB!}9AE?Q6rP zf$Z%D(PleLlKTiXc;KjYeBYa!R-UO)u$DX9gS?Mh%p$NCZgoPQPCyp9x{}uXRk?Ey znC-8vn}wr3c?4|F+}wjVpf-_O36MD3lsVM(%>Xx+0~tBp^CSJcu6%woSj}dF3WfDY zeIsXPD2FE9#L$-5qZICTZX!!f8!VKLRc8 z2ie{$VG_}F4o#{K^VbVU?%+HU*1F>>rc1Mv7#kVnZp*{z`Je9|%oo&_w>3PLSm6E6 zH*>v_Aa=S8_dZOG>r{5$^@^wkDUNB*8*0_*!4U?oP2@KGqTT$z4NX0l?)VNmc5=;3 zUB{9IBIMV3u5_st(R%yQ`BI=1!AO&=H|bF6Tc)~DAI4(#&LxoW3#>RBb!+Q2e@lPq zW^Tc&FGaOjDYh-^{zK^rKkbfQ&D-z3@T!$N+^Rn&8qiF)qq#m59Sm?q`xaKUM$w@_ z-2``JwH7l!GwG{C&Z#B&#QiF=$u3xl`N|?N0B`9C2K;GGCTsVcbE~SuCZ~ktv4Awd z1fL=bOyY&e?Tuo%o1>C^-5ushhLoD*sB_w?V}&w z^zn`dr5`Wh)Ec8+tt7AW5#eVPu>5J!W0ffu^!jF!N-Vz3TG^Y>U8VRlzU|#xzvrg{ z+kaDz{728nf97n$$%Px=VsI0=gXVS1>I6WV#+}z0{j$U!4BA6d8AZsKRFln;w<}&e z>55P&;&S57;!IEtXO3T?CED?YpMDVOPfE&b_shx3$YGsaNb6}gxSba2+a_ zk#v%m>d&t{n~DnWm~7x&%V#@g8)2=ZXE-++y|)B8FB`-*oc;j3SJd)pX`kF-Kxw-; z|1t)Z?{K2JBtuHIKoV8^<_dTzmKZ5~J*WkM8D;8JIjjrh9EE7Wm&~70g*Uok$Ji z^jC|_)v9y{dcpV&tHQgaiPK(BM)egMZE!!x@J!5|K#vKsRTXnh2M`<#e!>%{dHZ2hoYS8Wv}Ug^2Rkk zP!d)pJwkCse#PZ-a><+RI6wwO(kv1%6p$FpZoIU0V!}1Eu=c9)-sff@$H1((rT<6? zs%;@|RqQDA2Y|EhlpQmm;d#ln zQgoIRgQuf3LJ45coxbJDsPz6aVdRRQfbegvR1rQtVS^8@$s-N7&7hP|^wc%`6dj<%kmFLI$6x_k?^_CyU(l%9*;4nAj)d$|&e}!^CBsih- zGTdjhDrA&rtexus&o$h)4?ymdh|T}9C9eA4Jv`0R|IuJ>1iR@U>)L;HG$0FIyyNef zhUXO-`F8`)XZsl*RN{3(vH@0Vydvue`x4!+x+?q3Bp1Ab3r~%K2!o3_cYhm@%hN7@ z8{rhT;_dpMSb>C&lEOb5T+(ccc!TmCk$*>RZDil(OdGApqb88r+duOa?Wj zjF^?Cr4R-D=Uno~*q zo{njg-wyFruPGY&EA8Tk4c<$tls{$1HgJ(B-*o7sJ~7oo>jd~h{{G_s_3}T4Wt*lM zzi>%lB@JH+42#;M@|u<4QpP@`&+rl)+;MODUMgQA)5<_w^>G0W5m*Ji1a^lKAhtUB zhxwI4t8P!@XLWb}#SQYHu(;3|NM&|TmH))ufp0HCn1*O)S4k_7$mnDn(U37(HHtCa zB(QO{QLDKe*aPX~v>Lhzn|9yYx|~$ZsOZo4OW=ISAuG4U_yw~$l0I_)i- zN%W0vIEY(j59{Qav)iRbaR=fWNe&o?zMA(ZC-@9|r3_{wAJ>O;g?|?TLz7pjzd{77Ut^u*oq~vr^DhD=ZMD z`1m=Y2Z6kq!FPvcT9fnh1+!RL09KxI_CxnEhtq3=;V^DGQ&G(v&8Yb4sn8UIX5KMJnM?<$eO5NEQiyH83Xt1h;$`=p0($h9ET3IqkBRAd3s zL|+1_@^{q?(J$o^@7Vc9f0l3I=)Ithxv%!b$)iN4(HBbvlC5+h2&oQ-M&Bu`;fiE7 zw+GQI0Q~kWp87X)3`%itA`*i777kiZNelk!GCfnJuazRta6XQDWZ#S2CpG%JaXPJM zG$@9)QuQ3BKXhWM12?muNgnAxvvt*0(4G_wG$+;!Y(%xk5|Jmz$0=yfrj85<##o?U zk4^6%#bQiHG}S~0u*I%UD#qu{ZBvB+Ghfng0l>M_-lqD}gTtPGU3L`Sg84;)kuqpD zt!BtzLh}TyYtH%Kx^VvQM*07~=Kn|M>@i0>l4}>?r_8C!9JOlQYVep}#x4IYvBI5=hsj`menT+`056!zZNVny%$ZJ9i`#3Qc%*;es;3=k@;?VnO9GnZVV3FyN^6TI$(i? zz|$QY(MjNUrqSBt`_`eJs!+q@226{s z92Mn`3%26z^+8u-_(oMj@aI+=~e(p~Ab3(;KfC-%{)FPl|B z{;3~o#cy`7c!e)~zxWCl0hY@)>DcZk#$jF=YMLpk6XjYeOl7f*c$AA`B1_xU94`WD`0rU>_t0KW=W=UU=s~@~vv;u|&e}CC?c_eTK)v z>Wb&4J79~+E9Y^BLGFM%uqo&dpt#uVBHFw3j$Xw*yz}ny8KA zH8$13nCTccKclz0(c)i{L@`=34-cr=S~x&tCn*X4YB;;Exg9tlz6{s>ahTDyR~S=d zI_v+&m)}%)s`vSWj)CVJ@VWU*SXlSAX``#qa`$rAu~L%TX!7-KbjMjSa|m@29rd>a z`*f?7czJdLIRcY}`t;lt+ZYm)q{WJD)UT`1Y>YBMaLm! zIDdgAXraMo%ORfQ_1<+8;z;!5%CyvR%P?JPKFAt*)r=)}Dh-P#!Uc?(2s zI-^;shoOJhl+f%~4@cItp_ASM{s1=hk$0KIBHudbQ$?ORQ@4t3+-f`S0ZCUy7hI-9 zb|t}LQ)8|x<4!i2e0)j+(hY$Sw53c1SUk8b#nD=G^x4GLz$%-o#N+Y%VK-pmxc6f; z=WW`bF>fZ%Y~Lp+3gLhLB{u~VBVrZZpG;qt^jl2Yjciof5WC0>OcBJ^(>CbuyNimc z7i$_z;+bQ1lC`|3_TSS{zJ*V3J*UJzEGs9EBi^=>T1S)ABc-~%W4GM-s!q#BxY(-)QuME%kzb0=CJjNJ?Fuy z9$%)`uY(B>7K^3Ulm*QE`el!pL%l68mCn?FeaT46IrgB%JxRS2qfEf0gz%DdI32yg zlq=`nS5N6gBF-1x!f)5Id%12JJ)L?C3W;*%3nuT3<#>#wuuT zVs5bJ262Cuxo}^CnmCJ6fA$krVHQgBlBHjHE=kD)Xj>L-+K6XqSTibc-q;+(vP9(X zKwVX|uWoC9Ejg4Il)4d-c*k5`o+J8Ky}whZl8W~(T7cCrGd@_kH+O#WYr5j>uWJ8) z6z1_i>r_prG0a-Y`MOoy*t$KmfhOjk?x-8@V~NF< zU!*`bgXRhr;i~QJzwGxmUd)G0rJmBLGjzVLfGxccGSAe!By*s*F7DULJ za-rh^MnuSMqQBs3rfmvibkN3D_oIg{+A0%8EBzqsK9U1x4KB^lYqRsC88gR*fyp1} z#OkJ$v^)fxEVZnqY+RZM)(YvgwgT%!hGyg}!{Z)X&s1ngnIN9=nYh}Zyh_X0hp8Yj zFvkG4`eH~q8aEb>O1voCboS(UzkI&G=QC?z@#|f({{5C@_>Tt&d+45{$nk>AYMiiO zt;qQ0y8_O%-$DgtQs3G_$5KzPo@B`YM6ka+VUR3^B5O8JV_uGFgs3n=CP_o-bZ|(u zKPCVu+fGZpw8bV~l{33VGmv|s!Dm_2Q>?JRwt9jcnXGbuLN-SqgEbp+& z{s0cH{{Uj>|K$u|o?8rdb^xC~tNsIM-MJU|mlF)1D!~jwOY7zFD_@mfhL{Q94Nwg| zVNS}r`vb5%%KMkIY}`F=3n{zbXWDCKtZnZXg;Dc}QUYBNj&9VmZ6V3*6eN1%GtN^<$}R=1*gX#EDSl6Sf@Ah4b;e}a>^^Wyr>I^hL-JnU zK&RcWr})ZrP=;ezQdpD3a89hIaJX)3(_YRg9_dCaZtb0#R{c1KpjtgQ7F$&hN7IjC zEW73aQja10XKQwe5jh-^N*bm)l=8)ffMSa07cdu7oICt2m{->POl7~E|1IwwupqZU zoMz9g`%~AsQ(XhZh{l_uwBjuo!@J0c4PTRE6OE#!{BbBA-zmK`o`vZLaj9-z$bK>RRBylHM@Bj_QU}s z_6LwRi;EvCqMIH~wabPxL#?hly625!?^3=>;a4cL-7!m*yzc;frXb+iUiI^3A9ZwT zo-NNAmhR5*t4q7U?Ah9djyOWiT-$wTz71hA4bYe4tV80{E)hRe*EN0r@-ViVTh!!< zLgJA8!CKZgkZ3-3_NWDK_C0c*qICjVHMs{9C@}v1vd+DpQnrmlthh%zmK`N9XV76> zsW!-VOx^E$Q`Nro^*L*Saq%3 zPWpvpoa-Shv&yl%c=lL2BKe+bTriTOaP*C#77Mmsl{@ZXf^<0MI%b^&)v}koP?il@ zc#&$=+!illYK@kg!#Glfe2(G=16--uS4yH9C{3aAg1)gSLOFcaO~L&-CVi}j;P{w^9EFY z>%Os;1bTd5yGeGPcR89%W|dQ8M(H^&>;^9If1eRZx|VN9wvITKy@?}hJ)EncMN^6-z3AB!*%K$3RD1B#uTyP=^^f> zfr5Pla}|L>kvbRS&ND|sr=0H>I>l&VrM3zAkw4EFPDYHKR(Y&b7<=ZNlW~8_D(#^KI{Pl{3bPe@MOpnors`k0;?4IoLbx56Y~mX8?Eut z$yVr1r`_9VyaI8%ih!Nv`s9;kpO2pIm^1xCp^R}jAyh(KpNp}-$S&9TStuPy{Q(3R z-bCnsO+uwCBkwepAuKTB?7xPjtcgEK2OEF7Q}vTT^hdkp3FKH$wH_(mzO^ty&d_rnfH|mi=|v@@ls@d6su2#3lAj0+N^vI7+kmb# z&ZSNE-d|b;J_;`~cT9OAF=AQ|51FMQ!#p5f*fML0f|~io9@7e|>I}5`BR?LuXY@(6 zOw8w}j%H=+^jK}Ps&dao&fKyD8rh6jxV$o&U)QFQjpZv6+nCKlHj5Cpk&0sHkVX=G zt|=1Or)qw_IOe8&%yF=zlP^dFTR0R69IQhe*h*G8P-EMBKh@oG6NqWpAyHAX+W%o ztnkG+qc|s|BJKw_`G6ZLw9a^v5Wa;uxJhwztyedTByOl5yev=a@^N_d{r!Pt<5j#` zET{+F%7(gO`Bzpc7f{f`pdtB&%?c!L-Oot$WJmSh8opQ_#*8s|KQ55)V?O?-Q-X29 zQvSd~(dvUYvRHFjsWEAt?bEq_F!?2&v~RU~Q<`4j`Z8)h*iJ^p`vLpdD*& zF-eWdX9X^k%X|>&hhgLwkMauCNoz)%gt=ed?{@!^{kiG?lQiY9 zd$>Em^%I9T9)^EuA)GoTGk**A#pEhE4u^Lr6AESZfkL%(0-{?&xL`%k@e zRuip+)`DSQ+DG=VX9N+~3Bxp0&<{;2Kz1G-D+D3~wT2!IGKZ$Chv;rnAK~n@XZYIfYxWGAohrnC$PJsa)I_Kmohdg6s|l>Bk2K0#B(=4g`uaid zhDd8`kPGNA>*(%dD1cst27cuJTR0OYeP6c>CS1wjrE4=Zlw!1kvwL zlhwOH3<2I&#h2lx&IbDwbU4oNek`|eiucn``svM3+6s;(gGx!x_qKI~SFw%@Kkt<~ zJeM>PE`xb9pwhmc`QgE8-_|!1rde3#1w$#I+?{aEaSjf$pKCz<@2?_kN}lm}@c)Fh zKq_2y_7*?^-po^SeWuRR6?Lnptq;wNfF`j}Z>_x@K?L+I^NM3xb)OH?ru`ZlFM zd1Bew$*R??=&fb3ef+v-ZAcLcdZpvZ2dysXh^~%=;x4ay1e#>}A@HwE)Bkci{OwvS zwV(5aKVc8qO7ia<50D1yuCk7v?aBuYeluw3tw-umxpxlGfX{yVEQhJA@NK*ONoTyY zlI|4*u)|WeK)3^+ryAj^jk;Vw1GeXcbar}c!g42)CIf``I(S~ud zhbU9A+h37J*AW?vh~b{=EW_fvl}XEF)Z4cWt(?ic4}>UK-rwe%6p@X_+onwxr@9#OZ4bvC^W6||KW6$M*I z4r1!%p{>`Y_Q`_GE!dXb5M)$rhU#EtP3Z`f%H;qoPk&DoIdk{EfIS$zqAKwMLo{LA zKw8#>TQ=21D-TII=8_e2Lt2%IX2%DFYk1^-Du@px*j zg3QHar8#F!3K7RHUu6nL+>*?UnnwUj?#F-dr4%hZlU%pbOL%GqV|&Q;K#@27I0={^ z$KCf*S6Uv-T9^fKBLeq?fG;nI-&Co}kki>l-+cAPIl&kgK)BvpqV{{Y+whjzRwxT+}p6G5%qM;;bJ9=hk^`F%Gf1W#G zm_eh@K(B_DJ2Z|I(@3Tzuihmb2it_mNNIJ@*rf|8vX>TZ{2Sez_jkIv!9zR2`Kd~+ zt-tf~W-#rzSH-1a!%p+SMSQ1M8gRuEw=(f0;WK;|H)D#TT$*B=d$EEH?AjihJHcTw zF^-Sm4H>y-PJ$u-7Jg5q&aGBe(0ZapmBO1{;o80_a3EENcYm*ssu*nrockR+CvPpC?gKh71 z#a%U~xeMh9qw~P(^;%?kv%kz`#gY)fg=hYOdmo+AazlTVI1nB&XWAtiCd;At8gH#` zvp+L@R+}lK2B9^l=1M#(L!rR`oSQ;|7FskMWMFCJJog6>_y_RHfPKp0-P)yirm0PN zO)}N$xyex6QmH0<=v|2NVAptDI+^pnsFT}+EXTpH<@SXoM90;cdEtOyeyn+-*cgT8y|!)}dIkWK0Bo5S9b{TB`OwZ^Lr$x|H> zi>x0g;tbFOa)V`}#jJ?QLcJTA=H@S5b*|I;ZohZp0!~Zum^F||D77E8RmS*crtOts zQc!!!LY15?zS&`St`yr7vu^851B4thXS>WjnFgPr;BDzo6IT0_9Zw4tVc&(*%65}k zQ-^tp!?X2TMz`tyYz`sMH!=!oY(}<+6croD2ImK8`$OxIQ0=SGCxIW=aAtaAt&%>7 z>k+yc;n&e#3D$ZR8d*%?i1)JxM(G0{GesH$Xr**`yQGl6P?~VlKpcDLx0;P4yq~pC zF(Kb4qfw@NYt884M3Z*m%4P7=&T+JaIPD*=+y7}-|1bBsfr*@)yuYEdk*;yTBwPOV zW50P8>O5?ua>6NK2(xj;#GJYL%paY`O2vEED2M`M$T1Go8inVUAn^Iv=m!kfg4b$| zVtTK7tZk8wjX%vt?L=VlNHVJnU2`RCSQFCL z+9r-z?#WH|q0FsAnQd_OnNI^|n{;uRDwtt}B{#&44_J5d0Mdt)P_cqp%+G8MAVD!n zwuvqziJ9T8@`kZF_JZ6$C`Zh{-TuT6wyy66t3t^#z7rKwvO{yMAuPolEVDjhusF=W*1{Z^mefpG6#bfd`}X0Zu( zYySLVTCqc%^##y>J(5w+pUAvrI~kBwpakNZG18QXzshCguBM#(n64GW24xYXG#!Xj zZA`vVK{RA?zf#fSXD%>B8(ij@T?y~9Re-A$%Fjz}3Bl!F0+JNk#Z+I%bN5N~Xe-$Z*9DeUhjIZSe8Uf)14EuWQey${qz zC*%j&yj!MDoAE>SrXuC?`9o+uKeSC=WRTyUYTj&Wv(z>6w&y!4rE+b8NQT9@uQszT zM)HQD?gNpay zj!yehnC#D0CCKz>kZge;1P%R6fA23FOn1pXB*@VXAs9DI_haam@0fq$F7>h8{O7~L zj^>Gi1_`4RicGanJ;CfY9iA#^x#Sb(*nMZ@^sjrYR2=&PV@f2Wv^Xh~uW+`6XX+`t zRn9|p=E7;H@ATwV`cW}6W2^iKO9eH$_lomrP&ywy%FINiNgV9*+DbHeA2$uRHqxF% zqs;x6IW;?}Ce!l$9JjniJ)y7ycfZ3(6OrE^ic0+Ym=l1ai`%j{3$Wy?U_%;UyD8XQ zx-_Gf^yhZwizE6^L4u{juy2BxE=Pq~!TL7D2O@XNtSb&X(Y9V?EG^GlQv7Xrk210% zp7|r$)0nCVzfHsncemN@4J!^Q5Vq^3HhJHi?aZiO|7J4bxvaTwEsrxU;&v^snW1#$ z-cSqlLx)WCx{(-?rAV=&BYoD#gBv*m_%jJ;Ne=D4)IOg)&6xHSDcz6BP%t``m(;MyE#23_GT7 zmCEfKf*Z?s8T+D@T3k?)M z{tY9LW|i$U*oE(gPFDKUmNCJLoe~B&OZZgIAEat%^qM*jK4p)e46<|ZUz7yu9k%3F zxWB`soi`>RKA(-{lO{6pwO}6eQpf*wV$q_E*Xe*@GtRfh5Z9JvuCOw>p^8LDWV_^_ z^wc91teB%u#bX}r7*w?o&rBb2{Z4NXa>INj91AKPDr;e~lwT5@*stxbZ)&WI#LF3b z+T&QLnFb8K8d}itKNBpF;5138nkqo~ye^ev6~DL@mV`h;y3g@cyQ%wy>_XnnM)&qA zU-MuEzl6fxr~14vE`)_ZSn_%Psn7YFP_04?30~ql5`tDpLZi~u5tS3KY9JVe9bBYt zN&_;|U9lkcQDA=%?~*KRyM?t*Y#jl0LUA!1*W8Rf6`m7)j;|>R3iV#9*GU64l%APW z#sH^TeGgosH=8)iwnr}$4Gs??Twk=NXhK9+T4G<2odz9li(m!%lqUZH5H_)U7ZrS^ z>Qra%Tg!8Dr$$`7{_MoO2^2Rv{7l9CK?1GtM=qQ0^xHYuB^x9=^wR!5_UlFajHFs& zevSu=sNcIV=XVZHD`AsJp+~Yp0R6|JxKy=zTN5dKnqDv##6A1z_;{5&XW9h^aI-^+R1?!(cM)vaeabDAqz1SSs2DJZdz z%CM7F)(}m;H#9U#h+vEZNW6sI(v<+&q|LZ?*)v&^gc8x?-(Pt8WBU7OWp>PZB3(H4 zgUk!fwk%$n)1wVSWETlNH1HJO1M1v9*XB&WJ*Fy1q!u}!$US$^zI{m9x+n*iqMwmtKC*2wNl7uQGqjdE{=h?kU_KEjP$Ws*Xat%|IyFe8 z-gW3Rv74467H7~{wrT4cy|bGAMJCsKj6ym$3+eGyE-i!KlXj9=pKt9QrdnFjEf4U ziaqHx5Nqm1h{pb!vr!A`^y8q6WTt(xHee3R;?QpH$l`-Iwa^X%O_|ieI&B*l~ z+v6bu+9AJ3l}&i%eqar>Z>>;LaUVIk=!hb4{)q>28wehywy@uWq4|X zO_x#VlKfu8zwodTl(FKp79H+@y#pyP1X>WrE#YNkmRO}e-a}Hyh*CV6c2it?@@3{? z_57@Cf}$oPZAJmO1Lo$=a<8v_1#4YO_ygca78T0KBBDRrK72%ae|$7@Gqw|&fun~O z;k@`?c+}uM6gK0qz6N))4z3wR?DY>;ZxKz4T%TwkQE6Y>E+Zh%bKvgKY(Kw z?K}3$!?5)0sP4s>brDmm4oy{n+*{c*xvO z?$rW5#4mw#Ms)=Q-W@U8&3PJ(S&Z3kd1YVoysRVe8-dBKbWIB1bJZVq{d#FiVa)h_ zb*8?Tp5=C@&|IFuiE#Q+A&bf*!4!}AAoYLE9)VnW>QODIa*2< zdiw$!BG}kpF1MC|<3LDaQAL9peNMW|HmX3O@J?U*1OJF_IYoD$>-aF%hDcwn{i)^m zflJ*E;Rf2Xfr}LOm%WYpdWo5yb(3+BGnG>?&isN2*ZWxMFDoL<(f6eJuAC_cX0N4S zY|Y(yg%aGPgPqS5S8zXvVFeHwGnY?|QL$w(Yh;GUD4^p{9kH5Qo4?MPDpHwuF#QOq zaMe`Zx+mWW2+J$k)>5@i=!;CkgdeaZZ)Fu&UG}^g-4%?#3n_^=H|eD%On*ePY6qqN z)kuKj_eo={i5=SEVDEeNWZeni5){;VeUiFcv{*dP?%C6>T~sMi70?6buMU~S(PwwY zb${R?$3j^Q{GQT@3iTw*)T4{OT2HUOOxVNhj&IMm4a&P-uj2weh^bL5Y|N9b*sR_{ zyy>TD3+kUx7R5dxT^0*V^j@0Tvs6|~4&rcpnvioL7T<$x7F)UTL7O*J8WMQ7p_r!? zf+)LWbzj}2ecTF~!){x~^u?P$fJXB=WAVCEi6CUUsEt%(P}$H$hcvFh4UR#gv9xoh z7XHXU6#c3P(DR;96CXSxjy65^q05qO*T&Az`6y6@D)foa$Ee!peh~cwcxX>9MRV9YRxwZc z?qvPTw@5bm`nRH(m8WHU%s&&id`%|5Rmt5`j*q5_JMS{WeJQ-Pqn$*DEhNQIHVRHV z>h;p=v#|=$_;#hG1!~z`X!zb)j2NesAYox{)XTXt`Ru1s=D$~b(5BOgTft(bvMFFN zRl^5IvdIOTB*6y*DZIAy-@^weh|sRXZ8YeQM1Z9B4x0P)a+6}Yz}Sw{=ZWb37;4?P z4&FGs(tfvj7$3c+`bgATlkq4uULcFg(a=B}Xi}Kv@Xnd7pM5J#KXu1d!D;bj<|46q zC<(|q*wSZCfZ%`9K=7L=OY0o;%Ta3P^LLYY)rW-EH_Ok0zmvQ{^QWrl7M0^nVjEmRl;U?+m zSZvJ%kIH1Iv;4KkcG0b%D zVy^?d9dfZ#G?nGzG<|3ySWsU3#g&cJCZQ`W+2)jVa#U-luM|Dq#Ww-r4FGCzO7-AP!D49Q~hCBlLc3N;Dn>dA1Q^(`50%Cfm^3p5zM|*qc z7dkSNN`38|!A_GGlZUftHD>_p7cs;*9lMFHR?w}BI!0ymseW4mxNH78UcroSo74z zy3_8`Ihdc<&)E*k=TMO+{`E(FrEJWPIw<1WQ(;3my0y~zYxfFtX7RuM=?P65 zz~9?AIO9I@$F2I7hqEraJs6w6Qe0dFb$gma7Md?#M1AR1%5p%F;G#2!^*OUuh~IEC zM+i==vadTX5F>1$pl+yH+l)?Wum5O^b2^Ks%E9?C?ssU}gE5!k>MQillUwL*4@&}A z)_rQ*e_ik|bX*IhdN>^9|E~RN`xP$KYonu~f)kCelDj+v#CxU&#+@;m(TKW_BZ<%f zc+N6E2eHdRVE*1#zoMm|Zt8Brh4~!$x}*xgZYaY9xdN^Fw(0r}djLjOMZhGY8eVe} z-0w<4=M`ogRD;tzY#_3&iT*CRx}MIJEcs=+Vp89{7Is&lCe&Qu(WcPjE76;}9^F*u zmhk*(4jf>?lpK8Df{3<(B{JgYySBynS^aHRPqn(;SVNgZ^V)pFKY$sy3FtJ@(R1Fk zx|8vEqL(4rf;!gy3Z>%v4hVLfL`{y&&a~o@jD{Orf9g_K&?`~Z*o#!)gut)OL(_{n z3u0=XG0Hvk%h6}25PkfZhUIii8=U~SB|U99Ej{I9=`w^~hP_Ta@6Y)Si;CBi4&0~E zl%USvyTv|~x8)Dhi0t)s3qpM5i|ohVG4Ef0GjhlSjio2Js`Q;5LRyXIapi>)r$4knxok3D5K=336=Nge#mo{~&-@lJN`tIt0B zhyZ8cnk{d{j+@lU;Z+ovn~lcqoX8WOULB;#}5q%via#mXjG zebUT@iw<+t9paw&LD2C_MvHazrk@x*7Fe7x&St@Er{d+ZoN_a!6=_^Tp)|{-+hts^ zXE?UpVl>H?OLR%EqJLbuxZi()5;T^(iY26)&EcDf1bCUQZYz+5A^7~KA*HE7L{61fr(aeg)9m*8)gmuDax=M^PRfUV-`EZ(=S98upbwsCEk z*GHuq%Bb}#;FH;WbM{Z6EJosrG`kmTPktKgD%F)c46+su1t|!N>3j~zi#YVIEgzgL zOX26ENEOo8?tcfqo^Gx6 zYUK~{$rOOhIbhzUB@f**2=!;9HDmhzSswEzy;8S{=K1kTd`?NEsONiD@jWlA&P>&e zI69P`b$2}d_y};zo-?^#SX|?-gwq${J+=UyyO+1N+>_KU2!8u{Ny)~PG{)(IN>6P= zp@D3*>P!a&G@;DPJbCzbR`oN!%hTmQrJGb}^WvRjGDEUkp=SsoQeEWx!~P!+S?{K+{CA8zl@HC* zW<(f`h8N~}s24~<4|g{!HeN$I7#!CV6$%Z;CB*IglVe^#Lu*YRVO&FotzAVWO;h0!&lm& zdLY>;*h{jdb;X=CX;t{VR01zaQppX3XP+@JT`5VT{YLh`nVDgWt{gl;le#~}!Xmvx z9rDm+8eEd(uXuZXLu_Wad$U5T)J69sbJ)mLXhr#(Nn~GNyn8pO=MAd#KZzXuk9MDxiO)vNMFbCUy0IC_6bU{u8(WuewE zHTw@t_cga1<dToZUInmRsMO`rDd)n>Y{A z&~W4Na5u*b%PK}g7~-1bMsv(JsiywPrsf=j4eQUF$moo;Yd`#C^z9BXd}qcXnm-1w zLiDvBU(wX;s3i!a6u?<$C*tTVnVdGl;j*oxNQzwY2HE%*oWuJ#LjBnf7QoK#?6`l>V^`ZRljtX#bXP$>`#`;MxT#z-=y71>smT+p=WtG=shDD3~O4D zE?lS1a@|WditIe!B-q;Q*564GSUR>f9Zj?(dM?eVYh0-EIcldg)Il~MTWeFk-AcUp z|0=uAaJbrTJxMSL5d_gq-i(A-C)xz@Dr3|TWppCyC?R?$qDG4rAxiX?K`=%yAtXZx zqlH0qCR&uyN;4=5v7G zjUq*}#wcHHK#qsr5|+=7ZZ#CEDpPt-kDdlC=jOd~TRdxSJwCJj#Bkqq3HX2cn=y3X zEGD8R#i;sbhwftSQ(m?@M4>t$6%%axJ>is0tnGMfW*+gS83=FWsfEr?BoD?^;UG7x z-TOLZrOZV93r6tC)}z42QDl12L9uSbo${%1im7eiL=i{`I%d_vVibuxAJluPw`!9Z z{JJ8M?t1urkdl0eEV)wpV}vZpg&C;XQ3M$JZPaJtuEng=FfXI%82^IqKTmwKZs`qSK67&d8r=+QTLT z=#{ca#nOD!cY2e(AU@JnO%cQdCIgtV8pEOp&JjEPwR7U+5$RTiJ-n#8=Mx1(EY^nF zJa)2%PYC$Ax7cVbuT5KX&#fGRF71Tr59NDe5vqE7RCUovW{TCtSwwLQl=%F{3o2*{ zQD5rR0bP*nJK#Y|Qsv6=*aBHNpe|MKrDjaP6I!5d>qG~tNrYq1g=te6arlXqq7a9(JS{B&Fl}%SDb8@Xf(2@LeF+smxF+l1 zZ(@VN8YW|Pi6|#ooB`6!Se;aUE;jI>M}c<{vtS@`=8$=A|H&C^cGc!Xx@nbBVz)S` zzsyGlPcUKXN}kT_(h&9v2pHJiW%RfVx`V%-1hX`VWV<-eaHL4t2W$|{k@*mJcDdW9 z4kPPdF_8%w$XTXgyvH|1mrE$$48?|l-TXez#%mwQ9&Gc8E{%^TyW7r*oZNpqTekiQ zXL8Ggzb9ONBo5A4u6{jx=Q2RBfouVZ74xvmp==Ep_BULAmG9T|1~EQj7YxVIc3PKq=B}ZRsD`r!trg-$-w4CCt8DAC)(T;hJ4%7b zTHeCFCY2a5$hSP0MX-a46BZ74cExI6qJKJ=Lb{tO@JG~2H53|x>m=jc>m6X2W*t`A zvRNyVve_k76y?`rE6&-I!)U z0T;E^Z&tMb3|{=x^SR$u*H1+(LWdu=qPCVwOU?#nXe(-6t}57VEmcY!Y|fh-R0VhD z3(d)11P|@%A+g%#6Yt$oyqaWXoG{u&L(VYP7a^29VblD;{VW%h=~QS#I@b7%WF{2iSl!jWTJ8|cEFPADs<+K`tbpD!pEIn@=}|%?D}Zk z?g{18E9qi!;$6nCtsF>$hp)C8E^w$ig@C}x6E!4pF}&Mpfqxb?iT2;oxYLia(x4{nbPMVD>=h&rSv-JcD zFq$$PA!u?w)4PE%XsRC>i{8}&`lvxYZUG<_PpSbn;z(5k9j&;9l|+oW?^D7g=(yLiH)ls^*^={ zguUF=c5~Hw&~6)TNmRZk(GpSCQJw|=>7suDccuV#@gzXJ%ptDC0(D>2ZGnYZn)d@j zfp`&lA`d`cCH>moF;y8lNLzIu6L|2IqVQWe>={m0m&n4Z%-uV1X#3232sJ&6YhrG62@?~z zF+`lysxln*cig3qBDtI$*}W?BiFcvj&9htr3*J*C6cDHin*GBQD-4Bjxpztkz5`~M@j>|2 zl7*x-%0DIE0d-`!e<0U&jqNFi!{c&3xl4i}@8r7Cj;ytQHdu(*W*6!-C{;^gD2oL{ z^#hc#WUdF9h0LoqQS+sF1d4d{UeB@#vUbs3McQH1qZx30`oT|SAZRVPwWDQN51GOzToy zdJ1Qg{rR^Uj-z}|0rm17Y3x}1iG$&r*HH_%mNo`Hir*=LQ|A9_Sa3dW{8)1s`E-LO zTf@(b(dY7I$_Tz-L0=nzhJw3U;6lxx9o1DV^@YAZ$-Awh;q>$doX9(+P4_Wa$~UEe zE+>@+dABHop3ew3cu!EA{DHOhRRyUFN67X+LU`SPss$GP;c}4P9+{r7F_kZK*BVrA zyKnm(Dc^NG38f_595<-!z{aCCu_T|CwiMi*?t6S3<&049{&=>semIhDlyu8la}7)> z{h7j-y3q6y`-F-wgdon9@l->@i!R;dcsZP*{NcX9l11yLO3JLk2`Ywt!V$v5f{yN= znbRtiB`*~~N2YP-xScsIA0)t!^kbq&r|F@j6|hd=pH*g^kIzawze|0M$1rYRrq!c^n9!Hd?g^Cu)sp6mW1(A0+?Fq>ViR-FAiv^H7qIGA3$h7&{EVu< zn6EF#%T%PSKG=qFWDpMW^^QDooZYfWkxb`fu=+Gelq+UOLdpv+vBGv0IHV;(bYPBKP~ z0n%7~_{5!^;>r`UHpRX-dXgu+pWEq3UNS&8OSFsT575oLv9ora;FVo~u`mC&Jk&I zH1v*RqGrmQC@x&+FrW3uKZH-3v>u(7kK3b>qWvc=E^THq&CM;Ushp2mEd+uG&VXzp zW4i~iUw`h|e?9Jxcp7^21`h0~%sd7Ishe9ETJ^&V@M?$| zx}@@heY6_S1sb)_8L-yL(j1A!_%)Fax@Y1Ia7G#IpK2XIZg~MB1!R`p!2kXXGv{OT z$?@w3&Nbf#IkaKkGnnE_z_GwCWd5gf{2W;$*2jhg!7LvAx^OzjQ#`y&^ra#_d()Mf zfobeYncBNYvv3C8UuvMg^Ifw+eB(Zm=3B!6Ce_D(rBVvp-28#cnL>17f8nzK?xq<{ r*oi=~k7TD;qeLR-hwJ8s2^q>8J%kK^ZT=4@`Tx$7{P(@wk1ziPxzf(= literal 0 HcmV?d00001 diff --git a/README.md b/README.md index ac2d3e00..6a5713d7 100644 --- a/README.md +++ b/README.md @@ -29,27 +29,30 @@ For a gentle introduction to *AI in the Cloud* topic you may consider taking [Ge # Content - + - + - - + + + + + + + - + @@ -69,7 +72,7 @@ For a gentle introduction to *AI in the Cloud* topic you may consider taking [Ge - +
NoLessonIntroPyTorchTensorflowLab
NoLessonIntroPyTorchKeras/TensorflowLab
IIntroduction to AIPAT
1Introduction and History of AIText
IISymbolic AIPAT
2 Knowledge Representation and Expert SystemsText
IIIIntroduction to Neural NetworksPAT
IIIIntroduction to Neural NetworksPAT
3Perceptron Text - Perceptron
- Own Framework
4 Intro to Frameworks (PyTorch/Tensorflow)TextPyTorchTensorflow
5 Multi-Layered PerceptronTextPyTorchTensorflow
Notebook
4 Multi-Layered Perceptron and Creating our own FrameworkTextNotebook
5Intro to Frameworks (PyTorch/Tensorflow)TextPyTorchKeras/Tensorflow
IVComputer Vision MS Learn MS Learn PAT
6Intro to Computer Vision. OpenCVTextNotebook
7Convolutional Neural NetworksTextPyTorchTensorflow
8Pre-trained Networks and Transfer LearningTextPyTorchTensorflow
8Pre-trained Networks and Transfer LearningTextPyTorchTensorflow
9Autoencoders and VAEsTextPyTorchTensorflow
10 Generative Adversarial NetworksTextPyTorchTensorflow
11Object DetectionTextPyTorchTensorflow
VIOther AI TechniquesPAT
21Genetic AlgorithmsTextNotebook
22Deep Reinforcement LearningTextPyTorchTensorflow
23Multi-Agent SystemsText
23Multi-Agent SystemsText
VIIAI EthicsPAT
24AI Ethics and Responsible AIText