diff --git a/3-NeuralNetworks/03-Perceptron/Perceptron.ipynb b/3-NeuralNetworks/03-Perceptron/Perceptron.ipynb new file mode 100644 index 00000000..8b4aad8e --- /dev/null +++ b/3-NeuralNetworks/03-Perceptron/Perceptron.ipynb @@ -0,0 +1,1085 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "## Perceptron\r\n", + "\r\n", + "> This notebook is a part of [AI for Beginners Curricula](http://github.com/microsoft/ai-for-beginners). Visit the repository for complete set of learning materials.\r\n", + "\r\n", + "As we have discussed, perceptron allows you to solve **binary classification problem**, i.e. to classify input examples into two classes - we can call them **positive** and **negative**.\r\n", + "\r\n", + "First, let's import some required libraries." + ], + "metadata": { + "collapsed": true, + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 2, + "source": [ + "import pylab\r\n", + "from matplotlib import gridspec\r\n", + "from sklearn.datasets import make_classification\r\n", + "import numpy as np\r\n", + "from ipywidgets import interact, interactive, fixed\r\n", + "import ipywidgets as widgets\r\n", + "import pickle\r\n", + "import os\r\n", + "import gzip\r\n", + "\r\n", + "# pick the seed for reproducability - change it to explore the effects of random variations\r\n", + "np.random.seed(1)\r\n", + "import random" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "## Toy Problem\r\n", + "\r\n", + "To begin with, let's start with a toy problem, where we have two input features. For example, in medicine we may want to classify tumours into benign and malignant, depending on its size and age.\r\n", + "\r\n", + "We will generate a random classification dataset using `make_classification` function from SciKit Learn library:" + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 3, + "source": [ + "n = 50\r\n", + "X, Y = make_classification(n_samples = n, n_features=2,\r\n", + " n_redundant=0, n_informative=2, flip_y=0)\r\n", + "Y = Y*2-1 # convert initial 0/1 values into -1/1\r\n", + "X = X.astype(np.float32); Y = Y.astype(np.int32) # features - float, label - int\r\n", + "\r\n", + "# Split the dataset into training and test\r\n", + "train_x, test_x = np.split(X, [ n*8//10])\r\n", + "train_labels, test_labels = np.split(Y, [n*8//10])\r\n", + "print(\"Features:\\n\",train_x[0:4])\r\n", + "print(\"Labels:\\n\",train_labels[0:4])" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Features:\n", + " [[-1.7441838 -1.3952037 ]\n", + " [ 2.5921783 -0.08124504]\n", + " [ 0.9218062 0.91789985]\n", + " [-0.8437018 -0.18738253]]\n", + "Labels:\n", + " [-1 -1 1 -1]\n" + ] + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Let's also plot the dataset:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 4, + "source": [ + "def plot_dataset(suptitle, features, labels):\r\n", + " # prepare the plot\r\n", + " fig, ax = pylab.subplots(1, 1)\r\n", + " #pylab.subplots_adjust(bottom=0.2, wspace=0.4)\r\n", + " fig.suptitle(suptitle, fontsize = 16)\r\n", + " ax.set_xlabel('$x_i[0]$ -- (feature 1)')\r\n", + " ax.set_ylabel('$x_i[1]$ -- (feature 2)')\r\n", + "\r\n", + " colors = ['r' if l>0 else 'b' for l in labels]\r\n", + " ax.scatter(features[:, 0], features[:, 1], marker='o', c=colors, s=100, alpha = 0.5)\r\n", + " fig.show()\r\n", + "\r\n", + "plot_dataset('Training data', train_x, train_labels)" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + ":11: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n", + " fig.show()\n" + ] + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEnCAYAAACpNTSTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA9CklEQVR4nO3dd5ycdbX48c/ZlrqbuqmbZEMICUmAAEuTFnox0hQviCAXNSpNUAQUf4IFG15FvIKCBe8VFASlJICh90ASSEICJATY9N7L9jm/P87M3clm6u70Oe/Xa1/JzDwzz3c3m+c833aOqCrOOedcSbYb4JxzLjd4QHDOOQd4QHDOORfkAcE55xzgAcE551yQBwTnnHOABwSXYSKiCXzVd/EclwY/p7YT7723q+dPJRF5QURe6MT7pojILSLi/8ddwsT3IbhMEpEjOzz1L2A+cEvYc02q+nYXzlENjAHeVtWmJN87BqjqyvlTKRQMVHVKku+7BbgZKFfV1pQ3zBWksmw3wBUXVZ0V/lhEmoCNHZ/vcEwpdvOS0IVNVTcAGzrZvg878z7nCoF3J13OCQ733CoiN4rIx0AzcICIdBeRX4nIQhHZKSJrReRxERnf4f17DRmJSL2I/FVELhCR90Rkl4jMEZFjOrx3jyEjEakNftZXROQHIrJGRLYGz1vT4b09ReQuEdkkIjtE5F8i8ong+y9N4Pu+QETeF5EmEVkkIudGOCbuzyCsdwDQEhqKC3v9+yLylohsE5GNIvJchJ6bK0LeQ3C56lLgI+A6YBewGugGVAI/AtYA/YHLgVkiMl5V18b5zGOBccD/AxqBHwLTRaRWVbfGee+3gdeAy4BBwH8B9wHHhx1zN3A+Nvw1BzgpeExcInIycD8wA/gmUA38GigHFocdmsjP4A9ADfBF4BigrcPphgO/AlYCvYDPAy+JSJ2qLkikva5Aqap/+VfWvoB64K8dnlMsAPSI895SoCewA7g27PlLg59R2+E8W4B+Yc/VBY/7XNhz9wL1YY9rg8e82OHc1wWfHxZ8PA4IANd3OO6O4HGXxvleXgXeBUrCnjsi+N4XOvEzuCX43rIEfoZlWND5dbZ/H/wru18+ZORy1VOq2tDxSRH5rIi8ISJbgVas99AbuyDH87qqbgl7/E7wz5EJvHdGh8cd33sEIMA/Ohz3ULwPDs6RHAY8pKqB0POq+gYWyDoe35WfASJysog8LyKbgu9vAfZL9P2ucHlAcLlqTccnRORTwAPAe8DnsIvwYdgEcvcEPnNz+ANtX4GU9HuBju8dGvxzfYfj1iXw2QOxoaFIx+7xXFd/BiJyCPAEsBMbUjoy+P75ibzfFTafQ3C5KtJ66AuApap6aegJESnHxtGzLRTABgEfhz0/OIH3bsTu0iMdOxhYFva4qz+DT2O9gvNUtSXsM/oBWxP8DFegvIfg8klP7GIW7mJsHDzb3sCC2Pkdnu/4eC+q2gbMBj4TvpFMRI7A5jDCJfozCPVgekR4fxthAVdETiSxYTNX4LyH4PLJU8A5IvIrYDpwKHA1OXBnq6qLReR+4IfBi/pc4ETgU8FDAlHfbG4GZgKPiMjvsVVG3wc6rpxK9GfwbvDPb4rIk0Cbqs4Jvv8a4F4R+TM2d/D/gFWJf7euUHkPweWTe4Bbgf8AHgc+iV1wt2WzUWGmAX8Crsd2YE8Ergi+FrONqvoMcBE2sftP4FvYhXtxh0MT/RlMB+7ElqS+jvVAUNV/YwHk6OAxlwGXAEuT+D5dgfLUFc6lkYh8C/gZtgR2ebbb41wsPmTkXIqIyFRgEjAPGyI6Ftuv8KAHA5cPPCA4lzo7gHOAG7EdwKuwjWk3x3iPcznDh4ycc84BPqnsnHMuyAOCc845wAOCc865IA8IzjnnAA8IzjnngjwgOOecAzwgOOecC/KA4JxzDvCA4JxzLsgDgnPOOcADgnPOuSAPCM455wAPCM4554LyOv31wIEDtba2NtvNcM65vDJ37tyNqlrd8fm8Dgi1tbXMmTMn281wzrm8IiLLIj3vQ0bOOecADwjO5afWVmhuBi9w5VIor4eMnCsqqvDuuzBzJixYYI/794czz4SjjoJevbLdQpfnPCA4lw9U4R//gOnToXdvGDECSkpg1y743/+FZ5+Fb33LAoRzneRDRs7lg9deg8cfh1GjYNAgCwZgvYLRo2HzZvjNb3wIyXWJBwTncl0gAI89ZoGgtDTyMUOGQH09LF2a0aa5wuJDRs5FogrLlsHzz8PChfbc+PFwwgkwZgyIZK4tq1fDhg0wcmT0Y0SgrAxmz4axYzPXNldQPCA411EgAA89BDNmQEWFjcuLwJw58MorFhQuvjj63XqqNTQkFoAqKmD79vS3xxUsDwjOdfT00zZeX1u750V/6FALFs89B1VVcN556W+Lqs0PrF4NLS123gEDIgeIpiYYODD9bXIFywOCc+Gam+HRR2H48Mg9gJISW+Hz5JNw2mnpXeq5fDn84Q82dLV6tc0RdOsGPXvCIYfsefFXtb0JRxyRvva4gueTys6Fe/99G6Lp3j36MeXldrcemltIhxUr4NZbrXdQWwuf+IQNCfXubb2UV16B9evtWFU7fvJkqKmJ/bmqsGWLvbehIX3td3nJewjOhdu5M7HjRNI3Xq9qewtKS9t7AYMH2wV/wQJ7XFZmcxp1dbYXYcIEmDYt+lxDIGATzjNmWPAoKbGvY4+1ns7gwen5Xlxe8YDgXLhYPYNwqjZ0kw6rV8OSJbbnINzo0RYgQkNIW7ZAjx7wla/AxIkWJCIJBCzAPPOMzT+MHGmBo7UVXn7Z9jjccIN9vitqOTNkJCIjROR5EXlPRBaJyNez3SZXhMaNswtrS0v0Y9ra7IK6//7pacPq1Xb3Huluv7ISJk2CU0+FI4+0Pw86KHowAJg1y4LB6NHQt2/755aV2VxJt27w61/bpLQrajkTEIBW4Juquj9wJHCFiEzIcptcsenVC046CVaujLzrV9VeO/ro/EgToWrpLqqr23c3d9SvH2zd2j4c5YpWzgQEVV2jqm8F/74DeA8Ynt1WuaJ03nm2iuejj+xCqWpf27fDxx9bL+Kii9J3/poaG+aJl4ZCNfZmNbANbWvWWM8ill694M03k2unKzg5OYcgIrXAwcAbWW6KK0YVFXDFFTZp++STNmYvYvsQpk2Dww+3Y9Jl6FAbjqqvjz7Zu3Wr3fWPGxf7s1paog8/hSsrg927O9NaV0ByLiCISG/gYeAaVd1rGYeITAOmAYyMd3fkXGeVldkY/RFHtNcd6NYtcykrLr4YfvQjWLfOchiFzhtaNrprF9x4Y/RhoJCqKntPW1vsndW7d1sgckUtpwKCiJRjweA+Vf1npGNU9W7gboC6ujpP7VjsWlrgnXfg3/+2O/myMhvuOeGE1KyaEbFAEMm2bfDGGzb2HgjAfvvZfoFBg7p+3mHD4LvfhXvvtRVH4QGhpgauvhr23Tf+51RW2tLU+fOjX/BV7ed4zDFdb7fLa6I5ki5XRAT4C7BZVa9J5D11dXXqNZWL2O7dlvL53XftTrhPH7swb9xoK2Y+8xmYOjU9d/Uvvwx/+YvdeVdV2Tl27LDzn3UWnHNOas6rCqtW2d4BVctqOnp0cp+9YgV8//v28+k4lxBK4nfooXDllZlN2ueyRkTmqmpdx+dzqYdwNHAx8I6IzAs+9x1VfSJ7TXI57Y9/hMWLbSdv+IVs2DBbY//gg7Zu/6ijUnvet96Ce+6x84TvW+jb18778MPWqzjzzK6fS8R6BPF2IMcyYgRcdx3ccYftfK6stJ7Uzp0WOA89FL78ZQ8GLncCgqq+AvhvpEvMqlUwd65t3op0ISsrs6GbRx6xeYB4Y+2JCgTg73+3Cd1Im9jKymzlzyOPwJQp6du8lqzx4+G22+xn9sYb0NgIBxwAxx0H++zjwcABORQQnEvK3Lk2SRrrQlZZacMhK1bsveu3sz76yJZyxvq8igqbiF6wwCamc0WvXhYAjjsu2y1xOSpn9iE4l5TNmxNb+hmqO5wqW7cmdjddWmqBw7k84j0E1zltbbBoETz1lJVtFLF8OqecYsMT6R6C6NPH7sLjUbV8P6lSXp5Y3eJAIPrqJOdylAcEl7zmZvj97y17ZlWVbZ5Shffes81cJ50En/98eiuKHXqo1S1QjR58du609BKpGi4CG28vLbXJ42j5g0I7m9OV68i5NPEhI5e8Bx6wC38o+2ZpqV0cBw+2FT/PPms9h3QaMcImRVeujPx6W5vl/D/rrNRNKIPNSxx3nE1qR+sprF1rvaSurAxyLgs8ILjkbNtmhedDKZQ7KimxDJrTp6c3e6aIpZEYOdLyC23b1r4jd906m0yeOtXy/afa+efDmDF23vAiM83Ndt7KSl/G6fKSDxm55CxcGD8NQrdutqxx8WI48MDox6m258/p2TP5C2hlpaVvCM85VFJihWROPdXy/KTjotyjh63rf+456wlt2GDnKS+3vQennmpzHM7lGQ8ILjnbtyc2NyASPVlaUxO8/jo88UT7SpzBg+GTn7Q9A8kkjuvWzVJRH320TeSKZObOvHv39ov/xo0W3Pr3z+xE8tq18Oqr1lMpL7eUHYcemjt7H1ze8YDgklNVZRfeeKJVFNu9G26/3WoXV1fbhK+qpX24+24r5nLVVYlXLguXyrmCRJWVWTqJTGprs81xzzxj33OozvK8eXD//fDVr1rRHOeS5HMILjkTJ9pFqK0t+jFNTXanvN9+e792//3wwQc2IR3KqyNigWaffSwv0YMPpqftheIf/7BkfjU1Nrner5+Vxhw1yoLD7bfbUmDnkuQBwSWnb184/vj2ZGsdBQK2AufMM/e+y9+82YY4amoiD+uI2IT0iy+mr4B9vtu82YLBqFGRh+569bKvf/wj821zec+HjFzyLrjALkxvv22Tp/362fObNtnQz/HH23xAR++/b0Ek1hxEWZkFlcWL4bDD0tP+fBaqahbrZzhggKXMXrcueoGdrmhrs57cK69YbYY+fSx19sSJsWs7u5zn/3oued262Tj/O+/Y6p4PP7S7+wkTbJJ1woTI4/mNjYnt8gUv+B7N6tXx51dELGBs2ZL6gLBpkw1JrVhhq626d7e9IHPmWL2Fa65JTT0IlxUeEFznlJXBwQfbV+giH291T6huQCLi1QAuVj162C7peAIBW3mUSg0N8ItfWO+wtnbv19ets4yqt9xiw1Yu7/gcguu6RJd6TpzYngk0msZGu+h52ofIDjrIqpvFEvoZjhiR2nPPnQtr1kSvvDZ4sC0jfsNLoecrDwguc3r0gE99yoYbIq1Sam21Cemzz05vEft8Nm6cLXPduDHy66p20T7ttNT/DP/97/b5omgGDkx/2hKXNh4QXGZ98pNwxhkWFFatsn0Ju3bZ31eutGBw6qnZbmXuKi21+Ruwn2H48NGuXVav4aCD4PTTU3/utWttWWssvXpZLyFHSvO65PgcgsuskhK48ELbWfzCC7byCGxl0nHH2TCH5wCKbfhwuPlm2+n98svtF9+qKrjkEqvUluj8wfr1MH++rQ7r08fSfgwYEPnYbt0sAMVa4dTaaj0T/zfMS6J5HMnr6up0zpw52W6Gc9mze7etJiottZ3fiaYc370b7r3XUpiDva+tzS7kxx4Ln/vc3mk4/vY3y2QbK4vrqlUW7C+9tDPfjcsQEZmrqnUdn/cegnP5rGfP5HMXNTe372YeMWLPJcKBgG0M3LEDrrhizwBz/PHw9NM2aR1p6Wtzs014n3hip74Vl30+h+BcsZk714bqOgYDsMejRtm+gnff3fO1YcPgssts0nr9+vacVqo2b7ByJXzhC5aS3OUl7yE4V2yeeMIys0Yb5xexfSAzZ1oRonDHHGMbz6ZPt42JJSUWECZMsAUDvlw4r3lAcK6YtLXZ6qR4ZUX79rUVS5Hstx984xuWb2rXLhuy8voPBcEDgnPFJLSJMFYtarDX46UTr6qyr0SoWtqN5cttqGnIEMtu66uRcooHBOeKSUmJDe/U19uqpGg2bkxdcsE1a2xF0+LFewaAoUNtNVKkNOkuK3xS2blic9pptoooWqGjtjZLLpiK1UJr18Ktt1rPYNSo9q+RI2246ac/bd+L4rLOA4JzxWbSJNu8Vl9vS0jD7d5tz0+dakM6XXX//bZZbfDgPXsHIpYGo08fq5QXq+CSy5icGjISkT8BU4H1qjop2+1xriCVlNjy0CFDYMYMy1IaulhXVtrS0ilTYo/v79gBixbZn7172zBUx4nl9ethwYLYy1D79LEA9P77lvzQZVVOBQTgXuC/gf/JcjucK2ylpVbV7uSTrZ5FQ4PlIRozJnaRm9ZWePhh26AWyqMUKno0ZYoVTwol1Vu50oJPvMnpkhL4+GMPCDkgpwKCqr4kIrXZbodzRaOiIvG9A4EA/OEP8NprdtcfHjja2uC556yAzlVXtb+WSGocX2mUM/JuDkFEponIHBGZs2HDhmw3x7ni8d578PrrMHr03r2I0lKbLH7rLUuWB7azWTV+UGhr893NOSLvAoKq3q2qdapaVx1r2ZzrurY2WwkSq6CNKx5PP23DSrF2OPfrZ2VVweYoJkywuYRoduywTXATJqS8uS55OTVk5HLE5s2WmvqZZ2xsGaxU5mmnWYEWV5wWL7aUF7H062dzEoGAzQ1cdBH86EeW62jgwD2Dyfbt9rt23XWx5y1cxvi/gtvTihXws5/Z8sPBgy1vTSBgq0DmzoXPftYmI33ct/gk828eOnb4cLjpJpt7qK/f8/UBA+Bb3/LJ5BySUwFBRP4GTAEGishK4GZV/WN2W1VEmprgV7+y/6zhY7olJRYYWlrggQdsrHiSrwouOhMnwsKFdqMQzaZNMH78nsGjpsYK+ixbtmfqiv32i78CyWVUTgUEVb0w220oavPnWxe+tjby6+Xltm58xgwPCMXo5JPhzTfbh4M6UrVhoEjlO0Xs9yra75bLCR6eXbtXXrGNSbH0729jydu2ZaZNLnfst5+ls6iv33uhwY4dMG+eXfCHDctC41wq5FQPwWXZzp3xa/GGsmU2NWWmTS53iMDnP283BTNm2O9AU5MFiE2bbL5g+XK44QY45BA4//zYw0su53hAcO2qqy0zZayUxqGcM716ZaZNLreUlsKnPgWnnAJvvAF33WXzS8cd115WMxCwlBXvvw/f+Y73GPKIDxm5dscd177MNJr16+Hwwz0gFLvu3W2IqKoKJk/es8ZySYkFgUAA7rknsd3KLid4QHDtxo+Hffe1HDSR7NplK43OOCOz7XK5Z8MGCwhDhkQ/prrachQtW5axZrmuSTogiEgvESlNR2NclpWWWh6aESPsP/LGjZYeeccO+0+9bRtcc42nGXDtewpiLRsNLT2NVorT5Zy4cwgiUgJcAFwEHAY0Ad1EZAPwBHC3qn6Q1la6zOnTx8Z9Fy2yVAXr1tlwwGc/C0ceaTtRnUu0foFIe1ZUl/MSmVR+HngG+DawUFUDACLSHzgB+KmI/EtV/5q+ZrqMKi+3ceHJk7PdEperqqvbE9fFq80ca1jJ5ZREAsLJqtrS8UlV3Qw8DDwsInHWKrq8sX69pTeur7f8MpMn2xLCnj2z3TKXS/bZxyaOt22z5HSR7Nplk86euC5vxA0IqtoiIuOB4cAbqroz9JqInK6qT0UKGC7PBALw0EPwxBM2Lty7tz03Z44NGV1+ORx4YLZb6XJFaE/Cz39uNw69e+/5ekODDTeG10ZwOS/upLKIXA08ClwFLBSRs8Ne/nG6GuYy7JFH4PHHbUJ55EjbfDRwoO08ray0HEcf+FSRCzNhAnzzm7bwoL4eVq+2fSz19dZzuPxyOOywbLfSJSGR0P1l4FBV3RmsZvaQiNSq6q8BT3lZCLZuhenTLWldaYQFZL162R3fgw/ahLNnOnUhBxwAv/ylJb1bssR6lfvss/feBJcXEgkIpaFhIlWtF5EpWFAYhQeEwjB3rk3+xeraDxgAS5fC2rUwdGjm2uZyX0WFzTMdcki2W+K6KJF9CGtFZHLoQTA4TAUGAgekqV0uk1auhG7dYh8jYnMLW7Zkpk3OuYxLJCBcAqwNf0JVW1X1EuC4tLTKZVaPHomtK4/Xi3DO5bW4AUFVV6rq2iivvZr6JrmMO+AAS0kRS1OTDQ2MGpWZNjnnMs5zGTmrkzxkiKWqiETVVpCcemr8oSXnXN7ygOBsbuCqq+zvK1fumWpg927LazRhAnzyk9lpn3MuIxIeEBYRwfIZ7aOqPxCRkcAQVX0zba1zmTN8OHzve1b45NVX29MSVFbCBRdY+cSKimy30jmXRqIJ5ioXkbuAAHCiqu4vIv2AmaqatZ0ndXV1OmfOnGydvnDt2mUVsEpLreKVTyS7OFRtT9ratdbhHDHCViq73CQic1W1ruPzyfxPP0JVDxGRtwFUdYuI+C1jIerVywvguIQtWwb33Wcb2UN7FlXh0EPhwgs9MOSTZAJCS7AOggKISDXWY3DOFamPPoKf/tQS5I4c2R4QQlU0P/oIbrrJg0K+SGZS+Q7gX8AgEbkVeAXPZeRc0QoE4He/s20s1dV7ZjQJVdHcsQPuvz97bXTJSaiHEJxQfgmYC5yEpaw4R1XfS2PbnHM5bMkSy5ZeWwuoUtbWBKq0lnX/v+gwdCi89ZZNSXkvIfclFBBUVUXkEVU9FHg/zW1yzuWBDz+EMloZtvItxi59gr5brXbyzl6D+WDsGayqORLKuiECy5d7QMgHyQwZzRIRz2XrnAMg0NTClPfv4vDZ/02Phi1s6zOSbX1HURpo4ZC3/8gnXvsF5c27AJtkdrkvmYBwAvC6iHwoIgtE5B0RWZDKxojI6SKyWESWisiNqfxs51xqTfzwUWo3zmFr39E09uj3f8NEzd0q2dp3NP22fMiB8/+HQMBWL7vUCASguTk9QTaZVUZnpP707YIrmH4LnAKsBGaLyGOq+m46z+uc64SGBkYtnsniqhpKW2TvPYsibK+qYeCHb3LQQZ9h2LDqrDSzkNTXw7PPwuuvWzKByko47TQ45pjoVUyTlXBAUNVlqTllVIcDS1X1IwAR+TtwNuABwblcs2QJpW0tHFRXzhtvWOegvENl9YbGEsrblAsOXISVUXGd9cILcO+9lixgyBDbK9rQAA8/DDNnwvXXQ01N18+TTOqK70V6XlV/0PVmAFazeUXY45XAESn6bOdcKjU2Ara09IgjYN482+AevjGtZ0844KASqip3Za+dBeCDD+DPf7afdXhuyR49LPnwpk1WtO7HP+56kbpkhozC/1W7Y0VyUrnsNFL1tb1GyURkGjANYOTIkSk8vct1gYAlXW1qsu7yoEHZblERq6z8v78OG2ZzBOvXw/btFhT69bOS3LIsYA9cpz3xhCUOiJZoeMAA2y0+f74F565IZsjov8Ifi8gvgMe6dvo9rARGhD2uAVZHaMfdwN1guYxSeH6XowIBePllePxx2LzZLjiBAIwdC+edB+PHZ7uFRWjsWOjd28YtevSgtNT2HOxRXbW11fJhHeCFFTtr92670McbDqqshJde6npA6Er6657APl07/R5mA2NFZHQwR9IFpDbguDykCn/9K/zhD/b3kSMtcdrIkZZM7Sc/gVmzst3KIlRebtF49eo906WHBAK2+eC00/boTbjkNDTYnyVxrtQVFdY766pk5hDeoX0IpxSoBn7Y9SYYVW0VkSuBfwc//0+quihVn+/y07x58PTTMHr0nv8pRKyr3LOnBYv99oP+/bPWzOI0ZYrV2H7sMQsQ/fvbP8yWLTbHcPzxFjSKkCosXWqrgt5+2yrU1tZafJw8ee8J+Gh69rQfaVubdbaiaWy0DPZdlcwcwtSwv7cC61Q1wq1B56nqE8ATqfxMl79UrTxDv37R75B69LCb0ddeg6lTIx/j0kQEzj0X6upsGcw777SnOT3xRBgzZs8ER0VCFR56CKZPb8/zVFJicyy/+Y3Vmrr6arvYx9Ojhw0DzZ3bYTiug507LT53VTIB4XJVvSH8CRH5WcfnnEuVhgZLjxBv7UC/fvDmmx4QskLE/oEuuSTbLckZL71knaba2j3v6vv3t9/VJUvgT3+CK69M7PNOP92GRXfvjhxE1qyx3sGkSV1vezJzCKdEeC6tm9VccWtttetNvJvM0lLbuelctrW1waOP2l6BSEM8IjYHNmeOXcgTMXKkVbjdssWmZXbvhpYW2LrVNqv17w/XXpv4MFQscQOCiHwtOH8wLpiyIvT1MfBO15vgXGS9elmXObjkPaodO+L3IpzLhPp6u1DHqi8Vusl5663EP/fgg20BxTnn2BDp1q3W25g2DW6+2Zb4pkIiQ0b3A08CPwHC8wvtUNXNqWmGc3srLYVTT4VHHrENOJGo2h3TiSdmtGnORbR7d2LHVVTYRT0ZAwfCWWfZV7rE7SGo6jZVrVfVC4HtwGBgFDBJRI5LX9Ocs4myfv1sQq4jVduQM3myrTJyLtsSmSgGG/JJVf6hVEpm2emXgK9jG8bmAUcCrwN+b+bSpk8fuOEGuOMO645XVNhXQ4ON19bVwZe+FH+dtnOZUFtrv7O7dkUfNlK1YZ9DDslo0xKSzCqjrwOHAbNU9QQRGQ98Pz3Ncq7d4MHwgx/Y6ozZs22J3aBBcOSRtrqiCFc2uhxVWgpnn22riEaN2ntiWRVWrLAbmVjLSLMlmYDQqKqNIoKIdFPV90VkXNpa5lyY0lLYf3/7ci6XHX+8DXHOmGGLIgYOtB7stm22Umj//eGyy7LdysiSCQgrRaQv8AjwtIhsIUKuIeecK2YicP75Nrf1zDO2276tzXoMn/98cjuVMy2Z5HbnBv96i4g8D/QBnkpLq5xzLo+J2EKH0GIH1fwY2kx4Kk7M50Xke6r6IjaxPDldDXPOuUKRD8EAktupfCdwFHBh8PEOrOSlc865ApDMHMIRqnqIiLwNoKpbgmmqnXPOFYBkeggtIlJKMAW2iFQDgbS0yjnnXMYl00O4A/gXMEhEbgU+A3w3La1Kt7Y2G9Tz3UxFZ9Mm2LjR/ulramxZoHPOxA0IIvK/qnoxMBC4HjgJq398jqqmsqZyerW2WqWKJ5+Ejz6ygLDvvnDGGXDQQbGrT7i8t3IlPPywLQEsKbFVH2VllgPprLMSTzmQa3butPXtZWXtefed6yxRjV2WWETexdJcPwZMwYLB/8lmgru6ujqdM2dO/AObmuDOOy0g9OvXnkRkyxb7OvJISxuYq4uDs0XV8u3Om2cpRQcOtOIn1dXZbllSPv4YfvpTuwcYNKj9otncbCmIR42C66/Pr6CwerVtfAqVD1W1f5ZPfhKOPtrvb1xsIjJXVes6Pp/IkNHvsP0G+wBzwz8Tm09IZV3l9HjgAatUPXr0nuu/QhUr3nzT/jd99rPZa2Ou2boV7roLFi+2K2hFhV1BH3gAjjnGdth065btVsbV1ga//S10727/1OEqKiwYLFtmOewvvDDyZ+SaDz+En//cgsCwYe0X/x074J57LMXHf/6nBwWXvESynd6hqvtjNY73Cfsaraq5Hwy2b4cXX7SqFJEWA4vYYPLTT1tGKmc/h9tus6G1UaOs2MCQIe0V7l96Ce6+2zJ05bj33rM5g47BINywYfD884mnLs6mpib49a9t7iM8GIDVsh892n7dX3kle210+SuRAjkCoKpfi3dMTnr33fgVqsvLbY7h/fcz165c9uqrNugeKXNcSYmldJwzx25Fc9yiRfFHAsvL7Vdk+fLMtKkr5s+3nkC01MklJTYsNmNGXsRrl2MSmYJ6XkSuEpE9alKJSIWInCgifwG+kJ7mpUAyt30NDelrR74IBGzifdCg6MeI2C3qs89mrl2d1NKS+ERrW1t625IKs2fHn+uorLReUaQaEs7FksgcwunAZcDfRGQ0sBXoDpQCM4Ffqeq8dDWwy3r3TvzYWHXvikVjo80fRCtRFtKnjxUoyHE1NfHrLataMMiHufLGRltRFI+IBUPnkpHIHEKjqt6pqkdjldJOAg5R1VGq+uWcDgYAEyfa/6BY/zuam22G0XMrt99Ox1l9RiCQF7OWdXXWzFhBYcMGmDAhdqeoM1pbbcw/3o8yGcOHx5/qamuzc+ZiRS6X25JatayqLaq6RlW3pqk9qderF5x+ulWliDSoGgjYePnUqbYUpdh16wZjxthy3Fg2b87Nkk8d9O4Nn/60/fNHuifYts2CRaoWmKnatNWvfgVf/jJ89avwzW/CzJm2Z6Crjj7aAk2sILNuHRx+uA0dOZeMZHYq569zzrFhkJdftgHY0JKTzZtt3uCkk+DMM7PZwtwhYpv1br/dbjEjDcA3N1sgPfbYTLeuU04/3f58+GG7e+7e3Zrf1NReorO2tuvnUYWHHoLHH7dANGKE/fh27YK//c2mXK6/HgYM6Pw5amrgqKPgtdeszR3n/Ldvt3ZMndqlb8UVqbgb03JZwhvTwK4AS5bY8tLFi+25CRPg5JNh7Nj8yU+bCYEA/PnPtn5x6ND2/A6qtsRlwwa45BI45ZTstjNJ27fb4qjly20UceJEmDQpdfsRX3/d9j/W1kYeTVu71sqB3nxz13YUNzdbicZZs6ztvXpZr2HnTusVXH21/Uo7F020jWmdCggiMkRV16akZV2QVEBwyWlrs3JPM2bYlUbEAsWQITYGU7fX71JRU4XvfMd6HVVV0Y+rr4dvfxvGdbH4bKg2b2iFcEUFHHGEVePykU8XT1d2KkfyBJCyAWQROR+4BdgfOFxV/SqfbaWlcNppNpxWX2/LWyorbXOa96b2snq1jd2PGBH7uIoK2xjf1YAgYv8UI0fGP9a5RHU2IKT6irAQOA/4fYo/13XR9t1lbCnfl9Lu1jko81gQUWOjXaTjxcqKChu6ci4XdTYg3JPKRoSypubyhudis3q1TY6+8Ub7Ra53b5tvPukkzwPYUe/eNqIWr3ZuY6PlCHQuF3U2IPwzpa1IgohMA6YBjPT+clqEsoOq2rr30ATp7t1w//2WH+jKKz0ohBs0CPbZx+bb+/ePfIyqTf4eeWRm2+Zcojq71uGJZN8gIs+IyMIIX2cn8zmqereq1qlqXXU+bC3NMy0tcMcdkZOn9expydPeftvW1afa1q22Uue55yxFQz4kmwsRgXPPte+hqWnv10OTwAce6OP+LndlbA5BVU/u5LlcBi1aZHvSoq3LF7GVqE89BaeemppeQmOjrdN/+eX2fEIi9tlnnmkFbPJgUzSTJsEXvwh/+YstK+3f39q9Y4d97b8/fOUrPifvcldOzCG43PHWW/GXLfboYUMjq1fHT3kUT0sL/OY3FohGjNjzwt/SAv/8J6xaZa/Nnm3PjRxplc7Gjct8hbDdu60NPXtGDobHH2/tevllm39parLgesop7VlUnMtVnfr1VNU7U9kIETkX+A1QDcwQkXmqeloqz+ES09SU2N24iI2Hd9Xs2fDOO3vXLgK74JaVWf7/yZMt+JSUwMKFtnRz8mT42tfSv+4+lI7iySctcInYaqEpU2yCvePI5ZAhcP759uVcPsmJCqyq+i9VrVHVbqo62INB9tTUxM8CHgjYV6yiM4lQhSeesFQOkYZR1q+3HkvPnjas1Lu3/X3wYLvrnj8f/vjHrrUhkTY++ij87Ge2HSO09r9/f9v0fvPNeZH01bmE5ERAcLnjyCPb00FHs2GD3Z1HW02TqOZmGw6KtLNX1eoVde/ent8/nIj1GGbPts9IlwULLAfSyJG2XDQUuCoqbBirvNzSPjU2pq8NzmVKIhXT+ifw1TcDbXUZUF1t493LlkUOCtu32xj6ueem5nzRMqfs2mW5B2MNB4nY8NYbb6SmLZHaNn265fiLNvbfr59lTJ03Lz1tcC6TEplDWB38irU2ohTwxXQ5KLTccdMmu3jW1sbOtQPtqaBnzrT39OxpwWH3bhu2uf761CydDBW537Zt79z9zc3tO38bGmwoK5Lu3dNXGWzbNli6NP732ru3ZR/1/QUu3yUSEN5T1YNjHSAib6eoPUWtrc02fb34ol3Aq6rgmGNs7XpFRfKft3ixbSRbtqx9NY6Ifeb550cvJldWBp/7nE2Yvv66ZQetqLDyBwcdZCUTUiGUafvOOy0Ndfg8QmhiOxCwn0u0ZbAtLckVxUtGc7P93OItEy0vj1+0Jhe0tdkw3Pz5FmSHD7e6CV0d+nOFI5GAcFSKjnExbNliY9HLltkdeejOd948G8a59lrbKJaoBQvgl7+0C+2oUe0XtdZWWxL50Udw442xq4YOHmylJNKprg4OPtguUjU17UMzVVV2oQ1VM4tW/aulJX2JV0OBpq0t9sqr3bu7nqwu3VautA2H69dbcC8rs0ypDz5oOQw/85n82Ovh0iuhEprRXhOR/4x3jIuvqQn+679gzRq7Ex40yC6IAwfa49274ec/tyGMRDQ2wu9+Z4GkX78973DLymwIZOVKGx/PtrIyuOIKm7dYu9YC4rJl1isZMsQ2wU2YEPkufd06CyL77ZeetvXsacNA69ZFP0bVft7HH5+eNqTC+vXwk59Yr6C21m4sBg2y34Phwy3D+QMPZLuVLhd0dZXR91PSiiI3b55doKP1AKqrLRi8+mpinzd/vgWRWHf/Q4daioh4S0wzoaICLrrIyk5ecYXt9v361+0idemltqxz06b2CeiGhvae1FVXpXdz2hlnWA8hUvlLVft322+/9AWlVJgxw246IlVqKy21HuTMmRaQXXGLO2QkIguivQQMTm1zitPTT9vQTiyDBlm6iDPOiD+mvXBhe5GzaCoqbPhozRpLypYLKiv3Hv65+GI44ADbr7B0qX3vvXpZjZ7jjos/Qd5VNTXwjW/YbupNm2zoqrzcAsSuXRYI0h2UumLXLruRGDIk+jGBgC3r/fnP4ROfgPHjLc2GDyEVn0TmEAYDpwEdq64L8FrKW1SE1q2LXxC9Rw/r+re0xJ9gbmtLPF9OIJDYcdkiYnMMkydbz6CtzXoGmbxYTZxoF8vZs+GVV2yIaNw4m3TP9Qvn5s3Wk4m0bFbVel8LF1qPcuNGW1Y8Y4b1Jr72NRgzJuNNdlmUSECYDvRW1XkdXxCRF1LdoGLUrZvdrce60Le12V1oIrlwxoyx1UGxtLXZBWHQoOTami0iFgiyparKAsBJJ2WvDZ0RK1jV11vm2qoqO65fv/blvVu32u7s737Xs7MWk0Qmlb+oqq9Eee1zqW9S8Tn6aBuOiGX9equZm8jQxGGH2XHNzdGPWbsWjjoq/UMuLrsGD7beZ8dU4i0t1jOoqrKbjOZmm1cK6dvXgsSDD2a0uS7LcnTks7gcc4zdAUfL/9/cbMMUid6dVlXZ5rLly/fOza9qSzm7dYOzk6pE4fJRaamlEF+7ds9d4WvWWC8xFAzKyvYMCGC9x4ULY6+ycoUlkUnlt1T1kK4e46KrroavfhV++1u7UA8aZHf4qjauu2OHTa4mM5576ql2MfjHP+w/fGlpe4nHmhobH86X4aJ0UrUKca++ahe+nj2t5zRpUuFUhDvhBLuwz59vF/3u3W3VWkmJTTqHqrh1/H5DqUE2bLCehit8otGSyYQOEGkAPoh1CNBHVTM+0lhXV6dz5szJ9GnT5uOPbTXN3Ln2nzEQsDX4U6fa5GVnNDTYhWDNGvsPv//+tqrIi7RYj+z3v7efT3m5BYOWFnt+wAC45hpLYFcImpvh3/+2lWqhZbsrVtj8wIQJ0XcrL18O111nx7jCISJzVXWvLZ2JBIRESqC0qerKzjauswotIITs3GkXpe7dfYw/XQIB28m9aJFdFDsGyNC+h+9/P/L6/XzV3GyBYNEiuO++2DcaoWXJt98efxWcyy/RAkIik8rLwr+AL2JF7g8HKoLPZzwYFLLevdt3K7v0WLLECvMMHWrBt7FxzzH2AQPs+WefzV4b06GiwoYezzzThg63bo1+7OrVcOyxHgyKSdKTyqr6PeAOYAfwaRHxcpouKYGAjV1ns4bAo4/acMhTT8Ezz9hwyosv2kUwFBiGDLHd3C0t2WtnupSVweWX27/D+vV7BsPW1vbUIZ/+dPba6DIv4RKaInI7cK2adcBTwS/nErJ9O7z0kqVJ2LHDnhs3znZfH3hg5uY1PvwQ/v53u/D169c+gd/QALNmwdixNqlcUWHBYOfOrleHy0Vjxtg+gwcesCy74RlxjzvOgoH3DopLMjWVdwKPicgFqrpLRE4FblbVo9PUNldANmyw3b4bN9pwWP/+dhFetQp+8Qs4/XRLuZ3uoNDYaGPi3bvb+cMvgj162CqvDz6w9g0bZr2ZQlltFMmoUVbfYt066ymUlNicigeC4pRwQFDV74rI54AXRKQJ2AXcmLaWuYIRCFjq5R077AIUImJj9X372tDNyJE2Zp1Ob71l7dh3X5tD6FiRraTEAsOSJfbavvvGThJYKAYP9qWlLok5BBE5CfgyFgiqgatV9eV0NcwVjiVLLCtotARrpaXWa3j88fTnVpo1y+5+a2rsvJF2c3fvbpOt69fb5Ksv0XXFIplJ5ZuA/6eqU4DPAA+IyIlpaZUrKLNnxx92qay04aTVq9PblsZGm1Dt1s1SfDQ02BxBaFJV1XZ379plG7omT05ve5zLJQkHBFU9MZTTSFXfAc4AfpSuhrnCsXNnYiVAS0rSv/Jo6ND2cpdDhtjkaXW1TXhv22Z/lpXZ+vyLL/begSsuyUwq70FV1wSHkZyLqbo6/oVe1XLrpHsy89hj4YUX7HwitnroiCOsfaEUH1u3WiqHYpg7cC5cl5LbqWoO1Ntyue6II2yJZ6xN8Vu22DLIdOdXGjMGDjrI1tmHtye0K7ytzR6ffnp62+FcLvJspy7tamrg0EP3vgiHNDbaXfl556V/iEbEEglOmmT5fNassWGizZvtcVMTfOtbVmvYuWLT6SGjVBKR24BPAc3Ah8B/qurWrDbKpYwIfOlLcOedsGCBDcVUVtrd+ObN9vpXvpK5BGo9e8K119oGtdDu5B49bJjo0EOzW4jHuWyKm9wuI42wTW7PqWqriPwMQFVviPe+Qk1uV6ja2mDxYssPtHy5rTw67DCrB1Fdne3WOVc8oiW3y4kegqrODHs4C1vW6gpMaan1AjyVsnO5KRfnEC4Dnsx2I5xzrthkrIcgIs8Akfaq3qSqjwaPuQloBe6L8TnTsPTbjPTq3845lzIZCwiqenKs10XkC8BU4CSNMbGhqncDd4PNIaS0kc45V8RyYg5BRE4HbgCOV9Uopeadc86lU67MIfw3UAk8LSLzROR32W6Qc84Vm5zoIajqvtlug3POFbucCAjOhGr79uhhX845l0keEHLA0qXw5JNWvEXEvg4/3PLphBeUSYeGBnj7baivt30C48fDxImW8dM5V1z8v32Wvfwy/OEPli5hxAhLAd3WZsFh1iy48kpLp5Bqqnbu++6z/D3dullxmieftApmX/ua1Tt2zhWPXJlULkorVsCf/mS1ewcPbq/vW1pqefurq+Guu6xyV6q9+ircc4+lf66ttfMNH25/Lymx+scffpj68zrncpcHhCx67rn26l2R9Oxpd/KvvJLa8zY1Wc9g2LC9awqDpYHu2RP+9rfYKaudc4XFA0KWhC708QqbDxyY+oCwYIHNHcSauO7f33oIq1al9tzOudzlASFLAgFoabHhoVjKy231USqtXBl/0jg0ub1hQ2rP7ZzLXR4QsqS01MbvG+LUnNu1K/WpocvLLSAlosR/Q5wrGv7fPYtOPTX+HfiWLXDaaak979ix8ecGQqUk073s1TmXOzwgZNHRR0OfPrBxY+TX1661OYZULzsdOxaGDIFNm6Ifs3q1VRDr2ze153bO5S4PCFlUVQXXX2+rjOrr7QK9c6cFiPp6uxhfd13qdy2XlNg+g5YWCzrhw0etrbYcduBA+I//SO15nXO5LSdKaHZWoZTQbGqC+fOtvu+2bbbCZ8oUOOAAG+9Pl1Wr4MEHbdVRSYkNI5WUwCc+AZ/+tPVenHOFJ1oJTQ8Ijg0bbPNbSYltTquqynaLnHPplNM1lV12VVd7kXvnnM8hOOecC/KA4JxzDvCA4JxzLsgDgnPOOcADgnPOuSAPCM455wAPCM4554I8IDjnnAM8IDjnnAvygOCccw7wgOCccy7IA4JzzjnAA4JzzrmgnAgIIvJDEVkgIvNEZKaIDMt2m5xzrtjkREAAblPVA1V1MjAd+F6W2+Occ0UnJwKCqm4Pe9gLyN+qPc45l6dypkCOiNwKXAJsA06Icdw0YBrAyJEjM9M455wrAhkroSkizwBDIrx0k6o+Gnbct4HuqnpzvM/0EprOOZe8rJfQVNWTEzz0fmAGEDcgOOecS52cmEMQkbFhD88C3s9WW5xzrljlyhzCT0VkHBAAlgFfzXJ7nHOu6OREQFDVT2e7Dc4lY9s2WLYMAgGoroZhw0Ak261yrmtyIiA4ly+2bYMHHoBZs9qfCwRg7Fi48ELYZ5/stc25rvKA4FyCtm2DH/8YNm6E4cOhtNSeV4U1a+DWW+GGG2C//bLbTuc6KycmlZ3LBw8/DBs2wIgR7cEAbKho4ECoqoK77oLW1uy10bmu8IDgXAJ27IBXX7W5gmj69IGtW+HddzPWLOdSygOCcwlYscL+LIszyFpWBu/7ommXpzwgOJeAQCCx40SgrS29bXEuXTwgOJeAwYMtKMQLDC0tMGpUZtrkXKp5QHAuAdXVcMABNqkcTWMjVFTAwQdnrl3OpZIHBOcS9NnPWg9h8+a9X2tshFWr4KKLoEePzLfNuVTwgOBcgmpq4Nvftgv+xx/D8uWwciXU19vqomnT4Nhjs91K5zrPN6Y5l4TaWvjJT2DJEvjgA5szqKmBAw+E7t2z3TrnusYDgnNJKimB8ePty7lC4kNGzjnnAA8IzjnngjwgOOecAzJYUzkdRGQDVlAn3EBgYxaakymF/v1B4X+P/v3lt0L4/kapanXHJ/M6IEQiInMiFY8uFIX+/UHhf4/+/eW3Qv7+fMjIOecc4AHBOedcUCEGhLuz3YA0K/TvDwr/e/TvL78V7PdXcHMIzjnnOqcQewjOOec6oSADgoj8UEQWiMg8EZkpIjEKH+YfEblNRN4Pfo//EpG+2W5TKonI+SKySEQCIlIwqzlE5HQRWSwiS0Xkxmy3J9VE5E8isl5EFma7LekgIiNE5HkReS/4+/n1bLcp1QoyIAC3qeqBqjoZmA58L8vtSbWngUmqeiCwBPh2ltuTaguB84CXst2QVBGRUuC3wBnABOBCEZmQ3Val3L3A6dluRBq1At9U1f2BI4ErCu3fsCADgqpuD3vYCyioiRJVnamqrcGHs4CabLYn1VT1PVVdnO12pNjhwFJV/UhVm4G/A2dnuU0ppaovARGqRRQGVV2jqm8F/74DeA8Ynt1WpVbBZjsVkVuBS4BtwAlZbk46XQY8kO1GuLiGAyvCHq8EjshSW1wXiUgtcDDwRpabklJ5GxBE5BlgSISXblLVR1X1JuAmEfk2cCVwc0Yb2EXxvr/gMTdh3dj7Mtm2VEjk+yswEuG5guq5FgsR6Q08DFzTYTQi7+VtQFDVkxM89H5gBnkWEOJ9fyLyBWAqcJLm4drhJP79CsVKYETY4xpgdZba4jpJRMqxYHCfqv4z2+1JtYKcQxCRsWEPzwLez1Zb0kFETgduAM5S1d3Zbo9LyGxgrIiMFpEK4ALgsSy3ySVBRAT4I/Ceqv4y2+1Jh4LcmCYiDwPjgACWDfWrqroqu61KHRFZCnQDNgWfmqWqX81ik1JKRM4FfgNUA1uBeap6WlYblQIiciZwO1AK/ElVb81ui1JLRP4GTMGyga4DblbVP2a1USkkIscALwPvYNcWgO+o6hPZa1VqFWRAcM45l7yCHDJyzjmXPA8IzjnnAA8IzjnngjwgOOecAzwgOOecC/KA4JxzDvCA4JxzLsgDgisIIlIrIg0iMi/sub3qD4hIj2CdjGYRGdjFc/YQkReDqa0RkauDufKTyi0lIn1F5PKutCWBc+xVq0BEKkTkJRHJ2xQ2LrU8ILhC8mGwBkbU+gOq2hA8JhV5hC4D/qmqbcHHlwNnqupFSX5O3+B7kyIm0f/D99KhVkEwDfezwH8ke25XmDwguLwRrFZ1SvDvPxKRO2Icnon6AxcBocyzvwP2AR4TkWtF5PMi8mawN/L7sF7EIyIyN1hxa1rwc34KjAkee1uwtxN+J3+diNwS/HttsBdyJ/AWMCLaucLFqFXwSPD7cM4DgssrN2MpzS/CctFfG+PYSPUHUlbMJJigbh9VrQcI5pJajdXeeAq76z462Btpo/2ie5mqHgrUAVeLyADgRoK9G1X9VgKnHwf8j6oeDPSMca5ELAQOS+J4V8B87NDlDVV9KZhx8hvAlNBQjYj8EMtCGS7d9QcGYon3IjkJOBSYbc2lB7A++NrVweR9YOmwxwJrkzz3MlWdlcC54lLVtuB8SmWwCpgrYh4QXN4QkQOAocDG0MVLRIYQ+fc46foDInIF8OXgwzOBc8Mfq2r4+xuA7tE+CviLqu5R61pEpgAnA0ep6m4ReSHKZ7SyZ++94zG74p0rSd2Axi683xUIHzJyeUFEhmKV4c4GdolIKB32wcC8CG9Juv6Aqv42OGwzWVVXd3zc4dgtQKmIRLqgPwt8RkQGBdveX0RGAX2ALcFgMB4r1A6wA6gMe/86YJCIDBCRblghpGiinSshwSGrDarakuh7XOHygOBynoj0BP4JfFNV3wN+CNwSfHkyEQKCqrZipVP/jRVDf1BVF6W4aTOBYyKc+13gu8BMEVkAPI31bJ4CyoLP/RCYFTx+E/CqiCwUkduCF+cfYPV6pxOjwFOMc+0hWKvgdWCciKwUkS8GXzoBKJh8/q5rvB6Cy2si8kdsWGckMF1VJyX4vnqgTlU3duHcBwPfUNWLO/sZ2SYi/wS+raqLs90Wl33eQ3B5TVW/qKoBbHVNn/CNaZGENqYB5bRXversud8Gno+0zDMfBIfSHvFg4EK8h+Cccw7wHoJzzrkgDwjOOecADwjOOeeCPCA455wDPCA455wL8oDgnHMO8IDgnHMuyAOCc845AP4/znfK4PpkDfYAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "slideshow": { + "slide_type": "skip" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Perceptron\r\n", + "\r\n", + "Since perceptron is a binary classifier, for each input vector $x$ the output of our perceptron would be either +1 or -1, depending on the class. The output will be computed using the formula\r\n", + "\r\n", + "$$y(\\mathbf{x}) = f(\\mathbf{w}^{\\mathrm{T}}\\mathbf{x})$$\r\n", + "\r\n", + "where $\\mathbf{w}$ is a weight vector, $f$ is a step activation function:\r\n", + "$$\r\n", + "f(x) = \\begin{cases}\r\n", + " +1 & x \\geq 0 \\\\\r\n", + " -1 & x < 0\r\n", + " \\end{cases} \\\\\r\n", + "$$\r\n", + "\r\n", + "However, a generic linear model should also have a bias, i.e. ideally we should compute $y$ as $y=f(\\mathbf{w}^{\\mathrm{T}}\\mathbf{x})+\\mathbf{b}$. To simplify our model, we can get rid of this bias term by adding one more dimension to our input features, which always equals to 1:" + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 5, + "source": [ + "pos_examples = np.array([ [t[0], t[1], 1] for i,t in enumerate(train_x) \r\n", + " if train_labels[i]>0])\r\n", + "neg_examples = np.array([ [t[0], t[1], 1] for i,t in enumerate(train_x) \r\n", + " if train_labels[i]<0])\r\n", + "print(pos_examples[0:3])" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[[ 0.92180622 0.91789985 1. ]\n", + " [-1.06435513 1.49764717 1. ]\n", + " [ 0.32839951 2.25677919 1. ]]\n" + ] + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Training Algorithm\r\n", + "\r\n", + "In order to train the perceptron, we need to find out weights $\\mathbf{w}$ that will minimize the error. The error is defined using **perceptron criteria**:\r\n", + "\r\n", + "$$E(\\mathbf{w}) = -\\sum_{n \\in \\mathcal{M}}\\mathbf{w}^{\\mathrm{T}}\\mathbf{x}_{n}t_{n}$$\r\n", + " \r\n", + " * $t_{n} \\in \\{-1, +1\\}$ for negative and positive training samples, respectively\r\n", + " * $\\mathcal{M}$ - a set of wrongly classified examples\r\n", + " \r\n", + "We will use the process of **graident descent**. Starting with some initial random weights $\\mathbf{w}^{(0)}$, we will adjust weights on each step of the training using the gradient of $E$:\r\n", + "\r\n", + "$$\\mathbf{w}^{\\tau + 1}=\\mathbf{w}^{\\tau} - \\eta \\nabla E(\\mathbf{w}) = \\mathbf{w}^{\\tau} + \\eta \\mathbf{x}_{n} t_{n}$$\r\n", + "\r\n", + "where $\\eta$ is a **learning rate**, and $\\tau\\in\\mathbb{N}$ - number of iteration.\r\n", + "\r\n", + "Let's define this algorithm in Python:" + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 6, + "source": [ + "def train(positive_examples, negative_examples, num_iterations = 100):\r\n", + " num_dims = positive_examples.shape[1]\r\n", + " \r\n", + " # Initialize weights. \r\n", + " # We initialize with 0 for simplicity, but random initialization is also a good idea\r\n", + " weights = np.zeros((num_dims,1)) \r\n", + " \r\n", + " pos_count = positive_examples.shape[0]\r\n", + " neg_count = negative_examples.shape[0]\r\n", + " \r\n", + " report_frequency = 10\r\n", + " \r\n", + " for i in range(num_iterations):\r\n", + " # Pick one positive and one negative example\r\n", + " pos = random.choice(positive_examples)\r\n", + " neg = random.choice(negative_examples)\r\n", + "\r\n", + " z = np.dot(pos, weights) \r\n", + " if z < 0: # positive example was classified as negative\r\n", + " weights = weights + pos.reshape(weights.shape)\r\n", + "\r\n", + " z = np.dot(neg, weights)\r\n", + " if z >= 0: # negative example was classified as positive\r\n", + " weights = weights - neg.reshape(weights.shape)\r\n", + " \r\n", + " # Periodically, print out the current accuracy on all examples \r\n", + " if i % report_frequency == 0: \r\n", + " pos_out = np.dot(positive_examples, weights)\r\n", + " neg_out = np.dot(negative_examples, weights) \r\n", + " pos_correct = (pos_out >= 0).sum() / float(pos_count)\r\n", + " neg_correct = (neg_out < 0).sum() / float(neg_count)\r\n", + " print(\"Iteration={}, pos correct={}, neg correct={}\".format(i,pos_correct,neg_correct))\r\n", + "\r\n", + " return weights" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "skip" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Now let's run the training on our dataset:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 7, + "source": [ + "wts = train(pos_examples,neg_examples)\r\n", + "print(wts.transpose())" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Iteration=0, pos correct=0.2631578947368421, neg correct=0.6190476190476191\n", + "Iteration=10, pos correct=0.8947368421052632, neg correct=0.8571428571428571\n", + "Iteration=20, pos correct=0.8421052631578947, neg correct=1.0\n", + "Iteration=30, pos correct=0.8947368421052632, neg correct=0.9523809523809523\n", + "Iteration=40, pos correct=0.8947368421052632, neg correct=0.9523809523809523\n", + "Iteration=50, pos correct=0.9473684210526315, neg correct=0.9047619047619048\n", + "Iteration=60, pos correct=0.8947368421052632, neg correct=0.9523809523809523\n", + "Iteration=70, pos correct=0.8947368421052632, neg correct=0.9047619047619048\n", + "Iteration=80, pos correct=0.8947368421052632, neg correct=0.6190476190476191\n", + "Iteration=90, pos correct=0.8421052631578947, neg correct=1.0\n", + "[[-0.66042328 4.90850882 -1. ]]\n" + ] + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "As you can see, initial accuracy is around 50%, but it quickly increases to higher values close to 90%.\r\n", + "\r\n", + "Let's visualize how classes are separated. Our classification function looks like $\\mathbf{w}^Tx$, and it is greater than 0 for one class, and is below 0 for another. Thus, class separation line is defined by $\\mathbf{w}^Tx = 0$. Since we have only two dimensions $x_0$ and $x_1$, the equation for the line would be $w_0x_0+w_1x_1+w_2 = 0$ (remember that we have explicitly defined an extra dimension $x_2=1$). Let's plot this line:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 8, + "source": [ + "def plot_boundary(positive_examples, negative_examples, weights):\r\n", + " if np.isclose(weights[1], 0):\r\n", + " if np.isclose(weights[0], 0):\r\n", + " x = y = np.array([-6, 6], dtype = 'float32')\r\n", + " else:\r\n", + " y = np.array([-6, 6], dtype='float32')\r\n", + " x = -(weights[1] * y + weights[2])/weights[0]\r\n", + " else:\r\n", + " x = np.array([-6, 6], dtype='float32')\r\n", + " y = -(weights[0] * x + weights[2])/weights[1]\r\n", + "\r\n", + " pylab.xlim(-6, 6)\r\n", + " pylab.ylim(-6, 6) \r\n", + " pylab.plot(positive_examples[:,0], positive_examples[:,1], 'bo')\r\n", + " pylab.plot(negative_examples[:,0], negative_examples[:,1], 'ro')\r\n", + " pylab.plot(x, y, 'g', linewidth=2.0)\r\n", + " pylab.show()" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "skip" + } + } + }, + { + "cell_type": "code", + "execution_count": 9, + "source": [ + "plot_boundary(pos_examples,neg_examples,wts)" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVa0lEQVR4nO3db4xcV3nH8d+zaxt711gJ9q4R2eyaSECgEApeUlDaqiUEpRAFCbVS2k0UUYkFSiNXAhESixd9sRUqCLAEVbUK4QVeQavwtyilhAKVaEnIGhJSMEUQ2Y4TyK4NUZLdxH92n764M/Z4PXfmztxz59458/1Iq92ZvXPn3LH3N2eee8655u4CAMRjqOwGAADCItgBIDIEOwBEhmAHgMgQ7AAQGYIdACITJNjN7BIzu8fMfm5mh83sTSH2CwDo3KZA+zkg6Zvu/udmtkXSSKD9AgA6ZHknKJnZDkkPS7rCme0EAKUL0WO/QtKypM+Z2WslHZK0z91XGjcys1lJs5I0Ojq698orrwzw1AAwOA4dOnTC3cfabReixz4t6X5J17j7A2Z2QNLT7v6RtMdMT0/74uJirucFgEFjZofcfbrddiFOnh6XdNzdH6jdvkfS6wPsFwDQhdzB7u6/kfSYmb2idte1kn6Wd78AgO6EGhVzm6SF2oiYRyW9K9B+AQAdChLs7v6QpLZ1HwBA8Zh5CgCRIdgBIDIEOwBEhmAHgMgQ7AAQGYIdACJDsANAZAh2AIgMwQ4AkSHYASAyBDsARIZgB4DIEOwAEBmCHQAiQ7ADQGQIdgCIDMEOAJEh2AEgMgQ7AESGYAeAyBDsABAZgh0AIkOwA0BkCHYAiAzBDgCRIdgBIDIEOwBEJliwm9mwmf3YzL4Rap8AgM6F7LHvk3Q44P4AAF0IEuxmNiHp7ZLuCrE/AED3QvXYPyXpQ5LWA+0PANCl3MFuZjdIWnL3Q222mzWzRTNbXF5ezvu0AIAUIXrs10i60cyOSPqipDeb2cGNG7n7vLtPu/v02NhYgKcFADSTO9jd/Q53n3D3PZJukvQdd785d8sAAF1hHDsARGZTyJ25+/ckfS/kPgEAnaHHDgCRIdgBIDIEOwBEhmAHgMgQ7AAQGYIdACJDsANAZAh2AIgMwQ4AkSHYASAyBDsARIZgB4DIEOwAEBmCHQAiQ7ADQGQIdgCIDMEOAJEh2AEgMgQ7AESGYAeAyBDsABAZgh0AIkOwA0BkCHagAwsL0p490tBQ8n1hoewWARfbVHYDgH6xsCDNzkqrq8nto0eT25I0M1Neu4CN6LEjeqF62fv3nw/1utXV5H6gSuixI2ohe9nHjnV2P1AWeuyIWohedr3H797895OTXTcPKETuYDezy83su2Z22Mx+amb7QjQMCCFvL7ve4z96tPnvR0akubnz23JiFVUQosd+VtIH3P2Vkt4o6f1m9qoA+wVyS+tNZ+1lN+vx101NSfPzSUmn8Q3A/XzJh3BHGXIHu7v/2t1/VPv5GUmHJV2Wd79ACHNzSa+6UWMvu520nr2ZdOTI+To9J1ZRJUFr7Ga2R9LrJD3Q5HezZrZoZovLy8shnxYDKGvZY2Ym6VVPTSVhPDUl3XprErhZSiZZe/ycWEWluHuQL0nbJR2S9M522+7du9eBbh086D4y4p4UPZKvkZHk/tCPzbr91NSF29S/pqbyHi1wnqRFz5DHQXrsZrZZ0pckLbj7l0PsE0iTp+zR6WOb9fjrdfVGeUs+QEghRsWYpM9KOuzun8jfJMQu7+iRPGWPbh47M5PU09fXL6yrb9wmyxsA0AshJihdI+kWSY+Y2UO1++5093sD7BuRCTFhaHKy+fDDLCNd8jy2nZkZghzVEGJUzPfd3dz9Knf//doXoY6mQoweyVP2oGSCQcDMU/RUllJIu1JNnrJHiJIJE5FQdeZp86QLND097YuLiz1/XpRvz57mpZCpqaR+vbFUIyU96qrUq6vePsTNzA65+3S77eixo6falUKqPtEnb/vo7aMXCHb0VLtSSNUn+uRpH8sOoFcoxaBS2pVqypanfVU/NlQfpRj0paqPWsnSvrRyS9U/jSAeXGgDldK4qNaxY8n48rm56pyYbNe+VuP0ixxDDzSiFAME1KrcMjfHiBrkQykGKEGrckurE8eMlkFIlGKAgNqVW5otOxDyuqyARI8dCKrdydVmPfOqj91Hedxdz5x6Rr/67a/0g8d+kPlx9NiBgFqdXE3rmaddeo/RMnFaW1/TidUTWlpZ0tLKkp5ceTL5/mzyfWm14eeVJT139rmOn4NgBwJLW+UxrWc+PCytrV28PaNl+sfqmdULwrge1s2C+8TqCbmyD1rZtmmbdm/frfHRcf1QP8z0GIIdhauXG6o4fLGX0nrga2tJuWbjaJmqjN0fROu+rt8+99vMYb1yZqWj/e/ctvNcWI+Pjmv36O4Lvo+Pjp/7/ejmUSWXvZDs3ZZp/wQ7CsWJwfPSTqzWh0Ly5les588+f2HJo0VYn1g9oTVv8jEqxZbhLdo9urtpWF8Q3Nt3a9fILm0aKjZ6CXYUqtWJwUELrmbj2CXp2WeT7ywr0Bl31++e/935YK4FdrOwXlpZ0tOnnu5o/5duvfSCnvP4yHjzXvb23Xrhlhee61VXAcGOQjGN/rz6G9m+fdLJk+fvP3lycD/FbHR67fS5IE7rXTeG9dn1s5n3vXlo80VljrSwHhsd05bhLQUeabGYeYpCsfDVxQbpNXF3PX3q6Qt70S1KIU89/1RH+9/xgh2pJY+N912y9ZJK9aq7kXXmKT12FCptGv0gnxjs908xZ9bO6MTqieZhvWGo3tLKkk6tncq872Eb1tjoWKawHhsd09ZNWws80v5FsKNw27adD/adO6UDBwa75FC1xcDcXc+efjbzuOqTz51sv9MG27dsTz2Z2BjYu0d369Jtl2rImDeZF8GOwjS7jNxznc+1iE4vPsWkTYI5F9g5JsEM2ZB2jezKFNbjo+Ma2TzSfqcIiho7CjNIteROdTO2v1eTYNqNq965baeGh4ZzvgLoRtYaO8GOwgwNJZeA28hMWl/vfXuqpj4JJstQvSeffTLYJJhmdevGSTCoLk6eonRVqyX3Qn0STJawXl5Z7utJMKgu/uVRmBhGxKRNgkkbV93tJJhzveeUcdXjo+Pa8YId9KqRCcGOwlT1MndZJsE0/tztJJjGsN44+mP8W9/X2N9/XFuOPCZN7pDm3iP9xQAPFUJQ1NjR9+qTYJoO1StwEsxFYZ11Ekyz4UJcIw8Z9PTkqZldL+mApGFJd7n7R1ttT7CjnfokmCxD9bqdBNNu9Ef9K/gkGIYLoUs9O3lqZsOSPiPpOknHJT1oZl9395/l3TfikTYJJi2su50EkyWsX7TtRaVOgvGjx9SsT592P9CpEDX2qyX90t0flSQz+6Kkd0gi2CO3tr6mk8+dbD76oxbWjWWRTibBmOxcr7rdUL1+mwTz+PCkJtYu7rE/PjypiRLakxkL6/eNEMF+maTHGm4fl/QHGzcys1lJs5I0GfN4tz63emY187jqUJNgmvWum06CWViQ3rsxWK4O/AoU7/a1Oc1rVqM6X2Nf0YhuX5vTQontaomF9ftKiGBv+qnyojvc5yXNS0mNPcDzIoNmk2DShuoVNQmmHta5JsFEFCz/PTWjdx+V/kH7NaljOqZJ3ak5/c9UhY+DhfX7SohgPy7p8obbE5KeCLBfpGg2CSYtrLudBNNsxMfGn8dGx3o3CSaiYEnG98/oC6vn2z0yIs2XNb4/S4ml35ekHDAh/ioflPQyM3uppMcl3STprwLsd2C4u556/qlMQ/VCTYJJC+7KToKJKFgqNb4/6yehQZxG3MdCDXd8m6RPKRnueLe7t+x7DMJwx9Nrp7W8spwprJdXlnVm/UzmfW8a2nTxScSRi0d/1Nes7ucrwZzDEMFiZH1dGXtfCT1dK8bd75V0b4h9VVWzSTCtTjJ2Mwkmy1C93aO7o7gSTMdiWJ+girJ+EqrUxwy0M9BLCrSbBLOxp91Xk2BiQ7AUo5MSy8wMr3efiCrY3V0rZ1Yyj6sOMQkmbVx12ZNgokSwhMcnoShVPtg3ToK5aD2QgifB1AN7bGRMo1tGCzxSoAR8EopSKcG+7us68tSRTOOqO50Es3XT1gvWrO54EgwwaPgkFJ1SVne0l5jrPdm337lt54WljpGUMdZ5J8H0uYGf8T3wLwBiV+krKJmZJnZMZArrXSO7tHl4cxnN7CtVnpjZk7zt5QvAGwgqjvXYI1HVYd49G/7cqxeA8dwoERezHjBVvXB0z95wevUCVPUdFAMha7AzHi8SaTO7y57x3bOVALp9ARYWkrAeGkq+L7RZXzGipQ0QL4I9EnNzSUWgURWGI/fsDaebF6BeVjl6NOnt1+vyrcK9qu+gQAOCPRIzM0mZd2oqqT5MTVWj7NuzN5xuXoBWK0amqeo7KNCAGjsK9/2/WdCe+f16ydoxPTE8qSOzc/rDf6rAicZu6/KMikFJOHmKaqjyKBJOhKLPcPIU1dBNuaOu0xObnaKsgkgR7ChWt6NImp3YvOWWpEzSbchvfKOQqnliAsip8ouAoc91e+WdZj39etmwm1mlaTNT5+cpuyA69NgRRlrZpNtyR7sefdZyTl2ekhDQZwh2tJalzt1qPHi34zCzjAvvZFJQLyYWFX1OAMjK3Xv+tXfvXkcfOHjQfWTEPYnr5GtkJLm/0dTUhdvUv6amwj53nv0X0cZ27W32WgE5SFr0DBlLjx3pspYviugNN/b0paS336jT0StFj4BJe61uvpneO3qOYEe6rIFd1DT7mZnkxKa79PnP5xu9UvTU3FZvYlmWKgACItiRLmtg92I8eD3k19eT7+0CuVm9u9N9dKLdmxgnatFDBDvSZQ3sqi1U083iXnk1e602YgVI9AhLCqC1flwXZdcu6eTJi+8veqmA+mvVbNx+L54f0WNJAYRRZPmimbxDBhcWmoe6VHyPuf5aHTzIUgUoFcGO6ghRQmlVx+7VmulVK01h4FCKQXWEWG0xbSleKelJE67oYz0pxZjZx8zs52b2EzP7ipldkmd/GHAhxsOn9cp37iTUMTDylmLuk/Rqd79K0i8k3ZG/SRhYIcbDp43kOXCg+3YBfSZXsLv7t9z9bO3m/ZIm8jcJAyvEeHjq20C4GruZ/Zukf3H3gym/n5U0K0mTk5N7j6YNCcNg68fhlUCPBLs0npl9W9KLm/xqv7t/rbbNfknTkt7pGd4pOHkKAJ3LGuxtL7Th7m9p80S3SrpB0rVZQh0AUKy8o2Kul3S7pBvdfbXd9ugzrC8O9KW8o2I+LemFku4zs4fM7J8DtAlVkHeyEG8KQGmYoITm8kwW2nh9USkZ3cLoFCAX1opBPnkmC3F9UaBUBDuayzNZqBfXFwWQimBHc3kmCxV1RSUAmRDsaC7PDM5eXFEJQKq249gxwGZmujvZWX8MM0iBUhDsKEa3bwoAcqMUAwCRIdgBIDIEOwBEhmAHgMgQ7AAQGYIdACJDsANAZAh2AIgMwQ4AkSHYMVi4AAgGAEsKYHBsvABI/apQEssfICr02DE4uAAIBgTBjsHBBUAwIAh2DA4uAIIBQbBjcHABEAwIgh2DI89VoYA+wqgYDBYuAIIBQI8dACJDsANAZAh2AIgMwQ4AkQkS7Gb2QTNzM9sVYn8AgO7lDnYzu1zSdZKYvgcAFRCix/5JSR+S5AH2BQDIKVewm9mNkh5394cDtQcAkFPbCUpm9m1JL27yq/2S7pT01ixPZGazkmYlaZK1OQCgMObeXQXFzF4j6T8l1ddBnZD0hKSr3f03rR47PT3ti4uLXT0vAAwqMzvk7tPttut6SQF3f0TSeMMTHpE07e4nut0nACA/xrEDQGSCLQLm7ntC7QsA0D167AAQGYIdACJDsANAZAh2AIgMwQ4AkSHYASAyBDsARIZgB4DIEOwAEBmCHQAiQ7ADQGQIdgCIDMEOAJEh2AEgMgQ7AESGYAeAyBDsABAZgh0AIkOwA0BkCHYAiAzBDgCRIdgBIDIEOwBEhmAHgMgQ7AAQGYIdACJDsANAZAh2AIhM7mA3s9vM7P/M7Kdm9o8hGgUA6N6mPA82sz+V9A5JV7n7KTMbD9MsAEC38vbY3yfpo+5+SpLcfSl/kwAAeeTqsUt6uaQ/MrM5Sc9L+qC7P9hsQzOblTRbu3nKzP4353NX2S5JJ8puRIFiPr6Yj03i+PrdK7Js1DbYzezbkl7c5Ff7a4+/VNIbJb1B0r+a2RXu7hs3dvd5SfO1fS66+3SWBvYjjq9/xXxsEsfX78xsMct2bYPd3d/S4kneJ+nLtSD/oZmtK3nHXM7aUABAWHlr7F+V9GZJMrOXS9qiuD8GAUDl5a2x3y3p7lq9/LSkW5uVYZqYz/m8Vcfx9a+Yj03i+PpdpuOzbDkMAOgXzDwFgMgQ7AAQmVKDPfblCMzsg2bmZrar7LaEZGYfM7Ofm9lPzOwrZnZJ2W0Kwcyur/1//KWZfbjs9oRkZpeb2XfN7HDt721f2W0KzcyGzezHZvaNstsSmpldYmb31P7uDpvZm1ptX1qwb1iO4PckfbysthTBzC6XdJ2kY2W3pQD3SXq1u18l6ReS7ii5PbmZ2bCkz0j6M0mvkvSXZvaqclsV1FlJH3D3VyqZd/L+yI5PkvZJOlx2IwpyQNI33f1KSa9Vm+Mss8ce+3IEn5T0IUnRnZ1292+5+9nazfslTZTZnkCulvRLd3/U3U9L+qKSjkcU3P3X7v6j2s/PKAmGy8ptVThmNiHp7ZLuKrstoZnZDkl/LOmzkuTup939qVaPKTPY68sRPGBm/2VmbyixLUGZ2Y2SHnf3h8tuSw/8taR/L7sRAVwm6bGG28cVUfA1MrM9kl4n6YGSmxLSp5R0pNZLbkcRrlAy6fNztVLTXWY22uoBecextxRqOYIqanNsd0p6a29bFFar43P3r9W22a/kI/5CL9tWEGtyX1/8X+yEmW2X9CVJf+fuT5fdnhDM7AZJS+5+yMz+pOTmFGGTpNdLus3dHzCzA5I+LOkjrR5QmJiXI0g7NjN7jaSXSnrYzKSkTPEjM7va3X/Twybm0urfTpLM7FZJN0i6tl/ejNs4LunyhtsTkp4oqS2FMLPNSkJ9wd2/XHZ7ArpG0o1m9jZJWyXtMLOD7n5zye0K5bik4+5e/4R1j5JgT1VmKearinA5And/xN3H3X2Pu+9R8o/y+n4K9XbM7HpJt0u60d1Xy25PIA9KepmZvdTMtki6SdLXS25TMJb0Mj4r6bC7f6Ls9oTk7ne4+0Tt7+0mSd+JKNRVy47HzKy+suO1kn7W6jGF9tjb6HY5ApTv05JeIOm+2qeS+939veU2KR93P2tmfyvpPyQNS7rb3X9acrNCukbSLZIeMbOHavfd6e73ltckdOA2SQu1Tsejkt7VamOWFACAyDDzFAAiQ7ADQGQIdgCIDMEOAJEh2AEgMgQ7AESGYAeAyPw/SugVn8RyP80AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Evaluate on Test Dataset\r\n", + "\r\n", + "In the beginning, we have put apart some data to the test dataset. Let's see how accurate our classifier is on this test dataset. In order to do this, we also expand the test dataset with an extra dimension, multiply by weights matrix, and make sure that the obtained value is of the same sign as the label (+1 or -1). We then add together all boolean values and divide by the length of test sample, to obtain the accuracy:" + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 10, + "source": [ + "def accuracy(weights, test_x, test_labels):\r\n", + " res = np.dot(np.c_[test_x,np.ones(len(test_x))],weights)\r\n", + " return (res.reshape(test_labels.shape)*test_labels>=0).sum()/float(len(test_labels))\r\n", + "\r\n", + "accuracy(wts, test_x, test_labels)" + ], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "1.0" + ] + }, + "metadata": {}, + "execution_count": 10 + } + ], + "metadata": { + "slideshow": { + "slide_type": "fragment" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Observing the training process\r\n", + "\r\n", + "We have seen before how the accuracy decreases during training. It would be nice to see how the separation line behaves during training. The code below will visualize everything on one graph, and you should be able to move the slider to \"time-travel\" through the training process. " + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 11, + "source": [ + "def train_graph(positive_examples, negative_examples, num_iterations = 100):\r\n", + " num_dims = positive_examples.shape[1]\r\n", + " weights = np.zeros((num_dims,1)) # инициализируем веса\r\n", + " \r\n", + " pos_count = positive_examples.shape[0]\r\n", + " neg_count = negative_examples.shape[0]\r\n", + " \r\n", + " report_frequency = 15;\r\n", + " snapshots = []\r\n", + " \r\n", + " for i in range(num_iterations):\r\n", + " pos = random.choice(positive_examples)\r\n", + " neg = random.choice(negative_examples)\r\n", + "\r\n", + " z = np.dot(pos, weights) \r\n", + " if z < 0:\r\n", + " weights = weights + pos.reshape(weights.shape)\r\n", + "\r\n", + " z = np.dot(neg, weights)\r\n", + " if z >= 0:\r\n", + " weights = weights - neg.reshape(weights.shape)\r\n", + " \r\n", + " if i % report_frequency == 0: \r\n", + " pos_out = np.dot(positive_examples, weights)\r\n", + " neg_out = np.dot(negative_examples, weights) \r\n", + " pos_correct = (pos_out >= 0).sum() / float(pos_count)\r\n", + " neg_correct = (neg_out < 0).sum() / float(neg_count)\r\n", + " snapshots.append((np.copy(weights),(pos_correct+neg_correct)/2.0))\r\n", + "\r\n", + " return np.array(snapshots)\r\n", + "\r\n", + "snapshots = train_graph(pos_examples,neg_examples)\r\n", + "\r\n", + "def plotit(pos_examples,neg_examples,snapshots,step):\r\n", + " fig = pylab.figure(figsize=(10,4))\r\n", + " fig.add_subplot(1, 2, 1)\r\n", + " plot_boundary(pos_examples, neg_examples, snapshots[step][0])\r\n", + " fig.add_subplot(1, 2, 2)\r\n", + " pylab.plot(np.arange(len(snapshots[:,1])), snapshots[:,1])\r\n", + " pylab.ylabel('Accuracy')\r\n", + " pylab.xlabel('Iteration')\r\n", + " pylab.plot(step, snapshots[step,1], \"bo\")\r\n", + " pylab.show()\r\n", + "def pl1(step): plotit(pos_examples,neg_examples,snapshots,step)" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "skip" + } + } + }, + { + "cell_type": "code", + "execution_count": 12, + "source": [ + "interact(pl1, step=widgets.IntSlider(value=0, min=0, max=len(snapshots)-1))" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "8561af1ae77c421f9ca068fe0bdac566" + }, + "text/plain": [ + "interactive(children=(IntSlider(value=0, description='step', max=6), Output()), _dom_classes=('widget-interact…" + ] + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 12 + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Limitations of the Perceptron\r\n", + "\r\n", + "As you have seen above, perceptron is a **linear classifier**. It can distinguish between two classes well if they are **linearly separable**, i.e. can be separated by a straight line. Otherwise, perceptron training process will not converge.\r\n", + "\r\n", + "A most obvious example of a problem that cannot be solved by a perceptron is so-called **XOR problem**. We want our perceptron to learn the XOR boolean function, which has the following truth table:\r\n", + "\r\n", + "| | 0 | 1 |\r\n", + "|---|---|---|\r\n", + "| 0 | 0 | 1 | \r\n", + "| 1 | 1 | 0 |\r\n", + "\r\n", + "Let's try and do that! We will manually populate all positive and negative training samples, and then call our train function defined above:" + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 13, + "source": [ + "pos_examples_xor = np.array([[1,0,1],[0,1,1]])\r\n", + "neg_examples_xor = np.array([[1,1,1],[0,0,1]])\r\n", + "\r\n", + "snapshots_xor = train_graph(pos_examples_xor,neg_examples_xor,1000)\r\n", + "def pl2(step): plotit(pos_examples_xor,neg_examples_xor,snapshots_xor,step)" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 14, + "source": [ + "interact(pl2, step=widgets.IntSlider(value=0, min=0, max=len(snapshots)-1))" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "8f45bf78e4c8471fbc6eea233dde2bf6" + }, + "text/plain": [ + "interactive(children=(IntSlider(value=0, description='step', max=6), Output()), _dom_classes=('widget-interact…" + ] + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 14 + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "As you can see from the graph above, the accuracy never goes above 75%, because it is impossible to draw a straight line in such a way as to get all possible examples right.\r\n", + "\r\n", + "The XOR problem is a classical example of perceptron limitations, and it was pointed out by Marvin Minsky and Seymour Papert in 1969 in their book [Perceptrons](https://en.wikipedia.org/wiki/Perceptrons_(book)). This observation limited research in the area of neural networks for almost 10 years, even though - and we will see this in the next section of our course - multi-layered perceptrons are perfectly capable of solving such problems.\r\n", + "\r\n", + "## Complex Example - MNIST\r\n", + "\r\n", + "Even though perceptron cannot solve XOR problem, it can solve many more complex problems, such as handwritten character recognition.\r\n", + "\r\n", + "A dataset that is often used when mastering machine learning is called [MNIST](https://en.wikipedia.org/wiki/MNIST_database). It has been created by Modified National Institute of Standards and Technology, and contains a training set of 60000 handwritten digits, collected from around 250 students and employees of the institute. There is also a test dataset of 10000 digits, collected from different individuals.\r\n", + "\r\n", + "All digits are represented by grayscale images of size 28x28 pixels.\r\n", + "\r\n", + "> MNIST Dataset is available as a training competition on [Kaggle](https://www.kaggle.com/c/digit-recognizer), a site that hosts machine learning competitions and contests. Once you learn how to classify MNIST digits, you can submit your solution to Kaggle to see how it is rated among other participants. \r\n", + "\r\n", + "We start by loading MNIST dataset:" + ], + "metadata": { + "collapsed": true, + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 15, + "source": [ + "# If you are not running this notebook from a cloned repository, you may need to grab the binary dataset file first\r\n", + "# !wget https://github.com/microsoft/AI-For-Beginners/blob/main/data/mnist.pkl.gz?raw=true\r\n", + "# In this case correct the link to the dataset below as well.\r\n", + "\r\n", + "with gzip.open('../../data/mnist.pkl.gz', 'rb') as mnist_pickle:\r\n", + " MNIST = pickle.load(mnist_pickle)" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Let's now plot the dataset:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 16, + "source": [ + "print(MNIST['Train']['Features'][0][130:180])\r\n", + "print(MNIST['Train']['Labels'][0])\r\n", + "features = MNIST['Train']['Features'].astype(np.float32) / 256.0\r\n", + "labels = MNIST['Train']['Labels']\r\n", + "fig = pylab.figure(figsize=(10,5))\r\n", + "for i in range(10):\r\n", + " ax = fig.add_subplot(1,10,i+1)\r\n", + " pylab.imshow(features[i].reshape(28,28))\r\n", + "pylab.show()" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[ 0 0 188 255 94 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 191 250 253 93 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n", + "1\n" + ] + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlIAAABRCAYAAAAZ1Ej0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABJF0lEQVR4nO2dd3Rc133nP/eV6R29EyABgmBv6t2qlhzJsuPY2Y1LsnaKnbWTbE682STr3XU2PtmUdbKbeB3HiRU7cZEtybZkqzdSIilS7A0s6L0MBtNn3nt3/xiwSSwAiDKE3uccHBIzr9wv3i2/d+/v/n5CSomNjY2NjY2Njc3MURa7ADY2NjY2NjY21yq2IWVjY2NjY2NjM0tsQ8rGxsbGxsbGZpbYhpSNjY2NjY2NzSyxDSkbGxsbGxsbm1liG1I2NjY2NjY2NrPkqgwpIcT9QojjQoiTQogvzlWhiglb47XPUtcHtsalwlLXuNT1ga3xPYmUclY/gAqcApoAB7AfaJvt9Yrxx9Z47f8sdX22xsUvm63R1mdrXFoaZ/pzNTNS1wEnpZSnpZQ54LvAw1dxvWLE1njts9T1ga1xqbDUNS51fWBrfE+iXcW5NUDPeb/3Ate/8yAhxGeAzwCoqJs9BK7ilguLCy8meQIiIjMkAT7Oe1DjUtE39VEceOydxy0Vje/legpLX+NS0Tf1kd0WsTUWOxmS5GRWXO6YqzGkLnbhd+WbkVJ+Hfg6QEBE5PXifVdxy4VlSPYyxiBtYgs75YvkycF7UONS0Qfwgnx8lPfgMwRbY7Fjt8UCS0Xje7mewrWt8Xx2yheveMzVLO31AnXn/V4L9F/F9YoOJ24ypM//yNZ4jXERfQ6WkD5Y+s8QbI1LAbstLg3eCxpnytUYUm8BzUKIRiGEA/go8OO5KVZxECBMmgRpmUQWDG5b4zXG+fosaQFEWEL6YOk/Q7A1LgXstrg0eC9onCmzXtqTUhpCiM8Bz1Lw4v+mlPLwnJWsCFCEwkq5gb28TpoUwPdtjdcW5+ubavTjS0kfLP1nCLbGpYDdFpcG7wWNM+VqfKSQUj4DPDNHZVkw1BWNWCEvht+Bmsyj9Y1hTcSwksl3HVsqqiilip3yRSbl+J8uSAEVFaGqCFVB+P0Q8hc+FwLL60LJGYixCaRpgbQwxyfAMmd9u0XR+A602hqkxwWASKYx+uZupviMPoAX5OODc3bhImIxnqHQHQiXE6WsBKmpoKkYYQ+mU73gOD2eQ0lkYGAYK5lG5nOzul8x1FOYqqteN5bfheVQMV0aasZASRsoPYPIVBorlZrVtYtF43xht8WlwUJoFLoDxedF+LxIl2PqQ1EYB/0u8gFn4SMpUZN5lEweJZZExiaxEkmkJUFaIN/lvjXnXJUhda3S/psVbLmhnf9e+xP+Zvgutn9rMxVvxmBPcRjVaiSEcDqRXjfjW8sYvqFQEaQuWdnSx4nBMoIvRdAyEi0tCbx6EnN0bJFLPUuEQKgqvb/YwGSLgbAEgeMqlX83jDTNBWkENrNACNTyUvINZZx82IMRMFEDef7zpp/xAd+pCw79g777efV4M42PBXCdGsHo7luwDm5OOb+urs5x46pTbAl28bHAAb4XX8PPh1Yz8f+aCR6egMPHrz19NjZFgtAdqJXlJNdUMbZGJ1k3NVGggHRabF11mq/WP4UqBHFL8tfD72N7fyPZXXVU7M7h2d+DTKWRuRxWJjPv5X1PGVLKhjZGtgapW9fPB0r3U6vqhPQUlgooi5gtRwiEw4FaXYlRFqDrfX5yQYnllKhVKW6oK0SZcKoGtwRPsMe/jJ9l1iAUiZSQCbXg7zNw7+lEJpOzfhteFISC0DQmmw1u3XAMSyq8Ya2kUlXBkiBnP9M23yQ/fD3xGhXHpMTfl0d/Ye+sDQShaSgeD9RVISbiczojN2dMGRJKYz25mhDdt7nIVBusXdVByJEi4khxg7uDctV7wWkPl+zF25bj6Y+txdlXS/X2clxdE5jHTy6SkFmgqChtzUQ3hEhfn+DWhk5UIenORvhRYhUeJccHKg/wlw9WEG2NsExtRRkaxxgcWuyS27xHUcNhRNBPYk0lplMgFTAdAtMhSFUKrKlJHi0NelziGzBxTBg4O0eR8QTm2PiCl1nxeFACfuI3NDBZrxHbkKOsYoTVocJEgYLEq+W4I3SMctUDgE8YPBDeT6UzxnOuVXQ2laDf0UTpfom3L4uy/cBVrdhMh/eUITWyNcjaXzvEr5e/wjqHiS5ULCkQkkV9exSajuLzkmopI7rSwR984vt82NePU1z88dzjPcl1/tO0OfuoVLN8pOKTdB8vpXmoHHVw7JoypIRSMCJXtvbxT/WvAHBvKoDQNDBNCj6pRYgQDD2a5fc2PM8zI2s58tYyml/TsXL5WRl/wuFAREKMbQwTOOVCFKUhpSDcbhKrSxlZp/EXH/8m97vfWdecmO94aB/wTPIBz5t8tfpNfp728DvOT1H+VgneEx3z3sHNCUIgdI3xjWHyvzjO37Q9ySbnOL/V+TAnJsp4Md3Cr7W8waeCR3n0jqN8e9N6ftB1L+EjGtiGVPEgLhMKaKnNHgoB5SWk64J0f0Ci+nNomknIl6bKm+Avlv2Q5ZobgO1ZhZ9MbOTx/ZtwdbmoeqMcV58bxqML/ndRAn7M2jJ63i9Zv/IUP1hxac+hM72MLlTudSe5132YPyg5DGsgYWXZUvtZPPvd1OzSkFnbkLpqtMoKhh5sInpbhk+Vv06TniEh4X8O3cRT+zbQ+moU0T/KgnfpQqD6/eQ3LufEI04qVw3zcNVx7vJ04hSeS55Wqji4y3Mav1BwCgd/uvIJjjTU8r1Vmxnb0UTDUyUFgyqRxEokll4nUQxISfW/Ovi/+x7mM594mlONJQifFxFPzKrRKgE/2cZSJh9JkHk1QMWb81Dmq0BxucjdvJqOT0jWNHTzSEk7W51jgHtG19nqHOPzv/BT/rr+fZS6t1Lyeh9GV8+VT1xEtOoq+h5dRvLGFH/b9iQ/jm7kj4fqcf5zGGfUoMKUfPWX7yF7o87Hgnt5yHeQ9t+s5LUX1rFioBZrZHRBlhdsLkTxelH8PkbvaSJVIUjWmyilWRrKz820mJbCUMyPcdqHIybQE+AesVDzhT7T25tGPdaFlc4g80ZxG/5CoHg85K9byeD1LoxNcdoqu/mv5fvxKxlUYeERWQJKhmr1nB9jm56kJPIGG27qpn9riD33NHB4uBJj/43UvphGeX3vgkmI3tFItFXhrnUHuCl4kpiVwSN0VCF4NhVk3PSRkxoJ08Vo3sdYvjD7/UD4IMu0MVY7CiaNU2g8uOoQOyLL6HZtpnxvHufTb81buZe8IaW4XFjlYcY3WGxa1sN6RxpQ6DXg+c6VeE454PRxrHT6iteac4SC8PuI1zrZvPUEn6jcPvWGf2kjCgqVpEY99+jucOW53nmcmzwn+NXMJ5g4GSSkCtQhDSuZKurlsYuhKyZKKIgVnUAaxmIX55L4dnfhiFfT8OkRakMxqChFkRIzm53ZhYQAXScX1HiwaS9PnbihUG9z+eLouIVAuJwkqh38zuZnuN17nNW6g5kaUQAlipvfCHZxYMVxXm7bQOBUBHV8omgNfsXjwSoLEVuf446mU9zomuBLI7XEDpWw4uXjZ5c/3DffxP7VtXwwsI8GTeO/Vz3L7cuXY1aEUBJJWERDSnG5QNcRLhdYJjKdAcs6kzcNLIk0TYQiQCigXHrmRnE6QdcQbvfUjLGFjCewMtnFratnlp09nkL5XC6skgDZiIexdaDUJrlneTu3Bdv5oHfg7GlZafBWNsjfld/F6WiEyQkPqQEHSq7g6pENeIlY9ajxDCKTQ07GkZnsRTcmLSqKiuL1QGMNY6tcWFsm+f3VL3CP9yQ6oAqBKSXq1Kxc3JLEOde3RhSTu9xd4O7ilwL7eb20gb913slEdxllR0uwJmIL0hfn/IJs2MKpGmSkzmnDgYokLxUeH93CaMaHaSkk8g4m0y7SKScISbpRZ62/D7+yn4ii4FF0bg8cJ6BleHKTg9h4kPJ5LPfSNqQUlfRdaxlZr/MP7/9/NOsxPMLNz1J+nhrbRPVf6zg6OjEWqRNXXE4SG2sZWy/4ccOP8Qmd2T4St3CwxmHy803/QM96J7/07GeJvO2n/N9iWPH43BZ8nrm+pJMnful2qrbFYdfBxS7OJTGGhtEDPiZMD/dVHOFvf+cu6p8M4vrprlldL+9W+I2S13mhbSWZO9bi2dddFD42QlWhNEK6XPDx4DE8wnHV1/xU6TYqH5zkO85bCbWtofyHRzBjk0VlTAlNI33naoY36Hzrrr8jZTl5LNaK9b0ymp89jTEePXusnoTTsRKOlFegOgap19xsqeth/72raHgqD9HoZe40v+RuWs1kg4OxjRaOCYWyvRZ63EDNFAwfNWui9Y8h3U4svwfLpSHVixhTqmBorZtUpaR0yxDjcS+ZUTf1T4Pv8BBGV++iGVNqeRlEggzdWkqiHgIbxriv9jAPBPZTpqbRkagCRkwHL2fOpSrRMQmpKb7R9CMyUmIBGSkwZUF/SmokpYOnopvYF61l4NVWQu0WgR/uLp7NMEKg1VUT31jFuv+yn08EnuUmdw9BRSVlwZ8M3Ec05yZj6iiiUF5LXtxY9mg5Hi17mxotyj+1PcbnfvmjtK9upuXro1inu2e943a6lH33EBVeDyeXt7Jr1Ub+z30JjNM+vL2CqtcnUKJxNMMkZKUIySRSSoSiMFxZx49WtfF399zBpzdv4/dLjnCfZ5itrn5WuIb40xMfsg2p2SCcThSfl/FVOumVWdr0GB5FJWZleHz0TrafWk5r3zjW2PiiNAa1JAIlYcZXaYj6BCXK5d/un0yGOJ6pImp4qHFGuct7jGrVJHzeeRoq5aqHchVWtvRx3KqhbE89Wv8YxsC1s9O4xhElscwkd9jJ1Q/Z84iUiGyO7ZPNhPQ01bXjZIPluISYXZ0S4FcELkce0yVAVa98zjwjnE6UQICRWyuJr8zjE853HWMh6TbSHM6V06yPElIK3gu6EPgVBxrv1lGtpbnZ286Lq1fS5y6l/LUSFNMqHqN/agPI2CqdzMoMrXqSb8ea+Gb7jVT05Aq7ZM97xoEuk+E9ZfxR5mHaygf5q/qnqPNEebM5ixFyI2ZbJ+aAbFgjVSmobx1iMuOk3x9GzTpQpsZEJSdwD/swXYK8FywHSPXdZZUKGA1pIuEkj9TupytdytFwBclgNV51ETfrAKnNDcQadSY25SitmOTh+gOscA6RkTovJesYyIfYP1FLfyLAeKywHKQISSSYZEtZD39ZvY2daT+7k010pyOkTR2AiCNFpTOGUzFYFRxiaKOfEV8AX+8a9M7hxd8UMlVPY1urGV2r8gvht2nWo1SpbvbmLN5Or+DF3WtQkwrC5Fxit0tURanC3to6SsNxbqs8VTC4KrNYHidC1+bdkLLicaxkCl0IwqKSTImfYK+FdyCH6OzDuNjLlqKiqSraMh8OTx6PWlgRcAodv7CIqAmkNr9tb8kaUkooiKwqJXDPIJ9r2EFEdRKzcvQYOm9sb6NyF1hDI4vmu2A21xJb7qHy/h4eqLhy2IUv7n4U/bAH96gk3gid95TyaHg3Nzsv7o39WPP3eau+hC92/Colhz04ryFDarWzl3XrO+nf01TchhRANsdzh9toqBnjgeojPB6pJORwIHO5qxo45eUcYxcQJRTEbKxkza8f4uGSd/tKWEiyMs9T8XX804kbeLRpP+vcBZ+nMm2SNj2JR+jv2jhRo3qocWd539rH2d6i8MfPfAavlFAkhpTQdITfR+DuQb7Y+CphxcW/dW6h9BseXEe6Md6xzOF5YifLnhSoKxrp2drMzi9Vs9nbQcXWSX74zD34HQ7kTJd854h0iUKq3uDrK79Do+bC2lzoMxQULCxiVo43MhVE1ATLtAQRRcMpdCwuPO7M/wEsLEYCWbpKPHwu/LlF0XUWIej6sOTz1z/Dp4JHzxr7fxVt5mtdt9NzpBL3gELNqwnKBqKEu04AU35/N63m+feVkfr4y/zR4UcwtkUoOZJHTxSeb0eNk8kGBccN49xee5LXt/4DP1i1gj/zP0TDT2pwLrIhJTQdxe8j+rEEH2/eze3uFBoeLCR/P3QXr7Y3s+rLXRhDI9ObLRQCNRjAbKnn6TtvIr0mzbKqMcxACbrTCQuxkckyMQYGUQYGqdl27uPLlt6hky5V+GjrHm5wn7rckfPCkjOkFL8fUVvJwJ1lTKw1+E/1r3Oz+xQKDr4R3cQ/HriJyp2S4IExrNz8WteXY2SDl+h6k09Xv8VmVxdQeAOykGzP6HTmS9kea8aSAguBftRDuN3E35EkeMrJq73X8WzJdeTCFh+7Yzt3+w9zsyuPMvXK4RM6y7QoydVZ1IyTikVTurSRhoES1clUaNzrP8i/Bu9C8fsKQVKn65smJRgGSl4yYipk81rRNMzkdcsYXaPxqfAh1jsGOd9/r89MsSNTwx/teQRx0kPpQclPIrfxpKtQB3NByDZmuX/1YX6t9DVW6Vx0J2qdmqDvl3NoJ6qoeS2C6+Twojugy40rGV3l497KbZSpk/z+4PVMHCmhrH0QK564xEkSkcqgZgsGdL02znJ9hO9E7iNUWoIxMLRoS19KWuFDb3+abFbDGHajlOQIBpJIKTAthXTagaJa6LqJplgoioWcWv4RQtIYGqfFN8xvlbxBleoBFLal6/jnvpvw9xowOs6ibbGVEv9hB1/lbv61YiuJtJNsrw9vj4Jn0KJ+1ECfzKN1DhV8Rs/DdCm4hgU37fh1nK/5qX4rgTY8Cbk8AOFhD/5OD5M9IZ6vu45bP9WOS+RYsXKAyR21OBV1UX3D8reuZXS9k4ebtnGf/yAaGtuzCk9FN/HGi2uoOCSxJuPTL6OUWMk0Wvcw1a8pZI47SfqqKTnVg1mku8EVh05mWQnJGsED/gM0aGlm4795NRRLfz0nCE1DiYSIt4SJXZ/hd7e8yKO+o0RUJ1mZ5+WRFsKvugjtGcA43bmoZU0sg1WtvXzAd+rssl5WGiRknm3J1eyL1bKnfRlYhc6s8pSF/3QCDpzAIS2qdnkQAT9WxM/TjavxN2W40XUEZWoZxSk0ytQ8rQ0DnBxoQGha8azpLyWkRI8r5AyVzQ4VwycRLhdCVZEz6GClYaLmJSOml7y5+Et6Z4g2a2TXprjJ3UONes6IspAMmk62TbYQfM5D5HACdh3Ed965Wl0tsetreM7RygZfN3XaiYsaUhWqgy9vfop/rryJ/lgDVfEQdPcual2NN3oZ2yC5038Ev5LhmRNtBE6Bebr7soOSzOVRcxJVWFSoaWo1N7mAQPo8CEUsiq0hVRCmwNoVIjAsKd0fJ9biI1l1brBxnXe8ybvf/vcsD9FRF+Gj4V1UqYUZqT3JZRw/XkNLfxpzIrYQUi5JuN1ATTtIBsvwjEoadkYR/SMXBCp+l6u0omDpAveYhfqSj/IdMeTew+eOO+PArqpETgUJNlZy4KN1LHcOcVPpaX7sr5txO59rxlud5G+e5EOh3axzqGSlwZvJ1fzkxBqq3zDw7OqcsQEk8zmMwSHE4BBuCiZJ8W75AVSVdLlOttRko9NCWWAjCpaQIaV4PMTfv5bhzQq/+/CP2eDqpkFLk5HwZsbJ1wbvoGtnLSu2jWKNLHIUcCGQFNboz/csePj4h+h6qxbXiMA5Llm1YxTMqZ53fKKQemJqjdpMJBHpDEoiQepgG/8qt/Ab4QP4xLlB2CkUbi89QWdzhNSDm/AdHsE82bFwOt8DyFyewGnJSGXgygdfBisaxTVczePjW0mMewjNTfFmz5TvRXxVnk+u3kVQudC4s7DYmVrB633LqX6hB2s8yjttBHNwmMDLKXLelXxl9AMk7nqeu7xHWevQLzjOKTTu8/RR2fAzvvWRm3nbWkvtUC1m/9C8+2S8C0VFcbsYWyN44Ja3adZjPJNYScX33fjao1iXGzSFQPg8ZAMK6x2DVGvv9idbcIRgYpVF8+o+ul+tR09JxNEOwp0uIs7pl+/0f1hGVVscHYuoleaFVC2P791M82NZ1FN9Cx865h14t53A53SAriNzuUJAyczll1KtdBrfaydA0xCqgjUZP+s6pHi9yFWNxJp9RFsVwtcNcVvlAT4d3slfj9zGC4/dQNWu+MLXz3cQX27x5bVP06QZnMwbfOHUR+jYWceyn6TQTnZiRRc+FtSCoqgIl5NsUCAdJifzBhWqRVBxXfncOWRJGFJC0xB+H9EWFb0lxqeDPYDAwMmraQ/bky3sPLScklPA8BjyCg1sXsuqO1C8boygyTLvOCrnfGE6h0qIHAb3WB7neBbz2MlLNwLLRFomVhL8HTDhDvLtlha2uk+z2VEY9HRUVroGqI9EGawP4OnzXvxai4yUkomMm24jRa3mBorDP2haWBZ6ykLkrs7ZVhoGSs5kNOcFRZIJKQTeYXAsJGooBGURAmUJtnhPo7/DYTwvTV6PrmBiIEDlSOdFfQ1lPoc5Nk7wdAbD7eJfGq7jdHUZv1f+IiFFIXBeZxdQXKzUJ3m4ZC+vNrcxuakafyK54NGVFa8HUVdFrsLglkA7PYaHtyYb8Z2OI0auXBapKkhV4FcEGirWpbx6FxDpMan1TtDhrkOqFLbuz2T7vqIirGWUuRI4hMW4Jfjp2HpcPQ60k11Yk5dY6lxAzNnsipTy3HlCoJZE0CrLMUv8ZMNOoiscJOolzpYYj9btY72rm8cmNvPM6dVUHc6iDk8s2kyN0B0oQT9myGC9sw+n0Og33Zw4VkPpcdCOdhbyzZ3x5ZuaXSv8Xzm7DFvM4WUuiaKiRkIQCWFGvKRDThJ14ApnSEoNiwuNW6+SxfRaaI0NWMOjhYDVc2xcLglDSgmHMRsqWPvgMR4pPecQm5cmf9V9L+0H61j1X49iJZKYi1xx1PJSco3l3LjuBH9ZvQ3tvEl1OeCiZPcosquvENdqGg9b5nOUfustKqoq+Wb7Q3zt/hj7rvs2MBWUzBMjU/MGf3RDOcleL57d8yZtVkhLIkyT4dMlfKX0Hr5S9SIXLjQUOVKi5iXCmLtdS5WVEwzeECFyNAiLNIGYW99I750ufm3Fs9zrTqKc11VkpcGIaXDwhZXU7TMK8a4ug7L9AOU7VJQXa9m3fgN/+QfwQHj/u6Kil6seHvTE8N//DZ67aS17f2s9yluTC9vZN9XS8cEwd607wAPefh45+lG6D1bRcuowZrI4fUSmg1vNY1RnSQ+7CM7gvDOpi3Ihiy2BTvyK4JV0NfufbKNqXw5zZGTeyrxgKCqKQ2fifc2MbBTccecBbg0e535vFzkpSUnBE5Mb+F9d96N+MURj9xDmyBjGIqZdUKsriN5YQ13tEHWagi5UtiVW0vrVURgefddSq9B0lFAQoSqF3cCWhcznr70crYqKGg4yeccK+u6W3LyuneuDHejCIKIlcLxjXtwpNOq0STavPs3BX2um8akAysETc77J7Jo2pISmIRwOxu9bznib4CORI7Q5BwAHWWkwbhm091bg7VEKWeeLwPqWHhfZEgchR/rstnALScLKIvICkc1jzdCXSRoGMh4n0Jmnc9TLpJXBpzhRECgIdGEihCzOiR5pIfMG3m6Vl0ubSVY+v9glmhmWhZKVKHmBMUcLHA2BKIEVWfKBShZrTmqywUndLT1c7zl1dgPDGToMk7cyTQROS7wnJ7GuNKBMzZ4yMob/pIvnXtzEnvW1+Ff+kDY9eUEIDwVBkxbjRt9J3nZsnA9pl0UqCqZT4lbzqAiGJ304owrkrxwcVagqE1sqGF8NulAwMIlbOZQ8iLxRyEa/gCgeD0o4hDuUock9AgkdLTmzMgi3GyrLMMMGq519vJSq5ftDW4gcNXD1Ti76kt47EZqGWlpCbkUVqOJdu1+VnImSNbBcOpZDIe/VyAYVklUKibYcKxqGcCoGO+Ir+EbXrYwnPaQmXWiDDlwjgrre0wUjZTGDjyoqRlWYoevhA6U96FPuHHmpIiYTmMlCcGnF40F4vWTX1ZOs1Ik1K5gOidRAWKAlBCVHTLwdCeT+Y8UR/PcKCEUgdJ14jcqt6w/xUMl+lusjmAhcwiSiGJgSolYGc2oMDSmCB0oPEb/RRadZT6BtI6U7RmBiEnNoeE7KdW0bUg4HSsDP6ANZfmfjC3zY33F262tC5uk33DhPuQidMguO1kWA9DhJl6j4tXMWcVbmGbEkigFYViFZ7wyx0hk87SPoo9WMWBJd5HGfFzhRiMVfYrgoUiLzOcLHDUZ1H7EbisfRejpIKdEyJkpeKzTcOfgzbwz0sKaih/8Z+uSiGVLxRsGuVT8BwHyHpoPZan4+tobwwUmsQ8emfU1zIoY4mqH561V0/WI1T5VvoqRkO+F3TObVax50MYipK6higeMTqQJLA00xMZGkxj2EoiDNK88+CE1j6AZYsbYHHZWUlWfIVAq7+K4wazcfKMEARl0pTaUjrHH1oEcVnLEZGlIeN9naEJGySTY6k3x64E72Hl3Gqh0dWJOT81Ty2SMcDszaMvpu9yC1Quyr89FS4JiUZEMCwyvJlRmEKif4QssrbHB1U6Hm+GLvQ+zqaqD0KTdVvRkcHQNYY+NYmUxROF0rDp14rZsP3LabR0Jvnw1JkZcqVjJV8NsSAiUYwKwsoeceB+7WCb69/lvUaXlKFDcJmeXNTIjPvvnLhF8LUHZUx8pR/MaUUMDlJFkr+cf6l6c+PDNmKIBOr5FmxHRgSQWXMFjlUPh4oI+PB/r4n6VreW6glclEGf5TDrANqULHhdtFdekE7/cexXVesMBbtv8moZ97WbY3ijIawyiSCiJyBnpSng34BtBlSL4TvRH3sEBOxGZl9MlcDmtoBF9XNV/qfYjfq36WDVN2VKtjiA+37uVn1bfg83rnZY141kyt3cdrNdKNOTzCZGKxyzQDZC6H41AP/hUr2JV14Vo1QcenGmj8tjbjSM/KZJqdB1eQWaVzd/1h8m6B4vUWXTqKbw/cwPFdy2iZ6J/xwCJzOazBYWpf8PL60PUMfSbAf6v5KfWa+4KZLxXIlOi4K8sxenrntPyXI1vqZvmmHm7ynyAvLfxHdUr3p6fXJhUFSrOsCfWjCsHbWT8/HN+Kd9DCGh1btEEqmnHTmS9DzGIlKru6jtH/mOI3l2/HkpLdpxvwndCRyWQhVloRIZxO8te3MrzRRdv97TR6x6hyXLjElbF0UpYDv5pBERbDuQCHY1V8+bUPoI9pOKOC8AmDhrE8js5eZDpdyFm6SDHA3oUQCJ+XdEThk5HtVGsGeanztYlW3hxuJPcLFcQbFHJrUoQCKSLuGL9VuZ1m5yBNmoFHKQwKLqGx3jHGn13/I77sfz9D2kYqXxjCPHF6kQVeHmmayFicwCn44ImHWBfso9xxoUH/f/bfgeOIBzULphPSjTnuWH2cr9e9wkeCu1nl7uMPHvxFgntDVOybm3Jdu4aUoiKCAXK1EZYFTlGvFbZmG5hkpIFy3Efpk4cxJxOX32mzkAiBdOrk/AK3eu4NNSNVRnJ+lLwsJMaczdq7lFipFO4xi7f7aukrD7HBUXACLVMt7vAf5Sn/LQiHA9KZ4sq/p6pkSqG0chJXMS4/Xg4pMUdGcI828XJiFfWhCU6u1TAjPpQBHSsz/b+zyOVxDmmMLPMSUnKYLhA+byEIXrEYvkBfLIinX8xulmWqnirt3ZSNhTn+4XLGqpzUYp0N3QGF3GDZkMAqCUDvAkUFV1TyXoU7Sk9To0WJWxLPkIU+MIE5nTYpBE5XnnJHHAWFvnyYvaM1eCaNRQn8Ky0LkTUZGg/wWqAFNS1QczPrWzIlGr/T+iJbXV2YSJjQcY3Lqw44Ox8IVSUX0Mj7oNodo9U9QLPz4oGIS5Q0urA4kS9hIBPEOaQROSIJnJhEHO3ASqWKYvbpoqgqli6o1oyptGLQmSkhbymMrxG4VkX5StvTLNdHCCp5/IrAAuJSkjQLBmFI0YioTj7oHWdPw1G+v/Y6Svf5EacWNy7WFZEWMp3GN2hy5HA9XdVhwp4L8+S693mofDONmsph+hwMpdwcKK+COlihOwkpfaxoGKJ7oHbOinVNGlJn1sH7PlhP2y8d5XOVL3Jmeu9k3uDfJq7DPVLYmr5oQeLeiaKiBgN03xfiP37ySe73tnMmuOEah+DPq5/nuoZ1VFWUIfsGZh0FOXAiTuaFIDuaV/CgZx9QSBR7uzuF4ZOFrb6LFM/mYghVRTgcRG4c5PG2xyhV3Zwu2h7s0gT2D/H039yG5yOD/POWf+bzbZ+jJFEL7aemP+DkDVxjkM7p1GtuJhshsLEB1yuTixaBf76wEglkOk08tYoJ0wNc6MjtFArRG3MYnhBVh/X532auqGgVZaTKFR4K7GPCcvN8qgXPiAHDY9N+hkJIlKmpn/3JeoaPldEcSyzK3j1zeAQxPkHLf6tn1FdDQ2IYkZiZgZD3KHzQ14WOSqJYOo1LYKVSeJ89gG+7l5M/WMaxcBuG7+KL43236hiNGT6zfhutvkEyd2oczbTg69bOJvYtSqTEmojhHTL58tAdPBrew82uPH9S8Qr5ckm8TeARkpCioQuVuCX4p9g6jiWqeHuoYDhoqsUfr/wpG5zDVKluPh5+k1V39fN/d3+I8s6y6UdBXwykxMpk8P78AK3bvIVVqXek0vIlD2OlM2BJ9GW1ZO92UfEOY2uuuSYNKSUYYPLGZUyuNPmF0n1Uq1kMnJzMG3w/toXv7LiR+i5jananON6ahCJAVTA8cKv7JBHl3J9eQyWgqEhNIq8yZ5VI53CPW0waFwYl01CRgstmdl8shBD4HVnKzwv4eK0hJ+NEjoXoT3qIqBkmVoKWKcF/qnPamxxkNot3wGRw0kNemoWt6nrxPa85QUqkaZIddfNsbC3Xu7ZfEAMNQHMamDOIdXTVqCpSLTitjhgBjqcqUVPG9JawhEDoGk7dwK8UjN6JvAc9LhA5Y3GCIEz5H8r+IZSpUBpX2mF5FkVFKy8lFxTnklTLPHpMwRmzFtxxfrpYmQzk8iiJJNqoH91z8R3ApaEaEmNuvpa9Hd2bI+RPk64y6b/NS7hiDe6hDMquw0WxQeldmCausTzPHF9DzboJbnQdIay4sZAEpclJw2JbuoqXY62cjpdysqMCZVLDNVoYWzI6HKivp16LUqVChWqxwdlTCBwb8iNGRovmRftSWJkMXOHlUi0twSgPYC1Psy7UN6/luSYNKVlVzsTH4/xK034+5BsF3MSsDI9Fb+b7O6+j9fcOI7PZomsEQtMwnZIWff6294tUBtdonqRZ9FnqlhTm6BhidIzUL1/PiOnmlrsP8nLtSgI/0aZdD62JGKFtXYxsbCRm5Wbl03JNISWhQxo/0jfzuXtfw1cMvdGUfXAoXcsbw42EJjPTmg0UDgfC7abMm6TOUdhSHs25cY4LRHZx+6HZJIJWHDqp9XUka+RZ3zVTSvyd4D8eu/JOzcXEMgtL6pd5br6uHnxCUO10ItesoOvBMlpu7+J3H3iOr/XfUXCoP+LFnEwU3eyMNAyc7YPUfLeWH4Y28NnwQTShYmExZOb4TvQWHj+8kaonHQT2j7CyY98FfZDi9/PcTatY5epntSNKWHETdEgyZZJsVQD9tApFNnbOGEXFaKllZKOHf7j+a6zQJ5nPtDHF0HXNCMXvxwi7aQgPUj/VYeWlyZCp8P1dWwnvVwtGVJHs0juDNE2seAJvv+APhzfxyfAb82pQ2SwuccOJnGGQTmmaWIkkzgnB86llOFZMMiADNO8OIUdGi+7FYC6wdBAOE/UiE2/5qBP/uFy45Xldw9IhqJjUOsap8k6SUsundaridCJ9HsKuCcrUOAoq0awH15hEZIvLKXta6Dqpch0jaGIheTHt4eex6wh05RCDI0Uz06+GglAagVgCmclgJRLTL5uUWLk8as8wtS/pDI028FuNv8Z1Nx7n+jWn2POHbUQOQunL3VijY0W1vG5NxPAedZJ8spL1x79QSAMkQc0I3IOCmk4D35FRGBl791hoWVjyXINLyCzjpokeF+ixzKx2jRcbQhFEWzxMLreo0yYJKfNr6lxbhpQQKKEg2RKd1f4RKvUJoBDq4ES+gtB+ncixTHEOOFNOtp5hi6c7VnOb/xgt+oV+IVID6XYU8jfN9j6aiulU0M6bzrCQZGW+MMOxBBpJUWNBRuqkDAeYl1mWEwKEUog2rExFHRYCoSooOTiYqmNjVR9HHXlkOICSTGEW4XbzWTO1W9Nygu4yeKfJaUmJHlNxTSxMfRWKQGoqlgZeoVCmxal0xzmtTjPdt9OJ9LqIOFKElBzgJplz4I6acA0aUkJVyEQEqr+wFLgt0cILXSupH0gUVRBHEQ6RbirB1aWixJSCITUTLBNzaBhlaJjq7jrytSXkrld5qOwAq+4b5FveWwmdKEHNZCCbLRoD0kom4WQHFeksZTsDSFVFWBYikz8bH+lSUwlCVVGmwuFYWMQtkx7Th54AJZbCLLJJiJlyJohsolag1yQoVVWcU075FhZ5IJ3XEcbcuU1cM4aUGgggIiGOfaGamlVD/HbpK3gERC3Bnbs+g3UgSNPP+5Gj40UXJO4sUuLtyzLxdpAjLTXc7z5xwdcfu2M7T9avo/bPlqMcPlUIUzBdppzZB++ppvSXeviVsu1nv3o57eJP2n+J8DGJFY0Wp6G5RPB1qPxR+yO0hYc4XZYq7JI8vwNWVLSqCmTIT6ohQKJGI10myK5J4fdm0DWT+6vf4LdLtpGXcLy8hN/++K9ScjBC8Ds7FlfcHKKuaCS+pozK+3r448afUKZe6AuVkRbluy2C2zsxFqC+SksiDBPFgKS0yMuZxTMzWmoYW+3ml/ynqVV10jJHPOUiMpRe1JRUs8bpZLLVYH1tPwqC5/paYWcQEeta7JIB5zYc9Xywhts+toc3v7GJ8u0aYnRs1v2b2T+EFpuk7+9X8ZXWZv79Iy/zizftpGdDmJP/0Erp9iGsju6i6j/NoWHE2DhCKbyKWKYJlzGEtJpqjJoS1kXaWescQMHFoVwJ3xu9Dn+PgdXTX3RLmdNCURF6IUB3+rZWxtp0nFvHubG684L0VjuzOs9NbsL653KWnYjNme/iNWNIiXCQfE2EspZRHq3dS63mZNzM0mk4yJ0KUHHIxBocnpnxsQhosTS+XhftyUrGggcoOS+q893+w6jLLZ5ZexthrRntRG8hUfEVNAlNKyx5rqpnsgk+X7ODJi0BeLCQHM9WM3q0lPpho6g6gaWIe1Qy2FXCTeUd1IRjyKZa1EQaYZhYPg+WRydW7yETVEhVCbIRCyuQpyKUwOsozFxYUnDa8NGmJ2nSxzF9FoZrgQNTXoZyX4KO8jDS50E4nTPbYSoEitOJUeZnsl7l1kgXWxw5tPO6osP5HDvSLbiHsgs/+yEK+3+VaTqoCd2BEvAx3uhiolVSp4+Rx2RbJkxm3IUSGy+6eEvTQSgKeihLrWcCgFjCTXBIIotkdk04HFgVEbIlki3+DrarmxGmeVUDo8znMGN5QkcmEVaAp7euZnNpLx8q28MfLm/FPVaKp3+osFRWJDNT0phZn27UlTK+ykuTe4TQVJfSmStl/3A1kUlj1rvFFxOhO1Cal2EGXKRLXQxv0sg1p7mrsod13p5CKBWZJ24ZPD5+Jy91t1B7PI7Se+lZu5lyzRhSybZKRjbq/NcVT/CwdxQFhf1TlnT16yaup/cUT7yoy3Gym7KRCV66u4Xv+Xr5VPD42QjkN7vybHHuQfm85One1ej/vIJA+yTsP3rZSyolEfItNXT8Brx/5W4+5h9CmQqtELMy/HRwLc3/MonoGyne2bolQvjgJErez8rbB7kneJjPfvJTqKkgWlog18VZXTnAn9Q8R5maxi8k34uv4aXRVjp+2oQclDgnTZ5rquXxti18+bYf0eoYINCuEuwong7uvzT+lJdL23j+7VsJmCbmqc5pDyyK04lorGN8tYfkdWk2eztxigu7od869suMb6uksadnQWajYGppz6FjOqBUdeNVpvf3VivKmNxaS+rDMZ7Y8E+s0BR259x87uVfoXy7VghwWCSD7ozQNFqqhtni78BCkou6CHRmkUXiJ6QEA/TfEsbSJH9+6F6qj2Qun+R9ukiJtf8owVM+OFrL8x/exLoP93DLPQfZvbYO3+GrC0+z2HTf7+Oeh97ig/4DZ1MzvRJdSfbNEhxDo9fc+CB0B2plOUc/G2Z5az/faf43XEKgCwUdFVUIFBRO5i1eT63m9X/aSv0rY1jtHZhzGFLlioaUEKIOeAyoBCzg61LKrwohIsD3gGVAJ/ARKeUsUnBfHjUcxlpew9BWHc/WUVr0YVKWyds5P/+7+2463qynqXviqoyojExxmLfIkkEgqKGRetFMXuY4yA7SpMiRQQgRvlqNMpfDSiTxHHDzVeVOmq8bZKU+Rr3mQUHgFDp3+o+Qr1H57g03k/eEKBufChwmz70RCk0tRFJ26AzcV028CR5qfYu7AkfO7rJJyxxPJZZz/LBO8vA/ks0XHDEvps+NB8nC78RRVi5nfEOYm4O7ruo603mG7oJxOa85aNRoHF+/g+OpSioDEyxb008qr5PNa7gdeUbTPv6s+0HShs5owku8O4CnX6V8fw5HNIuSzKLkg2DpnLyuglbHAMICIeWC1lM1A7uyeZq1PAHlwk0RdWqC672n+G7bHQizHM/p7ssGeD0/51eqQmdsrYDGJO9fcZRmxzBQeJHoNFJ8q72Fvb/1Q8TgJL2ZPDWyYd40XrK8CHRM3EoOqSmgXCRI4dQSbWJDDb0PmXys4TCVqsnenIMfRbcQ2qcT6Hh34vHp1tPFaItnUMvKyNeWcGN4N23OvrOBUsU0jZQFaYuKglQKDtbZjI6St+bOYJUSmU6jDI4RPBXifx++i3sbj3F77UkONa3DLSWJ00cXrC3OBWpZGbk1deRWpLkvdJCQojBqpvm3yXXsPN5Ew4E8InqhD+ZC9jdnyxkIIPw+UmuqUXMWjsE4jIxjjk8U2qCiogZ8CK8X6fMQW1dCvE5l07p27i45SoV64c48C4tRM83XR+/m5zvW03gsC8PjSGNuUzZNZ0bKAH5PSvm2EMIP7BFCPA98EnhRSvkVIcQXgS8CfzCnpQMoizB0XYDKW/v4Rst3qFIdDJkWT4xv4fSuepq/0Y81dHUZyAWCZtYREGEMmWcXLxKRFQzQSYRylolWXpNPY2JctcYzCYarX4kzMeDjp80byIUOUq8VnCQVBDc7LerCbzJxm4dn9A0Eugo7h4Qp0eJZpBBIp4qlKRhejbJf7OELNTumZqLOOdClpMn3+7eg9nlYkWst6OPi+jrlMbo5cdEyzyexthATDye5PXDsvLxqMx9EpvMMO+UxxhmunFsFF2KNjOEA9kVrWe/t5r83PXn2uz/teoj2gXIcRz24RiSlJ3LUnuzH6OoBCjvvTcAzVo5ruJzjH6ngocDMNM5VPdXj8LPJ9YRCbxG4SC68MjWGvinKiBqm8WeXieA+lfPLKg/Tc7cDrTnOP216jAYtRZXq4YwRBbAvW823T9xAc2OWio5+DCs3rxovhYXEpeQJaBksXUXVNWT2Qn1C18gvK2d4s8ZLd/85EVXFJRw8F1/Lcx2tLHtpBIbH3vWGP916uhht8QyyupT4MjcPBvax2TFzW2ch26IwwMqoYM3twCgNA3NomMj+MkxnAM9v5PhQaDefXrEFNRtBnF64tjgnemrK6L3Tyd0t+7jfnQJcHM9Kvn7kZsJv6bie24PxDuNiIfubs/eMhMjXROi+T0NLCUoOOQgeUVESSaxcISwHZSXkywMkq50MPZLjzhXt/Fn1cwSVd++Cz0iDHtPJzw6tZtXfjSN7BzFnEQ7kSlzRkJJSDgADU/+PCyGOAjXAw8AdU4d9C3iFuawwQqD4fMRXl1Lx4S5+pWYHFWohWuvBXDnPvbiJsj0W1tAI1lU6czqFG+dUjAlN6HiknyxpRuhnM7cDoOMgR/oR5kijeqKXklgJL/xsE6+ta+KGzd/AJ/SzyxwVqoPPlL3Kxnu62HZdMwB5S6U3EUIIiVfP4dOz+LUsv1K2nSYtcXY57wxjpuDUvlpK+wUBEb6svioaOMWRuZA2Iwy3Qk0kRkg95wc2mz2L03mGVTRwkkPhuSn5xbHSGRgeRX5lFV8PfghLO2fYOicMGpMG2sQ4IpOFWOLiMX7SGdTxBBnzwua5kPW06pUxfjZxG698spnPL3uRe93jFyzBOYXOl1c/xY6GFXy3cQuOTieh4xc+t3SZwmRbHk9JispgnN+o3MlKVz9tegbneQm1s9LgtYyfvz59N7XPleLrHsWUcsHaIhRCTzA8hre/jL+NNrPe3cVDgX08seEOyq1ViDcPnp2VUvx+lEiIga0ecs1pylSNnVkvO5Mr+MHjtxM+ZkJfd6EuvIPp1tPFaIvvxJIKFvKCl7PpsBBtUebyOCckiQbJ2uZeJirq8fr9Mwt/MA2U7gEqsnmefaiVZa5RoutNhHRR+vrCjxmzQegOWN9C/60BPvPoz7nbdwRVuBg1k7ya3EDoKS+hY5OFGZp3/N0Wsr9RPB6UcIjjn6umbv0AX6jeh4ng9P1lvNzTTGJgHUpOwXJY+KoSVAbG2BgY5rbgMZbrI3jEuQj2P0yU8kZ8BbcE2unPh/k/++8gvMuB7B3ESs6PD/WMfKSEEMuAjcBOoGLKyEJKOSCEuGjAFSHEZ4DPALiYQeRqoaCEgqTKFb5Q+zobnP04ReGhTpgefJ3g7c/OuXN5WiaJM0GQCDmyZ+85lWF7zjSa0ShKLkfJoVKGPEEOrg2wUo9RpRYeiVNorNZhtd7PJwP9QOFN+Wg+j4okopj4Fe2sfxXn3ddC0mukeTvbgK9LwTtwztC8lD6ncCMv0QHN+hlOA0uDsDOFS+SBi6dzmCmX04i8eJ2fM42WiZVKob+w55JqrjTfJg0Dkc1hXGbn2HzXU3mqi8jwOEdvW8brkRZud227wJBSENznibHJ+Qb1W8b414rrGHBeOMFgVaf55LqdbPGeplUfpVZzoqEC594cs9Jg1Mrx4+hG+jpKadvdhxWdWBCNFwqWWIkkrqjJK2MtNFUPs8kRJ1EvcU64KTnoLWQF0B3IihIyFV7iy00aKsZRUDiSqeXF4ZWU7TPwHR7GiMevOKAXW1ucD+atLVomzkkThMK6YB/PBRvxBwOFgXIO84ia0RhKMsXExCoGciF8lQnSZaFpaZyXejpdpiYilICf8RYfk80Gnw+fRBUuTGlxPO/mrYkGwgcnEANjmLOsq3OlUTh0pNdNuHWcLy9/go0OA12oEDrFE4F2ttW1EDdc+LUMdwcPs1wfo0U/M/Zd2E/uSCznxa4WRit9DKX9uA54CHbkZxWYdrpM25ASQviAHwJfkFJOimnmI5JSfh34OkBARKb9qqB4PYzcVcf4RpP7PMPo4tz2aIcwMbwC06XOqbe8IQ0O8CYr2YAmdKY7MTJbjVYySeCnB/D2tfDr4tM8fMcu/rxy9yWPVxCs0vULfn8nCZllyLS478UvENznoPqZARibwFwEfbPFnOEb8PlcKxqvhoXQeCbVRs1zTTzTfwP/4VPb3rXEp6FSobr594FTPNrWzkTrhd+7hCSoqOio6MJ90fr6WsbPj6MbOfIna1l1agyjfxAsc1Geo8zn8PTEOfFSE0+/P86Dta/xnx96gh9s3kzUaCNVrpBotNi6tZ17Ike4yX0agN05D1/dfyf+7R6q93Vj9A1c0Yi6VurpTGejzmc+NVqxSXxvdOBY18xtvmN8d80taNk6gk9Pzu2slGViZS2suE5vJsz9DUd5fGjr2a+L8TkKTUO43Yx/cA3RVvj4Qy9zo/cEFhJD5hk3s/z7l75A8ICDqpP7MNOXz0O3IBp1B9LjJJOHnnwJ6x0DnDGQ3u8Z4i53/9lDXUJFF5ce+Z/au4GGHwlGJhvQk3nqO48i0+l59Tqclh0ihNApGFHfkVL+aOrjISFE1dRsVBUwPNeFs3RAt9CFesbyBUAXJnkfmC5lzgwpS1oc4E0qqadc1ADgwElWpnEKN1bhMcy9xlQKfWCC0JEqXlu5grfCb7HGkT9vpulCLtexZaXBC6kKvjN4Pf5DDiJHsjA2gZVKXVFfVqYRV9FpzgXmVUawno5GKN6k7mexLGQuz0gqQGe+FMMFhktFZYHrqWXi60piOnz8j74Hub/kIP/Of+GlFQRu4cAtoGQaERoMTPqNLAdz5exIrOCnXatJ9ARoPTUKg4VkqYvVFgGUeBpfd4jORISszLPZ1UW8wsXf3FKLEkrTVDnKo2V7uMHVR5XqZldW8Bc996Mf9xA+kUPGJq8Yh+daaItnSMscI6aBGlfRYonLxig6n/lui9I0kfE47mHJN4duhZo0Q6oL93ALzr4Y5omOOYmHpFaUY1WXoUcylDnivB2tQ5tQp6VxPuupcDpRwyFwOkBRMHsHEKqCaKonV+EjWeVg5HqL0mXj3O07TIOWBtzszOpsS6zBf9RBuD1fcIu5jNG5YBrzOUQyQ7K7nG+Hb6Bt2RNUqxmCigun0M8G1IRCJpOTeYPDuUpaHUNElEI16jXcvJRow93pwNM1hogVEqOb0fn39Z/Orj0B/CNwVEr5V+d99WPgE8BXpv59aq4LJ0zAFJhSns21m5cmLiVHpsIgG1SZi5SmUkqOsBsvfhpEy9nPy6hmgC6W0UqeHMyDRgCzp5/yJyfpqGnlzwP3878anmCZNrNceQYm41aOr7Tfj/4vEere6Mbo6cVkevoG6EKbo6W1xWC6GoGJxSrjdLFyeZR4nMHBel4sbSMXkmRDKo5FqKdy9yHC7QHavat588bl/Lv7v35V14tZOZ5PtfDVI3fieClIzavjcPog5tQS/WK3RWtkjLIdOu33lDDebLDK4WSt4zS//cjfv+PIQoy2H0W30PedRpbtmsDad+SK28evtbbYb5o8n1yFZ0DAqR6sK8xewAK1RSmxMhlK9yY46mrlg594k4/euJMP+z5LaH85lT39BSPhKo2p7Oo6et/n4OGWHdzgO8n3X7yJkiOLX0/VcIj0ujqyIQ3DJSj5WQrhdNDzQCnJDWk+u/E5HvIfpFE7s4xeWIr7m767efvtFbQ+NYB5suOy91hIjeZkApHOUPtSKZ39jTz7K6u51dPO5ncM8BYWMSvHv03cyPeObObRVfu4zluYGf7ByBb2P9tKzbYMVnsHcg7DG1yJ6Uzo3Az8CnBQCLFv6rM/pGBAfV8I8WtAN/CLc1kwmclSuidKNhLhzfe5cYk8eanxpVMfoquzjIanJZ6OuYliHmOMQbrxEWSHfB6AFayhgZUcZAd9shOTPBQ0zznSyGMlktS8lqFrcAV3b/4dPKUpNlf3cF/kMB/1jZy3o+0cj02W8vOxtezY24I2qaDmBIFTktC+EayJ2Iz0uXDj5N27HuabYEeWg68288YHT7LZUWgQb6SaOfpaE/Wd049ZM12NTG2cKGosE2kIyKhM5NzkKvOkhx1Yi1RPrXSGsrcmcI8FWDX0WWhKsqZ6gA9VvE2zY/CSO7sO53N8a+wmhrIBJnJu+mJBYhMe/PtchPos/KdiiIFhzPM2iyx6W8xkEYOjlP2khLs7f598qYEjmOWepuM0uEdpcozQnw/TnY3wo8MbcLa7adg1geib3s7hYm6LZxhfF2JsHQSVLK+nlvMXrz1A/QmjEKF9GktmC9kWte5hqiQ8WXMDz7W0cvf6IxxvKOdkw3oiByF8LIFyug8rkZx27CetphqrLMTI5iDjGy3uuW4fJ+LlvNjbQtUbEt/pScYXuZ6i62SDKoM3QWXrEEdvawBVsrX5OFtDndzlPUqZIshLk0N5wVvpJn7Qu5nh16ppeCsHo+NXvMWCtkXLROZBS5q4h1X+4Sf38v/qb+PT67YxnPMzkvPx9kAdyZgLR68D97Cgss/kueqbeNp7EwCucUnt0RSOzpF37UCcb6aza28bXHKe+X1zW5zz7pvPIQ8cI7j8el5LtOIUBlmpMbijispjEtezuzHnKFhfSJRyNx++6HdndifslC8yKcevXPtmg5TIbBZt+yEq3nbh628lURPg9Y2tpFY7uN395Fl3ujOGowo8OXwPew810vJYCq17uBBaIZ3BTCZnrA8KGhcaR0+Uyp1l7LiziQ/7DxXKEW2kcqeJoyc67bn/6Wp8QT5+TcSck6aJyCmkDAeecJqc30HFItVTmc8h9x/Fd9xF4M0gI/c1sX/9cpStkq2hLiqDe9+VKw9gR7qFn7SvJR9zok2qePoElYMWwafexsrlkZb5rhehxW6LMp/DjOYI//w4kTf9pFvKSVR7eeau1ZSWxFldMkjHZAkD0QCVP3Hi74hj7Zv+DrtibotniDcIAivH8CsW+5N1lO1Q8XREsab5hr+QbdEYHILBIapqrmNsIswjv/oE3tIc26pa+KbndsBHSaoMdUzHmoghTQukhZzKOSqUMzkvFVAUhKZh1JUSb/AwfmuO21vb+R9VL3D/3l8lcShC1a5uzJHRRa+naCqGW6GqdYh/bXuMiVYNHYsW3YWFxEIhIy3GrRw7U6v58eA6Bt+spu71NMqre6c1AbHgGi0TLWXgHlcJPmsQXenih6ENTEx6yE86CO/TiAyY+HecxoonsJJJfF4vQi+YMTKXx0qlFsV3o+gjm/tfaWfHpzed/X35UC8ymZozI6qYkPkcpmni3X4Cn9NBxUteEr4KPhn8jxc9XotlWZUYRfYPYWazhc7hKv2MFhqrpx/v2AQTn6s+q1OLZfF2nji73POeYypvlPQbVLgnOXyiiUjP4vu/W9kscnSM0mcsyl73kghW8JKzmhectyAvsvlEyZqsmEiBEUcYJmSyyHwes4iSv14KMzaJSCRxjU/gcrsofTOAdGoM6PW4DYvl+QwM9SIz2TnL11UUCEGuLcWfrHwOv6JyNFZJ6c9OFXy/ihjfqyfwHwjypYFPkWgQ+LeO0NLWS2hjYSlyKOWn60gjvm6FyLE8Sr7w1NKlGpmIQqJeojQkaa4YYWt4Dx4lx7jhZdtQEzfs/l3qn7GoOtKHOTQy58EcZ0U6g7c/x2jGiQ40aBJ9ajhvz+fYnl7ONztuYqgvTOVLKt7BHMtP92CNTyxiqNcrox48jU/TkIZB5TEP1vYwpWYKzARiMoHM5jDj8UK4Ego+xpxZrVnEsa/oDSkzGoVd55zFlp759A4s85xz3GDhn0ttgj8TwPFaRmazhYE1Gj2rcynomgv0QZ3X/CvwdQs8Q0WQ40zKQqDCkREYObeUdTkf82v2OVpmYcYsn4PJSRg650+7pAyn8xEC4XAQ9KdY7+xDRyOd11GGeha7ZFfEjEYRiSSle1w4J4MMBEqYqPJQXRLjprLT1LmjxBpdRF0BTKeOMtWc0hUS02/gLU/SVj7Iav8AES3JuOHlzeFGBk6WEdmn4D02hNHZvbgiz0NmcziiGZIdQf5Hzd141SyKkKhYtCfKOTVeSvJomGCfIPz2MIxFMRY6b+UsOD9EgRWPw+DQ5U+Qck7DXcyWojekbGzec1gmMmvS9N/eRqgqMm+cfQOzsZkvFI8HJRKmLhBjlcODKS0Uce2YjTKfQx48jveQQvNPtUJqoqCfJz5xC5mGHJubO1nbcJBb7j7OmOkD4C53P8fzbr45fBs7+xt468Bygkc1fAMmgVdPszIzikynMYqs/ZnRKLw9QfMhBx2aBorv3JdWmkrZjTQ7wJKYFwm2aTO32IaUjU2RIrNLbNnIprhRzs0t5qXJ0Xye8aSHmkUs0oyZmqGQWRPLNBG5HBW7K0h16Rw90cJBXzPf8t+KMAQIsHwmIqPgGlZxjkFFzMLfm0aLpgs+VcVshEz51l6rCZSXErYhZWNjY2MDVsHHxJAKCSvLG6lmkrHF2z14tUjDQBoGzqffwgnMJB9NMfsR2RQftiFlY2NjY1PIDTgySubP1vBA2e+ipyTNg9MPQWJj817FNqRsbGxsbApR5TMmjmd3M7NwwDY2723EpZJjzsvNhBgBksDogt109pRyYTkbpJRlVzppqWu8xvTB0tdo19NLsNQ1XuP6YOlrtOvpFEtd44IaUgBCiN1Syi0LetNZcDXlXOoarxV9sPQ12vV0/s5dSOx6Oj/nLiS2xvk7dyGZTTmnkWbUxsbGxsbGxsbmYtiGlI2NjY2NjY3NLFkMQ+rq0scvHFdTzqWu8VrRB0tfo11P5+/chcSup/Nz7kJia5y/cxeSGZdzwX2kbGxsbGxsbGyWCvbSno2NjY2NjY3NLLENKRsbGxsbGxubWbJghpQQ4n4hxHEhxEkhxBcX6r5XQghRJ4R4WQhxVAhxWAjx+anPvySE6BNC7Jv6ef80rmVrXCTmSmOx6oOlr9Gup7bGd1ynKPXB0tdo19OZaURKOe8/gAqcApoAB7AfaFuIe0+jbFXApqn/+4F2oA34EvCfbI3vHY3FrO+9oNGup7bGa0Hfe0GjXU+nr1FKuWAzUtcBJ6WUp6WUOeC7wMMLdO/LIqUckFK+PfX/OHAUZpXw3Na4iMyRxqLVB0tfo11PZ8RS11i0+mDpa7Tr6cxYKEOqBug57/deZlng+UQIsQzYCOyc+uhzQogDQohvCiGulDzc1lgkXIXGa0IfLH2Ndj19z2u8JvTB0tdo19MralwwQ0pc5LOiirsghPABPwS+IKWcBP4eWA5sAAaAv7zSJS7yma1xgblKjUWvD5a+Rrue2hq5BvTB0tdo19NpaVwwQ6oXqDvv91qgf4HufUWEEDqFP+R3pJQ/ApBSDkkpTSmlBfwDhSnKy2FrXGTmQGNR64Olr9Gup7bGKYpaHyx9jXY9nbbGBTOk3gKahRCNQggH8FHgxwt078sihBDAPwJHpZR/dd7nVecd9kHg0BUuZWtcROZIY9Hqg6Wv0a6nZ7E1FrE+WPoa7Xp6luloXJhde7LgFf9+Cl7xp4D/slD3nUa5bqEw1XgA2Df1837gX4CDU5//GKiyNS59jcWq772g0a6ntsZrQd97QaNdT2em0U4RY2NjY2NjY2MzS+zI5jY2NjY2NjY2s8Q2pGxsbGxsbGxsZoltSNnY2NjY2NjYzBLbkLKxsbGxsbGxmSW2IWVjY2NjY2NjM0tsQ8rGxsbGxsbGZpbYhpSNjY2NjY2NzSz5/zlHSon0X6T7AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Because perceptron is a binary classifier, we will limit our problem to recognizing only two digits. The function below will populate positive and negative sample arrays with two given digits (and will also show samples of those digits for clarity)." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 17, + "source": [ + "def set_mnist_pos_neg(positive_label, negative_label):\r\n", + " positive_indices = [i for i, j in enumerate(MNIST['Train']['Labels']) \r\n", + " if j == positive_label]\r\n", + " negative_indices = [i for i, j in enumerate(MNIST['Train']['Labels']) \r\n", + " if j == negative_label]\r\n", + "\r\n", + " positive_images = MNIST['Train']['Features'][positive_indices]\r\n", + " negative_images = MNIST['Train']['Features'][negative_indices]\r\n", + "\r\n", + " fig = pylab.figure()\r\n", + " ax = fig.add_subplot(1, 2, 1)\r\n", + " pylab.imshow(positive_images[0].reshape(28,28), cmap='gray', interpolation='nearest')\r\n", + " ax.set_xticks([])\r\n", + " ax.set_yticks([])\r\n", + " ax = fig.add_subplot(1, 2, 2)\r\n", + " pylab.imshow(negative_images[0].reshape(28,28), cmap='gray', interpolation='nearest')\r\n", + " ax.set_xticks([])\r\n", + " ax.set_yticks([])\r\n", + " pylab.show()\r\n", + " \r\n", + " return positive_images, negative_images" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "We will start by trying to classify between 0 and 1:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 19, + "source": [ + "pos1,neg1 = set_mnist_pos_neg(1,0)" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAACqCAYAAACTZZUqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAI8UlEQVR4nO3dS6hV1R8H8H3kKuHzopZIiIgEooYRUYN0IGoZCkqiGSIiDq5Y6ERwEBENGog5UtTIBtee+IAok3yEICImSoMiqFEiKb3z+iqse/+TKP6steUc9z33dzz38xl+WXedlZ77ddPae69aX19fAcDAGxK9AIDBSgEDBFHAAEEUMEAQBQwQRAEDBOloZHCtVnPPGk3V19dXG+jP9L2m2cq+166AAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCNLQoZzce06cOJHN582bl2Rr1qzJjt23b1+/romBM3bs2CQbOXJkduwLL7xQ15xPPPFENt+1a1eS9fT0ZMcePXo0yfr6Bt/ZqK6AAYIoYIAgChggiAIGCGITro2cPHkyyZ588sns2N7e3iQbjJsg96JRo0Yl2TPPPJMd+8477yRZR0dzfu0nTpyYZJMmTcqO7e7uTrKtW7dmx3733XeV1tXKXAEDBFHAAEEUMEAQBQwQRAEDBKk1svNdq9Vsk7eAl156KZu//PLLSTZ06NDs2P379yfZunXrsmNv3rzZwOqq6evrqw3Yh/2jVb/XnZ2d2fztt99OskWLFjV5Nc33ww8/ZPMlS5Yk2TfffJMde/Xq1X5dU38p+167AgYIooABgihggCAKGCCITbgWt3Tp0iR7//33s2OHDRuWZF9++WV27Jw5c5Ls2rVrjS2uCWzC/WfhwoXZ/MiRIwO8ktazYcOGbL5nz54BXkl9bMIBtBgFDBBEAQMEUcAAQRQwQBAvZG8RZS+ufuWVV5Isd7dDURTFr7/+mmS5x5OLojXueOA/s2fPTrItW7YErOT/bdq0KckuX76cHbt58+YkKztBuapt27Zl819++SXJDhw40JQ19AdXwABBFDBAEAUMEEQBAwTxKHKAxx9/PMnefPPN7NiZM2fWPe+qVauS7IMPPqh/YS1gsD6KfPDgwSR79tlnK897/vz5JPv888/r/vk33ngjyb766qvs2BEjRiTZ2LFjs2NzG2O534tGHTp0KMmWL19eed6qPIoM0GIUMEAQBQwQRAEDBFHAAEE8itxEq1evzubd3d1JVnY3Su6U1xMnTmTHHj16tIHVEaFWy9/kMWRItWuh3B0wRVEUP/74Y5J99tlnlT6rzI0bN+rKiqIoPv300yR77LHHsmMb+bOZNm1aki1evDg79vDhw3XP2yyugAGCKGCAIAoYIIgCBgjiUeR+MmHChCQ7fvx4dmzu8eKyv4d9+/Yl2dq1axtc3b2j3R9FnjVrVjb/4osvKs07efLkbH7p0qVK8w6kZcuWZfOq7/Mte8y/q6ur0ryN8CgyQItRwABBFDBAEAUMEEQBAwTxKPJd6OzsTLJjx44l2YwZM+qes+yU4o8++qjuOWh9U6ZMqTxHT09Pkt2+fbvyvNHOnDmTzXP/vaNHj272cgaEK2CAIAoYIIgCBgiigAGC2IS7C7nTXxs5vThn0qRJ2bxsc4570++//155jnPnziXZb7/9VnneaFeuXMnmR44cSbKVK1fWPe/TTz+dzUeOHJlk169fr3ve/uAKGCCIAgYIooABgihggCDeB3wH48ePz+a5p94eeeSRuuc9e/Zsks2dOzc79s8//6x73nbQTu8Dzj2t9e2332bHPvDAA5U+qx3eB1xm0aJFSfbxxx9XnnfcuHFJ1qzNTO8DBmgxChggiAIGCKKAAYIoYIAgHkW+g507d2bz3Mm2ubtJyt5vOn/+/CQbbHc7DAYdHemvV9W7HQaj77//PnoJTeMKGCCIAgYIooABgihggCA24f6Re+x46tSpdf987lDErVu3ZsfacBsccu/+fffdd7NjV61a1eTV0IpcAQMEUcAAQRQwQBAFDBBEAQMEGXR3QZQ9Cvree+8l2aOPPpod+8cffyTZ+vXrk+zw4cMNro520tvbm2THjx/Pjq16F8SBAweyee6x94E++bdenZ2d2by7u7vSvHv27Mnm/XFCdVWugAGCKGCAIAoYIIgCBggy6E5F7urqyua7du2qe45Tp04lWdmpxjSmnU5FzhkzZkw2P3nyZJI1ctJ2mfPnzyfZli1b6l5Ds9x///1J9vrrr2fHrl69uu55b926lWTTp0/Pjr148WLd81blVGSAFqOAAYIoYIAgChggiAIGCNLWd0E8//zzSbZ79+7s2FGjRiVZ2anGK1asSLIrV640uDpy2v0uiDKzZ89OsrLv6owZMyp91unTp7P5xo0b6/r5np6ebD5s2LAku++++7Jjc48XP/zww3V9/p0cOnQoyZYvX1553qrcBQHQYhQwQBAFDBBEAQMEaYtNuLLHOy9cuJBkU6ZMqXveZcuWZfMPP/yw7jlozGDdhMvJbfYWRVG89dZbSTZixIhmL+dfP/30UzYfPnx4kg3kuoqiKFauXJlk+/fvH9A15NiEA2gxChggiAIGCKKAAYIoYIAgbXEq8pIlS7J5I3c85IwePbrSz0MVZbv3Dz74YJJt37692cv5V+5l6s109erVJCs7WOGTTz5p9nL6lStggCAKGCCIAgYIooABgrTFJtzt27ezeW9vb5INGZL/N+fvv/9OsoceeqjawqAJ9u7dm2QLFizIjl24cGGzl9Nvbty4kc2fe+65JDt27FizlzMgXAEDBFHAAEEUMEAQBQwQRAEDBGmLF7KX+frrr5OsoyN/48drr72WZLmTW2kuL2S/O2WnD8+fPz/JnnrqqezYF198MclqtfSvo6wzcmN37NiRHfvqq68m2V9//ZUdm3sU+V7jhewALUYBAwRRwABBFDBAkLbehOPeYxOOdmQTDqDFKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCdDQ4/ueiKC42YyFQFMXkoM/1vaaZSr/XDR1LD0D/8b8gAIIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYL8D+KUFeSspSmeAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 20, + "source": [ + "def plotit2(snapshots_mn,step):\r\n", + " fig = pylab.figure(figsize=(10,4))\r\n", + " ax = fig.add_subplot(1, 2, 1)\r\n", + " pylab.imshow(snapshots_mn[step][0].reshape(28, 28), interpolation='nearest')\r\n", + " ax.set_xticks([])\r\n", + " ax.set_yticks([])\r\n", + " pylab.colorbar()\r\n", + " ax = fig.add_subplot(1, 2, 2)\r\n", + " ax.set_ylim([0,1])\r\n", + " pylab.plot(np.arange(len(snapshots_mn[:,1])), snapshots_mn[:,1])\r\n", + " pylab.plot(step, snapshots_mn[step,1], \"bo\")\r\n", + " pylab.show()\r\n", + "def pl3(step): plotit2(snapshots_mn,step)\r\n", + "def pl4(step): plotit2(snapshots_mn2,step) " + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "skip" + } + } + }, + { + "cell_type": "code", + "execution_count": 21, + "source": [ + "snapshots_mn = train_graph(pos1,neg1,1000) \r\n", + "interact(pl3, step=widgets.IntSlider(value=0, min=0, max=len(snapshots_mn) - 1))" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "afbc754ef0d04c95a1af039574b46a6b" + }, + "text/plain": [ + "interactive(children=(IntSlider(value=0, description='step', max=66), Output()), _dom_classes=('widget-interac…" + ] + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 21 + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Please note how accuracy goes up to almost 100% very fast.\r\n", + "\r\n", + "Please, move the slider to some position towards the end of the training, and observe the weight matrix plotted on the left. This matrix will allow you to understand how perceptron actually works. You can see the high weight values in the middle of the field, which correspond to pixels that are typically present for digit 1, and low negative values by the sides, where parts of 0 digit are. So, if the digit presented to the perceptron is in fact 1, middle part of it will be mupliplied by high values, producing positive result. On the contrary, when perceptron observes 0, corresponding pixels will be multiplied by negative numbers.\r\n", + "\r\n", + "> You may notice that if we give our perceptron a digit 1 slightly shifted horizontally, so that its pixels occupy the place where there are vertical parts of 0, we may receive incorrect result. Since the nature of our MNIST dataset is such that all digits are centered and positioned properly, and perceptron relies on this to distinguish between digits.\r\n", + "\r\n", + "Now let's try different digits: " + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 22, + "source": [ + "pos2,neg2 = set_mnist_pos_neg(2,5)" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAACqCAYAAACTZZUqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAI90lEQVR4nO3dT4hVZR8H8HPfphJxLKzQippFZIOGTFEZFNEuxE3YIjAKFxYoKkqMItGqiCjEhSARWRkVgqCbyAKFslqJC/9ubKOgKP6JcuHoQPfdvBAvz+/Uvc2993e98/ksvzxzzsPM8cvB8zznNJrNZgVA7/0newIA05UCBkiigAGSKGCAJAoYIIkCBkgy1M7gRqNhzRpd1Ww2G70+p+uabqu7rt0BAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyQZyp5ANz3yyCNF9sQTT/Ts/CtXrgzz559/vsi2bt0ajj18+HCR7d+/Pxx74cKF1icHLRgdHS2ytWvXhmNvv/32Ips7d244dunSpS3P4dChQ0W2Z8+ecOy+ffuK7OjRoy2fq9fcAQMkUcAASRQwQBIFDJBEAQMkaTSbzdYHNxqtD+6huiet0WqBBQsWdHs6Xffdd9+FeTtPlvtVs9ls9Pqc/Xpdd8vw8HCYv/fee0X22muvFdmsWbNaPlejEf852+mddkxMTBTZ7t27w7ErVqzoyhwidde1O2CAJAoYIIkCBkiigAGSDMRW5E8++STMB+GBW2TRokXZU+AmMTIyUmQ//vhjOPaBBx5o6ZjffvttmE9OThZZrx/CPfbYY0X28ssvh2N///33IhsfHw/H3rhxY2oTq+EOGCCJAgZIooABkihggCQKGCDJQKyCgOkuehl6VVXV119/XWQPPvhgODZambBr164ie/XVV8Of//PPP/9uij0RbZNevnx5OHbZsmVFNnPmzHCsVRAAA0YBAyRRwABJFDBAkoF4H/Drr78e5h999FGPZ9Ib586dC/NWt5L2M+8D/nfqrvXo30bd9uAvv/yyyNavX19kV65caW9yeB8wQL9RwABJFDBAEgUMkEQBAyQZiFUQs2fPDvOjR48WWTsrBU6ePBnmn376aZH99ttvRbZjx46Wz9UOqyA6q1+v63ZcvHgxzO+6664i+/zzz8OxGzZsKLLopeW0zyoIgD6jgAGSKGCAJAoYIMlAvA/4jz/+CPMXX3yxyOq+oPzVV18V2c6dO8Ox0VbMuuNCpy1ZsqTI7rjjjnBs9JA9ethWVVN/4HbnnXcW2dBQXDHRvC5fvjyl89+M3AEDJFHAAEkUMEASBQyQRAEDJBmIrcjtqHtaPNUnwNH24Llz507pmHX2798f5i+88EJXztdLtiL/pe5Lxz/88EORPfXUUy0f95Zbbml57L333ltkq1atCsdGebQVuqqq6vr160X28ccfh2PHx8eLrFtfKe4WW5EB+owCBkiigAGSKGCAJAOxFbkd7TxsGxkZCfO33367yIaHh//1nP7OTz/9VGQrVqzoyrnoL3XXVDsP3L755psiW7lyZTh206ZNRXbPPfe0PK923HbbbUW2Zs2acOylS5eK7J133pnyHPqBO2CAJAoYIIkCBkiigAGSTLudcO04cuRImD/66KMdP9fExESYR+9YnZyc7Pj5+4WdcH+59dZbwzzaCfnss8+2fNxGI/4Vt9oFhw4dCvNjx461PIeXXnqpyOp2qZ4/f77IHn/88XDshQsXWp5DL9kJB9BnFDBAEgUMkEQBAyRRwABJpt1W5DrRE+dou2QnRCsetmzZEo4d5BUP/L26v/1bb71VZAcOHAjHRtf11atXw7HRl8Hff//9Ijtz5kz48+2IVm1EK36qKn4n8UMPPRSO7ddVEHXcAQMkUcAASRQwQBIFDJDEQ7j/id6FOn/+/K6ca86cOUUWfaQQIj///HORLVy4MBwbfYDz2rVr4dhOPFxrVbTtuW4r9OXLl4vs7NmzHZ9TBnfAAEkUMEASBQyQRAEDJFHAAEmm3SqI0dHRMO/Gl4brvsDczkvwoRW//vpr9hRCdf/e7r///paPcfjw4SI7ffr0v55TP3EHDJBEAQMkUcAASRQwQJKBfgg3NjZWZHv27AnHjoyMdPz8b775ZpjfuHGj4+eCfrRz584wnzVrVsvH2Lt3b6em03fcAQMkUcAASRQwQBIFDJBEAQMkGehVEM8880yRdWO1Q1VV1YkTJ4rs4MGDXTlX9KXbqqqqhx9+uCvnO3/+fJFduXKlK+fi5rVhw4Yie/LJJ8Ox0Xb8HTt2hGM/++yzqU2sj7kDBkiigAGSKGCAJAoYIMlAP4TrpeirtFu3bg3HHjlyZErnqtvGuW7duikdt873339fZMuWLQvHTkxMdGUO9I/nnnsuzLds2VJkjUYjHHv16tUie/fdd8Oxk5OTbczu5uIOGCCJAgZIooABkihggCQKGCDJQK+CiLbQRk9fq6qqhoeHO37+pUuXtpVna+d3MzQ00JfOtDRz5swiW7NmTZGNj4+HPx9tL65bwbBx48YiO3PmzD9NceC4AwZIooABkihggCQKGCBJI/qP89rBjUbrg/vUL7/8EuZPP/10j2fSG9GDyKqKt33Wje3lV2mbzWa8d7WLbrbrevHixUV23333hWOjv90bb7wRjl27dm2RLViwoM3Z/b8PPvggzDdv3jyl495s6q5rd8AASRQwQBIFDJBEAQMkUcAASabdftJXXnklzLdt21ZkdS+ernsheq9cv349zA8cOFBkdU+bjx8/3tE50Tvz5s0rsi+++CIce+3atSK7++67w7Gtrog6depUmEdfNf7www9bOuZ05Q4YIIkCBkiigAGSKGCAJNNuK3I76t7bO3v27CJbvXp1kW3fvr3jc6qq+i8P93LLcLfYivzPxsbGiqxui/2MGTOKrO5LxdHXuqNrKnrYVlVVdfbs2TDHVmSAvqOAAZIoYIAkChggiQIGSGIVBH3FKggGkVUQAH1GAQMkUcAASRQwQBIFDJBEAQMkUcAASRQwQBIFDJBEAQMkUcAASRQwQBIFDJBEAQMkUcAASYbaHH+pqqrT3ZgIVFU1knRe1zXdVHtdt/VCdgA6x39BACRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyT5L95QHHzClR5NAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {} + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 23, + "source": [ + "snapshots_mn2 = train_graph(pos2,neg2,1000)\r\n", + "interact(pl4, step=widgets.IntSlider(value=0, min=0, max=len(snapshots_mn2) - 1))" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "version_major": 2, + "version_minor": 0, + "model_id": "bf6f25d14c3b4d548ac026af97f70e2a" + }, + "text/plain": [ + "interactive(children=(IntSlider(value=0, description='step', max=66), Output()), _dom_classes=('widget-interac…" + ] + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 23 + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Discussion\r\n", + "\r\n", + "For some reason, 2 and 5 are not as easily separable. Even though we get relatively high accuracy (above 85%), we can clearly see how perceptron stops learning at some point.\r\n", + "\r\n", + "To understand why this happens, we can try to use [Principal Component Analysis](https://en.wikipedia.org/wiki/Principal_component_analysis) (PCA). It is a machine learning technique used to lower the dimensionality of the input dataset, in such a way as to obtain the best separability between classes. \r\n", + "\r\n", + "In our case, an input image has 784 pixels (input features), and we want to use PCA to reduce the number of parameter to just 2, so that we can plot them on the graph. Those two parameters would be a linear combination of original features, and we can view this procedure as \"rotating\" our original 784-dimensional space and observing it's projection to our 2D-space, until we get the best view that separates the classes." + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 24, + "source": [ + "from sklearn.decomposition import PCA\r\n", + "\r\n", + "def pca_analysis(positive_label, negative_label):\r\n", + " positive_images, negative_images = set_mnist_pos_neg(positive_label, negative_label)\r\n", + " M = np.append(positive_images, negative_images, 0)\r\n", + "\r\n", + " mypca = PCA(n_components=2)\r\n", + " mypca.fit(M)\r\n", + " \r\n", + " pos_points = mypca.transform(positive_images[:200])\r\n", + " neg_points = mypca.transform(negative_images[:200])\r\n", + "\r\n", + " pylab.plot(pos_points[:,0], pos_points[:,1], 'bo')\r\n", + " pylab.plot(neg_points[:,0], neg_points[:,1], 'ro')" + ], + "outputs": [], + "metadata": { + "slideshow": { + "slide_type": "fragment" + } + } + }, + { + "cell_type": "code", + "execution_count": 25, + "source": [ + "pca_analysis(1,0)" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAACqCAYAAACTZZUqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAI8UlEQVR4nO3dS6hV1R8H8H3kKuHzopZIiIgEooYRUYN0IGoZCkqiGSIiDq5Y6ERwEBENGog5UtTIBtee+IAok3yEICImSoMiqFEiKb3z+iqse/+TKP6steUc9z33dzz38xl+WXedlZ77ddPae69aX19fAcDAGxK9AIDBSgEDBFHAAEEUMEAQBQwQRAEDBOloZHCtVnPPGk3V19dXG+jP9L2m2cq+166AAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCNLQoZzce06cOJHN582bl2Rr1qzJjt23b1+/romBM3bs2CQbOXJkduwLL7xQ15xPPPFENt+1a1eS9fT0ZMcePXo0yfr6Bt/ZqK6AAYIoYIAgChggiAIGCGITro2cPHkyyZ588sns2N7e3iQbjJsg96JRo0Yl2TPPPJMd+8477yRZR0dzfu0nTpyYZJMmTcqO7e7uTrKtW7dmx3733XeV1tXKXAEDBFHAAEEUMEAQBQwQRAEDBKk1svNdq9Vsk7eAl156KZu//PLLSTZ06NDs2P379yfZunXrsmNv3rzZwOqq6evrqw3Yh/2jVb/XnZ2d2fztt99OskWLFjV5Nc33ww8/ZPMlS5Yk2TfffJMde/Xq1X5dU38p+167AgYIooABgihggCAKGCCITbgWt3Tp0iR7//33s2OHDRuWZF9++WV27Jw5c5Ls2rVrjS2uCWzC/WfhwoXZ/MiRIwO8ktazYcOGbL5nz54BXkl9bMIBtBgFDBBEAQMEUcAAQRQwQBAvZG8RZS+ufuWVV5Isd7dDURTFr7/+mmS5x5OLojXueOA/s2fPTrItW7YErOT/bdq0KckuX76cHbt58+YkKztBuapt27Zl819++SXJDhw40JQ19AdXwABBFDBAEAUMEEQBAwTxKHKAxx9/PMnefPPN7NiZM2fWPe+qVauS7IMPPqh/YS1gsD6KfPDgwSR79tlnK897/vz5JPv888/r/vk33ngjyb766qvs2BEjRiTZ2LFjs2NzG2O534tGHTp0KMmWL19eed6qPIoM0GIUMEAQBQwQRAEDBFHAAEE8itxEq1evzubd3d1JVnY3Su6U1xMnTmTHHj16tIHVEaFWy9/kMWRItWuh3B0wRVEUP/74Y5J99tlnlT6rzI0bN+rKiqIoPv300yR77LHHsmMb+bOZNm1aki1evDg79vDhw3XP2yyugAGCKGCAIAoYIIgCBgjiUeR+MmHChCQ7fvx4dmzu8eKyv4d9+/Yl2dq1axtc3b2j3R9FnjVrVjb/4osvKs07efLkbH7p0qVK8w6kZcuWZfOq7/Mte8y/q6ur0ryN8CgyQItRwABBFDBAEAUMEEQBAwTxKPJd6OzsTLJjx44l2YwZM+qes+yU4o8++qjuOWh9U6ZMqTxHT09Pkt2+fbvyvNHOnDmTzXP/vaNHj272cgaEK2CAIAoYIIgCBgiigAGC2IS7C7nTXxs5vThn0qRJ2bxsc4570++//155jnPnziXZb7/9VnneaFeuXMnmR44cSbKVK1fWPe/TTz+dzUeOHJlk169fr3ve/uAKGCCIAgYIooABgihggCDeB3wH48ePz+a5p94eeeSRuuc9e/Zsks2dOzc79s8//6x73nbQTu8Dzj2t9e2332bHPvDAA5U+qx3eB1xm0aJFSfbxxx9XnnfcuHFJ1qzNTO8DBmgxChggiAIGCKKAAYIoYIAgHkW+g507d2bz3Mm2ubtJyt5vOn/+/CQbbHc7DAYdHemvV9W7HQaj77//PnoJTeMKGCCIAgYIooABgihggCA24f6Re+x46tSpdf987lDErVu3ZsfacBsccu/+fffdd7NjV61a1eTV0IpcAQMEUcAAQRQwQBAFDBBEAQMEGXR3QZQ9Cvree+8l2aOPPpod+8cffyTZ+vXrk+zw4cMNro520tvbm2THjx/Pjq16F8SBAweyee6x94E++bdenZ2d2by7u7vSvHv27Mnm/XFCdVWugAGCKGCAIAoYIIgCBggy6E5F7urqyua7du2qe45Tp04lWdmpxjSmnU5FzhkzZkw2P3nyZJI1ctJ2mfPnzyfZli1b6l5Ds9x///1J9vrrr2fHrl69uu55b926lWTTp0/Pjr148WLd81blVGSAFqOAAYIoYIAgChggiAIGCNLWd0E8//zzSbZ79+7s2FGjRiVZ2anGK1asSLIrV640uDpy2v0uiDKzZ89OsrLv6owZMyp91unTp7P5xo0b6/r5np6ebD5s2LAku++++7Jjc48XP/zww3V9/p0cOnQoyZYvX1553qrcBQHQYhQwQBAFDBBEAQMEaYtNuLLHOy9cuJBkU6ZMqXveZcuWZfMPP/yw7jlozGDdhMvJbfYWRVG89dZbSTZixIhmL+dfP/30UzYfPnx4kg3kuoqiKFauXJlk+/fvH9A15NiEA2gxChggiAIGCKKAAYIoYIAgbXEq8pIlS7J5I3c85IwePbrSz0MVZbv3Dz74YJJt37692cv5V+5l6s109erVJCs7WOGTTz5p9nL6lStggCAKGCCIAgYIooABgrTFJtzt27ezeW9vb5INGZL/N+fvv/9OsoceeqjawqAJ9u7dm2QLFizIjl24cGGzl9Nvbty4kc2fe+65JDt27FizlzMgXAEDBFHAAEEUMEAQBQwQRAEDBGmLF7KX+frrr5OsoyN/48drr72WZLmTW2kuL2S/O2WnD8+fPz/JnnrqqezYF198MclqtfSvo6wzcmN37NiRHfvqq68m2V9//ZUdm3sU+V7jhewALUYBAwRRwABBFDBAkLbehOPeYxOOdmQTDqDFKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCKGCAIAoYIIgCBgiigAGCdDQ4/ueiKC42YyFQFMXkoM/1vaaZSr/XDR1LD0D/8b8gAIIoYIAgChggiAIGCKKAAYIoYIAgChggiAIGCKKAAYL8D+KUFeSspSmeAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAow0lEQVR4nO2dX4wkx33fv7XDXcZ7ywS8ISVcJO0sGQtBqJdE3AgiEutFQSTeixxYBmQM6QMt4KBlAjAB9MDDvujlHqwgAajQVHiJjrpwFpadf7AswKAlJkEAJzC9DPhPkM8k5dszLULk3dkJqZV11F3lobu1vb1V1VXdVd1V3d8P0NjZmpnu6p6qb/3qV7+qElJKEEIIGRdLfWeAEEJI91D8CSFkhFD8CSFkhFD8CSFkhFD8CSFkhNzSdwZsueOOO+TGxkbf2SCEkKR4/vnnr0gp76ymJyP+Gxsb2N3d7TsbhBCSFEKIPVU63T6EEDJCKP6EEDJCKP6EEDJCKP6EEDJCKP6EEDJCKP49srMDbGwAS0vZ352dvnNECBkLyYR6Do2dHeD0aWB/P/t/by/7HwDm8/7yRQgZB7T8e2J7+0D4C/b3s3RCCAkNxb8nLl92SyeEEJ9Q/Htifd0tnRBCfELxD0TdYO7Zs8Dq6uG01dUsnRBCQkPxD0AxmLu3B0h5MJhbbgDmc+DcOWA2A4TI/p47x8FeQkg3iFT28N3c3JSpLOy2sZEJfpXZDLh0qevcEELGjBDieSnlZjWdln8AOJhLCIkdin8AOJhLCIkdin8AOJhLCIkdin8AOJhLCIkdLu8QiPmcYk8IiRda/oQQMkIo/j1hmgTG1T4JIaGh26cHTCt6AlztkxASHk7y6gHTJDCAE8QIIf7gJK+eKbtyVOIOZJPAOEGMENIFdPt0QNXNo6OYBKZqHDhBjBDiE4p/B6g2bqlSngRWbSg4QYwQ4hu6fTrA5LIpTwIDgEceOSz80ykniBFC/GMt/kKI80KIt4QQr5TSjgshvi2EeDX/e3vpvTNCiNeEEBeFEJ8qpd8rhHg5f+8rQgjh73bi5Phxdfp0Cty8eTCQ+9BDwNWrhz/zzjtBs0YIGSkulv/XAXy6kvYogGellB8G8Gz+P4QQ9wD4HICP5N95Qggxyb/zVQCnAXw4P6rnTJK2sfnb28B77x1Nv36d+/qSnD4ngHDyyfCQUlofADYAvFL6/yKAE/nrEwAu5q/PADhT+twzAO7LP/PHpfRfAfCkzbXvvfdeGSuLhZSrq1JmW7dkx+pqli6llEIcfq84hDg4h+4z1c/1wWIh5WyW5WM2O7gv0iF1hSyWa7OwRAeAXanSc1Wi7lCI/19W3v+L/O/jAB4opX8NwGcBbAL4Tin9FwB8y+baMYv/dKoW7dkse382M79v+kz1c13Tp+aQEjaFqO9rs7BEiU78Qw34qvz40pCuPokQp4UQu0KI3bfffttb5nyys3PUT1+wt5e9b7PE89mzwPLy0XOsrPQb6aOKVNrfpyuqc/qcAGJ7bRaWpGgr/j8UQpwAgPzvW3n6GwA+VPrcBwH8IE//oCJdiZTynJRyU0q5eeedd7bMahjqynWxNEPdEs/zOfDUU9kgcMF0Cpw/32+kDyedRUKfOwTZXpuFJSnaiv83AZzKX58C8Dul9M8JIW4VQtyFbGD3OSnlmwDeEUJ8PI/y+dXSd5KkrlwXhs98nkX1FNE9KkGfz4ErVw76zFeu9B/iyV3JIqHPHYJsr83CkhYqX5DqAPCbAN4E8B4yC/7zAKbIonxezf8eL31+G8DryAaF7y+lbwJ4JX/vceTrC9Udsfr8Tb76WAZs20A3bkT0OZhqc22bwsIB4c6BjwHfPo9YxV9V3mMasPUB6yuxxlRYaEn0gk78uapnS3Z2slm5ukHf1VXO0CUEgHk5Wy5ZGwyu6tmQuk1XTp8+LPwrK9lALffuJaQCB4Sjggu7GTBtujKfqyPbrl8H1taywVpCSIn1dS5ZGxG0/A3UhS37NmQ4g54Mmj4jlsgRKP4G6sTdZ2Rb0cvY28tGwopeBhsAMhjm8/oJL6QzKP4G6sTdpyHDyZFkFNhMeCGdQPE3cPKkOv3nfz7769OQ0fUy9vboBiIdQ//jKKD4G/jt31anP/vsQX1wNWR09crkKircQL/2a6yHJDD0P44Gir8BXew+cNgdUwi6EMAtt2R/NzaAhx8+LPQPP6yvVyoXUpXr17M5BSQxUrKk6X8cD6qZXzEefczwrVu2oVjawbQWf3WZB9MM4PLkSNN56uCM3IhIbVarzeYTJCnQ8ZLOyfPww+b3hTgIWbadJK37XOHvL7uQmsJee2SkZkmPdXG2lHpnnqD4K3j4YeCrXzV/xueqGKp6VV7a2Sa9IDWtGTypzWodYyz+SC0mir+Cc+f079WJbx3V7ep19eqxx45u7rK8nKWbSE1rBk9qlvQYY/FHajFR/BXcuKF/77HHsvrQlC98wa5eFZu7lD/71FP1dTA1rRk8sVjSLm6NscXij9ViUg0ExHh0OeA7mZgHeW2WcVYd02n4vKc2vjgK+h6BZ6Ew0+f+yB0Arudvz9aWXsCLoIdyfZ5OpVxeNgt/l3Wtb60hkTFwcWvNwBtHnfhzPf8Kdevzq5Ye1y1TPplkPef19ayXP/TeM4mUpSV1hIIQ7ULLhsTOTubjv3x5cBWW6/lboFqfv8q77x51l+pcgzdvjsdtmgQjDOcDEH4gaAjPdWzjHKD4H0I16F/l6tWjUWAcZO0RW+EZaTgfgLCDzmN+rqmj8gXFeHTh87edqVsM3tb5/KfTA7ch/fABcPHVjt3vHaoAjv25JgDo869H57u3YWUli8P/0Y8Op6+uAqdOARcuHO5VcG9fD7jsCUu/dxj4XKOHPn8LbBZX03H9OvBXf3U0fX8/E/kRziEJj0t8dte+uVB+cF/n9XUe+jzTRdUdiPHoKtRzsZByacne/dPm4FpZLXFxOXQZzhfqWr7O6zN/Aw+THAJgnL8906lesGcz8/uqQzdpjG7RlrgKT1cDL6H84L7O6zt/HNCKGoq/A3Wr2rrM8F1dzSaN6TSK9aYlMT7AUMsi+zovl20eFTrxp88/p+wCXdI8lcKNWax9ZcO5c8ATT6jXygIYJdeaGOOzjx93S7fFl3+dfnoCDvgCOBqqrFrYrRoWPZ/XL/C2tGTWopEuJtg9Q5iEBPiL149lsTnSL6ruQIxHSLePzgU6mZi9CXXun2PH9J8zfY+9b4/0MSAZ0q3iy80Vo7uMBAGM89fTJlR5Zwd44AH1e8X3TWv/qHoZqjB10hCXuQApX5MQDYzzN6BzdUppt/S5zv1TnFcXjn7jBnvfweljrXa6VUgCUPxhntxlMwhbV9d1jUsx8DumTZM6p4/BzS52wxrKOAbpD5UvKMYjdKhn4QI1xffbfF/lQuU8mB6JdS5AG1igiANgnL8dpsXdmurBYnF4Ylh5wTfSAbaCnoqocjE14oBO/On2qWDyBkiHWPyiVy4E8OCDh/cI+PGP2+WRPX5HbOcCxBx7W/7RdasPDn3PWeIVin8Fm8Xd9vezlTptlo4HjkYStdETLp8ekFg38q7+6DpcxzHKFsott2R/aU2MB1V3IMaj6z18TZu41y3RYLP2T9OQb/b4AxLrwzUNRjV1T5kmqcTo6iKNAX3+9VR98zbHdGq/zo8PPeGyLAEoj/ZXH3AMQhhiIKquQem7wSPe0In/KN0+Kp+5zf69Kq5erd/6sUqbkG8uy+IZlY9OiOx1LLG3plhh1zWNisJft2tR364uEh5Vi+B6ALgE4GUALyBvZQAcB/BtAK/mf28vff4MgNcAXATwKZtr+LL8dQEdrhZ/00OIzK3kO/99G6fJEqurp0zIdfxp+Q8ehHT75OJ/RyXtywAezV8/CuDX89f3AHgRwK0A7gLwOoBJ3TV8ib+N+9THMZ2GW8c/hVD0ZEjFj+bjR7ct/LQmBoVO/EO6fT4D4EL++gKAXyylf0NK+RMp5Z/mPYCPBczHIbroza6sAI89pl8XqG0eYlzFOFlS8aP5+NFNBW8yyf725epi/HLn+BJ/CeD3hRDPCyFO52nvl1K+CQD53/fl6R8A8Gel776Rpx1BCHFaCLErhNh9++23vWRUV6cLN68Pbrstqzup6MqoGdM6PKaxg5/+NLP7+7AmVPHLDz6YRuhpyo2WqjvgegD4m/nf9yFz6XwCwF9WPvMX+d/fAPBAKf1rAH6p7hqhff6+/fqma7FHHYimrpGx+NFiLZB17qgY8qgi1udZAV2FegL4EoAvIhvMPZGnnQBwMX99BsCZ0uefAXBf3Xl9hnqq6rrPsYCyT38sutI7XVTEIfyYMd6DKZS1OCaTOPJaJoVgARlQ/AEcA3Bb6fX/AvBpAP8Shwd8v5y//ggOD/h+Hx0O+OrY2vIn/tOpvm7FWPcGQeiKmIiVlySpDkQnEiwQUvzvzsX8RQDfBbCdp08BPIss1PNZAMdL39lGFuVzEcD9NtcJLf4u5c8lLLRcXqkfAQldEbu08mK0EELmKdUQ1LFb/l0docTfxuVT6EpR5hcLKVdW7BuAoiwkUlbSJPTD7crKi9FC6NKlVq5wuiOGxlDKOH8rBRR/BW0MDtUyzabGQ8pkeolpEnrd/q5abt/XCTk/IJTVsliYF9eKSWBj7KVVoPgrcBnktRHoujpCyz8wIdft78rK82kh+Mqzrzy5CGWdZcZKY83oxV9V7myCDFRlrdxDLQyU2SwbNDbVtUR6icOnaSvsw8qrO4dPC6HJuVxC4VTdYZ/b2S0W+grJ7rI1oxZ/Xbk7dsxO+OsEvPy5rS1z3U6glzh8+vK/qQaLVlbai6QO1/vUXbvOqrHJd9NGjd3l1oxa/NvE8FfDi7kS7gDoQlBUrbxuYGg6rf9uE1zv0/T5tj2WpstSs7vcmlGLv4t7x1T2bc7F3mgChBaUJtPIfaNrbEz32aZHVPfdNhvSsLvcilGLv4/Zu0W57MryZ3kPTMgH3KTAtclr9X2Vm6bcy9Dda5seUd13bUPr2HX2zqjFX1XumvQGinoXevc79nQTQjX673pU3T7V87tGEdQVbpOF3bTg2Xy33Ejp8saus3dGLf5SuhlHpqOY5FXtUZsMKlc4xpUILhNFikKyvHw4bXnZXHCaxg83tbDb9IhcvstC3hmjF38VdXNJdEaTTfBDGzgZLBFchLcoIK7iWlcYmg5o9V2Y2L3tDIp/BRv/ve4ItUNXAY2iRLAR3tARO7r36/JmW5hCjI3oJspQ+INA8S/h2lvv2piiUZQIXYz+N/H5F91TXWhpdbEql2uvrJiXrW17P67nYlRELRT/Em2jf5aWwtX1ApbrBOhi9L+4juvMQVsLx5TPNuGZOnx1a2khWUPxL9E27v/YMZY7Io+O/Nta1F3gYuFMJuqGxbaiuAh30wEt26gq+kaPoBP/kBu4R8vx4+2+v7+f7XE9m2XbjPa15zXpkWLf2atXD9J+7ueAxaKffXCrmDZrr3LjRiade3vZPRX70NpuNu1yrSYbW5f3+C3y6yMvI2eU4m9iNgO2toC1Nf1n1tezun3pEnDzZhx1nXTM9nZmBZTZ38/SY8BWuKuU7+Hkycy68Xmts2eB1dXDaaurWboO1bP2kZeRM0rxv3ZNnS5EJuRPPAG8807WCFTLfl05JSNBZ2H2aXnu7AAbG8DSEvDuu8Dy8uH3l5eBlZX681y+nJ3rwoWsR1BmqSIZrhViPnfvNts+05UVVk4HRin+tj3PJ54Ann6a7p1RUxbUjY16l0hflmfZNSJl5o4SAphODwrvU08B588fFOjJRH2u9XW9tX377foKoXtWVVy7zbbPtNpQETOqgYAYj9ChnhywJUcwFZTYClHTtft19+BrOehQ651wwNcaMNrnMLroOIZXkp9hs1hZLAWmbRRN9R58LgftA9ton75nLkcIxb+G2Aw5EgEprbPRVnxtFr8KsRz01taBmE8m2f9d3O+IoPjXwLJEjpBSoQixImfdtnRlmjyrrS31d2waAFpr1lD8a0jJyCMdkZrA6Fw4bXfhsr2267PSuW8mk3b3Sw5B8a8hJSOPdEjqAmMjyr4sH9dnpfPbA653SQzoxH+UoZ4qmsw9IZ6xDRXskpRn8+3sAKdO1U9G8xW2avusit9Zhy4EtY4Yy0/MqFqEGI9Q6/lXx7hSNvKSJjUXS+zUhUeWrfoun73Ldo62FVK1uxLLz88A3T6HodZEho3fbQgumK7y77rcdFd5a7OkrqqCbm2ZF6CL3W/bwXOn+OfUbeISe1kZLHV+59Rb667zbxLEPp5bm92TTMZA3cqjMUdsdFQmKP7SrscZc1kZNHWWv25jklRa66YRBU13vdJdbzLpbp+B8ud87Z5UrqA2jUnM5aOjKBOKv0y/rAyauqUUUm+tm0TUtNksJrRV6XL+uoq3uqpv3E0VtM7qj71n2FF8OcVfNi8rTd1yqbuoO8d1qYEuW+umMfQFTay8tttEhiyApvupXtd0D9Op/c5j1QpqOndx3pih5R+H5a+rG00NqNRd1FFharW7eKCmGbC2P7LqHHX74cbsz64bU6jm06YBcw2/Uz1TIeyXiOgb+vzDiX/b5UqkbN44c9KYR3QPczrt9/q6Gaq6H7lcIKdTKZeXzYWxreUfEtdnEqoB66p7Heo6jPbxL/4+liuRsrlbjstFeCSEheRS6Vw3fbb5kW1DW219/l37GHW/ictz6rsBsyXxbvzoxN+X5b22pj7P2lo31yc5PsXNtTK7Wrk2PRJb68Am2qcvcVL9JqZeWqoC2ldl9lTmRyf+vizvOnelS3RbKmV98LhWZlM3suq6ATJfftNQzCaCEpOlURe1lWIERIhufN2z8CggoxN/X/WhrtdqqueplvWC1POvpWnYpephNJ1/4CKSdb7K2HyMQys4bcRE9SxshN1jgz468XdtOHXl1Wb8qqtxxy4J1nOJQRh8WspthNdWGKpHQKFwyutY8B3yZ2MweGzQoxN/AJ8GcBHAawAerfu8j2ifJi4a3X4T1WNoBNGTWHxhPvPh+0HVRfiozh/6ucbyu/VJk8bP9rdUCftQLX8AEwCvA7gbwAqAFwHcY/pOyPX8655zeae5sYh/EE9CbL5pH5aszlJvOsnINrJINzCsup+292oa8B5jT8AWF+EP2KDHJv73AXim9P8ZAGdM3wkp/rb1S9dbG6LbJ4hOx+ab9sVi4W9J4SaWf13e2oqITYM0tp5AHYuF/rnZRj4NMdoHwGcB/PvS/w8CeFzxudMAdgHsrq+vN7rxOky/UdWwWSyOBncsLw+zzAfp6cdk+fvG17018fmHzpfvBmkM6J6ZEJ1HPsUm/r+sEP9/Y/pOKMvftlynHq3WBO/3OmTfsc9ejWu0T+h82TRIQ+jB+cTUW+qY2MS/F7ePSsxcJm/SsPHAUFvPWHs1Pnskxe/muqyFy7mHUiYiKg+xif8tAL4P4K7SgO9HTN9pK/4q40UIKY8dsxd/GjZES4hejQ9RDJUvX+ccam8wovuKSvyz/OAkgD/Jo3626z7fVvxN7h3VJM1IGu2kSN6Aa3sDxfcBt41XdOeqisfysnkV0Lp8+fxhbM5p85mILGTvRFIhohN/16Ot+JvcO9Op3V4SxUqxkfymURGRodMMXzfg6zw2g1G2E436KKy2z8FmTMLnPYyw8o5e/E11SQj9Am7lozDmquU1KZELRJIGXAg/tq8HYTsYZTpvn+FppudQfu5LS+rPFfHTfbuYBtBYjF7868JubepZ0zo4BpIL4Q8VwWI6j4uQ2IahmfLX58QUl81eTHn0aVW4nquv7qznBmf04i9l5rZRWe0+xD9akeuI5Cz/ELHrPib2lM9lI5Km/Jm+FxrTrGCXCuXTqnB1MfmOarIhQIND8c9RNapthT9qkeuI5Hz+IWatmib2NFn9s1xYp9NsCVmX/HUp/qrBbpWl5VqhurT8Y5jPEMCKovhrsIn1rzNWoha5DvHaWw3ta62z/Jtc01SQfE22cnkmXbl9TKJZ3HfZ1+/S6Hbp849hJnMA/ynFX4Pp9y5H9+j2pE50DChuuuhGmFr9ppXbZLX14RdbLI72FlZW2s0YVmHTkJbz5LqRfchon/KzsBH+0JYeLf/uxN92FvYABv3ToSuhNFnpTTA1WrEMHm5t+QtptRVNlxVIu8TWzdPl6qX0+fdv+Y/dh98rXYUOhfjxTcIWg+j5uGdb0Yy9MvmaS+EbRvt0I/7JDVSOga5a5KH/+CoR8dGw2vrGY3+edWM0bYQ3hoY+h+JvIKLfKRhJ3WOXohyzpd7m+q5xzS4Nq22EROwFLZSR0WRcIyAUf0v6ru8hSNLA7fuH6PuhuVxf5dfXCfTS0tFZv75CWmN17+jQRXIUkR5NicydRPG3oO/6Hoqh1NVOCf3Q6ho32+ubQtF0R1sr1LaidNGAt72GrofUJq8+lubwCMXfgqGKZHJLL8RAyIdmI56213fxv6sKdVPxrPteV+G6ba8RotLb/iYdVUCKvwWm+mYKD47dPTTURi0oIR+azbltr++yG5GqUIcS6C4KnY9rhGjkbaOhaPnHI/5t1viJ2T00VHdWUEI+NNs1Zmyu38byDynQXXQ3Q0Yu+Rj0LazD6bT9OEsLKP4Kqr+P7ZpTPTfkjeh7/DRJQj00F39+3fVdY+7LwhNSoPu0/KdT+9+tK8uoxwpI8a/QpM7UHfShEyt8D5hWP2fqwhbnWSzCrloZSlSrFptq+QpXK3vglhHFv0LT3nKqlj+JjJADpovFUVEEDjZyMVk+PoXSt6iq8l3d2tLHPIaBQfGv0HScrK4nHSsDN26GR1u3iUkEdeeeTOJwkeiweSaurqwRVAyKf4U2lr8QUn7yk+mUmb7rLKkhxDIMpu/brmZYpe+wMZtn4pLHEDH+EULxr9DU5x+70Kvou84SA7qW2cZvb8L0o+veK8I/dfQ9YcSmILuMp/he0jtSKP4KVLH7pgYh1TLRd50lBkwRK6bCWLehhEkEmwpf3/50nwPlpq7/wCoGxd+SxUJdxlPuDdLyjxibmYV13VHTmj/lwjydHnzOVfgWi6NRNEA2sNxlxfDloze5vgZWMSj+GnRlaUjjQPT5R0ybQcw6wTL98K4WgamHkiJNXV8JQvFXMCZRHFJjNihsCqGN9a+y2E0C71r4m/gOY14TRXX/Plb0jBCKvwIfEwQJaU2TmH8bi71OsF26va49BZs8921pjcQiovgrsI3177uMkoHQRmzKAmwbnthksEfXI3Dd+9c2lnpg/vUYofgrcIn1ZxkltZjE3aeP0WXZB9dr1rmKbHsKtpbVwCJrYoTir8Al1p9llBipE9q+Qq5cextNZsi6zlOo+lib5JNYoxN/kb0XP5ubm3J3d9f7eXd2gAcesPvsbAacPQvM596zQVJnYwPY2zuaPpsBly4BS0uZ3FURArh5M3Tu7LnjDuDq1aPp0ylw5crRdN19T6fAj38M7O+br7eyAnz+88CFC4c/u7oKnDvHyuYBIcTzUsrNavpSH5lJlb094KGHsgaDkENcvmxOX19Xv69LTwXdfV+7lon3bJY1cLMZsLZ29HPXr2efqzYS+/vA9rb//JKfMXrxdy1f770HPPJImLyQhKkT97NnM2u2zOpqlh4T1665pZvuez7Pej03b2Z/f/Qj9Wdv3FCn6xoW4oXRi7+qx1qHqlccEzs7WW98aSn7y55KIMoP+t13MxdGmZWVLH1pKbMyTp06bAnH6NbQifnx4+p0l0ZNd+7JxC0vxA+qgYAYj1ADvrbRPtUjVsY0ca1X6taW73nrvsY0WcahbfSRaxgpcQKM9jlK02Wd19a8Z8UbXMfHgTYRJnUPOuUfQhepM5m0j8bpYz2VtudOPBKJ4q+g6YYupuVM+i4nXMHTkrZdpLoH7fOHCF2oque3qQTVHbRiFcS2v/MAutIUfwVNLX/XkOcuy0nKBmentH1QXVn+NoWq7cxh1Ro3rpVC5xbq2xoK/TsX9H2fBij+ChYL9VanTS3/GIQ3hgYoCdpa5nUP2tcPUVeo2l7HtLpl24oRsjDaim3IHdHKeYm40gURfwBfAvDnAF7Ij5Ol984AeA3ARQCfKqXfC+Dl/L2vANlEs7oj5JLOtpMR68Q/FpdLxEZIPPhoqW0WZGv7Q9QVqrb3UbeuvRCZr9+2cpQJZQ25iG0Xln8MVp+BkOL/RUX6PQBeBHArgLsAvA5gkr/3HID7AAgAvwfgfptrhd7MxcUFpBPzyMsAKRO5tfYz6gpVW4vDptC6rINSJpQ15FLRuvD5x2L1aeha/M8AOFP6/5lc8E8A+ONS+q8AeNLmWqHF36V8u6xiG6OehCDJ3kYKmVYVqpWVg8FWnVVua3HYFtrys9KJXbVLHMoaarL+UMhon8itvpDifwnASwDOA7g9T38cwAOlz30NwGcBbAL4Tin9FwB8y3D+0wB2Aeyur68Hf0hbW/WuzjoxT0FPfDPmRq8TyoVKNX/AtZCazm9TaFVzAZaX1Q2GbcFwyUNsYht5BWgs/gC+A+AVxfEZAO8HMEE2U/gsgPP5d35DIf6/BODvK8T/d+vyIGU3e/jauH6Wlg7KWSS/be/EVhcHgU4MdQ/bRwy+j/w1+ZyreMYothFbfcGjfQBsAHglf52c20dK9wCHvstbLETu8kwPk7i1edixClQT6yHWe4mQUG6fE6XX/wLAN/LXH6kM+H6/NOD7RwA+XhrwPWlzrS7E3zXqp1xGx1z2aPl7xvRAmz7sLqzlpoIc2noYeUMRSvyfzsM2XwLwzUpjsJ1H+VwsR/Tkfv9X8vceR8+hnmWaiv/YewEx9sKTxiSGpklZJmEL3UK3KQQhXVksnOHdPqGPUJO8XGe11/UAxsrIjSu/2EzsKj5ju59vaOu6TeNiE2rXVLDZLaX4V/E1qz1EPSIDokmrqCucW1uHP+cibKFF0MeM6bpJZU3yWteLGoHFQvGv4HNW+wiNCWJDG5eDKu7YZXJRVdhCL5vss3Hx2UvR5Ws6HY07iOJfwSTyLrPZ68rNSIwLoqKNILZZVkAnbFtb4Qpjk4bONZy1SUOiy5dugG+AFhzFv4KpfC0WdoJfNBKm8OXqwnGmPTHIwGhjwbZZUKwvYXOxdEyNhe9BWlW+RhSfTPGvUFf2islcpqN6vmr50tVB034AZECEtvylTFfYbAe1Q3WZRzQQTPFXoCpftmv8TCaHz6NqSGwbDjJQbCxYnciFCJ0szh+DH7LvBmpEIaAUf0tsQz6PHTuoP03mB5io9hqm00GWyXFgEts6AWoq1LrzxrRXbgyWdywNYWAo/pa0Dfe0Oeq2gVRtMKNaN4skTkgBVAlbFxO9fPj8iVco/pbUWf5tG4c6ETddf4DuyHHTtesj5PV8RvsQr+jEX2Tvxc/m5qbc3d0Nfp2dHeD0aWB/3+95hQDW14GzZ4H5XP+5paWs5ujOcfOm33yRHtnYAPb2jqbPZsClS2ldr+t7IdYIIZ6XUm5W05f6yEzMzOfAuXNZmRUCmE4zQW7DbJaJ9qVLZuEHsgaiyXskQc6eBVZXD6etrmbpqV3v8mW3dNI7FH8F83km1DdvAmtr7axt17p19iywsnI0fXk5nCaQnqhaGrNZ9r/JQtjZyazspaXs785Os+sBwGSSdXG3t7PztDm3zjKhxRIvKl9QjEdXPv8qbXz8k8nh5Vhc9r9YWzvskq0u60JGQKglGlT++ZWVo7tz+d7rlvQCOODbjLarfS4vH4Rt2i7AuFjY7ZJHBozLyoOukQAuhdrl3BzAjRKd+HPAt4adHeChh4D33gtz/vJ42M5O1gNXjZsB2fjDlSth8kEiQzeAqsI1EsAUVdD23CQ6OODbkPkceOqprA6EoBgPK6KMTPX96tUweSAR4jJQ6upXd/k8ffaDheJvwXwOPP00cMst+s/MZpll7kpRt7a3/YeXkoTRiW7VCmkSraOK+llZyaIK2p6bJAPF35L5HPj617PonwIhgK2trAd96RLw2GNu5yzXLRtDr0njQhJFF5b5hS+4RQepUEUZnT+fdXHbnpskA33+Hih89Zcv27tSJxPgwoWDulXn4l1Zyeon6+KIKBcsmxmChCjQ+fwp/i1pOiO4Oo5mGlieTrNeBes9IcQVDvi2RDf/5ZFHmvnqjx8/fN4HHwRu3FB/dm2Nwk8I8YthCJMUVK37vb3s/z/4A3MEzmxmduVUz6vrhHGGPCHEN7T8LVBF4uzvA08+qf+OEMDJk/r3r12zj/BhtB0hxDcUfwt0lrdp7ouUWbCEjvV1O4ue0XaEkBBQ/C1oannrfPhAJui6804mjLYjhISF4m+BLuT62DHz9yYTdfp0mgm67rwXLtgvAU0IIU2g+FugW3n3ySfVyy8DmYifPq0W92IyWJMVfQkhxAcUf0vKa/wXFvl8nk28Ki+PDhyI+BNP1Iu76ryEjJY2ewoQJzjJixASB6oZk6ur7A63hJO8CCFxo4up3t7uJz8Dh+JPCIkD7gPcKRR/QkgccB/gTqH4E0LiQBf7zFmOQaD4E0LigLHPncKF3Qgh8VDEUJPg0PInhJARQvEnhJARQvEnhJARQvEnhJARQvEnhJARkszaPkKItwEYNkUMxh0ArvRw3bakmm+Aee+DVPMNpJv3rvI9k1LeWU1MRvz7Qgixq1oUKXZSzTfAvPdBqvkG0s173/mm24cQQkYIxZ8QQkYIxb8ewzbsUZNqvgHmvQ9SzTeQbt57zTd9/oQQMkJo+RNCyAih+BNCyAgZtfgLIX5ZCPFdIcRNIcRm5b0zQojXhBAXhRCfKqXfK4R4OX/vK0IIkaffKoT4rTz9D4UQGx3ex5eEEH8uhHghP042vY8+EUJ8Os/na0KIR/vOjwohxKX8ub0ghNjN044LIb4thHg1/3t76fPK599RXs8LId4SQrxSSnPOa9dlRZPv6Mu4EOJDQoj/LoT4Xq4rj+TpcT5zKeVoDwB/B8DfBvA/AGyW0u8B8CKAWwHcBeB1AJP8vecA3AdAAPg9APfn6Q8D+Lf5688B+K0O7+NLAL6oSHe+jx5/i0mev7sBrOT5vqfvMqLI5yUAd1TSvgzg0fz1owB+ve75d5TXTwD4KIBX2uS167KiyXf0ZRzACQAfzV/fBuBP8vxF+cxHbflLKb8npbyoeOszAL4hpfyJlPJPAbwG4GNCiBMA/rqU8n/L7Bf6DwB+sfSdC/nr/wTgkxFY003uoy8+BuA1KeX3pZTXAXwDWf5ToPzbX8DhMnHk+XeVKSnl/wRwrZLslNc+yoom3zpiyvebUsr/k79+B8D3AHwAkT7zUYu/gQ8A+LPS/2/kaR/IX1fTD31HSvlTAP8XwDR4Tg/4Z0KIl/Iuc9GtbHIffaHLa2xIAL8vhHheCHE6T3u/lPJNIBMAAO/L02O8J9e8xlRWkinjudv37wH4Q0T6zAcv/kKI7wghXlEcJqtSZbFLQ7rpO16ouY+vAvhbAP4ugDcB/KuaPAXNa0NizJOKfyCl/CiA+wH8UyHEJwyfTeWegPjLSjJlXAixBuA/A/jnUsr/Z/qoIq2zvA9+G0cp5T9q8LU3AHyo9P8HAfwgT/+gIr38nTeEELcA+Buw77rWYnsfQoh/B+BblTwV2NxHX+jyGhVSyh/kf98SQvxXZG6cHwohTkgp38y77G/lH4/xnlzzGkVZkVL+sHgdcxkXQiwjE/4dKeV/yZOjfOaDt/wb8k0AnxNZBM9dAD4M4Lm8y/aOEOLjuT//VwH8Tuk7p/LXnwXw33J/XXDyAlXwTwAUURJN7qMv/gjAh4UQdwkhVpANmn+z5zwdQghxTAhxW/EawD9G9qzLv/0pHC4TR55/t7k+glNeYykrKZTx/DpfA/A9KeW/Lr0V5zMPOfod+4GsEL0B4CcAfgjgmdJ728hG3y+iNNIOYBNZwXsdwOM4mCX91wD8R2SDNs8BuLvD+3gawMsAXsoL1Imm99Hz73ESWYTE6wC2+86PIn93I4vOeBHAd4s8IhvbeRbAq/nf43XPv6P8/iYyF8l7eTn/fJO8dl1WNPmOvowD+IfI3DMvAXghP07G+sy5vAMhhIwQun0IIWSEUPwJIWSEUPwJIWSEUPwJIWSEUPwJIWSEUPwJIWSEUPwJIWSE/H9eZaOb4ecC7gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "code", + "execution_count": 26, + "source": [ + "pca_analysis(2,5)" + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAACqCAYAAACTZZUqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAI90lEQVR4nO3dT4hVZR8H8HPfphJxLKzQippFZIOGTFEZFNEuxE3YIjAKFxYoKkqMItGqiCjEhSARWRkVgqCbyAKFslqJC/9ubKOgKP6JcuHoQPfdvBAvz+/Uvc2993e98/ksvzxzzsPM8cvB8zznNJrNZgVA7/0newIA05UCBkiigAGSKGCAJAoYIIkCBkgy1M7gRqNhzRpd1Ww2G70+p+uabqu7rt0BAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyQZyp5ANz3yyCNF9sQTT/Ts/CtXrgzz559/vsi2bt0ajj18+HCR7d+/Pxx74cKF1icHLRgdHS2ytWvXhmNvv/32Ips7d244dunSpS3P4dChQ0W2Z8+ecOy+ffuK7OjRoy2fq9fcAQMkUcAASRQwQBIFDJBEAQMkaTSbzdYHNxqtD+6huiet0WqBBQsWdHs6Xffdd9+FeTtPlvtVs9ls9Pqc/Xpdd8vw8HCYv/fee0X22muvFdmsWbNaPlejEf852+mddkxMTBTZ7t27w7ErVqzoyhwidde1O2CAJAoYIIkCBkiigAGSDMRW5E8++STMB+GBW2TRokXZU+AmMTIyUmQ//vhjOPaBBx5o6ZjffvttmE9OThZZrx/CPfbYY0X28ssvh2N///33IhsfHw/H3rhxY2oTq+EOGCCJAgZIooABkihggCQKGCDJQKyCgOkuehl6VVXV119/XWQPPvhgODZambBr164ie/XVV8Of//PPP/9uij0RbZNevnx5OHbZsmVFNnPmzHCsVRAAA0YBAyRRwABJFDBAkoF4H/Drr78e5h999FGPZ9Ib586dC/NWt5L2M+8D/nfqrvXo30bd9uAvv/yyyNavX19kV65caW9yeB8wQL9RwABJFDBAEgUMkEQBAyQZiFUQs2fPDvOjR48WWTsrBU6ePBnmn376aZH99ttvRbZjx46Wz9UOqyA6q1+v63ZcvHgxzO+6664i+/zzz8OxGzZsKLLopeW0zyoIgD6jgAGSKGCAJAoYIMlAvA/4jz/+CPMXX3yxyOq+oPzVV18V2c6dO8Ox0VbMuuNCpy1ZsqTI7rjjjnBs9JA9ethWVVN/4HbnnXcW2dBQXDHRvC5fvjyl89+M3AEDJFHAAEkUMEASBQyQRAEDJBmIrcjtqHtaPNUnwNH24Llz507pmHX2798f5i+88EJXztdLtiL/pe5Lxz/88EORPfXUUy0f95Zbbml57L333ltkq1atCsdGebQVuqqq6vr160X28ccfh2PHx8eLrFtfKe4WW5EB+owCBkiigAGSKGCAJAOxFbkd7TxsGxkZCfO33367yIaHh//1nP7OTz/9VGQrVqzoyrnoL3XXVDsP3L755psiW7lyZTh206ZNRXbPPfe0PK923HbbbUW2Zs2acOylS5eK7J133pnyHPqBO2CAJAoYIIkCBkiigAGSTLudcO04cuRImD/66KMdP9fExESYR+9YnZyc7Pj5+4WdcH+59dZbwzzaCfnss8+2fNxGI/4Vt9oFhw4dCvNjx461PIeXXnqpyOp2qZ4/f77IHn/88XDshQsXWp5DL9kJB9BnFDBAEgUMkEQBAyRRwABJpt1W5DrRE+dou2QnRCsetmzZEo4d5BUP/L26v/1bb71VZAcOHAjHRtf11atXw7HRl8Hff//9Ijtz5kz48+2IVm1EK36qKn4n8UMPPRSO7ddVEHXcAQMkUcAASRQwQBIFDJDEQ7j/id6FOn/+/K6ca86cOUUWfaQQIj///HORLVy4MBwbfYDz2rVr4dhOPFxrVbTtuW4r9OXLl4vs7NmzHZ9TBnfAAEkUMEASBQyQRAEDJFHAAEmm3SqI0dHRMO/Gl4brvsDczkvwoRW//vpr9hRCdf/e7r///paPcfjw4SI7ffr0v55TP3EHDJBEAQMkUcAASRQwQJKBfgg3NjZWZHv27AnHjoyMdPz8b775ZpjfuHGj4+eCfrRz584wnzVrVsvH2Lt3b6em03fcAQMkUcAASRQwQBIFDJBEAQMkGehVEM8880yRdWO1Q1VV1YkTJ4rs4MGDXTlX9KXbqqqqhx9+uCvnO3/+fJFduXKlK+fi5rVhw4Yie/LJJ8Ox0Xb8HTt2hGM/++yzqU2sj7kDBkiigAGSKGCAJAoYIMlAP4TrpeirtFu3bg3HHjlyZErnqtvGuW7duikdt873339fZMuWLQvHTkxMdGUO9I/nnnsuzLds2VJkjUYjHHv16tUie/fdd8Oxk5OTbczu5uIOGCCJAgZIooABkihggCQKGCDJQK+CiLbQRk9fq6qqhoeHO37+pUuXtpVna+d3MzQ00JfOtDRz5swiW7NmTZGNj4+HPx9tL65bwbBx48YiO3PmzD9NceC4AwZIooABkihggCQKGCBJI/qP89rBjUbrg/vUL7/8EuZPP/10j2fSG9GDyKqKt33Wje3lV2mbzWa8d7WLbrbrevHixUV23333hWOjv90bb7wRjl27dm2RLViwoM3Z/b8PPvggzDdv3jyl495s6q5rd8AASRQwQBIFDJBEAQMkUcAASabdftJXXnklzLdt21ZkdS+ernsheq9cv349zA8cOFBkdU+bjx8/3tE50Tvz5s0rsi+++CIce+3atSK7++67w7Gtrog6depUmEdfNf7www9bOuZ05Q4YIIkCBkiigAGSKGCAJNNuK3I76t7bO3v27CJbvXp1kW3fvr3jc6qq+i8P93LLcLfYivzPxsbGiqxui/2MGTOKrO5LxdHXuqNrKnrYVlVVdfbs2TDHVmSAvqOAAZIoYIAkChggiQIGSGIVBH3FKggGkVUQAH1GAQMkUcAASRQwQBIFDJBEAQMkUcAASRQwQBIFDJBEAQMkUcAASRQwQBIFDJBEAQMkUcAASYbaHH+pqqrT3ZgIVFU1knRe1zXdVHtdt/VCdgA6x39BACRRwABJFDBAEgUMkEQBAyRRwABJFDBAEgUMkEQBAyT5L95QHHzClR5NAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAD4CAYAAADCb7BPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAArd0lEQVR4nO2dX4wlx3Xev7PDGcZDOoE4pIQN5Z1dJkIQ6iUQN4aIIHrZIPzzojimANJjiYEFLDBy8pBAD7vYF73wQQqCwAodR4xFQdFdWEoCBGIcK4wkw7CBBKaHgSguE625FLkkLUIkLcegTMCydysPty/Z09P1v6qrqu/3Axoz03Nvd1V39fmqzjlVLUopEEIIIb4cK10AQgghbUIBIYQQEgQFhBBCSBAUEEIIIUFQQAghhARxQ+kCxHLrrbeqkydPli4GIYQ0xdNPP/2mUuq2mGM0LyAnT57EwcFB6WIQQkhTiMjV2GPQhUUIISQICgghhJAgKCCEEEKCoIAQQggJggJCCCEkCAoIITouXgROngSOHVv+vHixdIkIqQoKCGmLqYz6xYvA2bPA1auAUsufZ89SRAjpQQEh7TClUb9wAXj77cP73n57uZ8QAoACQlpiSqP+8st++wlZQyggpB2mNOonTvjtJ2QNoYCQdpjSqD/yCLC9fXjf9vZyPyEEAAWEtMSURn1vD3jsMWB3FxBZ/nzsseV+QgiAGSymSNaIlfG+cGHptjpxYikeuYz63h4FgxADFBDSFjTqhFQDXViEEEKCoIAQQqqECwHUD11YhJDqWM0ZXU37Wc0ZBejBrAmOQAgh1cGFANqAAkIIqQ4uBNAGFBBCSHVwIYA2oIC0CiOMZMZwIYA2oIC0CJcaJzOHCwG0AQWkRUpHGDn6IROwtwe89BJw/fryJ8WjPpjG2yIlI4zMrySEdHAE0iIlI4ylRz+EkGqggLRIyQgj8ysJIR0UkBYpGWFkfiUhpIMCUjOmYHWpCKPL6IdBdkLWAgbRa6XWYLXtnRy1lpsQkhxRSpUuQxSnT59WBwcHpYuRnpMnl8Z3yO7ucsRRK62Wm5A1Q0SeVkqdjjkGXVi10mqwutVyE0K8oYDUSqvB6lbLTYgDDO8dJomAiMjjIvK6iFzq7btFRL4pIs93P9/T+995EbkiIpdF5J7e/rtE5Nnuf58XEUlRvibxSdWtqVVzESMyU7iC0AhKqegNwEcAfAjApd6+zwE41/1+DsBnu9/vBPAMgBsBnALwAoCN7n9PAbgbgAD4BoD7bOe+66671GxZLJTa3VVKZPlzsRj/zPa2Uss2vdy2t8c/OxUu5SakMXZ3Dz9mq213t3TJwgBwoGJtf+wB3jkQcHIgIJcBHO9+Pw7gcvf7eQDne597shON4wC+19v/EIAv2M47awFxofVWXaHYlChShZeBDBAZf9RESpcsjBQCkjMG8j6l1GsA0P18b7f/dgCv9D73arfv9u734f4jiMhZETkQkYM33ngjecGbolTQOoXbrEKfQIkiVXgZyAgM7x2lRBB9LK6hDPuP7lTqMaXUaaXU6dtuuy1p4ZqjRKtOZfEqXFerRJEqvAyTUVP4zgbDe0fJKSA/FJHjAND9fL3b/yqAn+l97v0AftDtf//IfmKiRKtOZfEqTPktUaQKL8MktDby4jtKjpJTQJ4A8HD3+8MAvt7b/6CI3CgipwB8AMBTnZvrLRH5cJd99Yned4gOn1adyu00NlEQ8Ld4FfoEShSpwsswCS2OvPiOkgGxQZRlLAa/AeA1AH+B5UjikwB2AHwbwPPdz1t6n7+AZfbVZfQyrQCcBnCp+9+j6GbKm7a1D6K7kiJba+wYMYH7CjPIShSpwsswSupA/9yC0q2BmrKwSm0UEEdis7UWC6U2NvTiEWrxKkw/YhbWUXKIXOsJhK2TQkC4Fta6cOzY8vkcIrIcj5sYLpA4xmLB8fyMybHE2Viz2t5mXGEquBYWcSfG0T7mrO6zu8snfubkCPQzKN0+FJB1ISZby2QlSuQxtpT7ORNyBfoZlG4bCsi6ENPd01mJjY3pu4yt5X5moIR+cg4EGSU2iFJ6YxB9AqZKE3KJJK955LVkxlbtgX7iBxhEZxB9Mi5e1L+FMNXxXSKqMckAM4Dv6yKpYBCdTMfKWf2Vryz//vjH0/pPXGeVreusu451nbVug2GxMlBA1oFUT1fO+IPr7PY1dMb3b98xzRO7Jvo5Ss1hsdkLW6wPrPTGGIiGlcN6NbU3hdM8Zfyh71Df2Rk/ru7Ya+SMt03+r3XWuo2Ut1DXLDc2yr8Wp+YVBsCZ6GskID5PnIvVCTH6qdaecCnf6ri1PG2FMBnHVvUztWHVNcvSBrv2fI8UAsIgegv4TtnVRVr7hASdU0VwXcq3ovH2GcsccwZSJwLYmlOpBIPa7x2D6OuC77KlLhHVEKd5qviDa8R3d9fvuJHU6K+eImdg6nqnTgQYa5Zjx526nmuR7xE7hCm9rYULy9d1pBs7pxjXp3Be28pXwPdQq786d7lK1DuHa8e01ueqmXKV5cOAMZA1ERDfJ26x0ItO6cjiqnzDJ2traxlML+TYr9lfnTNnoES9cxlW03FNfZacza3mfA8KyLoISMgTZwpMu5wvtNW7freyJyv3uykqq+47lHonR67roTuuKdBe28hgKigg6yIgSvk/caFdy9Du4WIxno7byJOZsydesyuj5pFXSly8pnOrsw0KyDoJiC+hVivEoqR+U2H/uBN123Ma+ZqNdKoXVfan9KTwRKa+9S6Z4+v2JkQKCAXETMhTGOLTsHXvQp7MAt32XHpV+6tbfes9FIzNTf2tD7llOWMktljIOkEBoYCkJ6S7bHMwhzyZNXfbPZlRVZzngMbUM/f1qtmlOCUpBITzQFyocYJALkLmepgS20PXqapk1cAUt35Oy3fZXk45hu8ty33r+SbEhMQqUOkt+whkHbsrIT6NsW7pzk74dco1WcCjXilvfW1ZWKHlsQ02WxiBkCWgC2sCASnZmmuzOiamiHrGToD0PN5cDZnuUuzvh7/Pq1QMxKXZpWiaLT2KrlBAphCQkony6zbyGZLyqQ1Qg9qD36HoLoXLos22OaBTZmEtFvoA/uo7qbLM5vgoUkCmEJBc3VDbE6I7785O3HnXlQA1mOsIxMcNNVbXWnrjprcArIy87jM+93Cu7YACMoWA5Oh+uBzTJbOp9S7QlARYgbn2PH3cUDWPtlzrEFuvuY5EKSBTCIhS6btcLsbM5SkXWTquY6ilO5mbQDWY4+UZuxQ6I1nzCCRGQDgCoYBMJyCpcenSLBZuT0LMS5fm2sXWEWv5MlrOqY3y8Hz7+25NoaYmY3NhAcvPMAYyDgWkVQFx7dK4PCGmrlBonKX1rlUOMlqRWgyUi4jV1GQWi2UAX/dYrK4hs7DGoYC0KiCuFmOxcIt4jjljY+IsrTt3c5DRctZklG3kbDIhRnq4rErBNwI0RxMCAuAlAM8C+M6qwABuAfBNAM93P9/T+/x5AFcAXAZwj+34TQqIUu5Py/6+XUR2do4eKybO4mq5QrtlKbtzCY9lPFRGy9mSjrs2mRRzUefgJqqZlgTk1sG+zwE41/1+DsBnu9/vBPAMgBsBnALwAoAN0/GbFRAfTIKwtXU0Gd60WNEwzhL61IZ+N/X07kTHsh6KIxCllNslD7ktLV2DudCygFwGcLz7/TiAy93v5wGc733uSQB3m47fnICkDuTq4iSm93umKE/oE5/SUiQ8lvVQaxADcSVHaK2lUdhcaEVAXgTwvwE8DeBst+//DT7zJ93PRwH8Ym//FwE8MHLMswAOABycOHEi7VXNSQ5LYXJvuUwtTn1e2xOf0lIkPJbpUCuD+QtYqFc2dtV1tJ+F5UJomVK+EaD2EUiN982VVgTkr3c/39u5pz5iEJBfHRGQnzcdv6kRSI6nxHVdihRzRlzOO7MRSIo00KlIHVryrbfJ09q/LWPlbG0UplSbZe7ThIAcOhnwGQCfXlsXVo5xeuyssJTnnWEMJMVSGFOQ2pj5avPY+cfKYipna735VkdNK6oXEAA3Afjp3u//E8C9AP7lIIj+ue73Dw6C6N+fVRA9V4sbPnm6p9hFqHye4jXIwnLV/JhipKhC6qbl29exjTxWdarJ6MZe99bjNi0IyB2dIDwD4DkAF7r9OwC+3aXxfhvALb3vXOiyry4DuM92jqYEJMYv4NPKQ5/S1sfkGXC5lCWS2YakNma+Tcj1/LbPTTUKSXHdU4vh1COw6gVkiq0pAVHKrZX0nckhgfDQpyPkicjV6ivxZ7hcyhhDksoI5TBmPk3I9fymz03Zf/G5XrqmWKk31hkKSIsCYsPmTHa1CiEG2Lcbm6vVx3bpEwuP7ZAxvX9TEp1PmVzXsvKpn683M3YtrSndWz6uSVO9TNfI5/qVcO1RQOYoICZnso9lSnluXSvO1eobc8HlGIGY1sjUVdPljYKuxwrtRbucX/e5KWMKKUZMJnyva4l4CgVEzVBAXNa+ytUtqaXVhx63UIQ2xgjv7/vf5kozoaNJURYfEXPJBjM9hiF10b0PjiOQQtvsBMQ2Asndo65h3K057nVAvbKxq35vP17QUnu6Qo5n81bq9LLSuZjRjF2PVflcrqmvkI/dMxcPMrBc6MGESXx0YU/GQCgg8cQ+RVOS0pdiO25v+zG2x0XEUdBqSTaz9RXmPgLRGXCX/JGx78bUxfR93WbCdCxdeZiFVWCbnYAoVUf32PUYKaO5I+e7rnkKX9nYHf+Ow7lrcduYeqm2nnPL2T8u5w3J1tJdS9toynXU4dNWFovw8kwFBSRUQCpJEZ2kHCEWIkYUEltnnYBcgyEzzHJNndw2E9wb3aXa2LCfbn//3fUyNzbiVqkp8TiYYgSm3rtprqzr+qGuZQkR9z61r2JAAVEBAlKL/2KqcvgadJMLzeUYKZ3qi8VSKEaONzoCccR6SSa6N6GnyV28KQTFJVdEd49sIzff62Iry9ZW2IuqajE1OiggKkBAavFf+KZphJJyTQqXY0zgoL8G0QfSHbA+2D51iLS2IV/P2YRL92tMm8t8kZTXc9XEfUZ3Y4P3GpwdY1BAVICA1JJ24pumEYqvtfHpGo4c4/f2F+rPJJEF0pTlOuB/rAFGQ5NqllmmMqZuwv3z6NxAGxtpjaBv3GEYdkt52W1lcRXm2kccQyggaoYjkNRl8W3VunI5LKmyOtVDWKgXsauuQdRVMaTd2ih1r1zPm7F8ptuW8rQhAeRURnGx0AuWrV458kx053cV5lpMiysUENV4DCS2xfqcy3VcHZGam/wBqjVFaEXG0WxIFtKweC5GNsSVlMoomh6BEo9mbPutxbnhCgVENZ6FVSJNY3/fPpoIvD5ZHqBS9yrG+ia4f7Gr1sZqoG2LNYouIx9TtniIWIaUyUfAOAJpcGtiHoiudS8WSm1uHm5tm5v50mnGxCNhKy/1AK2qmfOVs9oTZxohhVxLl1jG8Pu6PozNtRR7T00jn7H+jG1BQ5dlSVyaRIwQ1eLccIUC0oKA2Fr31tbh/21tpXMwD4+ds0tpqWou+nGXH6OQuyvDCMn3WrrGMvq3eaz/0m+CJhFxHQH4JgKMHdsmpqaExhLtsQbnhgsUkBYExNT6c3bZdV3LXF3KjqkfoNUlfBG7WevlQo7AruvxXGMZ/cthMrxK2Rd6tI0ATPNPfZq+zZ3n64ar1aXkQ4q2RgFRDQiIqfXnjLr5PFEi4U7jwt2t1SXUTTicKoJZ2n3hYkSHt9ml+fVnvA83mxiY3Gg+1yt0BGK6Di2Tqq1RQFQDAlJqBOLzNIWsg1HaYnbUMgIpHUB1NaKhZTZ91ncEMJYIsLOjn+0dGgOpfSmRUFK1NQqIakBAbDGQXEZY9/TcdFOaUUNpi9lRPAbSUTqF0zWryfYdXUKeSQx0/z92LH05fLKwKunjJCdVW6OAtCAgSplbfy43UO4Mr9IWs0fOLCzX21ODnvaNvcO8z0Pfce35j9VPl6+xsXG0CY6VI+baTZHeWxscgdQoIHNqaX1LsnJEp65TDRYzMz492Np6uymas80tthKp3d3lwHbsMzffbC9H6Ko+tV3zqWAMpDYBmVNLzFGXdfIP9PDVyDn1QZQyG3afuIftOvjMCXH5Xsk+zFRtgFlYNQlIjS0xBFPyfmhdbDGcOVnMAbm8dK1cNt/sKt1ma3o2V5nu+773x/UdKqH3x3fEWroNUEBSCUhF/vxgbE9haF1MVqR2CxiJa7/CxxjUPHAb1uPMGT+hMG22a7NY+Dddn36fbk7LUERi7o9Pe6mhDVBAFEcg72BzWIfWxcVXkWr2fGW4ZgelWOzY5/a4CJZvcHls0p/u1vuOQMauzVj5QlyGsVlkGxvp7o9rP7QWc0MBUYyBvIPJ0MfUxXWCQeoXYVWCzRD7GoPYwW4KUXPJrrJtoYswrpqK78z1kPvjWs9U98fUFvrlNF3TKaGAqBlkYaU6r8nVFBtAd7U2qajBQeyIr8GJ7X26fN/2Gdc+gW5bjVhiRMTF0NomF9qaiEs9U45AdMJ95ozbteIIpMDWxDwQHSlHPjlHUS7dp1QCMuFoMHeqq851FFM9F8FKuXaUaT6JzT3kuw0Xeoydf+tSzzNn0t6fMdegSzkYAym0VSEgoZZI9/SFuoOm6LnrZrincmFN5CBOpVO2AdrYMWNuU8wIZJX34BrDWLmVQnv6OleVyxIjpnq6NhEXcRtrVikfI1sZmIWVeANwL4DLAK4AOGf7fHEBibFELjOoanPnLBazmOGeUqdsPfGU2ucaA/FZyX9MZFKlr441X5c6pFiD1MX7mjPuYMos820XOczA7AQEwAaAFwDcAWALwDMA7jR9p7iAxFgiF/+HLtpYUlRyiprn9QwtSg6dmiob3CXDauw9H7YtJplusTg8stjZCQ9+K5VmBNI/zxTiPjyvLas+hUDHMEcBuRvAk72/zwM4b/pOcQGJsRq25Hddy3dd6KhFPJ6WmAcrh6eslvTM0LiEbSDpm/EU0yxTxEBcjxeD7pqY7oHvAti52tUcBeQBAL/e+/vjAB4d+dxZAAcADk6cOBF3FWOJvbsmh7BvmksNLi4fdE+f47AiR8aMy+UzFbuGbPCY7CjdtbPVLYeRM41qQkaeqQfOpmsSur7XGLlGtnMUkI+NCMi/MX2n+AgkRdqG7vshXcmYLK4xZ3UuV1UCa5tiTkWIEbLNsXA5ZgkvoMvmm368Egnf4/XxEeT+Ao4x1yzF9U/lZos5TwxzFJD2XFhKxbfGvlj0V8/1mR4c07LGntTNzaOR2JTd6QRPRQmXUYpz5h6pjB1/a+vwnArfly3Zmp3u/7brEtN/Kp3xburApLzHjIG4Fga4AcD3AZzqBdE/aPpOFQKSAteA+ZiohHT7Vuf07a6mss4JxuUlXEYp3Ak5MsB8vYC+186lmYSE5mI9uCV79LbjpBxlMgvLtUDA/QD+sMvGumD7/GwEZOrUEpccx1hLmaq+lmpMmZBmLbZDgVL5tFN4T12vnSnfY2yzZWHZjmnKIYltjrVc/9LMUkB8tyYFZOzJDWnVY9NcY9OQcnT5dNegwadvrNj/ZHOh3trZffd+Weo0VQ84NbrRwtjmcittfSCXPk7pmMLUHZiUUEBUgwKiM5w+TulhekpfbFzzA0NSdVIb+Eafvv4AcE9G3sVuuYdT+OBD6mNzg/muf2UzyD7zaMc+EzpvpdG+S3IoIKpBAdE9Dbp1H1yc1yHdKNcRSKq0l8rx1bLVbXgRu27XMMH5hp/PGdDXrYzrujCgi5CZHoWxcqZcAMHl+udOQizdd6KAtCggttQNW6uyGX6fALprhlfOtKYKCOmRrm7DNWSK9DqUz2f5c13T0jUn04sth8fyzeiy1cun2ZvOEWOkc45SahkBUUBUgwIS2220df98u58uju2cCwZVQMgtWd0G6wgkgWUwlc+1Jx0y4c21KegGxa6BdBcj7+uuCzXSKfJTbJRIPx+DAqIaFJDY7oepdfsaK9dMrJm+LGpFSCxhdRsewkgMJLHbLzbWYTJYISOQMXR9kVQ9a1+jGzpimWLxxanWTLNBAVENCohS6cfWrt29Ia5xkJSr7VZIrLF5CAv1InbVNcgyGyvxtXItn65ZhUx4C3k7YM6etW+/K6ZTYOtLxcYuTLGfKeMiFJBWBSSWsfSYkJY397W2HIl1d+gWFwy5JWPfcymf6TM2w+6ahWWrQ+6etU95YtySpn5UisUZxu7V1tbRJIHccREKyLoKSJ8Yl1jIXJCZ5jumzIqJESTTCrSm8pkM5v7++P98VoR1wSerKjcxiRG66xiaLKArX4pEhBgoIBSQOL/BWGJ/rvUj1ojQWxJzK029/9xBW1vgWecBzZ3KGpqarROdnCOsEnERCggFJLzl6dJ4z5xxiyTWkMheKaG3JMaImEQii3Hq7v91iLoqu+ohLLz6HLWksg4xNeucQlwiM4sCQgHJ091dLMwCMrbV8PRXQokRSEwMxHbcIwZ15GQ/xrZRRIZiVUsqqw9zmxtCAaGAhLc8W7c0JD4y1dNf+egnRwzE9by6YHjK8ryz7tdgexG7zk2jllRWX3I2vambNQWEArIkpOW5pOb4rtY7xdNfq+9jQMosrBTfS9lEdLPvr0GcB6em+SeV3crZQgGhgITjmhvatzqpRyA5hG8NCdFUl9Rd3W3Wzb5fjUCGL6/SBdB1/RPXzLPaaK28FBAKiB+x80dMIrIKrPuUJYfrbQ3x1VSfyYNj2z/bOXqAP5Nt9QtYeI+adDPeXdcWrYVGBsaHoIBQQNxJ0cIXi6MzqVab76SCEpHmGTDWyzWNFsb6Byb3kU083mkyibrbIXNZa6TFZkkBoYC4k6KFLxZK3Xzz4e/fdFOY8UiZflxhVy+HO0NXdd0kNN27rXwMdv9YOdwyvrkaoQPN3O6lFgfGFBAKiDuxLVw3+ghdJyt5bmk95NI400xvn5X5XUYaU/WifUUxdNZ37j4HRyCNbhSQEcYMrE8L739/Z8e+5HutT3Uhci1L4fMqGVMP3nQrc85z0AXt+9drtSZoyuYxhXFvsTlTQCggR/GJkIa88TBmFDNW1opHEiGYXESx7gwfQ2j6rG0OaewtGcvVCGmSqZrHVO6l1pozBaQ1AZmihcVah5onEDaA6fLFXiafXq7pszl7y2PH1hlw33eOuJzbJ2lg3ZstBaQlARl7sjY37QnzvoR0t1yS/3WbaeZXbsGssMtnuoQ5XUG+n8116UL6HylGBqUEs2UoIC0JiMuTFdOqTXEOU3crxGXV33Rrded+aqc4foCFNQW61wGfPkjKEYhtlBErmBX2VaKhgLQkIK5PVqqAtKthje0y6rqLuf0GOY8fIU62nnDLRijGAzqWUhzy1kMdOeMccx3BUEBUQwLiaqhDWrxt5KFzkqfwN+gMdm5fTk6LESlOY4Z2MiPkYOVDhMy1/KYcDl0WVgpRzdmfmGsMhQKiGhIQV1dRSKv0NaYuZem/oHlnR6kbbjj6GdMcEJM45ZwUkeKpziBOkxghBysfKmShWeC5R1r9fpBu4mQsLU4SdIECohoSEKWOzq9I8YJlpfytk23kMSxHiGPfJlJTpiT5HDNTGlVsboOTIXZoB6GxghqNqCnja4rEAI5AKCBlSdVV8zWmtsWTht+LWXZEd55UzulUXd2YOJIDvkYoSB8d7pNtMmKOl1LlYqoyMQZCAZk/PsbUtxsaM7U69imfyh8SEkfywNcIBV22yBGI6X81GtEpR0WtJ0CMUbWAAPgMgD8C8J1uu7/3v/MArgC4DOCe3v67ADzb/e/zAMR2HgqI8m/dvknzm5vh7rYYyzOl1dJYo2uQZAbD5zYFGcfIGIjtnLUZ0RpHRS3RgoB8emT/nQCeAXAjgFMAXgCw0f3vKQB3AxAA3wBwn+08ay8goUZWZw1M8Y5Q6xEicJliEVo051u9JGnq3nawcYzIwmrNINc4KmqJVgXkPIDzvb+f7ETjOIDv9fY/BOALtvOsvYCkfupLR0tdMsRiy+KYZ/tjbKuHsEhmSH10tIRxnPqcKUY0tY2KWqIFAXkJwHcBPA7gPd3+RwH8Yu9zXwTwAIDTAL7V2//3Afym7TxrLyCpDb6p579ambfEWl7DcoTiMNPvGkS9iN1D4jG8pCm8hrZXv5YwjlOdk6OH8hQXEADfAnBpZPsogPcB2ABwDMAjAB7vvvOrIwLy8wD+7oiA/FfNec8COABwcOLEiWwXuAlSj0Bc56vonvhYC+QyY39rK+uMM5ccA1/j56KLIS+IbLX33Zq7bI4UFxDnkwAnAVzqfqcLKyUl5kPonvgUZQk5rw8OIzZbNUKMX+qVbFrvwZf2lJLKBQTA8d7v/xzAV7vfPzgIon+/F0T/AwAf7gXR77edZ+0FRKl8XVEXq9d/4lN0K11HQKlddIMymi6p6bL4nja0Wq334Fsv/xyoXUC+0qXkfhfAEwNBudBlX13uZ1p1cZBL3f8eRYtpvC37FYa4WL3+E5+qW9m/hjleGhHZddddltVkPNfTxlSr9R586yOoOVC1gEy1VSUgc3sqfGdn5+hW5nTRBYq8ac6Eqar90+7sLKfXhFZrDj34OfW1WoQCUpuAzOGpHrK/P24tVy+v7pNLQCu0NLZRhOs8yZipNXPqq5DpoYDUJiCt+xXG8BXFCo19DmzevSPGPMN1qfFS11gmMg4FpDYBmeMIJGYhxZYtiaX8LjGNd277mgwX1qSas4ECUpuAzPEJChFF23WoXVzGyj+yTrgt2/kdjZ1jx2KENanmbKCA1CYgSrVhHGOnUIfOmludz/XVdqWuo6d/ymo45+jaHGFNqjkbKCA1CkjN6Iy37n2j/e/5GHNTl9ylm1p6JOcy/6VXXmtx16RrvibVnA0UEAqIH6YJDKmMtS3H1aWbWtoSucx/GXSrjRobIYg27a5pwFta94kfFBAKiB+u62nEGGvbLDvdy6n6CySW9oV4Rcg9julp6V1CSbUZ7JoEjZihgFBA/HBdT6Nv8H27v6ZjKuUmIKVHIP266UZoNrdfAmyXYarLRFGYJxQQCsg4uifedT2NlUH37f4Op1aPWTWX0UXurnVIIkH/8/v7k3T9bZdqioFajaMckgYKCAXkKD4ptLp1plZrbYR0f02jGdP3ht3m/f13y7exsfx7iuvjwkRd/xpGIDUMBkkeKCAUkKP4PPGmmEho91e3rRgz4MM3K+Xq4S8WaRZnTN3114yIaoiBlA5HkXxQQCggR/F54k1iE9r9dTHOtlUFQ1YqtGFz3/lYxJhuuac7rHQWFkcg84UCQgE5is8Tb+rChnR/NzeXowmfLrGvKyz1dQmxiKFdf9MM90otNGMg84UCQgE5iu8Tb+rChnR/fbvEU6QW287jahGHIyff98NPJZaJYRbWPKGAUEDGmfKJjz3XFJMbTedZBehtdUjRFc8oljTyxBcKCAVkWnKks4YurxJSdt15XOqQIhiQSSzpZiIhUEAoINOR03/fn7S3ypIazl9JISZjx3EVhhTpSJnEkoFuEgIFhAJymNR+DJc5I6n892NvPvQZIYTiKgyprHQGXxNTbUkIFBAKyLuk9mP4zFpPYVR1VjDFvA0TrsJQsZ+IIxASQgoBOQYyDy5cAN5++/C+t99e7k91vDFEDv+9vQ088oj/uZQa/9+1a+P7X37Z7xw6HnlkWeY+Y3XY2wMeewzY3V3WeXd3+ffeXppyROBaBUKSE6tApbe1GIG4uD1S+zFcMoZSBbtN58o9AlFqFilM1irMoI4kLaALaw0ExNV1ktqPYZuzIJJufSpTdtJECxe+wxwNbcXuN1IOCsg6CEgpH71LDGRjI40R0mV4rQQqxqj7fHeuhpZBEjICBWQdBMTHNZUrC8vmxkolIql7/r6CEGJoY2enTwHTtMgIFJB1EJCpe49jhly3tHsNPVmT8PheO1MsxnV2eo2jF45AyAgUkHUQkCndKrpz3XSTWUBK9WRt18a3520abW1tHU0YsAlrLUa6YtfcHENOrUABWQcBUWq6p8xnsb/SRnKx0GdorWIzvj3vmLkvtYnrkAotdcW6thZQQNZFQKbC9yVRqZ54X+PmYuh1dbGVd7FIKyA1jEAqhZ61sqQQkKiJhCLyMRF5TkSui8jpwf/Oi8gVEbksIvf09t8lIs92//u8yHImmojcKCJf6/b/voicjCkbCeDEifH9OztHZ6qt9sdOprt4ETh7Frh6dWk/rl5d/n3xov47LpMclTq6TwR4+GFzeVNODORsPiO6uaCp5oiS/MTORL8E4B8D+N3+ThG5E8CDAD4I4F4A/1ZENrp//xqAswA+0G33dvs/CeBPlFJ/E8C/BvDZyLIRX3RTmn/lV47Owl4sgDffjDe4ITPoQy2MUsBv/Zb9czs77sfc2Xn3uuzsLLfKZqrXiq6/ottPKiR2CLMcCeF3AJzu/X0ewPne308CuBvAcQDf6+1/CMAX+p/pfr8BwJsAxHZuurASM7WvPCTFNDRW4xqTWCzcX7PrOpmywhhEaRgDKQtKu7AM3A7gld7fr3b7bu9+H+4/9B2l1F8C+FMAHl1BkoS9PeCll4Dr15c/c/egQ7qhYyOl2PP12dsDvvSlwyOuW24Z/6zLiCbETbcGVLy8GHHEKiAi8i0RuTSyfdT0tZF9yrDf9J2xMp0VkQMROXjjjTfMFSB1c//9fvuBccuzv7/8CRxd4HHF5ubyuCdPAseOLX/qjPhQSH/0o/HPDd1pFy8ePv6nPrWMu6Rc6HJGTN1fIYmJHcIourBIDDlScXSTHzc2lvM5QnwmLuX0TQOuJcWXrCWo2IX1BIAHu8yqU1gGy59SSr0G4C0R+XCXffUJAF/vfefh7vcHAPx2V0nSAsOet6t7Jkcqzt4ecPPNR/dfuwb85CeH97mOBFzWTHddAn8Fo8WkcWLTeH9ORF7FcnTx30TkSQBQSj0H4D8C+D8A/juAX1ZKrV7ssA/g1wFcAfACgG90+78IYEdErgD4FwDOxZSNTEiMjz9XKo6PAF296iZ8P/VT7/4+lsLsc06m+JI5EDuEKb3RhVUBMW6oXKk4oZlaY+eOXVJ/zJXGVCNSGFTswiLrRIwbKlcqzpjLaXMT2Noyf2/MpeU6V8UlO2x7G/jylxktJrOAAkLiiXVD5UjFGROmL30JePzxd/fpGAqfq0CassOG4hgaMyKkIkQ1Hqc+ffq0Ojg4KF2M9WYVA+n30re360/qP3lyGf8Ysru7FDLfz5m4eHE5Ynn55eWckrfeOhzQb+F6kVkhIk8rpU7bP6mHIxAST6szwlwyq3w+p2OYZPDHfxyeDUZIRVBASBpKzAiLdQO5Cl+sQLqm93IVQdIYdGGRNmnJbXbs2PjqwENsLrG+G+zEieUIqLa6kmagC4usLyGr+JbCJZlgzCXWH2HdeivwS7/E9bRIVVBASJu09DIJXUqxael3xk1IA1BASJu09DIJXUrxm2/qY0aMm5AGoICQNonNjJoa3yQDV2GoUTDJ2kABIW3SaupwH1MWWWjchJAJoYCQdmn5ZRK2BShD4iaETAwFhJAS2LLIQuImhEwM54EQUgLd3BCRpUAQkhnOAyGkVVrKIiNEAwWEkBK0lkVGyAgUEEJKMIcsMrL23FC6AISsLXt7FAzSNByBEEIICYICQgghJAgKCCGEkCAoIIQQQoKggBBCCAmi+ZnoIvIGgKuly5GBWwG8WboQGZlz/eZcN4D1a5l+3XaVUrfFHKx5AZkrInIQu8xAzcy5fnOuG8D6tUzqutGFRQghJAgKCCGEkCAoIPXyWOkCZGbO9Ztz3QDWr2WS1o0xEEIIIUFwBEIIISQICgghhJAgKCAFEJGPichzInJdRE4P/ndeRK6IyGURuae3/y4Rebb73+dFRLr9N4rI17r9vy8iJyeujhER+YyI/JGIfKfb7u/9z6uuLSAi93b1uSIi50qXJwQReam7/t8RkYNu3y0i8k0Reb77+Z7e50fvYy2IyOMi8rqIXOrt865Pre1SU79pnjulFLeJNwB/G8DfAvA7AE739t8J4BkANwI4BeAFABvd/54CcDcAAfANAPd1+z8F4N91vz8I4Gul6zeo62cAfHpkv3dda98AbHT1uAPAVle/O0uXK6AeLwG4dbDvcwDOdb+fA/BZ232sZQPwEQAfAnAppj61tktN/SZ57jgCKYBS6v8qpS6P/OujAL6qlPpzpdSLAK4A+FkROQ7gryql/pda3un/AOAf9b7z5e73/wzgTC09Iwshda2dnwVwRSn1faXUTwB8Fct6zoF+O/syDre/I/dx+uLpUUr9LoAfDXZ71afmdqmpn46k9aOA1MXtAF7p/f1qt+/27vfh/kPfUUr9JYA/BbCTvaR+/FMR+W431F65CkLqWju6OrWGAvA/RORpETnb7XufUuo1AOh+vrfb32qdfevTYrvM/txRQDIhIt8SkUsjm6lHOjZyUIb9pu9MhqWuvwbgbwD4OwBeA/CvVl8bOZStrrXTctn7/D2l1IcA3Afgl0XkI4bPzqXOK+bSLid57vhK20wopf5BwNdeBfAzvb/fD+AH3f73j+zvf+dVEbkBwF+D+3A2Ca51FZF/D+A3uz9D6lo7ujo1hVLqB93P10Xkv2DpkvqhiBxXSr3WuTte7z7eap1969NUu1RK/XD1e87njiOQungCwINdZtUpAB8A8FQ3xH5LRD7cxTc+AeDrve883P3+AIDf7nyYVdA9nCt+DsAqUySkrrXzBwA+ICKnRGQLy6SGJwqXyQsRuUlEfnr1O4B/iOU967ezh3G4/R25j9OWOgiv+rTWLid77kpnEKzj1t3QVwH8OYAfAniy978LWGZGXEYvCwLA6a4RvADgUby7isBfAfCfsAyGPQXgjtL1G9T1KwCeBfDdrvEeD61rCxuA+wH8YVf2C6XLE1D+O7DM0nkGwHOrOmAZV/s2gOe7n7fY7mMtG4DfwNKN8xfdc/fJkPrU2i419ZvkueNSJoQQQoKgC4sQQkgQFBBCCCFBUEAIIYQEQQEhhBASBAWEEEJIEBQQQgghQVBACCGEBPH/Ac/WVNwz6pe6AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "scrolled": false, + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "As you can see, 0 and 1 can be clearly separated by a straight line. This indicates that in the original 784-dimensional space dots corresponding to digits are also linearly separable. In the case of 2 and 5, we cannot find the good projection that will separate the digits clearly, and thus there are some cases of wrong classification.\r\n", + "\r\n", + "> Later on this course we will learn how to create non-linear classifiers using Neural Networks, and how to deal with a problem of digits not being properly aligned. Very soon we will reach above 99% accuracy in MNIST digit classification, while classifying them into 10 different classes.\r\n", + "\r\n", + "## Takeaway\r\n", + "\r\n", + " * We have leart about the simplest neural network architecture - one-layer perceptron.\r\n", + " * We have implemented the perceptron \"by hand\", using simple training procedure based on gradient descent\r\n", + " * Despite simplicity, one-layered perceptron can solve rather complex problems of handwritten digit recognition\r\n", + " * One-layered perceptron is a liner classifier, and thus it provides the same classification power as logistic regression.\r\n", + " * In the sample space, perceptron can separate two classes of input data using hyperplane." + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Credits\r\n", + "\r\n", + "This notebook is a part of [AI for Beginners Curricula](http://github.com/microsoft/ai-for-beginners), and has been prepared by [Dmitry Soshnikov](http://soshnikov.com). It is inspired by Neural Network Workshop at Microsoft Research Cambridge. Some code and illustrative materials are taken from presentations by [Katja Hoffmann](https://www.microsoft.com/en-us/research/people/kahofman/), [Matthew Johnson](https://www.microsoft.com/en-us/research/people/matjoh/) and [Ryoto Tomioka](https://www.microsoft.com/en-us/research/people/ryoto/), and from [NeuroWorkshop](http://github.com/shwars/NeuroWorkshop) repository." + ], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [], + "metadata": {} + } + ], + "metadata": { + "celltoolbar": "Slideshow", + "kernelspec": { + "name": "python3", + "display_name": "Python 3.8.0 64-bit (conda)" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + }, + "livereveal": { + "start_slideshow_at": "selected" + }, + "interpreter": { + "hash": "16aeaa504b544176258e5caf576fc030dfd6fff62d0c15825e7863ff13e121ff" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/3-NeuralNetworks/03-Perceptron/README.md b/3-NeuralNetworks/03-Perceptron/README.md new file mode 100644 index 00000000..6ba59682 --- /dev/null +++ b/3-NeuralNetworks/03-Perceptron/README.md @@ -0,0 +1,65 @@ +# Introduction to Neural Networks. Perceptron. + +One of the first attempts to implement something similar to a modern neural network was done by Frank Rosenblatt from Cornell Aeronautical Laboratory in 1957. It was hardware implementation called "Mark-1", designed to recognize primitive geometric figures, such as triangles, squares and circles. + +Frank Rosenblatt | The Mark 1 Perceptron +-----|----- + +An input image was represented by 20x20 photocell array, so the neural network had 400 inputs and one binary output. Simple network contained one neuron, also called **threshold logic unit**. Neural network weights were potentiometers that required manual adjustment during the training phase. + +> New York Times wrote about perceptron at that time: +> *the embryo of an electronic computer that [the Navy] expects will be able to walk, talk, see, write, reproduce itself and be conscious of its existence.* + +## Perceptron Model + +Suppose we have N features in our model, in which case the input vector would be a vector of size N. Perceptron is a **binary classification** model, i.e. it can distinguish between two classes of input data. We will assume that for each input vector x the output of our perceptron would be either +1 or -1, depending on the class. The output will be computed using the formula + +y(x) = f(wTx) + +where f is a step activation function + +f(x) = \begin{cases} +1 & x \geq 0 \\ -1 & x < 0 \end{cases} \\ + +## Training the Perceptron + +To train a perceptron we need to find weights vector w that classifies most of the values correctly, i.e. results in the smallest **error**. This error is defined by **perceptron criterion** on the following manner: + +E(w) = -∑wTxiti + +where +* the sum is taken on those training data points i that result in the wrong classification +* xi is the input data, and ti is either -1 or +1 for negative and positive examples accordingly. + +This criteria is considered as a function of weights w, and we need to minimize it. Often, a method called **gradient descent** is used, in which we start with some initial weights w(0), and then at each step update the weights according to the formula + +w(t+1) = w(t) - η∇E(w) + +Here η is so-called **learning rate**, and ∇E(w) denotes the **gradient** of E. After we calculate the gradient, we end up with + +w(t+1) = w(t) + ∑ηxiti + +The algorithm in Python looks like this: + +```python +def train(positive_examples, negative_examples, num_iterations = 100, eta = 1): + + weights = [0,0,0] # Initialize weights (almost randomly :) + + for i in range(num_iterations): + pos = random.choice(positive_examples) + neg = random.choice(negative_examples) + + z = np.dot(pos, weights) # compute perceptron output + if z < 0: # positive example classified as negative + weights = weights + eta*weights.shape + + z = np.dot(neg, weights) + if z >= 0: # negative example classified as positive + weights = weights - eta*weights.shape + + return weights +``` + +## Proceed in Notebook + +To see how we can use perceptron to solve some toy as well as real-life problems, and to continue learning - go to [Perceptron](Perceptron.ipynb) notebook. diff --git a/3-NeuralNetworks/03-Perceptron/images/Mark_I_perceptron_wikipedia.jpg b/3-NeuralNetworks/03-Perceptron/images/Mark_I_perceptron_wikipedia.jpg new file mode 100644 index 00000000..48600994 Binary files /dev/null and b/3-NeuralNetworks/03-Perceptron/images/Mark_I_perceptron_wikipedia.jpg differ diff --git a/3-NeuralNetworks/03-Perceptron/images/Rosenblatt-wikipedia.jpg b/3-NeuralNetworks/03-Perceptron/images/Rosenblatt-wikipedia.jpg new file mode 100644 index 00000000..30dbec1f Binary files /dev/null and b/3-NeuralNetworks/03-Perceptron/images/Rosenblatt-wikipedia.jpg differ diff --git a/3-NeuralNetworks/04-OwnFramework/OwnFramework.ipynb b/3-NeuralNetworks/04-OwnFramework/OwnFramework.ipynb new file mode 100644 index 00000000..48a8ac16 --- /dev/null +++ b/3-NeuralNetworks/04-OwnFramework/OwnFramework.ipynb @@ -0,0 +1,2809 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# Введение в нейронные сети\n", + "\n", + "## Эпизод 2: Многослойный персептрон\n", + "\n", + "Дмитрий Сошников | dmitri@soshnikov.com" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "notes" + } + }, + "source": [ + "Данная презентация представляет собой введение в современные нейронные сети на основе Microsoft Cognitive Toolkit (CNTK). Идея однодневного мастер-класса основана на Neural Network Workshop в Microsoft Research Cambridge. Материал и фрагменты кода частично взяты из презентаций [Katja Hoffmann](https://www.microsoft.com/en-us/research/people/kahofman/), [Matthew Johnson](https://www.microsoft.com/en-us/research/people/matjoh/) и [Ryoto Tomioka](https://www.microsoft.com/en-us/research/people/ryoto/) из Microsoft Research Cambridge. [NeuroWorkshop](http://github.com/shwars/NeuroWorkshop) подготовлен [Дмитрием Сошниковым](http://blog.soshnikov.com), Microsoft Russia." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "notes" + } + }, + "source": [ + "## Обучение с учителем\n", + "\n", + "**Дано:**\n", + " * Обучающая выборка $\\mathbf{X} \\in \\mathbb{R}^{n \\times k}$\n", + " * $n$ - размер выборки\n", + " * $x_i$ представлено вектором свойств размерности $k$\n", + " * Известные значения целевой функции $\\mathbf{Y}$ ($y_i$ соответствует вектору свойств $x_i$)\n", + " * $\\mathbf{Y} \\in \\mathbb{R}^{n \\times 1}$ (задачи регрессии)\n", + " * $\\mathbf{Y} \\in C^{n \\times 1}$, где $y_i \\in C$ (задачи классификации на $|C|$ классов)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "notes" + } + }, + "source": [ + "## Задача\n", + "\n", + "**Дано:**\n", + " * Обучающая выборка $\\mathbf{X} \\in \\mathbb{R}^{n \\times k}$\n", + " * Входные значение целевой функции $\\mathbf{Y}$\n", + "\n", + "**Необходимо построить:**\n", + " * Функцию $f : \\mathbf{X} \\rightarrow \\mathbf{Y}$ который _точно предсказывает_ значение целевой функции на новом наборе входных данных $\\mathbf{X}_{new}$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt \n", + "from matplotlib import gridspec\n", + "from sklearn.datasets import make_classification\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "# pick the seed for reproducability - change it to explore the effects of random variations\n", + "np.random.seed(0)\n", + "import random" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Пример\n", + "Рассмотрим пример двухмерной задачи классификации на 2 класса. Примером такой задачи может быть классификация опухоли на 2 типа - доброкачественная и злокачественная, в зависимости от её размера и возраста.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "scrolled": false, + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "n = 100\n", + "X, Y = make_classification(n_samples = n, n_features=2,\n", + " n_redundant=0, n_informative=2, flip_y=0.2)\n", + "X = X.astype(np.float32)\n", + "Y = Y.astype(np.int32)\n", + "\n", + "# Разбиваем на обучающую и тестовые выборки\n", + "train_x, test_x = np.split(X, [n*8//10])\n", + "train_labels, test_labels = np.split(Y, [n*8//10])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": false, + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "def plot_dataset(suptitle, features, labels):\n", + " # prepare the plot\n", + " fig, ax = plt.subplots(1, 1)\n", + " #pylab.subplots_adjust(bottom=0.2, wspace=0.4)\n", + " fig.suptitle(suptitle, fontsize = 16)\n", + " ax.set_xlabel('$x_i[0]$ -- (feature 1)')\n", + " ax.set_ylabel('$x_i[1]$ -- (feature 2)')\n", + "\n", + " colors = ['r' if l else 'b' for l in labels]\n", + " ax.scatter(features[:, 0], features[:, 1], marker='o', c=colors, s=100, alpha = 0.5)\n", + " fig.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "scrolled": false, + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\winapp\\Miniconda3\\lib\\site-packages\\ipykernel_launcher.py:11: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n", + " # This is added back by InteractiveShellApp.init_path()\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEnCAYAAACpNTSTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOydd3hb1fnHv6/kve14JHbiOAtIAkmALAghgx2gYRYoFChQoKUtlA7oD2gptJRO2jJaoIPSQVkFQoAQCBlAAsQhg+yB7cTxTLynLOn8/vjqIlnWtLVsn8/z6EksXd177tW973vOO0UpBY1Go9FoTNEegEaj0WhiA60QNBqNRgNAKwSNRqPRONAKQaPRaDQAtELQaDQajQOtEDQajUYDQCuEsCEiF4nIOhGpE5FOEakQkVdF5NwwHe8OEbnEyzjuDMcxA0VErhcRJSIlQX6vRETuF5HxIR7PSBFZJiINjnHd4WW7GY7j53j4TInIz0I5Lg/H8Pibhmjf5SLyTD++V+I49+tDP6rgEZGFjvEs7Md37xeRxWEY1qBFK4QwICLfAfAKgH0AbgRwPgBDeITrBrwDgCfhcRGAqCqEAVAC4CcAQqoQAPwYwALwtzkFwH+9bDfDcfw+CiFCePtNQ8HFAB7sx/eqwWv2RmiHExV+gvA9j4OSuGgPYIjyfQCvKqVudHnvPQBPi8igV8IikqiU6o72OAbAZABblVKvRHsgoSLY30Qptbk/x3Ec46P+fFczCFBK6VeIXwDaAPwpwG3HAfgngBoA3QA+B/AHl89nAXgJQCWATgB7ADwEINllm3IAyu31jOPl/n65y/dyAfwJwGHHsXcDuNltfNc7vnc6gBcBNAHY4vjsGce4TgWwEUCXYyzf9rKPEpf34sFVUzkAi+PfnwGId3y+0MPYFYCFPq6lAPiu4xpZwNnsYwAyHJ+XeNlniYd9Xe9rW8f/fwbgOwDKALQCWAtgqod9XQIK0Q7H9XsRQLGf+8Ljb+r47H7H38cDeBu8315zfHY2gDcd594BYDuA7wEwe9j/Mx7Ody6AfwNoAVAF4I8Akly2M67h9S7vGffBiQDedxx3H4BbPZzXmQA2O+6V/QBucny/3Nf1cHw3D8B/HGNrAvAsuALudV8Ecg28/Lb3B/rMDdWXXiGEh08AXCcin4MP6l5PG4nIOMe2HeDydR+AMeANbVAMYAv40LQCmAqaPMYDuNKxzcXgA7AVFBYAUO/4Nw+8wb/k+LvbcewMAB8CSHZ8pwzAOQD+5JhtPuo23H8DeA7AZei9sswA8DyAX4IP+JUA/igirUqpZzxeHfIPAF8GH7QPQDPEvY7z+gqATwHcBuBxUOhudHxvp499/hzAjxzfeR3AFNAsMl1EFsBp7ngSgA3ANx3fq/awrzdAgX8vgMtB4eC+7TWgsLgdQAKAXwN4TUSOU0pZAUBEbgWV7t8BPAAgHbzea0VkmlKq1cu5+PpNDV4D8Ffw2tsd740HsArAo6DQnen4fh6Au70cy5V/gr/zJeC1uh9AI3h/+iIDFNa/B8/za+C9tEcptRoARGQKeF0/Ae+TBAD3Ach0Gb8v/gdgOoD/A5+VKxzn6U4g1+AUABvA5+pJx3vGbxzIMzc0ibZGGoovAMcA2AbnzOMI+JCd7bbds+DsrjDA/QoojK8BH6ARLp+VA/iXh+88A6DSw/v3gQ/LJLf3n3aMN87x9/WOc3jEy74VgCvd3n8HQAUAcdtHiePv4+EyI3P53r2O96c5/l7o+PvMAK5NjuN8nnF7/xrHPr7k8t4HANYEsE9j3BM9fKZAoRTv8t5ljvdPdfydBqAZwN/cvlsCrmDu8HN8b7/p/Y7j3B7g/XIPKNRNbvt+xsO5/tRtH8sB7HUbu6cVggKwyOW9RMd99JTLe/8BlVqKy3ujHL9buZ9zOcvLvfYWfKwc/VwDBeBn/X3mhuJr0NuzYxHFFcGJoOPy5+Bs42IAb4vIvS6bng1guVKqytu+RCRDRH4pIgfA2X0POIsTAJMGMMxzAXwMoExE4owXaIIYAc6uXfFmb7cBeNntvf+Cs6wiL9853fHvv9zeN/5e4GfsnpgLCiH3ff4XgLWf+/THO0qpHpe/P3P8W+z49xRw5vxvt2tcCZrnTsfA6PObiMgoEXlSRCpApdMDrnSyAOQHsE93Z/FncJ6PLzqUYyUAfOFr2Of23bkA3lRKdbhsVw1gfQD7PwXe77VeDPQahPGZi3m0yShMKKVsANY5XhCRQgArAPxERB5XSjWCgrfS+14A0NRwJrhk3QKgHcBs0CySNIAh5gOYCN7snhjh9rcnswoANLoJRQCodfxbBM/nZ0TtuO+zxu3zYPC4T6WUVUSO9nOf/mhw+9tw6hq/iyF83vXy/cYBHr/XuToCFpYBKARXEbtBG/hF4Aw5kPvF0zklBvA9T+fS7XbMUQDqPGxXC/+RZKPg+14DELJrEK5nLubRCiFCKKWqROQvAP4AzjI+AZfU3mbREJEkAEtB08ofXN4/IQRDOgo+nLd7+XyP29/Ky3bZIhLv9qAWOP497OU7htAZCeCAy/sjXcYWLK773GG86ZiRj+jnPgeKcczrXcfkgjf/QaC4/yYTQHv5V5VSX6yUROTCAR4nVFTD8wy9wMN7nr7r614zGNA1CPMzF/Nok1EYEJExXj46zvGvMRNeCeACERnlZftEAGb0ncVf72HbbtBBHOj7KxzjOaiUKvXwClRYmQFc6vbelQAOwrtCWOuynStXO/5d5zJ2eBm/Ox85tnff5xXgxGdtn2/4J5jje2I9KPQnernG7krX0/GDOXaK498v7hcRiYfzukabjwAsERFjnHDc+/MC+O4GeL/XXAnmGljQ9/oG88wNOfQKITxsF5HVoI23DLQjLwFwK4AXlFIHHdv9BExaWy8iD4FROkUAzlVKXaOUahaRjwB8T0SqwRXFDfC8qtgJYL6IXAAqnCNKqXLH+zki8g0ApQC6lFKfAXgEFJbvi8gj4IogFVQS85VSSwM811YAvxKRXNBmfBW43L5eObxy7iildojIcwDud8zg14M24vsAPKeU2ubYdC9o/79BRBpAAbnHk7JSSjWIyO8A/EhE2sEIncmg7fgD9C+Ryohouk1E/gEKiW1KKUsgX1ZKtYjIDwA8LiJ5oAO0Gfz9FoCO7f/4Ob6n39Qbu0Bn/s9FxOYY73cDGWuE+BnoeH9bRH4DCt/7QLOPzygjpdQ7IvIBgCdd7rUrwAAFV4K5BjsBnC8iK0CTV5VjJR/oMzf0iLZXeyi+QMG/DLwxu0Ab5GYAPwSQ4LbtBDAC6QiceQiPuHxeAgqSVtDE8xioRNxjr4+DMwbcNWY91bH/RvTNQ8gGFUMZOFuqc+zjDpdtrof3SJtn0DcPoQLAd9y2M/ZR4vKekYdQAT60FXDJQ3DZ7hbHNbG6n7OH8XjKQ3gcjjwEl+0CijJybPsTcKVjcz0HeIhQgYcIHMf7SwCsBuPnO0HF/zcAU/wc29tver/j7zgP35nhOL8Ox2/zABjr7379y+E5ymii2/7uB6B8nSO8R7Ktcb/OYLTQFjjv9VvAidPmAH6LPPBeboUzD2Gp+30RxDWYB2ATeN+65iGUIIBnbii+jLBAjSZoHLVwzlRKjY72WDSDExFJAxXkG6p3Zr8mCmiTkUajiRgi8ihoIqwCI4FuB1eqf/D1PU1k0ApBo9FEkiQws7oANOt9Aq4yt/n8liYiaJORRqPRaADosFONRqPRONAKQaPRaDQAtELQaDQajQOtEDQajUYDQCsEjUaj0TjQCkGj0Wg0ALRC0Gg0Go0DrRA0Go1GA0ArBI1Go9E40ApBo9FoNAC0QtBoNBqNA60QNBqNRgNAKwSNRqPROBjU5a9zc3NVSUlJtIeh0Wg0g4pNmzYdUUrlub8/qBVCSUkJSktLoz0MjUajGVSISIWn97XJSKPRaDQAtELQaDQajYNBbTLSaDQhoLsbaGsDEhKA9PRoj2bgKAXU1QEdHUBqKpCXB4hEe1SDAq0QNJrhSk0NsHIl8P77gM1GQTppEnD++cC0aYNPiCoFbNkCvPoqcPAgYDbzvMaNA5YuBaZPj/YIY55B3VN55syZSjuVYxuLBaio4L+ZmUBR0eCTM0OSAweAX/0KsNuB/HwgPp4CtaEBaG4GLroIuPjiwfVjvfkm8N//Ajk5vNlEeE5NTTyva68Fzjor2qOMCURkk1Jqpvv7eoWgCQtWK/D223xGOzv5bNrtQHExcNllwAknRHuEw5iODuD3vwdSUig4DUSAESP43iuvABMmDJ5Z9eefA88/D4wZQ+VmIAJkZ9N09O9/A8cey5tQ45GYcSqLyBgRWS0iu0Rkh4jcHu0xafqH1Qo8+SSfz/R0Pn9jxvDf5mbg178GPvww2qMcxnz6KX0GrsrAlbg4ICsLWL6cM+zBwKpVQFJSb2XgSkICP1uzJqLDGmzEjEIAYAXwPaXUZABzAdwmIlOiPCZNP/joI77GjeMzaiBCOVNYCPz971zFa6LA+vX+ncc5OcD+/UBLS2TGNBCUAj75hM5jX+TlcTuNV2LGZKSUqgZQ7fh/q4jsAlAEYGdUB6YJCqVoJvIV2JGURPPRRx8BS5ZEdnwa0IbnbSZtIAKYTHT+RIKWFuDjj4F167h6yc0FzjgDOPFEIDHR//etVjqRfWE2R+58BimxtEL4AhEpAXAigI89fHaziJSKSGl9fX2kh6bxQ3MzUF3tfwKamQls2kQ5sH8/fZydnZEZ47Bn5Eigvd33NlYrFUJaWvjHs3cvcNddwH/+Q/9GYiLDRp98EvjJTwB/z7kIMGoU0Nrqe7vWVi5PNV6JmRWCgYikAXgZwB1KqT7rVaXUUwCeAhhlFOHhafxgt/P59Bec0tXF1fsddziDQeLigEWLgAsvjIwcGrYsWABs2MCL7u2Hqq0FTj0VSE4O71hqa4Hf/IY/eEGB8/2kJDqDa2qA3/4W+OlPfa8UzjkH+NvfgIwM79s0NgJXXhm6sQ9BYkohiEg8qAz+rZT6X7THowme9HQ+y11dvf0HrrS20jIwYgQwY4Zzpd/Tw7D4zZtpLfjgA8qD5GTKptNP1xO8kHDMMcBxx3FmPmZMX6XQ0sL3zj03/GN57z3OIrwJ8pEjgbIyYOtWYPZsJtFVVHAFk53Nz0X42cqVQFVV35tEKeDwYaCkhCYojVdiJg9BRATAPwA0KKXuCOQ7Og8hNnnlFWDZMmDs2L6fKQWsXs2J4Xnn0cnsSkcHw1VTU4G5c/mv1Uqrgd0O3HgjMG9eZM5jSNPWBjz2GLBrF8NPU1OpkZub+fcdd1BxhBOrFfjmN+kvSEjwvl1DA51SU6cC77xDpWDEMU+YAHz5yxxrYyPw6KMMQU1I4Iqiq4vndcwxwG23eY+sGmYMhjyEeQC+CuAzEdnieO//lFJvRnFMmn6weLFzdl9Q0HsC2tDAydrUqX2fTcPRbDbTn5CQwP+bzcDo0Xy2n3qKeVSTJkX2nIYcaWnAD37AVcKaNc6l2CWXADNnUkGEG0NY+1IGAAX7O+/Q2VRY6DQdKcWZxUMPUYHNmAHcey+3++gjJqRlZwOnnELFMZiS7KJEzCgEpdQHAPQvNgTIzATuvht44gmu9uPi+LJYgEOHmK08fXrf57O+ntaKrCxOVJuaaBEwSEqinHrjDT7/mgFiNgOTJ/MVDRIT6bi22XxHCB04wBuipKT3TWMk0iUlAX/+M30NqalcDYR7dTNEiRmFoBla5OUBP/4xV+9btjCopaAAOHKEZmNPz//Bg72jIe32vtvk5tKc3No6NOqwDWvi42n7//RTRgl5wm4Hdu9maru3GX5qKm+sTZvoaNL0G60QNGFDhCv1CROc75WWcvXvic5OriSU4islpe82JhNf7e1aIQwJzjqLEU/d3Z6jiGpqeDOMH+97P6mpnHlohTAgYjIPQTN0Of54ZxSSO4mJ9DN2dzPoxJP/z27ny5Oy0AxCxo8HbriBySvV1TQfAU77YkcHbxp/yWlmM28ezYDQCkETUZKSGApeVdU3aXTsWD7/XV3eLQRHj9Ih7SvcXDPIOP10JqCdfDJvjIoKRh+ccw7w859zZuBP2Le10cegGRDaZKSJOAsWcCL43HP8NzmZVoGODgacTJrkuSxNdzd9BxdcEPkxD2tqamjnb2ykx//EE0OfEDJuHHDzzVwtWCxcERiOpvnz2bNh9GjP37XZ+NLxyANGKwRNxBFh4tns2fQpHDjA96ZOpZx59FGgvJwOZCMPoa6O/95wA3OqNBGgs5NVCD/5hI6b+HiGib70EkM8b7op9OGpRkiaK0uW8Eapr+87U7BauaJYsqR3prOmX8RMYlp/0IlpQ5O2NsqgFSsoAxISmKm8aJEuZR8xrFbgd79j4lpxcW/7nVJAZSXt/z/4gf88glBQVcVEuqoqp9IwEtSWLGEzH3/F7TRfMBgS0zQaAMyZWryYL1/ldjRhZNs2YPt2mnLcfwARmm/27KEpae7c8I+nsBD42c+YSLdtGx1NhYVMonNPd/dFdTWwcSNnGmlp9FvopLUv0ApBE3Gqq2kCEuHk09fzrJ/TKPH22842lJ4QYc+Et96KjEIAaLY67rj+2Qy7uoBnnmEGs8nE6AaLheMfP55lLUaMCPmQBxtaIWgiRnk5Kxzv3ctn0mDuXJajCWaipwkz5eX+G85kZjKbMNaXcXY7S2lv3sxQNnfz1+HD7C99333DvsyuDjvVRIQDBxhBePgwn8niYr6KitgX5ec/Z6kKTYxgNvtvn2m3Dw67vWHaclcGgLOXQm0tC3ANc7RC0IQdmw34058YkOLeSc1sZgXmhgbgxRejN8aA6exk+KWnzLqhxIknshyEL44cAaZNi+3VAcB+yykpvseZn88oBk/1UoYR2mSkCTu7d1N2+MobKixkBYMvfzlGKxTv30+B8emnzg5Ap5wCnH02NdpQwyhZa7X2DQMFqOU7Onj+sU5Fhf9MxpQU3qSdnZGp9Bqj6BWCJuzs3etZprhiWB4OHQr/eIJm3TrgwQeBHTsYXTNmDM0MH3/MDNutW6M9wtAzfjywdCmFqXu7zY4O+hjOPRc49tioDC8o4uL8z/yNAlqDwQQWRvQKQRNy2tpYZ6y2lgmnVVWBfzfmVuwVFWzN6FqHH6DgKCyksHzsMeAXv2Am3VBBhLH9ubnAq6/yOhgml4wMZgguXBj75iIAmDWLNdN9zfwbGpgi763N3zBBKwRNyLDbGcX3yiu0NMTH07JQW0tncna292fSZmPJ+4YG9lAYO7Z3JFLUWLXK2X3LE6mpLLC0fj3wpS9FdmzhRoR1hubN44qgvZ2mlXHjBtdM+rTTgOXLvVdUtdsZ0XDjjZEfW4yhFYImZLz2GvC//zF6yLWvwahRFPKrV7NkhWvfdqUoazZvptz95z/5fOblARddRDN91CahSlHQu3bp8URuLs1KQ00hGJjNvWuYDzby84HrrgP+8hfmGrjmV7S3s1bTmWeya9MwRysETUior3f2UXb3F8THM9dg7Vqa4Wc6EuaVYjLsjh2cuM2f73Qot7YyMunoUeDCCyN7Ll9gtXLp4s8BEh+vY2ZjnQULmOjyv/8xd8Js5swjMxP42tcGj/krzGiFoAkJGzbwefImO4uLaXnYuJHPZXIyhf3WrZzAzZ7dO7ooPZ3m3JdfZmTj2LGROY9exMUxUamzs/eyxp32dv9JXJroM306b6aqKs44kpIYIDCYzF9hRisETUg4cMB/kuf48Xz2LrmERTOXL6e/75hjPPsL4uP5Wr0auP76sAzbNyKsyW/YwbzR1ARcfnlQu1aK7UVra3nuxcWhryit8YAIsyE1HtEKQRMSTCb/ia3GdiefTLP86697VwYGubkM/Q9aIbS3s2TqypW0ZyUm0rm4YEFwkve001jX5+hRz7VuampYdvmkkwLe5b59LKtz+LDTSqEUe91fd51/l4VGEy5iIY5DMwSYNo3hpr7o7GSQSl6eM+zbn9nWZHJ2VQyYujrg/vuBf/yDBcwKCxkquWoVcO+9dBQHSlYW8MMf0nxUXs4wqPZ2KojychZ4+/73fZuUXNi9mxGqbW00g7m+yspYwqO2Nsjz1WhChFYImpAwaxbNO52dnj9XilVOzzmH2xlh/K2tvvfb3Byk/6Cnh3X8jZaK6enUKomJTCrLzweeeoo2rkAZMwZ46CHglls46IQE7vs736HiCdB/YLPx0FlZDMF1VYYiXBlYLOwkp9FEA20y0oSEtDTg619njlZ2du/Ivp4e+vGOPZbRfQZLlgBPP+29qoBSVBjnnBPEQLZvpxnHW52MpCTO5t94gwI9UJKTGQN7yilBDKY3u3ZxgeGrhEdBAR3tnpqDaTThRq8QNCFj1iw20EpLY2Sf8aqroyK4887eiaCzZtHRXFnZ1/+gFL97wglsrRkw69b5r0WTl8dUaveSDGGmrMx/QIvJREVaWRmZMWk0rugVgiakHH88y/4cPMiioHFxTGz1JKMTE6kknnoK+OwzCsOEBJpN7HZgzhw6k/2lAfSisdF/+QFD6nZ0RLSQWaDdanU4vCZaxJRCEJG/AbgAQJ1S6vhoj0fTP0ScjlJ/pKdTKRw+zGii5mb6aSdMYFjmI4/Q9j5pEhPXCgv9CMzsbFatTE/3vo3dTukcoCM4VJSU+HeQK8VtRo2KyJA0ml7ElEIA8AyAxwA8G+VxaCKI0aJ39Gj+/f77wG9+40wkFQHefZfVp886C7jySh+ml9NPZx0MX4Xm6uuBGTMi3h1ryhSeT1ub90PX1XGVpUNPNdEgpnwISql1ABqiPQ5N9Ni8mY7m3Fwma2Vm0ulsVJ1esYLFN71iSNO6Os+fd3XRVLRkSVjG74u4ONZPO3IEaGnp/ZlS1FMAcNVVER+aRgMgxhRCIIjIzSJSKiKl9cYTpBkSKAW88ALzvzy5AcxmmqHefLOvQP2C+Hjgu9+lOai8nNNxpeiYqKxkkP/Xvw5MnBjOU/HKtGlMWxBhRenycr4qKujrvucenUiriR6iAvV0RQgRKQGwPBAfwsyZM1VpaWnYx6SJDGVlwAMPcGXgy09QUcGM3oULfeysvZ0NbN5+m1PvpCQWU1q4MCYkrs3GxkE1Nc7SFSUl2qGsiQwiskkpNdP9/VjzIWiGMc3NzgAgX8TFOc0rXklNZRvIxYsDS4mOMGYzS1VMnhztkWg0TgadyUgzdElICCw002oNMkAoxpSBRhOrxNQKQUSeA7AQQK6IVAL4iVLqr9Edlaa/2GwsZWE202LjTy6PG0cXgMVC5WAgyo4RR/cio/kQFIDutjE4Yeox0PMZjSa0xJRCUErp+IohQEsLE4bffpumfKWYR3DeecCM47ogFeWsZ5GT0yuxIDmZYaWvv+60p484uhcnbXoaKR20EXV2ADNSgeLH84Cbv85yqRqNJiTElELQDH7q64GHH2bNnvx8RgwpBdRXduOD25chxfwuJpX0wGQSJhqUlABXXAEcdxwAdqE8eJDhp1MT9mHep79Ed2I6jqaVoK0NSMkFZpwGiKUJ+OUvgbvvprbRaDQDRq+5NSHDbmdxu/Z2hocadv44WzeW7P09Tmtejt31I3BQFTOpoLiYZaQffpi1hUBT0be/DdzwNYVZ2/+Oo91pqOvOQnc3i+MtWMAS2sjKYnbX3/8eeE0IjUbjE71C0ISMffsYEupezXNc2Srk1e9EU1YJknsEe/dSYYgIzUaJiWyg/LvfAampiI8HFhUfgP24anTlF0OBPog+2clZWTzggQNRyyvQaIYSeoWgCRkbN/Z2BgOAyW7FpH1voS1tJCCChAQ6mnsllqWm0pPsmlNSXQ2TKKSkClJTvZSqMLzU1dWhPhWNZliiVwiakNHezighV1Lb6xDf0wFrXBJS29gKrNOWAavVLW40NZWNABYs4N/BhIoOJKzUamWjgg8/dFbWmzeP9qlYbr7e1UUz23vvscJrRgaT7k4+2WFT02iCRysETcjIy6OcciWttQp59dsRZ+0GIFAA8i0KqbuKgJOmOoWXe6/MMWP4r6+kMsN3YGwbLPX1wB/+wJIWyck0XZWVUTkYHdFycvq373BSXQ389rcsipSZSXva0aPAX/8KvPwy8L3v9f+aaIY12mSkCZq2NvYG3rmTMslgzhzKdENOZ7RUYmbpn5HU1QxLQjq6kzLRbma1usSGasamdnRw4/Z2JiIYFBfzb9cDuHPkCLcpLg7+JNrbgV//mvsoKWGrsqwsFsYrKWGLt9/+tq+Gizbt7SwF29HBcWZnU5llZfFvux341a+42tFogkQrBE3AtLayb/0dd1Dm/Pa37JD2yCOcZBcVAbNn08+r7AonbXoaNnMijuQeh3hLO+x2Wmhy80yQjAzmImzbxjftdppqDESAG26ghqmr6x1JpBTfs9m4TX9MRh9/zH14qjNtt9OEtXs3sGZN8PsOJxs3cjWQn+/585wcKo0PP4zsuDRDAm0yGiJYrZzMJib2teOHgpYWRofW1DCXzOhiZrcDe/YAP/sZ8KMfUT53dwOH3y9DQk0FmkeMRXt6KuLrq4GedowcnepsUpaWxpn4nj3Al7/ct4nwmDHAvfcCzz7LSnCG4FeKCWnXXutsohAsK1b07Zlgt9NktG8fT6Knh3kOe/YAS5f6boYcLqqrqWFtNiqBt99mcocv8vKAlSujUuJbM7jRCmGQU1MDrFoFrF1LpSBC083ZZ4dWfr30EitHu1tnTCZOshsagCeeAH7xC+D224FDWeXoOAwc6RI0NaWgPuU0nNC1ES2HmmA9KkhKMaG70wZ7Wxc2ZMxGzZFLsXgfo0d7TfhHj6amOXyYygOgRioq6r8z2W7n6sC1pZvdziinykoqqsxMvmfYx7ZtY93qSFWjq6ujIty+vfd5bt4MzJ3r7AinFBWXCLW0CH0KtbUcv0kbATSBoxXCIGbXLobuK0UTeHw8J5KbNgEbNgA33dTbCtNfWlpogfDV1jEnh3X99+yhzBxb2IPqbIGtmhPalJR01GIREjsa0FVRh+4mKxKy05BZaMPhCadj62bB3nfKsGB6E86/KB7miS6NmN1bqg0UEefFMpY6Bw9SGWRnOwWw3c7tCgpoL3v0UdrJwt1688gR4KGHGJ/LhA3nZ1u3Ah98AJxyCm+f9KYAACAASURBVFcx+/ZxO4CRRpMmcSWRmKiL+vUHpbhK3LqVfpqCAkZuZWdHe2QRQSuEQUpTEwNkMjJ6tw82mym4u7qAv/yFMjSQ3sa+qKjgc+Kv2b3ZTMvO5MlAlRqFA/sV0ke6fk9Q1TUCdTIC8VmA1aJwslQgo+corvn8HqS2VKN1m+DQWoWSCXEsXX3xxZ675QwEEc6yP/6Yqw2lOPC0tN5CtKPD6ehOT6ft/tNPQ6NlffHCC/QDeOrbMGkSVytvvEGTV1oaHcpKUUGUljJy69ZbtUIIliNHmCB54ABv5rg45sf8+99ccl9+uf+HYJCj15ODlA0b+Px76yWflMTJ7apVAz+W3R6YbDGZaL0AgLcPTYElKRPJ9vZe+zl6lJPXuDgg3daEurYUHL/9OcT1dKIlqxjWUcXY1jQWliyHvfyRR3iioeaMM/iwWywUvp2dvbPqrFYKWVe7W3p67+S5cNDQwGN4W46NHcvVSkcHx2uM2TAVpaYynNZiCe84hxqGk+zwYV7jMWP4G4wdy1nVW28B//rXkC+TErRCEJFUEYnhjJ3hwbp1/n2L+fnA+vUUxAMhL4/78Pcs9PTw2VEK+PDjOOyZc+0XiWkAZZhh1k62tiJLmvBey0y0ZRShOykTEIHZzG3qm+L5MO7aBaxePbAT8ERJCXDNNTQTuYa2KkUF0doKzJjRd/kVbkFbVUXh7s32b0RAmUxUHsaPa7c7FcW8eVQqhilJ459VqzhbGTmy7+zHbOb9smYNTYtDGL8KQURMIvIVEXlDROoA7AZQLSI7ROTXIqJLTUaB9va+ZSLciYujmdyYtfeXUaPo7D161Ps2RoTT9On4Iry0dvTJ+GT2txBvaUNmUwXSWw8jv+cwCrrKYYINb2RdjYNSAmtcX5NQdzf4YI4cydmZ1Tqwk/DEmWcCd91Fs1B7OzN+m5tpL54/v6+tra2tfzkPweBP69bW0odRVMQZQVsbx9zaSvPXwoU8n54eYP/+8I51qNDTA7zzjucQZAOTiQ/U2rWRG1cUCMQgthrAuwB+BGC7UsoOACKSA2ARgIdF5BWl1L/CN0yNO7m5nCD6Ugrd3ZQd/hSHP0SAr3yFoaXNzQzAcT9OVRVDTg1/a3Y2J6hVRbNQO3I68mu2QZWVY2+3oDl3IspTpyKjfAtykuo8HvOL0NmUFGqio0fp4As1U6fyNWIEl1PFxZ59FoaWO+200I/BlcJCHstbhFBPj/P9E0/kNbFaKazcS22Ew9Q2FGlp4bVKTPS9XWYmHc5DmEBMRmcqpR5USm0zlAEAKKUalFIvK6UuBfB8+Iao8cQZZ9Cx7IvaWk6CQ+FbHDeOk2mlGE1UVeUMkT96FPja15xliAD64Iy+xzZzAqqLZuLg7MuwJudS7EuZDpvEoc2SiDmZe3sdx/BXuKckDNju5Y/LL+cD78nMYrfzpE8/nQI7nIwYQUFfW+v58+RkCi8j+slspiBzVwZKeXcwaXpjMgV2f9ntsV3fKgT4XSEopXpE5DgARQA+Vkq1GZ+JyLlKqRVKqQEaJTTBctJJ9BHU1nqeODc1UU64CumBcswxrJqwYwdD861WWlVOOqlvPbV585gbVV/vFO4JCcD48YyUjIsDRmfF44T4XegGHahKcbI2dqzLJN1i4caOsL/mZgYH7djhzE879dQQlBwqKGAS2h//SOGfnMyHv7OTdrfFi4Grr45M5M4VVwAPPshchLy83sdMTaVCOOUU78KpvZ2RR7okeGBkZfH3b2lh2J43mpvZ0m8II8qPzVJEvgPgNgC7AMwAcLtS6jXHZ58qpU4K+yi9MHPmTFUa7qiPGKaujmHxdXW8j43JY1MToxG/+10K4GhRXc08ifp6jicxkbLq008pZ8+aUYevfnYXmtOL0NUTh85OKrk5c1yi+yoqgHPPBa64AmvXMlfLbndOftsc05PLLmOLzgHL654eapvSUl7MwkIKX1/25XBQXc244QMHnAX+RCi8Ro1iwlpJSV+zUk8PHZ8330w/iCYw1q0Dnn6aD4ynm6iri8EHv/71kMhJEJFNSqmZfd4PQCF8BuAUpVSbiJQAeAnAP5VSfxCRzUqpE8Mx4EAY7goBoMzasoVBEg0NFLwLFwIzZ/L/0cZiYdj8++9zApadTTN8Vxf9eDmrX8ZJh15Fz8hiTJwcj1GjHBNfpbj8SU0F7rsPH+/JwmOPMYrJ3dRryMDrr6cpbcigFHDoEFcsNhtnsccey/effZYOzqQkmrqUolPcaqV2PP98nYcQDFYrU+1LS+mwN24ypbgyaGgAbrkl/DkoEWIgCmGnUmqKy99poFLYCWCxUmpGqAcbKFohDG6UAnq67ZA3lyPuzWUQwzlqlEydOBG45RbYcvLw/e9TUXhTcl1dzmQ9f77BIYGRUfvee7TBmUwM8YqEn2Oo0tPDiLYVK3hDiaPvd1ERa21NmxbtEYYMbwohkCijGhGZoZTaAgCOlcIFAP4G4IQQj1MzjBABEpJMwCVfAs5ezKVOXR1nvVOmfFG2Yd9uCntfGddJSXyGd+ygT2PII0LzRjRtgkON+HjgS1+iibKsjMvbzEwmqQ2T1VYgCuFaAL2CwJVSVgDXisiTYRmVZviRluY1pLOxMbBdiPjOlfCJUb7i3Xd7e6zPOovKSReJGz4kJNA0NwwJJMqo0sdnuui6JuwEWs5bqX6ai2w2liV47z165o0U8H37WF30lFNYKTAcdcW9jaeqiiaMrKzY7NqmGZIM7UpNmiGBURLbZvMeaWm4wib1J29+xQquDMaN670SyM9n2OdHH1EwX3VVP3YeBDYbldIbb9ADb9iwjz8euOSS3h3lNJowoBWCJqQYUZtr1zJKLz2dfs4ZM/pftDQri5P0DRu8V46orgZOOMF3iW6PdHcDy5czfMmTWUiENuRVq4ALLghfspfNBjz5JJVPQYHzRO12hp4++CBw551UDhpNmAhYIYiIALgawHil1AMiUgxgpFLqk1ANRkTOBfAHAGYAf1FKPRyqfWvCT0MDi5MeOsRo0eRkRuz9+c8U6nfe6b8UkBFpuXYtLTZmM4M7zjyTdejKy5kSYCgXi4XKID+fpTOCZvduZyEmb8TFMSxx+3ZqpnDwwQfUeO5x8CYTT66tDXj8cWYGftFyTqMJLcGsEJ4AYAewGMADAFoBvAxgVigG4qig+jiAswBUAtgoIsuUUjtDsX9NeOnuZpKc0bPelZwcKotf/YoTXW95PTYbS8+/9x7N9UZ4/fLlwOuv02py4onMXzDKYsTHs1PkOef4TjL1SkdHYNuJ0IwTDux2mony871Hs6SlOfsx6IQzTZgIRiHMUUqdJCKbAUAp1SgiAyyb1ovZAPYrpT4HABH5L4ClYL6DJsbZsoWl5L217czJYfLY++8zss8Tr75KYe9uyk9P50rghReA227jKuTIESqL3NwB5h0E2v0snLWBjh7lCflbPqWlARs3aoWgCRvBxNL1OGbxCgBEJA9cMYSKIgCHXP6udLzXCxG5WURKRaS03pgmaqLOu+/2rYLqjtEj3lMuZGsr8OabzDXwZMpPSOD3X3yRFhyjrfKAk9COPZbLDF99Dmw2DmrKFO/bDASjGbY/zOaB1zLXaHwQzArhjwBeAZAvIj8HcBmAe0M4Fk9PRB/RoZR6CsBTADOVQ3h8zQCoq3OatltaaOs/dIiyNC0NmDCBQryjg+Yldwfzli20nPjqUJiWxtJGn38ewrptyclMRHr1Vc+1gQynxqJFdISEg8xMHtfI1PZGWxsd3MFgs9FHEhcX2hTu7m76VKqqqFAnTuSPHMoErs5O3kzx8b17XWvCRkAKweFQXgdgE4AzQOF9kVJqVwjHUgnA9W4fDaAqhPvXhJGkJE5eq6oo3E0mZ2Mvi4Xh/Hv20BzkKZz/yJHAKguL0FEdUr70JTolPvyQjoicHB6osZGvk04CrrwyxAd1ISWFSXnr1nkX+EY/hkDNRU1N9MyvXOn0k0ybRmfL5Mn9F65K8Tr95z/cr9HiTimO/ZZbGLE1EGpqWELiww+5X7ud5rTzzwdmzdKKIYwEpBCUUkpEXlVKnQx2TAsHGwFMEpFxAA4DuBLAV8J0LE2ImT8f+Pvf2aQrPb33RDcxka/aWu85VikpnMwGwkAnuobJ6gu5EhcHfP3rwNy5tGnt3s2NJkwArr2WMbPhbq5+3nms693Q0PciufZjCETYVlUBv/wlZ9cFBc4eqEai3SWXAEuX9k+wrlvHKqyFhbThGShFX8gvfgHcd1//q8MeOMDoA5uN+4iLcxaYe/RRNtq45hqtFMJEMHf5RyIySym1MRwDUUpZReRbAN4Gw07/ppTaEY5jaULPqacygsjoNOiOzeZUDLt3s0mZK8cfTwtBbS33kZnZt9ObxcLVxYQJwY+vsxP45BPK++pqjnH2bFZHHT8ezuJw06d70BgRID+/dz+GpCQO0ujHsGhRYIKwpwf4/e/5HdfiT0b4ak4O8PLLnHEHW/Spo4NhYEVFfW1+IvTwV1cDL70EfOtbwe0boGnrD3+gGc/VPGeU/c7I4Ipn0iQqb03ICUYhLAJwi4hUAGgHzUZKKRWyEoBKqTcBvBmq/Wkih8nECd3Bg3QQG+YipShHLBbK2tRUhty7KoSKCuD557m6aG3lakGE8mzyZCoGpRjFdMEFgQcGGTQ2Mny/spJVKYqLKS9LSzmWK65w66UQbkVgt3PGqxSFnKFBi4uBhx8Gdu7k4Do7KXznzg18xr19Ox063sK94uKoFF5/nTG8wZzrp59S4fjKMCwo4HaeVjr+2LKFqxpvYzeZ+AMuX86mGXqVEHKCUQjnhW0UgxBDQLW3U4AVFQ3v+mcdHZwgjh9PwV7pUgGroIA+x9xcmrZdi9Xt30/rRnw8Z+vr11MmpKSw4OTRo1QkTU2UX0uXBjcuu52Whvr63pUf4uKY1dzTA/z3v/z/ieHu7NHdTQ30xhvO/qfp6bTrL1zIk46Lo62/v6WW16/v277OnawsauGGBmfdpkAoK/NvrzOZKKjr6oJXCB9/7L+JR2YmZx1Hj/KG0oSUgBWCUqoinAMZLCjFydsrr3B1bLRjzc+nsDrllOExcWlqYmLt1q3O3i2trVSMJ59MId7TQ5+jq+mnu9tpDTB6kqSmOt+bP5+yav9+Xsfqak6if/hDXttg68vt20c55q10thHAsmwZXQU+fzuLhbPfFSsYeRQXx05Eixd777Rl0NnJBIpdu3qXpujo4PJowwae5EBzHdra+tra3BHhjdvVFdy+jSVfIPTnIejs9P8DG2PX4bdhIZjSFT/29L5S6oHQDSf2efNNzihzc/lMG/d9Swvwpz8xQOLii4e2UvjwQzqQbTZnxGRZGf2BjY0U3HFxnn0JXV3OKtc7d3J7V2GdkEAT8YQJlL89PbSunHxy/4qNfvSR/+9lZdFsX1/f20/ai9ZWCvQDB6hBioo4Eygt5QW5+GJGK3n74V98kdrJXXGkpHDpUlnJi/qd7wR/kq7k5zMu11eIrN3OV7At9Y47jpmDvjByKor6pBD5Z9Qojt1XQosReZCQwG3tdjrN/SXBaAIiGJNRu8v/kwBcAPZZHjaUlzNbtri4r5DJyOCz/dprdJAec0xUhhh2tm1jDbbCwt6mZMMUvnIl5cz06X2/W1XFa7R/P5XB9u18nj1hMnH/SUlUGocOUR4FS1OT/6J6xqTTaxULpXjSFRW97U4mE4WY1UpH7ciRtG2709rK6JyiIu8Ko7CQEUB1dT60UgDMm8dwU6MPsyfq67kcClaInnACf9zWVu8rmepqLvP60791/nxg9WrfY6+s5MP3f//nVD52O/0sS5dy9aXpN8GYjH7r+reI/AbAspCPKIZZvZoTE28zzrg4OjzffXdoKgSluDoaMcKzkB05ktFGH39MBTF6NK9JWxv9LZWVfG/ZMr6/ezejijo7Key9+WCMZ74/5OSw+qq/87LbfZjeDx6k9vJmd4qL45LxlVcYJ+9+Ivv2cWbrK3TVsD3u3j0whTBxIi/m3r0U2hUVzsJPubn8kbq76Z0PloQE4BvfoIe+p6d3spjRwyEvj6ul/jBuHBXV1q29l98GR47ws4kT+TJMYzYbS3ps20ZFoVuI9puBuEFTAAyr/n2bNvF+90VeHs3M/kytra3AZ58xsOLQocBNs9Hk4EGnTd8bEyfSOTtqFP1+Bw/yebXbKaeOPZbPemEhzUKJiUxY8ya0DeuGv+vujblzKf98Xd/GRlpyvB6jtJTOEF92wIwMzu4PH/Z8gOpqYM0a+h/WruWFcbeDm0zUjgPBZKLQ7uhgNM7evZxJWyz0X7z3Hm1y3iJ5/DF1KvCjH1EZVFTwPA4e5HnPns3P+mu+EWFi2/Tp3Hd1NW2xDQ38e9cuKo0pU3r7Scxmrr6UolNqMDxMMUowPoTP4CwlYQaQB+DBcAwqVunp8R9JZDI5e8R7kh9tbQzT/uADCjqj8cu4cey/Essri6YmZxCJL7KyqBSuuILX4aWXmHjqnlM1apRzVXXgAGWUuyWivp4muP4qhAkTeE3LyjzndFksPK+bbvJxXo2NgWXDifS1O1VU0Ddw4ACXK93d3F9FBZXIokVOAapUaLqjbdhAIXnGGTxOayvHP2kSNfHu3fxBzj+/f/s/5hjg/vupCOrreaySEu9lbIMhORm4/Xb6B9ato6JJSmLU1b//7bt0R24ux3TgQAhrmwwvgvEhuK4xrQBqHb2Vhw1jxviPpmtp4TPnSXG0tzMJs7KS2xgWBNckzzvvpKk2FjHyAfxhtTrNLxYLe8t4CqNPSKCw37KFsvTgwd75CU1NVJZf/nL/x2wyMUfqd7+jUsjOptKx2SjLrFYmI/uM8szOpiD3h1K97U6NjcCvf83vJyRQOLvS0sIIo4suohY1LshA6O6mI6u4mErAk/kkIYF2u8WLg0/qMDASRbyZ0QaCCDW5awbiZ59R8fgyu4nwtWuXVgj9JBiT0TeVUhWO12FHZvEvwzayGOSccyikfAnFhgYmOXli+XKah4qLe9/XIlQyI0awmUwgsicaGHWIfBUGNezxhlxrauLKylskpGE2ttk4sauu5qSwooLP/913B1/PzZ3MTJqWv/EN/r+2lrJ4/nzggQfYfMcnM2c6l33eaGmh7d81uubDD7li6Oriy2bjhTC85WlpvDgrV3LWvnRp/wW0wa5d/hv+JCTwR9w5iCrLB1rXRFeEHRDBrBDOAnCX23vneXhvyDJjBm3gBw5QSLmaGJSiT62khPLDnc5Omm99tXhMS3P6zWbPDvnwB0xSEkvJLFvG8/RkYqmt5cTOMFH7C10Xof0+LY3bzZ9PZTllCrOU/RW86+zkNROhPPameBIT6U/oV8WD4mIu23bupN3J/cStVg7ittt6Lw1XrqSmX7uW0S8pKU4HrzEjMIroJSd7n0kEQ2trYNspFfi2scCIEc4ier5sllZrP/qoagz8KgQR+QaAbwIYLyLbXD5KB7A+XAOLReLjad586ikGNMTFUQB1d3MCM3kyZ6GeInCqqnzPlA2Sk+lgjUWFAAAXXsjZ++bNlHFGdGF3N3MwcnJ4DYxnNieH1pD2dt+dH7u6WFD07LMDG0dTE83gq1c7J+9JScBZZ3El5y9ZNyhEgJtvZo2g/ftpAsrI4IGPHOHJX3pp7x/NbneWbu7p4cmPGEF7VXMzLwjgrAQYH09tOlBhFkzlv1CWww43o0dzluGrJEZ3N6+jp5hnTUAEskL4D4C3APwCwN0u77cqpRrCMqoYJi0N+O53afopLaXtPyuL0YZjx3qfvPib2BgYTuZYJSEB+Pa32fnszTdp9xfhc3jhhfRjugaZmM30Xf7jHzQPeboGXV2cWHsK4ffEkSP0tzQ00ERuhAF3ddF8vmUL8IMf9C8U3ivp6cBdd1ETvvWW/0xlEWqo+vreq4aEBHrIDS95Tw+Vh9kcGoVgxO/66q1gNPyZPHlgx4okIsBXvsIfPiGh749rsdA5d+21IZ4NDC/8KgSlVDOAZgBXiUg2gElgYhpEBEqpdeEdYuwhQiuCv46HruTlUSnYbL7NIJ2dvXOfYpH4eMrAhQtp7bDbPVcnNTj9dMrR7dspwI3tjKrGDQ3ArbcGFq2oFPD004zWcvdnJiVxEnnoEH21N944kLP0QEICtVYgmksEWLAA+NvffNvM2tspmAOdMfigqQnYuDEDSeYFKHr/PaRPHYvcPOm9W6UoOOfPD1/Dn3BxzDGcjT35JGdiSUm8Zp2dVHDXXBOAQ0jji2DCTm8CcDvYuGYLgLkANgBYHJ6hDS0yM2lR+PRT73kzFgvv61mzIju2/mIUn/RHfDwrMixbxsoHhs9PKfpgb7gh8FpulZXMW/AV3FJYyBpvl10W5YoGCxeyZEV1tefPLRbODgoLqRX76T232ZgT98Yb/DvV/GUs6KxF4dufYXdOLqafls7ckdZWLq+mTGGM82Bk2jSGjG3ZwhmGzUan1axZvhNkNAERjFP5dgCzAHyklFokIscB+Gl4hjU0ufhiRs8dOdK3UGN3N4XdNdcMzfs6IYEC+oILGP5psXCC6ikh1Rd79/JfX98xm6lsDhwIvuR/SBk1imFSN9zAHz0nx5moYvgQ5syhr2HWrH7nILz4IpXB2LGGlSgJu0bdgZbDH6No65vYseIgZswAkscWMOFi7lz/zqxYJjEx8JWaJiiCUQhdSqkuEYGIJCqldovIsWEb2RCkoIDhj088wbpIJhOFl9XKWfS118b+itdqZbDNhg2UaXl5LJ/jzT/gTlLSwEzX3d2BlRlXimONOrNns+rhPffQzBEf70zkGjuWtq+MjH636KyrY9OfkpLepki7OR6Hik/DoTHzUFveif2nAl+9OXloV13UDJhgFEKliGQBeBXAOyLSCN3zOGiKithZ7MABhp5bLLQYTJ8e+76w6moW/KyrYzRUfDzD3t99lyv5W2/1HUkUCvLzA69rFIrE2ZAwfz696s8+y7pGcXHUak1NLON61VX9Xh2sX08Z79UvJYIRY1KwdiNw8dX0W5eW0nqUk0N95avmnmZ4IaofdT9EZAGATAArlFI+0pTCy8yZM1VpaWm0Dj+saGoCfvITzrrdy0goRUfusccC3/++/9yBgdDVxdDfnBzvUZOtrRzDww/HWNMio6tSXZ0zMiGYBjUeeOQRmuD86ZMDB2jBqqvrGy594olsKR3rExJN6BCRTUqpPhlTAT8uQq4RkR8rpdaCjuUZoRykJnZZs4aC1lNNIRH6Q3fu5KonlHR3c1K9cydXKImJwOWX09/iKSG1q4uRnldeGWPKAOCFGj2ajo0TTxywMgC4SvO3YurpYd7M4cO0Uo0ezZXWmDH8e+tWdpWLCRObJqoEYzJ6AoAdjCp6AEArgJdBR7NmCGOzMTrIV1VmEZqLVq3qXY+ov1gszHNYuZJC3iiBPWECaxtdeSVbEADOLOe2Ns5+b701ys7kCDJjBqvw+uomuX8/r+eECX1NQ4Yy37GDL53TNbwJRiHMUUqdJCKbAUAp1SgigzhUQRMoHR0M9fZXcTQ93XP152CxWIA//pERWYWFTkWkFG3gDz3EcPTf/IbObaPd5vHHM1hnoF0oPdLTwzDHTZu4bCkqYmu4KDdkOekk4F//8p4Jbrdz2JMn++43kZFB5asVwvAmGIXQIyJmOEpgi0geuGLQDHHi4iiMAykjEwo79Jo1NHG4Ry6JOJvzPPEE7ef9reAcFGVl1FCNjTx4XBwVw6uvMjX7qqt8V+EMIykpwDe/yWvR2cnrY1wzI5Q5Odn/qi0jg1nnmuFNMHfxHwG8AiBfRH4O4DIA94ZlVJqYIjmZFREOHfJtmmhsZAbzQLDZaCoaOdK78klNZVh/aSmzoMNKdTW900YatCt2O6fVSjFmOEpMm8ZwZqNts1FQMCHB2ebZXxSR14ZubW3AJ5/wZbHQAbFgQd9SHZohQSDF7f6plPoqgFwAPwRwBgABcJFSalj1VB7OnHceTTTZ2Z6jiDo6+P4ppwzsOHV1zNPyVxYkLY3lMMKuEF5/ndLVUwyrycRlzHvvsapeFKtsTppEpVBTQ8UcF+dsiVBZSUXhywd09CgXO7347DPg8ce51DCaZh8+zFpOqal82e0873POoUNjMBXM0/QhkDiMk0VkLIAbANQCeA4seFcrIiFo76QZDJxwArOMy8tZf8iIVrbbKcTr6tj9cKANv2y2wCaeJlMEyt63trJBtC8/gZFduD42Cv+OHEl/waRJTtl89tmc6Hsrmmix8LNeyrWsjHaotDSGIhmdhSwWpot/+CG1TE4OtckTT9C509wc9nPUhI9AFMKfAawAcByATQBKHS/j/wNGRC4XkR0iYhcRD90ENNFGhOGe3/oWJ4aHDnHmeeiQs07T3r2cLA9EJhjVHfyFQLa10WoRVhob+a+/xIrU1Jg2wE+ZAixZQmXu2gJBKeaXHDrEQqKuvX3w2mvUKK5VRevrWYwrPZ3e/ro6Lg2zs7lSqqpi/GqgmYOamCOQaqd/BPBHEfmTUuobYRrHdgCXAHgyTPvXhAARlo+ZPZuyoLKSVUWrq50JT93djHpZsoS1m4JNUktJYWLvunWeeyADnM3abCyZEVYMb7o/jE5oMYoI+1sXFbHAYEUFla7dzmt83XVMi/iCxkZ69d1/gD17qCQMZ4PJxJvAqCBYVMQMuH37mKWoGXQE4kMQRbwqA2Ob/g7C8EWIdlINCkQo//75TyoA9z4QVisnmBYLZ57BsmQJsHGj5yKANhtnukuWRCDis6CAws5fd5/2dpagiGFEqGjnzaMM7+riKRUWejDRtbTwTdc4VaM1nWv52Pj4vl3XEhJoPvOnELq6uNr48EMu9/LyWB02kDZ5mrARSJTRahF5GcBrSqkv1sWOHITTAFwHYDWAZ8IyQk1M8vbb3p2/cXEMyFm5Eli0KHhfa24u8KMfAY89RuFv1IPr7qac+tKXqBA2bGAdpSNHuLI4ctdV9wAAHlpJREFU/XQ6tUNW5t9s5oGefdZ7VE1LCyXrjMGRtG8yBdDHIz6+78rIqM3ueg1sNmd3IoOkJPoUfFFRQf9EUxOd1QkJNEeVltL0dPvtg69XwxAhEIVwLuhQfk5ExgFoAhvkmAGsBPCIUmqLv52IyLsARnr46B6l1GuBDlhEbgZwMwAUB9OhRhMyurrYunKkp1/TgdnM1wcf0PcQLIWFwM9+Rr/E1q08ZmEhG5RZrSwQWFNDuZGSQgfzCy9wZfKd79BuHhIWLWIFv40bqdmMRAvDm261sj2bp76pg5WRI6mVW1udWX6GknBNRrFa3RwPoNb21YTi6FHgV79yzhoMDF9FZSVbld5zT19lowk7gfgQusCyFU+ISDwYftqplGoK5kBKqZAUdlZKPQXgKYDF7UKxT01wNDZSFvgzm6enA59/3v/jmEzMfzjuOOd73d0sstfU1FueGP7Plhbg/vuBr36VFp9x4/xnWPskLo6ZX+++y3DLI0ecBvgZM4ClS3136xmMmEwMKXv6aa5+TCYmo2Rl0YmcnEwTUkpK34vb3e3bubNmDbW7t2ZARUVcFm7f7ubY0ESCoNIrlVI9ALy0f9IMFwI18RqtgkPJtm1cGbjniAGcsG/bxn9rali7B6BcueaaAYTExsUB557LZhVVVdSG2dkRr69dU0N9ZDbT7BPWUuOnnUZtvmoV058zM+kX+OADCnSjSY3rD1xTQ0e0N/+BzUbF6ishAuBM4t13tUKIAtHJt3dDRC4G8CiAPABviMgWpdQ54TiWxcLKmc3NvKePOWbgsfPDDUM++PO1traG/pl+7z3PHeVqauhTSE7myqC9nbJJhPlVDz0E3HdtBTI3r6H5p6eHs9Szz+ZMP5AoISPbK8J8/jnNYXv2OLOQzWb6YC+6KEyKwWRi9vXkycDy5QyrNSrhtbdTIxtVBdvbqany82n/9zYL6OzkCsJf8lpKCotWaSJOTCgEpdQrYFmMMB6Dk53//Y+rXsBpCp07lzPIcDd3GSqYzcxc/uc/vXdK6+yk/Jw9O7THPnqUQt8Vq5WlhVJTnXLdbuf7iYlAUaFC5kcrUPuNF5A5JY5mDpOJjszHH2cW1+23h6kq3sDYs4cm96Sk3u1Ge3o4id63D/jhD8PUy8BkcsYZNzXxgmZk0M6/ahXLWVit9Ddcey239fUQJSTwQbTbfdcmD1VRLE3Q9EshiMhIpVRNqAcTTl5/nbVeiop6r1htNt7XNTV8sNyFjcYzCxfS2WtUJDUmfd3ddAQfPsyon48+ojwJVbP79PS+SqG2lgLSkEVGXpQxUR1V/SnmVP0H+8zFGJsbj0RjMZCdTbt4WRnw5JPA974XU/V5LBbqq8zMvqui+Hi6LsrLeW9fcYWfnXV0UGtu3+6sIz5nTmBmL5He202YwNfXvx6cXTAhgUvGXbt8xww3NLAUhibi9LeFyJshHUWYqakBXnmFD5B7MIjZzFXwgQNMhtIERnw8o3mWLqWT+eBBypuXX+a1HDeOMui554A772Q/hf5nqjiZP5+TVVeOHu0tk9rbqaTi4gAoheN2v4LO1BGwSnyfsPkvmtZs385wyBhi+3Y6yT2ZyAwKC2lG6+rysaNNm1gv/G9/40537WJG4fe+x0qC/f1hfPbu9MI55/DG8FZ3pK3N6Z/QRJz+KoTYmUYFwIcfcoXqq0LxyJEMIvFW70XTl4QE4JJLGCV4wQV874wzqCSOPZaWhOJiTgaffTY0Ctfod+CqFNwjIa1WYOJE/p3eVo30lsPoTvSxRBHhzbFx48AHGEK2bfMfzZqQwPOtrPSywfbtLN2dmckZUV4ef5ixYxlG+9xztD1FimOPZXejQ4dosjOUkc1Gh31jI/Dtb4duSakJiv4qhKdDOoows2OH//srJYWzMV2bK3gSEujQnTSJwt/d6pKQQFPd88/TpOSNo0c5mf3kE5pCPE1cU1O54uju5qqku5vWDIuFTuy2NiYNGxaOeEs7YDLDrjgo19I8vUhMpKkihrDZAmsDanST64NSFPjZ2Z5t8vHxXB29+CKdPpHivPOAu+/mMvLgQSqHqiquCn7609C03NP0i/46lf8X0lFoBjX79zPIxFc4flISTXc7dvRtb9nQAPznP1QGIs78pzFjmE9wzDG9tx83jklra9dyctvTQ4VQWMicBdckV2t8MmC3o61VoahIvM+4/SVURYGSEkZ5+sJu58tjrkV5ubORsjcSE3nxtm5ldEWkmDKFr7Y2Zx0N7cCLOsPChzB1qv+Zf0cHbbUxJhMGBYFOrEVoJXClsZEhoVu3UgEUF1N+jR3LFdvDD1OJuDNiBM1Vjz3G/Kmnn2awgHtya3NaEeqkAGloxeTJXgamFO0uoQ6JGiCzZvkv811b23tF1IuGhr7lJjwRF0dtHQ3S0mjC0sogJhgWPoR585xhiN6oqeFKVtfVCp5Au0fa7X3D/V96if6AoqLe5hEjsCU7mwFA3oSiycTVx/z5wG23cbJZXu58VRwUHJ59EeZOqEdqopcboKqKy5Bx4wI7kQiRkQFceimtKhZL388bGnidLr7Yyw4C/WFsNt3YRgOg/yajQeVDGDmSD40RdupqNrDZuKqeODEC3beGKBMmUDDZbN4VquEPmDTJ+V5zM30P7uVwXElPp2DfscN//bg5c2iO2rWLM+e4OI5tzOjZkGWXMtQsKYlLCZOJToejR7ksue22mAo5NTjvPA715ZepFBMTeZ2tVp7G975HU5lHxo93NpfwphwM+5xrfRDNsKVfCkEp9USoBxJuLryQZsqXX6awMASUCCtkXn21XrX2l+xsCuNPPvFeoqa2lkmvrsL/8OHAIhfj45mgFUhB0fh49hjujTD0aepUJlSVllKqjhoF3HQTbTMxWpxOhFUzTjuNPpbDh7nKmjKFATs+r116Or+4bp33DOu6Oq6MPNUC0Qw7YiJTORKIMCRy/nzOIJub+WDp0hWh4aqraNooL+eM1TAN9fSwgU5WFmWv6yQ80PB3r1E0wSDC5cmkSc5ZcSAhPDFCWhp72wfN5Zcz8a68nArQMA1Zrfxh0tKAW2+NydWRJvIMG4VgkJAATJ8e7VEMPTIy2MPgrbcY+WP4a0wmYPFi4Pzz+zo+CwqcUTK+ZLPFEuJ2mSLDRwCmpgJ33QWsWMEfpqvLef6nn84EkhEjoj1KTYwQSMe0QObP9mDLYWsCQylnFnVlJRXanDm0cniNqY8SaWmckF54odMsl5fnvbxNbi7NQDt3em+i09lJa45W4gMgJYUhWeedx25mGzdSy7a3O1tgBuqA1gxpxF/nSxHpAlAF35FFZqVUxMtAzpw5U5WWlkb6sBGjq4vhlKWlFIppaZx5t7Rw5X/bbcAJJ0R7lAOjupo5BQAVhOvEvaOD0V/f+Ab9PJoB0NTEjOXPP+fNlJREpdDRwaiL7343Aj1Je1NbywAvo3qIe7tUTfgQkU1KqZl93g9AIWxWSvksYhzINuFgKCsEpYA//Qn4+GP6+9wtHG1tDDv88Y8Hvz+wspKhpYcO8W+j/0x6OqvQ6rI2A8RiAX7+c0pfTyFddXVUED/9aUQqvh4+zETEHTucpkKluFq86ir/7RI0A8ebQghknRjI3EzP30LMoUOM2vGkDACuFtrbgWXLWGRuMDN6NPDAA5y8HjjgDACaOlV3UQwJW7awcJ+3mUN+Pj9fvz7sVUYrK6mbACYfGve23U7T4YMPAvfeG/HFisaB3zALRwtNj4jI1/xto+kf69dTGPryfebl8Vl3r/45GBFhzsDZZ9PUPWOGVgYh4513/Kfg5+UxIiAUJWm9oBRNoCZT35pXJhMnAT09LISoiQ4Djbv7aUhGoelDdbX/vAijKoEuyKfxiRFe6ouUFM4sfNXJGCDl5VyI+PIVFBTQlFStG/VGhUCijLZ5+wiAXtiFiZQU36U2DDyVg9BoepGYyJvJ15LLSDMPY7RReTn/9bXqNSJiDx70Hnk2LFGKduT16+mNT0lh7a2pU0P6mwWypwIA5wBodHtfAKwP2Ug0vZg9m2UdfNHaSvOvtrdqfHLqqWyE4y2NHKBj2aimFyaCsUYNOBFxKNHZCfzlL0xVj4+n6aCnh41e8vKAO+7wXf8lCAL59ZcDSFNKVbi9ygGsCckoNH04/njmC3mrJGq38xm+4IJBlXCriQbz5/Nfbz0PenoY43zmmWEdxsiR/vMBjSRyPclxYLczBG/TJnrhi4pYWqGggEECnZ0sCXz0aEgOF4hT+UallMeq7Eqpr4RkFJo+xMdT8dtsXCkapl2laOotK2MpjnnzojtOzSAgP5/9j2trOYswpt9KUZAcPMguZkabuTBx7LHMVu/TxtSFpiZGncVY4dnosX8/sHlz75AsV3JzGW743nshOZyeW8YwY8YA999PwV9fT8Vw8CADRr71LeDaa/XqQBMgc+cC99zDWk6HDjlvplGjgB/+kKFdYcZsBr72NTZTamvr+3lLC5XFddcNn8oiflmzhjkivi7IyJEs2hiCgIBAEtM+VUqdNNBtwsFQTkxzp7ubD0x8PBWCfmA0/aa5mbPK5GQvnXXCy9atwF//yvvZqNZqs9EScvPNuhJ3L+67j2Yhf1FiBw8Cv/1twL/nQBLTJvuINALoXNZ9xsJMYqKXNomaoU17O7B9O1vLJSezhvhAU3kzM6PaGnD6dMqunTsZeWQy0UQ0ebJuUNWHhATPyylXDMdLCKKNAtlDIPraNtCBaDQaF+x24PXXgeXLaQow6nmIADNn0q4Sa9UNgyA+nopBFy30w5w5wH//61uBNzVRo4bgfvCrEJRSFa5/i8gDAMwAtgDYopTaN+BRaDQaJ0pRCLz1FhvbuOYP2O3Ap5/SqXTXXbqr01Bnzhz2mW1v91w22G5nKOJXvxoSO3LQLkml1I8B/BFAK4BLRWTA7TRF5NcisltEtonIKyKSNdB9ajSDlkOHgJUrGVbonkxmMjHaoKwM+MBj8J9mKJGZCXzzm/TEu0aIAfTAl5UBZ50FnHxySA4XsEIQkd+LUAUppWqVUiuUUg8rpb4egnG8A+B4pdQ0AHsB/CgE+9RoBidr1tAe7Mugnp/PFYTO4Br6zJjBin/uEWLx8cAtt7AkcIiiTILxQrQBWCYiVyql2kXkbAA/UUoNOBJeKbXS5c+PAFw20H1qNIOWPXv8R4ukplIotLWxXZ1maDNhAntWNDYyPCshgclpIY47D1ghKKXuFZGvAFgjIt0A2gHcHdLRkBsAPB+G/Wo0gwORwOo8KKXjj4cb2dlhDRUOxmR0BoCvg4ogD8B3lFLvB/H9d0Vku4fXUpdt7gFgBfBvH/u5WURKRaS0vr4+0MNrNIOHE07gTNAXLS2cIQ7iSCNN7BGMyegeAPcppT4QkRMAPC8idyqlAsqZVkr5LJQiItcBuADAGcpHtpxS6ikATwFMTAt49BrNYOH004EVKxhu6qlCqVJ0Mt5449BfIbS3A9u2scSGkYdRWBjtUQ1ZgjEZLXb5/2cich6AlwGcOtBBiMi5AO4CsEAp1THQ/Wk0g5pRo4BLLwWef57/T0lxftbTwx6U06axiulQxW5nDsbrr/OczWamMwNcQd14I5ClgxFDjd/SFT6/LJKslPJSQjGo/ewHkAjAKNn3kVLqVn/fG06lKzTDDKWAdeuA//2P5iHDr2A2A4sXU2EkJkZ7lOHjhReoDNzzMJRib+i8POD//k+bzPrJQEpXeCUUysCxn/CWWdRoBhsiwIIFLGe7dy+VQmIicMwxnhOUhhKHD7N/Q0lJ39BbEZaALitjeO4FF0RjhEOW8LVH0mg0AycuDpgyJdqjiCzvv+8/D2PkSPpZzjlHN98OIbp4skajiS327fOfW5GcDHR0cOWkCRl6haDR+KGzk2ZrpZggrPPAwkygeRjGtpqQoRWCRuOFtjb6NVevZo96Q07NnQtcdNHAq1BrvDBtGvDqq74rfLa1MUErimW8hyLaZKTReKC1FfjFL1hjLjeXwS5jxtCfuXEj8OCDQHV1tEc5RDHCaS0Wz58rxUJvS5boBgohRisEjcYDL7wA1NSwlW1CgvN9s5lKwWYD/vznwC0bmiDIzQWuvhqorOzbHKanhzWcpk4F5s+PzviGMNpkpNG40dwMrF/vOyE2Lw+oqGD04/jxkRvbsGHxYobXvvACL7ThKzCbgTPPZB6Gq6bWhAStEDQaN8rKnDlg3hDha8+eYaQQOjuBzz/nLD0zk3kC4XLqitBZM2sWcOAAtXR8PDBxok5GCyNaIWiihs3G6r2xFihiC7AhrMnk3cw9pLBYgGXLgHfeoTIAqDELCoArrgBOPDF8xzabmYyniQhaIWgiSkMDG3298w4dt0lJTMhduJBle2KBESNYSsdfdWmbLXbGHDZ6eoBHH2WBuaKi3maalhbgkUeAm25iQT7NoEcrBE3EKCsDfvMbWh7y8yl4LRZg1Srg3XeB224DTjop2qOkI3nMGPYu91Z6vrublSSmTYvs2CLO++8DW7bQLuauHTMyqCCefZYF58JYp18TGXSUkSYitLYCv/sdzcDFxVwZAJQno0dTOTz+OMvYRBsR4Ctfodm6vb3v5xYLA2CuvNJ5HkMSu501hQoKvC+VkpK43YYNkR2bJixohaCJCJ98wghCbxWLU1Jok1+1KrLj8sbkyexY2NYGlJcDtbUMfa+o4L/XXUcz15CmsZE2Pn9O3MxMriI0gx5tMtJEhNWrgZwc39sUFNBCcfXVsZFvNH06TeRbtgA7dtCnMH48A1/S06M9uggQaItOkcA98ZqYRisETURobe3d58UTcXEsEWGxsHZZLJCUxOjHuXOjPZIokJlJR4nhMPFGSwu1pGbQo01GmoiQlQV0dfnexmKhT2Eo930ZVMTHA2edxZRtb9hs1OI6ymhIoBWCJiIsXsyoHV/U1QGLFtGXoIkRzjiDpSSqq/vW6bBa6WA56yzd53iIoB89TUSYNYurhCNHPH/e2kpT9KJFkR2Xxg8ZGcDddzMWt6KCr8OH/7+9+4/1qq7jOP58CcgPUZl4mYok2hil5mCaM3UNJia5pmFlOio3Xc6ZkySbGk1NcqOxtVazZZss28jGJhnTsotkqS1KMFQU8UeTgRKCv/ACegXe/XE+V75cv/fe871wv+ec7309tju+53vP93te4uX7vp/POefzzv7cvDlb9vWyy8p3d6H1ywH1VC6aeypXy2uvwcKF2Uhh7NjsPEFnZ1Ykhg2DOXMGX3OwyojIFpV79tls7q+tLbtD2c0hKqmnnsouCNZUHR3Z8tHLl++7onHatGzF476uQjKzg6OnguCrjKypRo/OpoU8NWRWPj6HYGZmgAuCmZklLghmZga4IJiZWeKCYGZmQEkKgqT5kp6RtEZSuyTf9mhm1mSlKAjAwog4LSKmAA8CtxYdyMxssClFQYiI7TWbhwHVvVvOzKyiSnNjmqQ7gW8B7wK+bcnMrMmaNkKQ9IiktXW+LgaIiHkRMQFYDFzXy/tcLWmVpFVbt25tVnwzs5ZXurWMJJ0APBQRp/a1r9cyMjNrXE9rGZXiHIKkSTWbFwEvFJXFzGywKss5hAWSJgN7gQ3ANQXnMTMbdEpRECLiK0VnMDMb7EoxZWRmZsVzQTAzM8AFwczMEhcEMzMDXBDMzCxxQTAzM8AFwczMEhcEMzMDXBDMzCwpxZ3K1hwffghvvQUSHHUUDPX/fTOr4Y+EQaCjA1asgPZ2eP/97LnRo2HmTJg+HUaMKDafmZWDC0KLe/ddWLAANm+GY46Btrbs+Z074b77YPVqmDsXRo0qNqeZFc/nEFrcokWwbRtMnLj/SGDUKDjxRHjlFViypLB4ZlYiLggtbPNmePppOO64+t+XYPx4ePxx2L69/j5mNni4ILSw9euzP6We9xk6FPbuhZdeak4mMysvF4QW9sEHvReDLlJ2BZKZDW4uCC1s7FjI0zI7AsaMGfg8ZlZuLggt7JRTYPhw6OzseZ8dO7JiMGlSz/uY2eDggtDCRo6EWbNg40bYvfvj3+/shC1b4NJLYciQ5uczs3LxfQgt7oILYNcuWLYMDjkEjjgimyLquqroiivgrLOKzWhm5eCC0OKkbJRwzjnwxBPw4ovZczNmwNlnZ0tYmJmBC8KgMW4cXHJJ0SnMrMx8DsHMzAAXBDMzS1wQzMwMcEEwM7NEkedW1pKStBXYMMCHORrYNsDHGAjO3VxVzQ3Vze7c/XdCRLR1f7LSBaEZJK2KiDOKztEo526uquaG6mZ37oPPU0ZmZga4IJiZWeKC0LdfFx2gn5y7uaqaG6qb3bkPMp9DMDMzwCMEMzNLXBBykDRf0jOS1khql9RDl+JykbRQ0gsp+x8kVaINjqSvSXpO0l5Jpbwao5akmZLWS3pZ0s1F58lL0iJJb0haW3SWvCRNkPSopHXpZ2RO0ZnykjRC0r8lPZ2y/6joTN15yigHSUdExPb0+Hrg5Ii4puBYfZL0BeCvEbFb0k8AIuKmgmP1SdKngb3A3cCNEbGq4Eg9kjQEeBE4H9gEPAlcHhHPFxosB0mfBzqA30bEqUXnyUPSscCxEfGUpMOB1cCXK/L3LeCwiOiQNAx4ApgTESsLjvYRjxBy6CoGyWFAJapoRLRHRFdrnJXA8UXmySsi1kXE+qJz5HQm8HJE/DciOoHfAxcXnCmXiHgMeKvoHI2IiM0R8VR6/B6wDhhfbKp8ItORNoelr1J9lrgg5CTpTkkbgdnArUXn6YcrgT8XHaIFjQc21mxvoiIfUFUnaSIwFfhXsUnykzRE0hrgDWB5RJQquwtCIukRSWvrfF0MEBHzImICsBi4rti0+/SVO+0zD9hNlr0U8uSuCNV5rlS/9bUiSaOB+4HvdhvBl1pE7ImIKWSj9TMllWqqzg1ykoiYkXPX3wEPAbcNYJzc+sot6QrgS8B5UaITRg38fZfdJmBCzfbxwOsFZRkU0vz7/cDiiFhadJ7+iIh3JP0NmAmU5qS+Rwg5SJpUs3kR8EJRWRohaSZwE3BRROwsOk+LehKYJOlESYcClwHLCs7UstKJ2XuAdRHx06LzNEJSW9eVfpJGAjMo2WeJrzLKQdL9wGSyK182ANdExGvFpuqbpJeB4cCb6amVFbk6ahbwC6ANeAdYExEXFJuqZ5IuBH4GDAEWRcSdBUfKRdJ9wDSy1Te3ALdFxD2FhuqDpHOBx4Fnyf49AvwgIv5UXKp8JJ0G3Ev2c3IIsCQi7ig21f5cEMzMDPCUkZmZJS4IZmYGuCCYmVnigmBmZoALgpmZJS4IZmYGuCCYmVnigmAtQdJESbvSwmFdz32sT4GkkamvRaekow/wmCMl/T0tgY2k69M6/Q2tGSVpjKRrDyRLjmN8rPeBpEMlPSbJS9gY4IJgreWVtHBYV5+Cu4AvAicDl0s6OSJ2pX0OxnpDVwJLI2JP2r4WuDAiZjf4PmPSaxuiTN5/w78hWzfnI2m57hXA1xs9trUmFwSrjNQp6/z0+MeSft7L7s3oUzAb+GPK8yvgJGCZpBskfSN1x1oj6e6aUcQDklanjllXp/dZAHwy7bswjXZqf5O/UdLt6fHENAr5JfAUMKGnY9XqpffBA+m/w8wFwSrlNmCepNlk6+Df0Mu+A9qnIC1kd1JEvAqQ1oh6HZgOPEz2W/c5aTSyh30fuldGxOnAGcD1ksYCN5NGNxHx/RyHn0zW5WwqMKqXY+WxFvhsA/tbC/PcoVVGRDyWVrucC0zrmqqRNJ9sBcxaA92n4GiyhffqOQ84HXgyi8tIsoYokBWBWenxBGAS8L8Gj72hpu1ib8fqU0TsSedTDk8dyGwQc0GwypD0GeBYYFvXh5ekY6j/c9xwnwJJ3wG+nTYvBGbVbkdE7et3ASN6eivg3oi4pdv7TyNb8vhzEbEzrYdf7z12s//ovfs+O/o6VoOGA+8fwOutRXjKyCpBWXP1xWTnAXZI6loOeyqwps5LGu5TEBF3pWmbKRHxevftbvu+DQyRVO8DfQXwVUnjUvajJJ0AHAm8nYrBp4Cz0v7vAYfXvH4LME7SWEnDyRoc9aSnY+WSpqy2RsSHeV9jrcsFwUpP0ihgKfC9iFgHzAduT9+eQp2CEBG7yVqd/oWsEfuSiHjuIEdrB86tc+zngR8C7ZKeAZaTjWweBoam5+YDK9P+bwL/SC1EF6YP5zvIegU/SC9NVHo51n5S74N/ApMlbZJ0VfrWdKD0vQSsOdwPwSpN0j1k0zqfAB6MiFw9aiW9CpwREdsO4NhTgbkR8c3+vkfRJC0FbomI9UVnseJ5hGCVFhFXRcResqtrjqy9Ma2erhvTgGHs67jV32P/B3i03mWeVZCm0h5wMbAuHiGYmRngEYKZmSUuCGZmBrggmJlZ4oJgZmaAC4KZmSUuCGZmBrggmJlZ4oJgZmYA/B84qOy0PfvCMwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_dataset('Scatterplot of the training data', train_x, train_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 1.3382818 -0.98613256]\n", + " [ 0.5128146 0.43299454]\n", + " [-0.4473693 -0.2680512 ]\n", + " [-0.9865851 -0.28692 ]\n", + " [-1.0693829 0.41718036]]\n", + "[1 1 0 0 0]\n" + ] + } + ], + "source": [ + "print(train_x[:5])\n", + "print(train_labels[:5])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Подход\n", + "\n", + " * Задаём функцию потерь (loss function) $\\mathcal{L}$\n", + " * Определяем модель $f_{\\theta}$ с параметрами $\\theta$\n", + " * Подстраиваем $\\theta$ для минимизации $\\mathcal{L_{\\theta}}$ на обучающей выборке\n", + "$\\theta = \\mathrm{argmin}_\\theta \\mathcal{L_\\theta}(X,Y)$\n", + " * Проверяем качество модели на тестовой выборке\n", + "\n", + "Результат: $f_{\\theta}$, которая делает предсказания на новых данных: $\\hat{Y} = f_{\\theta}(X_{new})$" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Функции потерь\n", + "\n", + "* Определяют (формализуют) цель обучения, т.е. фразу _\"точно предсказать\"_\n", + "* Выбор обусловлен требуемыми свойствами (непрерывность, дифференцируемость)\n", + "\n", + "**Часто используемые функции для регрессии**\n", + "\n", + "Абсолютная ошибка: $\\mathcal{L}_{abs}(\\theta) = \\sum_{i=1}^n |y_i - f_{\\theta}(x_i)|$\n", + "\n", + "Среднеквадратичная ошибка: $\\mathcal{L}_{sq}(\\theta) = \\sum_{i=1}^n (y_i - f_{\\theta}(x_i))^2$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "# helper function for plotting various loss functions\n", + "def plot_loss_functions(suptitle, functions, ylabels, xlabel):\n", + " fig, ax = plt.subplots(1,len(functions), figsize=(9, 3))\n", + " plt.subplots_adjust(bottom=0.2, wspace=0.4)\n", + " fig.suptitle(suptitle)\n", + " for i, fun in enumerate(functions):\n", + " ax[i].set_xlabel(xlabel)\n", + " if len(ylabels) > i:\n", + " ax[i].set_ylabel(ylabels[i])\n", + " ax[i].plot(x, fun)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAADZCAYAAADsdQBUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3hUZdrH8e+dSknoPYGE3iEUKVbEhhUQQbD31dUVlLWvvXfBsrvWtQIiiNhFBUGlSAmhQyiB0EJPKOn3+8cM+2ZjIJlkZs6U+3Nd52Imc8pvJuHMc57zFFFVjDHGGGMCXYTTAYwxxhhjKsIKLcYYY4wJClZoMcYYY0xQsEKLMcYYY4KCFVqMMcYYExSs0GKMMcaYoGCFFmOCiIioiLTx07Eai8hsEckRkRf9ccwSxz4oIq38eDwRkfdEZJ+ILPDXcQONvz93YzxlhRYTMkTkMhFZ6D7xbheRb0XkZKdzBbGbgN1ALVUd66uDiMgsEbmh5M9UNU5VN/jqmGU4GTgLSFTVPn48bkBx4HM3xiNWaDEhQUTuBF4BngIaAy2AN4DBTuYKcknASg2PESiTgE2qesjTDUUkyhvrePuYxoQkVbXFlqBegNrAQWD4cdaJxVWo2eZeXgFi3a8NADKBu4EsYDswBDgPWAvsBe4vsa9HgMnAR0AOsAxoB9zn3n4LcHaJ9ZsB0937SQduLLWvT4EP3PtaAfQ+zvtQoE2J9/0BsAvIAP4BRLhfawP8AhzAVVsyyf1zAV525zwApAFdyjjOf4ACIN/92Z7p/tkTJdYZAGSWeL4J+Lt7nweASUC1Eq8PBlKBbGA9MAh4EigCct3Hec3D93kN8CvwArAP2AicW+KY1wAb3J/tRuDyMt7r9e7jF7kzPOr++Y3u39de9++vWanfw63AOmBjGftMdq9zPbAZmO3+eT/gd2A/sBQYUGKblsBsd9YfgdeBj6qwvzLf+7H+Nrz5udtii68WxwPYYktVF/eXXyEQdZx1HgPmAY2Ahu4T/ePu1wa4t38IiHZ/We0CPgHigc7uL7VW7vUfcT8/B4hyn9g3Ag+U2H5jiWP/gqvWpxqQ4t73GaX2dR4QCTwNzDvO+yj5pfIB8IU7YzKuAtb17tcmuPNEuI97svvn5wCLgDq4CjAdgabHONZ/+N9CSunnA/hzoWUBrkJaPWAVcLP7tT7uL8mz3JkSgA7u12YBN1TyfV6Dq3B1o/vzuwVXoVSAmrgKSO3d6zYFOh/jvV4D/Fri+UBcX+g9cRV4X8VdUCiRb4b7fVYvY3/J7nU+cOeo7n7Pe9y/6wj3Z7EHaOjeZi6uQkAMrttV2fy50FKh/R3vvR/rb8Nbn7vT5wNbQntxPIAttlR1AS4HdpSzznrgvBLPz8F1OwBcX75HgEj383j3ybtvifUXAUPcjx8BZpR47UJcV+ilt68DNMd1BR9fYv2ngf+U2NePJV7rBBw5zvtQXFfKkUAe0KnEa38BZrkffwC8iauNRsntB7q/fPrhvmo+zrH+g+eFlitKPH8O+Jf78b+Bl49xnFkco9BSgfd5DZBe4rUa7m2b4Pri3g8Mo4yCRanjXcP/FlreAZ4r8TzO/SWdXCLfwOPsL9m9TqsSP7sH+LDUet8DV+O6nVkI1Cjx2kf8udBS0f0d870f62/DW5+7r/6f22KLqlqbFhMS9gANyrnP3wxXFfdRGe6f/XcfqlrkfnzE/e/OEq8fwfXFxTFe213G9nHuY+xV1ZxSx04o8XxHiceHgWoVaLPQANcVeen3dHS/d+OqbVggIitE5DoAVf0ZeA3XrYedIvKmiNQq51ieKP1ejn5mzXEVHD1V3vv8n2Oq6mH3wzh1tU+5FLgZ2C4iX4tIhwoe93/+XlT1IK6/s5LH3VKB/ZRcJwkYLiL7jy64alSa8v9/J4ePsa1H+yvnvZf5t1FKpT/3MvZljNdYocWEgrm4brEMOc4623Cd5I9q4f6Zr20D6olIfKljb63ifnfjuvIv/Z62AqjqDlW9UVWb4bpCfuNoV2lVHa+qvXDd9moH3FXBYx7CdUV9VBMP8m4BWh/jNT3Odsd9n+VR1e9V9SxcBYPVwFsV2Y5Sfy8iUhOoX+q4x8td1jpbcNWM1Cmx1FTVZ3C1o6onIiU/3+ZV2N8x3/vx/jZKqNLnboyvWKHFBD1VPYCrPcrrIjJERGqISLSInCsiz7lXmwD8Q0QaikgD9/of+SHbFlztZ54WkWoi0g1XY8qPq7jfIlwNeJ8UkXgRSQLuxP2eRGS4iCS6V9+H68uuSEROEJG+IhKNqxBytAFqRaQC54lIPRFpAozxIPI7wLUicoaIRIhIQokr/51AmWODlPc+j8c9zsxF7gJHHq5beBV9r5+486aISCyuXmnzVXVTBbcvy0fAhSJyjohEuv8eBohIoqpmAAuBR0QkRkT647rtWKn9He+9H+tvo+SOq/K5G+NLVmgxIUFVX8J1Uv0HroauW4DbgGnuVZ7A9aWQhqu3z2L3z/xhFK42CduAz4GHVXWGF/b7N1wFjw24enJ8Arzrfu0EYL6IHMTV82W0qm4EauG64t6Hq7p/D67GnxXxIa4eKpuAH3D1DqoQVV0AXIur59IBXI2Tj17FjwMucQ/sNt7D93k8EcBYXJ/7XuA04K8VzPsT8CAwBVctSGtgZEW2Pc4+t+DqQXU///83ehf/fx6+HOiP63fyBK7PN6+S+zveez/W30Zplf3cjfEZUa1IDacxxhh/EpFJwGpVfdjpLMYECqtpMcaYAOC+ddfafftsEK5alGnlbWdMOLFRFY0xJjA0AabiavCbCdyiqkucjWRMYLHbQ8YYY4wJCnZ7yBhjjDFBwQotxhhjjAkKVmgxxhhjTFCwQosxxhhjgoIVWowxxhgTFKzQYowxxpigYIUWY4wxxgSFoB9crkGDBpqcnOx0DGOC1qJFi3arakOnc4QKOycZUzXHOycFfaElOTmZhQsXOh3DmKAlIhlOZwgldk4ypmqOd06y20PGGGOMCQp+K7SISHMRmSkiq0RkhYiMLmMdEZHxIpIuImki0tNf+Ywx5lhEJFJElojIV05nMSac+bOmpRAYq6odgX7ArSLSqdQ65wJt3ctNwD/9mM8YY45lNLDK6RDGhDu/FVpUdbuqLnY/zsF1Akgotdpg4AN1mQfUEZGmVTlucbGydMv+quzCmKC0ZPM+pyOEBBFJBM4H3vbG/gqKilm+9YA3dmVM0Ch0/91XdZJmR9q0iEgy0AOYX+qlBGBLieeZ/Llg45FXf07nkn/9bicJE1a+W76DoW/8zvSl25yOEgpeAe4Gir2xs+e/X8PF//yd7NwCb+zOmKDw2/o9XPDqr8xas6tK+/F7oUVE4oApwBhVzS79chmb/KlYJiI3ichCEVm4a9fxP4Cr+idRr2YMoycu4Uh+UaVzGxMsdhzI5d6paXRNqM2gzk2cjhPUROQCIEtVF5WzXoXPSed2aUJ+YTHfLdvhzajGBLRpS7ZSq1oU/VvXr9J+/FpoEZFoXAWWj1V1ahmrZALNSzxPBP50qaiqb6pqb1Xt3bDh8YeXqFszhheHp7B+1yGe+HplFdIbE/iKi5Wxk1PJKyhm3MgUYqKsg2AVnQRcJCKbgInAQBH5qPRKnpyTUprXIbl+DT5fstUngY0JNIfyCvlu+Q7O79aUatGRVdqXP3sPCfAOsEpVXzrGatOBq9y9iPoBB1R1e1WPfXLbBtx4Sks+nr+ZH1furOrujAlYb/+6gd/S9/DwhZ1o1TDO6ThBT1XvU9VEVU0GRgI/q+oVVdmniDCkRwLzNu5h2/4jXslpTCCbsXInRwqKGJJSpdYegH9rWk4CrsR1pZLqXs4TkZtF5Gb3Ot8AG4B04C3gr946+N/PaU+nprW4e0oaWdm53tqtMQFj+dYDPP/9Gs7p3JhLT2he/gbGMUNSElDF2hyZsPD5kq0k1KnOCcn1qrwvf/Ye+lVVRVW7qWqKe/lGVf+lqv9yr6OqequqtlbVrqrqtWElY6MiGT8qhcP5hYydvJTi4qq1YDYmkBzJL2L0xCXUqxnDMxd3w1WxabxJVWep6gXe2Fdyg5r0aFGHzxfbLSIT2nbl5PFr+m4GpzQjIqLq56WwuuHdplE8/zi/E3PW7ea93zc5HccYr3ni65Vs2H2Il0akULdmjNNxTAVc3COBNTtzWLHNejaa0PVF6laKipWhPap+awjCrNACcHnfFpzZsRHPfruaVdtLd14yJvjMWLmTj+dv5qZTWnFSmwZOxzEVdEG3ZkRHClOttsWEsKmLt9ItsTZtG8d7ZX9hV2gREZ4d1o3aNaK5fcIScgusG7QJXlnZudwzJY3OzWpx59ntnI5jPFC3ZgxndGjMF6lbKSzyyhAwxgSUVduzWbk9m2E9E722z7ArtADUj4vlheHdWZd1kKe/sZG5TXBydW9eyuH8QsaNTCE2qmpdCY3/Xdwzgd0H85mzbrfTUYzxus+XbCUqQriwezOv7TMsCy0Ap7VryHUnteT9uRn8vNq6QZvg897vm5izbjf/OL8TbRp5p+rV+NeA9o2oWyOazxZnOh3FGK8qKlamLdnK6R0aUc+L7ezCttACcPeg9nRoEs/dn6WxKyfP6TjGVNiq7dk8++1qzurUmMv7tnA6jqmkmKgILurejBkrd3LgiA3rb0LHnHW7yMrJY1hP7zTAPSqsCy3VoiMZP6oHObmF3P3Z0ipP5GSMP+QWFHH7hCXUrhHNMxd3te7NQe6SXs3JLyzmqzQbs8WEjs8WZVK3RjQDOzT26n7DutAC0K5xPPef15GZa3bxvnWDNkHg6W9WsS7rIC+N6E79uFin45gq6pJQi/aN4/lskd0iMqHhwOECfli5k8EpCV6fSiTsCy3gmlTx9PYNeerb1azdmeN0HGOO6efVO3l/bgbXn9ySU9oef44bExxEhEt6JbJk837Ssw46HceYKvsybRv5hcVc0st7vYaOskILrpPGc5d0p1a1KOsGbQLWrpw87pqc5mqHNai903GMFw3u0YzICGGKNcg1IeCzRZl0aBJP52a1vL5vK7S4NYyP5flLurN6Rw7Pfrfa6TjG/A9V5e+Tl3Iwr5Dxo3pY9+YQ0yi+GgPaNWTq4kwbs8UEtXU7c0jdsp9hPRN90t7OCi0lnN6hEdecmMx7v21i1posp+MY81/v/76JX9bu4v7zOtLOSyNLmsAyvHdzdmbn2ZgtJqhNXpRJVIQw1Mu9ho6yQksp957bgXaN4/j75DT2HLRu0MZ5a3bk8NS3qxnYoRFX9U9yOo7xkYEdGlG/ZgyfLtzidBRjKqWgqJipizM5o2MjGviok4AVWkqpFh3JuJE9yD5SwD1T0qwbtHFUboFr9uZa1aJ57hKbvTmUxURFMLRHAj+u2mkXTCYozVydxe6D+Yzo3dxnx7BCSxk6Nq3FPed24MdVWXw8f7PTcUwYe+67NazekcPzw7v57MrFBI4RJzSnoEj5fIlNomiCz6cLt9AoPpbT2vmuZ6PHhRYRqSkiId8K8NoTkzm1XUOe+Hol6VnWDdr43y9rd/Hubxu55sRkTm/fyOk4xg/aNY4npXkdPl24xWp5TVDZmZ3LzDW7uLhnIlGRvqsPKXfPIhIhIpeJyNcikgWsBraLyAoReV5E2vosnYMiIoQXhnejRkwUt09IJa/QukEb/9lzMI+xny6lfeN47j23g9NxjB+NPKE5a3ceZMmW/U5HMabCPluUSVGxcukJvrs1BBWraZkJtAbuA5qoanNVbQScAswDnhGRK3yY0TGN4qvx3LBurNyezQvfr3E6jgkTqsrdn6WRnVvAuFEpVIsO+YpNU8IF3ZtRIyaSSQusQa4JDsXFyqcLt9CvVT1aNqjp02NVpNBypqo+rqppqvrfAQRUda+qTlHVYcAk30V01pmdGnNFvxa8NWcjv1pXROMHH83fzE+rs7jv3A50aOL9wZlMYIuLjeLCbs34Mm0bB/MKnY5jTLnmbdxDxp7DjDzB95O3lltoUdUCABEZLiLx7scPishUEelZcp1Q9cB5nWjTKI47P01l36F8p+OYEJaelcMTX63ktHYNuebEZKfjhIRgbId3aZ/mHM4v4sulNomiCXwTF2yhVrUoBnVp4vNjedJa5kFVzRGRk4GzgfeBf/omVmCpHhPJuJEp7Ducb92gjc/kFRZx+4RU4mKjeH64dW+urFBoh9ejeR3aN45nwgLrvWgC295D+Xy3fAdDeyT45Va2J4WWoy1Rzwf+qapfADHejxSYOjerzd3ndOCHlTuZ+Ifdazbe98L3a1i5PZvnLulGo/hqTscJZkHfDk9EGNWnOWmZB1i+9YDTcYw5pqmLM8kvKuayvv4Z+NKTQstWEfk3MAL4RkRiPdw+6F1/cktObtOAx75cyfpdNhur8Z5f1+3mrTkbubJfEmd0bOx0nGAXEu3whvZMJDYqgk+stsUEKFXlkwWb6ZVUl/ZN/DO9iCeFjhHA98AgVd0P1APu8kmqABURIbw4ojux0RGMnriE/EKb2MxU3d5D+dz5aSptGsXxwPkdnY4T9EKlHV7t6tFc0K0ZXyzZag1yTUCat2EvG3YdYlQf3zfAParChRZVPayqU1V1nfv5dlX9wXfRAlPjWtV4dlg3lm/N5qUZa52OY4KcqnLPlDT2Hy5g/Mge1r3Zu4K+Hd5lfVtwKL+I6anWINcEngkLNlOrWhQXdGvqt2NWuNBS6qrlHyWvWiq4/bsikiUiy4/x+gAROSAiqe7loYru29/O6dyEUX1a8O/Z6/l9vXWDNpU3YcEWZqzcyd2D2tOpmXVv9rIqt8MTkWoiskBElrob8j7q9ZTH0bNFHTo0iefj+RnWAcAElN0H8/h2+XYu7pno14utyvYeOgfPr1r+AwwqZ505qpriXh7zYN9+9+AFHWlZvyZ3TlrK/sPWDdp4bv2ugzz+1UpOaduA605q6XScUOSNdnh5wEBV7Q6kAINEpJ+Xcx6TiHB5vyRWbMsm1UbINQHk04VbKChSrujnv1tD4MfeQ6o6G9jrwfECWo2YKMaP6sGeQ3ncN3WZXQUZj+QXFjN64hKqRUfwwvDuRERY92YfKN0Ory4etsNTl6Ot7qPdi1//sw/tkUDNmEg+mmcNck1gKCpWPpm/mX6t6tGmkX8a4B4VaL2H+rurYb8Vkc7HWklEbhKRhSKycNeuXV6OUHFdEmoz9uz2fLt8B5MXZjqWwwSfF2esYfnWbJ4Z1o3Gtax7s4+cD8xQ1XUi8g/gDcDj+7kiEikiqUCWe3/zy1jHZ+ekuNgohvRI4Ku0bVarawLC7HW7yNx3hCv6+aebc0mB1HtoMZDkroZ9FZh2rBVV9U1V7a2qvRs29N0U2BVx0ymt6N+qPo98uYKNuw85msUEh9/Td/Pm7A2M6tOcczr7fgTJMFbVW9oAqGqRqqYAiUAfEelSxjo+PSdd3jeJvMJiPltkF0fGeR/NzaBBXCxnd/L/+cuj3kPAeuAcEbkNaOTN3kOqmn20GlZVvwGiRaSBt/bvKxERwkuXdic6MoIxE5dQUGTdoM2x7T+cz52fLqVlg5o8eEEnp+OEOq8OiOm+WJtF+W3zvK5Ts1r0TqrLh/MyKC62W9HGOVv2HubnNVmM6tOcmCj/D9XmSe+h0cDHQCP38pGI/M1bQUSkibjHLReRPu5se7y1f19qWrs6z1zclaWZB3jlR+sGbcqmqtw3dRl7DuUxfmQPasREOR0p1FX5lraINBSROu7H1YEzcU0L4HdX9k8iY89hflnn3C1xYz6al0GECJf19W8D3KM8+Q98PdBXVR9S1YeAfsCNFd1YRCYAc4H2IpIpIteLyM0icrN7lUuA5SKyFBgPjNQgat16btemjOidyBuz1jN/Q1CUtYyfTV6YybfLdzD27PZ0SajtdJxw4I1b2k2BmSKSBvyBq03LV96NWTHndmlKg7hYPpyb4cThjSG3oIhJC7dwdqfGNK1d3ZEMnlzqCf9f3Yr7cYW7PKjqqHJefw14zYM8AefhCzuzYONe7piUyrejT6V2jWinI5kAsXH3IR75cgUntq7PTae0cjpOWFDVwyJy9Jb2ObiGVPDolraqpgE9fBLQQzFREVzWpzmvzkwnY88hkurXdDqSCTPTl25j/+ECruzv/wa4R3lS0/IeMF9EHhGRR3BNPPaOT1IFqZqxUYwb2YOsnDzun2bdoI1LQZGre3N0ZAQvjrDuzf7i61vaTrisbxKRIlbbYvxOVXn/9020bRRH/1b1HcvhSUPcl4DrcI21sg+4VlVf8VWwYNW9eR3uOKsdX6dtZ+rirU7HMQHg5RlrScs8wDMXd3WsSjVMVemWdiBqUrsag7o0YdLCLRyy+YiMHy3M2MeKbdlcc1Iy7uanjvCoUZqqLlLV8ao6TlWX+CpUsLv5tNb0aVmPh75YTsYe6wYdzuZt2MM/f1nPiN6JnNvVf/NzGKCKt7QD1bUnJZOTW8jnS+yiyPjPf37fRK1qUQztkeBojnILLSKSIyLZZSw5IpLtj5DBJjJCePnSFCIihDGTUq0bdJg6cLiAOyelkly/Jg9feMyxEo3vhOQt7Z4t6tI1oTb/+X2T3YI2frH9wBG+W76DkX1aON7rsdxCi6rGq2qtMpZ4VbUZ3o4hoU51nhralSWb9/PqT+ucjmP8TFW5f9oysnLyeOXSFGrGWvdmfwvVW9oiwjUnJpOedZA562zCVuN7H851Tdh5pQMj4Jbm/5FhwsiF3ZsxrGcir81M549NITPtkqmAKYu38nXadu44qx3dm9dxOk7YCtVb2hd0d3V/fve3jU5HMSHuSH4RnyzYzNmdmtC8Xg2n41ihxdceHdyZxLo1GDMxlezcAqfjGD/I2HOIh79YTp+W9bj5tNZOxwk74XBLOzYqkiv7JTFrzS7Ssw6Wv4ExlTR1SSb7Dxdw3cmBMRO9FVp8LC42ildGprAjO5eHpi13Oo7xMVf35tT/tmuKtO7Nfhcut7Qv79eCmKgI3rPaFuMjxcXKu79upGtCbU5Irut0HMCzYfxFRK4QkYfcz1u4h9s35ejZoi6jz2jLtNRtTLMW/yHt1Z/WkbplP09d3JWEOta92fhOg7hYhqQ0Y8riTPYdstmfjff9sm4X63cd4rqTne3mXJInNS1vAP2BoyPb5gCvez1RiPrrgNb0TqrLg9OWs2XvYafjGB/4Y9NeXpuZzsU9E7igWzOn45gwcP3JrcgtKObj+TbYnPG+t+dsoHGtWM7vGjjnM08KLX1V9VYgF0BV91GFGVPDTVRkBC9fmgLAmEmpFFo36JCSnVvAmImpJNatwWODuzgdx4SJ9k3iOa1dQ/7zewa5BUXlb2BMBa3YdoDf0vdw7UktHZnN+Vg8SVIgIpGAgmv2U8C+eT3QvF4NnhjahUUZ+3h95nqn4xgvenDacnZk5/LKyBTirHuz8aMbT2nF7oN5TE/d5nQUE0LenrORmjGRjOrjzGzOx+LJ2XU88DnQSESexDUr84M+SRXCBqckMHN1FuN/XsfJbRvQKykwGjeZypu2ZCtfpG7jzrPa0bOF/T6dJiJ3Hu919/gtIeOkNvXp2LQWb83ZwCW9Em1uK1Nl2w8c4cul27iqfzK1qwfWxL+ezD30MXA38DSwHRiiqp/6Klgoe2xIF5rWrsaYSUvIsW7QQW3L3sP8Y9pyeifV5a8DrHtzgIh3L72BW4AE93Iz0MnBXD4hItx0akvWZR1k5posp+OYEPDOnI0orikjAo0nvYeeVdXVqvq6qr6mqqtE5FlfhgtVtapF88qlKWzdd4SHp69wOo6ppMKiYsZMSkWAly9NISoycO77hjNVfVRVHwUaAD1VdayqjgV6AYnOpvONC7o1I6FOdf79ywano5ggd+BwARMWbObCbk0DYjC50jw5y55Vxs/O9VaQcNM7uR63DWzL1MVbmb7U7kUHo9dnrmdRxj6eGNolIP9zG1oAJfsC5wPJzkTxrejICK4/uSULNu1lUcY+p+OYIPbR/AwO5Rdx06mBWXNckQkTbxGRZUAHEUlzL8tEZCOwzPcRQ9ftA9vQo0UdHvh8GZn7rBt0MFmUsY/xP69jaI8EBqc4O+upOaYPgQXuCRMfBuYDHzicyWdG9mlOnRrR/OsXa+RvKie3oIj3ftvIae0a0qlZYI7DWJGalk+AC4EvgAvcjy8Aeqnq5T7MFvKiIiMYd2kPVOHOSUspKrYZW4NBTm4BYyYtoWntajw62GZvDlSq+iRwLa7JEvfjmjDxKWdT+U6NmCiu6p/MjJU7Wbczx+k4JghNXpTJ7oP5/OW0Vk5HOaaKzPJ8QFU3AauBa4Cr3cttR0fHNZXXon4NHr2oMws27eWfs9KdjmMq4OEvVrB13xHGjUyhVrXAallv/p+4hvDsBNRW1XHAnlAfxfvaE5OpHh3JP2dZbYvxTGFRMf/+ZT09WtShf6v6Tsc5Jk/atBwEDrmXIlztWZJ9kCnsuEZQbcorP7qGgDeB64vUrUxdspW/DWxLr6R6Tscxxxd2o3jXrRnDZX1b8MXSbTbytvHIl2nbyNx3hFsHtAmYIfvL4kmX5xdLLE8CA3B1IzRVJCI8ObQrjWtVY/TEJRzKK3Q6kilD5j5X9+aeLerwt4FtnI5jyheWo3jfeEorIgT+PdtqW0zFFBcrb8xcT4cm8Qzs0MjpOMdVlT6aNYDAvfEVZGpXj+alEd3Zsvcwj1g36IBTVKzcOWkpqvDKpT2se3NwCMtRvJvUrsYlvRL5dGEmO7NznY5jgsAPK3ewLusgtwxoHfCDE3oyTsuyEr2HVgBrgHG+ixZ++raqz18HtGHyoky+TtvudBxTwj9npbNg014eG9yZFvWte3OQKD2K969AyDbELemW09pQVKy8OdvGbTHHp6q8+nM6LRvUDIqJXj0Zxv+CEo8LgZ2qavcxvGz0mW2Zs24X901No0eLOjSrU93pSGEvdct+Xv5xHRd0a8rQHnZHNBi4G+HOBhYBZwCCaxTvVY4G85MW9WswOKUZH8/P4JYBrWkQF+t0JBOgfl6dxYpt2Tx/STciA7yWBTxr05JRYtnqaYFFRN4VkSwRWX6M10VExotIurs2p6cn+w8V0ZERjBvZg8Ji5c5PU60btMMO5RUyeuISmtSqxpNDu/oks9oAACAASURBVAZ0AzXz/1RVgWmlR/H2dD8i0lxEZorIKhFZISKjfRDXJ249vQ15hcW8NcdqW0zZVJXxP6eTWLc6Q4Lkgqwig8vliEh2iSWn5L8eHOs/wKDjvH4u0Na93AT804N9h5TkBjV55KLOzNuw16p3HfbI9BVs2XuYl0Z0D7iJw0y55onICVXcRyEwVlU7Av2AW0UkKOYvat0wjou6N+PDuRnsOZjndBwTgGat3cXSLfu59fQ2RAdJO72KjNMSr6q1SizxJf+t6IFUdTaw9zirDAY+UJd5QB0RaVrR/Yea4b0SOa9rE178YQ1pmdYN2glfp21n8qJM/jqgDX0DeNwCc0ynA3NFZH2JkbzTPNmBqm5X1cXuxznAKoKo1+TfBrblSEERb83Z6HQUE2BUlVd+XEdi3eoM6xk8U3J5VLQSke4icpt76eblLAnAlhLPMznGyUFEbhKRhSKycNeuXV6OERhEhKeGdqVhfCxjJqZyON+aD/nTtv1HuG9qGt0TazP6zLZOxzGVcy7QGhjI/4/kfWFldyYiyUAPXNMBlH4tIM9JbRq5als+mLvJalvM/5i1xlXLctvpbYiJCo5aFvCs99Bo4GOgkXv5WET+5sUsZTUWKLNBh6q+qaq9VbV3w4YNvRghsNSpEcOLI7qzcc8hHv9qpdNxwkZRsXLHpFQKi5VxI3sETbWp+V+qmgFkA42BpBKLx0QkDpgCjFHVP90WD+Rz0t8GtiW3oIh/261m46aqvPzjWlctS6/gqWUBz2parsc1WNNDqvoQrvu7N3oxSybQvMTzRCDspz8+sXUD/nJqayYs2MJ3y60btD+8OXsD8zfu5ZGLOpPcoKbTcUwlicgNuHoQfQ886v73kUrsJxpXgeVjVZ3qzYz+0KZRHENSEvhg7iaybNwWA8xYuZO0zAPcPrBt0F2UeZJWcA3ff1QRZdeOVNZ04Cp3L6J+wAFVtW9p4M6z2tE1oTb3Tl3GjgN20vGltMz9vPjDGs7r2oThQXYFYv5kNHACkKGqp+O6tePRvRt31+l3gFWq+pL3I/rH6DPbUlCkvGFzEoW94mLlpRlradmgJhf3DJrmWf/lSaHlPWC+e5r3R4F5uP4zV4iITADmAu1FJFNErheRm0XkZvcq3wAbgHTgLeCvHmQLaTFREbwyMoW8gmLGTk6l2LpB+8Th/EJGT0ylYXwsT1n35lCQq6q5ACISq6qrgfYe7uMk4EpgoIikupfzvB3U15Lq12RE70Q+mb+ZrfuPOB3HOOjrZdtZvSOHMWe2DcqRvSs8uJyqviQis4CT3T+6RlVTPdh+VDmvK3BrRfcXblo3jOOhCztx39RlvPPrRm481WZQ8LbHv1rJpj2H+PiGvtSpEfJT1ISDTBGpA0wDZojIPjy85ayqv+LdGmXH3DawLVMWbWXcj2t57pLuTscxDigsKublGWtp3zieC4Ng9NuyeNIQdziwTlXHA7WBh0Skh8+SmT8ZeUJzzuncmOe+X83yrQecjhNSvlu+gwkLtvCXU1tzYusGTscxXqCqQ1V1v6o+AjyIq2Z4sLOpnJNQpzpX9Evis0WZpGcddDqOccDkRZls2H2Iv5/TPuDnGDoWT+qGHlTVHBE5GTgLeB/4l29imbKICM9c3I16NWMYPXEJR/KLyt/IlGvHgVzunZpG14Ta3HlWO6fjGC8RkYeOLsBpQApwn8OxHHXr6a2pHh3JC9+vcTqK8bMj+UW88uNaeraow5kdA3sm5+PxpNBy9BvyfOBfqvoFYTDNe6CpWzOGF4ensH7XIZ78xrpBV1VxsTJ2cip5BcW8MjIlqMYrMOU6VGIpwjVuS7KTgZxWPy6WG09txXcrdrBk8z6n4xg/en/uJnZm53HPoA5B3V7PkzP0VhH5NzAC+EZEYj3c3njJyW0bcOMpLflo3mZmrNzpdJyg9vavG/gtfQ8PXdiJ1g3jnI5jvEhVXyyxPAkMIIhGs/WVG05pRYO4GJ75djWupoQm1O07lM/rM9M5vX3DoB/d25NCxwhc4xwMUtX9QD3gLp+kMuX6+znt6dS0Fnd/ttTGXqik5VsP8Pz3azinc2NGntC8/A1MsKsBhH0L9rjYKEaf0Zb5G/fy8+osp+MYP3h9ZjqH8gq599yOTkepMk9meT6sqlNVdZ37+XZV/cF30czxxEZFMn5UCkcKihg7eal1g/bQkfwiRk9cQr2aMTxzcbegri41ZTs615B7WQGsAcY5nSsQjOzTglYNavLMt6spLCp2Oo7xoS17D/PB3AyG92pO+ybxTsepMk96D1UTkTtFZKqITBGRO0Skmi/DmeNr0yief5zfiTnrdvPe75ucjhNUnvh6Jet3HeKlESnUrWlNs0LU0bmGLgTOBpqp6mvORgoM0ZER3D2oA+uyDjJp4ZbyNzBB69nvVhMRAXeESCcDT24PfQB0Bl4FXgM6Ah/6IpSpuMv7tuDMjo149tvVrNz2pylRTBlmrNzJx/M3c9OprTipjXVvDlWqmlFi2aqqNutoCed0bkyflvV46Ye1ZOcWOB3H+MCijL18lbadm05tTZPaoVHHUOHB5YD2qlpyRKKZIrLU24GMZ0SEZ4d1Y9C4OYyeuIQv/3Yy1aIjnY4VsLKyc7lnShqdmtZi7NmhceVhyiYidx7v9WAelt8bRIQHz+/Eha/9yhsz13PvuR2cjmS8qLhYeeyrVTSKj+Xm00KnKZcnNS1L3HMCASAifYHfvB/JeKp+XCwvDO/OuqyDPP3NKqfjBCxX9+alHM4vZPyoFGKjrHAX4noDt+DqMZQA3Ax0AuLdS9jrmlibi3sm8O6vG9m857DTcYwXTV+6jaVb9nPXOe2pEeNJ/URgK7fQcrQxG9AX+F1ENonIJlzzCJ3q43ymgk5r15DrTmrJ+3Mz+Hm1dYMuy3u/b2LOut08cH4n2jSy76ww0ADoqapjVXUs0AtIVNVHVfVRh7MFjHsGdSAqUnjiaxv3KVQcyivk6W9X0TWhNsN6htbErxWpaTnamG0Q0BLXyJKnuR+f77toxlN3D2pPhybx3DU5jV05eU7HCSgrt2Xz7LerObNjY67o28LpOMY/WgD5JZ7nE+aDy5Wlca1q3Hp6G35YuZM56zyaBNsEqDdmpbMzO49HLuoctMP1H0u5hZaSjdmAbKAxkFRiMQGiWnQk40f14GBeIXd9ttQGjnLLLXB1b65dI5pnh9nszWHkQ2CBe2b6R4AFuDoUmFKuP7klLerV4NEvV1JgXaCDWsaeQ7w1eyNDeyTQK6mu03G8zpMuzzcAs3ENMPeo+99HfBPLVFa7xvHcf15HZq3ZxQdzM5yOExCe/mYV67IO8sLw7tSPi3U6jvET9yi41wL7gL3A1ar6lLOpAlO16EgeuqAT6VkHee+3jU7HMZWkqjwyfQXRkRKyDas9aYg7GjgByFDV04EegNUlBqCr+idxevuGPPnNKtbsyHE6jqN+Xr2T9+dmcN1JLTmtXUOn4xg/KjEz/ThsZvpyndmpMWd0aMQrP65j+4EjTscxlTBj5U5mrtnFHWe1o3Gt0OjiXJonhZZcVc0FEJFYVV0NtPdNLFMVIsLzw7tTq1oUoycuIbcgPGeD3pWTx12T0+jQJJ67B9mfahiymek99PCFnSkqVp742nohBpsj+UU8+uVK2jWO4+oTk52O4zOeFFoyRaQOMA2YISJfANt8E8tUVYO4WJ4f3p3VO3J4PgynoVdV7vpsKQfzChk/qoeNXROebGZ6D7WoX4NbT2/D12nbmbXG5iUKJuN+WsfW/Ud4fHAXoiNDdy5jT+YeGqqq+1X1EeBB4B1giK+Cmao7vX0jrjkxmXd+3cgva8PrTt77v29i1ppdPHB+R9o1tu7NYcpmpq+Ev5zWilYNa/LgF8s5kh+etbTBZvWObN6es4ERvRODfhbn8lRknJY/dbVQ1V9Udbqq5h9rHRMY7j23A+0bx/P3yUvZczA8ukGv2ZHDU9+uZmCHRlzZzzq4hTGbmb4SYqMieWpoV7bsPcK4n9Y5HceUo7hYuX/qMmpVj+a+EJjFuTwVueqYKSJ/E5H/GdxCRGJEZKCIvA9c7Zt4pqqqRUcyblQKB44UcM+UtJDvBp1bUMTtE5ZQq1oUz11iszeHo6MXUcebmd4utI6vX6v6DO+VyFtzNrBi2wGn45jj+HBeBos37+cf53cMi8lfK1JoGYTr3vAEEdkmIitFZAOwDhgFvKyq//FhRlNFHZrU4t5BHfhxVRYfzd/sdByfeva71azZmcPzw7vTwLo3hyu70PKCB87vSN0aMdwzJY1CG7slIGXuO8yz363mlLYNGNojwek4flGRweVyVfUNVT0J12ByZ+AaGjtJVW9U1VSfpzRVds2JyZzariFPfr2S9KzQ7AY9a00W7/22iWtOTOb09o2cjmOcYxdaXlCnRgyPDe7M8q3ZvDXHxm4JNKrKA58vB+CpoeEzaKZHjdJUtcBdxbrfV4GMb0RECC8M70aNmChun5BKXmFoNbDbfTCPv09Oo33j+JAdVMlUjF1oec+5XZpwTufGvPzj2pC92AlWkxdl8svaXdx1Tnua16vhdBy/sZb0YaRRfDWeG9aNlduzeSGEukGrKvd8lkZ2bgHjRqVY92bzX3ahVTUiwhNDulIzJpKxk+02UaDYtv8Ij3+5kj4t63F1/2Sn4/iVFVrCzJmdGnNFvxa8NWcjv67b7XQcr/ho/mZ+Wp3FvYM60KFJLafjGBNSGsbH8tjgLizdsp9/z97gdJywp6rcMyWNIlVeuKR7yE2IWJ5KFVpEZKD736YiUuHLWhEZJCJrRCRdRO4t4/VrRGSXiKS6lxsqk88c3wPndaJNozjGTk5l36H88jcIYOlZOTzx1UpOa9eQa09KdjqOCUEi8q6IZInIcqezOOWCbk05v2tTXvlxLcu3Wm8iJ30wN4M563Zz33kdaVE/fG4LHVXZmpZBIpKIa0jslyuygbtw8zpwLtAJGCUincpYdZKqpriXtyuZzxxH9ZhIxo1MYd+hAu6dGrzdoPMKi/jbhFRqxkbx/HDr3myOTUSqMuLWf3A17g1brttEXahbI4Yxk1LDdmoQp6Vn5fDUN6sY0L4hV/RtUf4GIaiyhZY6wD3A3UBFRyzrA6Sr6gb3oHQTgcGVPL6pos7NanP3oPZ8v2InE//Y4nScSnn+uzWs2p7N85d0o1F8aE4OZrzmSxH5QkTeFJG/i8iAim6oqrNxzRId1urWjOGF4d1JzzrI09/Y3ET+lldYxJhJqdSIieS5YeF7kVbZQstjwCxVXcP/z+9RngSg5LdjpvtnpQ0TkTQR+UxEmpe1IxG5SUQWisjCXbvCa3h6b7rupJac3KYBj325kvW7DjodxyNz1u3i7V83cmW/JM7o2NjpOCbw/aKqg4G/A82BJt7cebick05t15DrTmrJ+3Mz+HHlTqfjhJXnv1vD8q3ZPDusG41CdAbniqhsoeV54Cz3IE3fVHCbsoqFpe9LfAkkq2o34Edcs7L+eSPVN1W1t6r2btiwYUUzm1IiIoQXR3SnWnQEYyamkl8YHD0D9h7KZ+ynS2nTKI77zwv9YauNV8SJSE/gCFBPVSd6c+fhdE6659z2dGpai7s+W8qOA7lOxwkLM9dk8favG7mqfxJnd/ZqeTvoVLbQskpVb1bVq4GRFdwmE9cVzlGJlJolWlX3qOrR201vAb0qmc9UUONa1XhmWDeWbT3ASzPWOh2nXEdbzu8/XMC4kSlUj7HuzaZCpgP9ga+A7xzOEtRioyJ59bIe5BYUc/uEJdYN2se2HzjC2E+X0r5xvF2kUYlCi4i8hauW5Xb3feGKjn/wB9BWRFqKSAyuws70UvtuWuLpRYDdOPWDczo3YVSfFvx79np+Xx/Y3aAnLNjCjJU7uXtQezo3q+10HBM8zgK+AHKBvg5nCXqtG8bx1MVdWLBpb1Bc7ASrgiJXwTCvoIjXL+9pY1BRiUKLqt6I6wTwB9AdqFCrfFUtBG7DNevqKuBTVV0hIo+JyEXu1W4XkRUishS4HbjG03ymch68oCMtG9TkzklL2X84MLtBp2cd5LGvVnBymwZcd1JLp+OY4FKy84BH9zREZAIwF2gvIpkicr0P8gWdoT0SGdWnBW/MWs9Pq6x9iy88//0a/ti0j6cu7kqbRnFOxwkIlb09dA/wENANSKvoRqr6jaq2U9XWqvqk+2cPqep09+P7VLWzqnZX1dNVdXUl8xkP1YiJYvzIHuw5lMf9ny8LuG7Q+YXFjJm0hOrRkbw4IvwGVDJV9hjwhbvzgEf3M1R1lKo2VdVoVU1U1Xd8EzH4PHxhJzo3q8WYSals3H3I6Tgh5au0bbw5ewNX9kticEp4TIZYEZUttNQF5gFPAO29F8c4qUtCbcae3Z5vlu1g8qJMp+P8jxdnuFrOPzOsG43DuOW8qRxVzQS2uh//aWBLUznVoiP51xW9iIoQ/vLhQg7lFTodKSSs2ZHD3Z+l0SupLg9eUNZwZuGrsoWWvUAkkIWNXxBSbjqlFSe2rs8j01ewKUCunH5P382bszdwWd8WnBPmLedNxYjIXSLyu4i0KfHjTBG52bFQIap5vRqMH9WD9KyD3DEpleLiwKqlDTZ7D+Vzwwd/UDM2ijcu70lMlM22U1KlPg1VfQzXaLjjARvTOYQc7QYdHRnB6IlLKHC4Z8D+w/nc+elSWjaoyT/Ot5bzpsLaAHdQoqOAquYAFzqWKISd0rYhD5zfiR9W7uTlH61hbmXlFxZzy0eL2Jmdx5tX9rJa5TJUpQhXC7hBVSs0jL8JHk1rV+eZi7uyNPMA435c51gOVeW+qcvYcyiPcZf2oEZMlGNZTND5CVeHgYKjPxCRBsBJjiUKcdedlMzIE5rz6s/pTF0cWLeXg4Gq8o9py5i/cS/PDetGjxZ1nY4UkCpUaDlGVetW4C++iWWcdm7Xpozoncjrs9KZv2GPIxkmL8zk2+U7GHt2e7omWvdmU3Gq+imuWuB0EflDRJ4ETgTWOJssdIkIjw3uwomt63PPlLSAHz4h0Lw+M51PF2Zy+8A2DOlhDW+PpaI1LVbVGoYevrAzSfVqcOenSzlwpKD8Dbxo4+5DPPLlCk5sXZ+bTmnl12Ob0KCqrwItgIdxtcH7O5DjaKgQFxMVwT+v6EVy/Zr85cNFrN6R7XSkoDBlUSYv/LCWi3skcMdZ7ZyOE9AqWmixqtYwVDM2inEje7AzO5cH/NgNuqComNETlxAdGWHdm02VqOoR91AL96rqqcDjTmcKdbWrR/PetSdQMyaKq95ZwJa9h52OFNB+Xr2Tu6ekcVKb+jw9rGvYToRYURUqtFhVa/jq3rwOd5zVjq/StjN18Va/HPOVH9eSlnmAZy7uStPa1f1yTBMeVPUXpzOEg8S6NXj/uj7kFhRx5TvzycqxOYrKsmDjXv768WI6Na3Fv6/sTWyUjXhbngo3xLWq1vB182mt6duyHg9PX8HmPb69apq3YQ9vzFrPiN6JnNu1afkbGGMCUvsm8bx37Qlk5eRxxdvz2XsoMEfadsqSzfu49r0FJNSpznvXnkBcrHU0qAiPeg9ZVWt4iowQXro0hQiB0ZN8N0HagcMF3DkpleT6NXn4ws4+OYYxxn96JdXj7at7k7HnMJdbweW/lm7Zz9XvLqB+XCwf39CPBnGxTkcKGlUatcaqWsNHQp3qPHVxV5Zs3s/4n9O9vn9V5f5py8jKyeOVS1OoaVcdxoSEE1s34M2rerN+10Eue2seew7mOR3JUYs37+OKt+dTq3o0n9zYlya1bSwWT9hQe6bCLujWjGE9E3nt53Us3OTdgZCnLN7K12nbueOsdnRvXser+zbGOOu0dg159+oT2Lj7EJe+OY/tB444HckRc9fv4ap3FlAvLoZJf+lPYt0aTkcKOlZoMR555KJOJNatwZhJqWTneqcbdMaeQzz8xXL6tKzHzae19so+jTGB5eS2DXj/uj7sOJDLJf+cy4ZdB52O5Fc/rNjB1e8toGntaky6qT8JdayTQWVYocV4JL5aNK+MTGH7gVwemra8yvtzdW9OJTJCePnSFCKte7MxIatfq/pMuLEfRwqKGPbP31mUsc/pSH7x4bwMbv5oER2b1uLTv/S3W0JVYIUW47GeLeoy+oy2TEvdxrQlVesG/epP60jdsp8nh3a1Kw9jwkDXxNpMueVEaleP5rK35vFV2janI/lMUbHy1DereHDacga0b8QnN/Slbs0Yp2MFNSu0mEr564DW9E6qy4PTlld68Kg/Nu3ltZnpDOuZyIXdm3k5oTEmULVsUJMpt5xIl4Ta3PbJEl78YU3IzQ594EgB17//B2/O3sCV/ZJ488pe1sHAC6zQYiolKjKCly9NAeCOSaked4M+cKSAMRNTSaxbg0cHW/dmY8JN/bhYPrmxLyN6J/Lqz+lc//4f7AuRLtErt2Uz5PXf+HXdbp4Y0oXHh3QhKtK+br3BPkVTac3r1eDxIV1YmLGPN2at92jbh75Yzo7sXMaNTLFBlYwJU7FRkTw7rBuPD+nCb+l7OH/8HK/3TPQnVeWT+ZsZ+sZvHM4v5JMb+3FFvySnY4UUK7SYKhnSI4EhKc0Y99O6Cjeq+3xJJl+kbmPMGW1t+nVjwpyIcGW/JKbcciJRkRGM+Pdcnv9+NfmFvhnE0ld2H8zjxg8Wcv/ny+jTsh5f334KfVrWczpWyLFCi6myx4Z0oUmtaoyZtISccrpBb9l7mAenraB3Ul3+enobPyU0xgS6rom1+Wb0KVzSK5HXZ67nwld/JXXLfqdjlUtVmbo4kzNf+oXZ63bz0AWdeP/aPjbKrY9YocVUWS13N+it+47wyPSVx1yvsKiYMZNSEbDuzcaYP4mLjeK5S7rzztW9OXCkgKFv/Mb9ny8L2LYua3fmcPnb87nz06W0alCTb24/metObmkz0/uQNSYwXnFCcj1uO70N439OZ0D7hmX2Bnp95noWZexj3MgUmtezkSCNMWU7o2Nj+rSsx0sz1vLB3Ay+TtvOXwe05uoTk6kW7fxMyDuzcxn/0zom/rGFuNgoHh/cmcv6JtmFmB9YocV4ze1ntGVO+m7u/3wZPZPq/s+4K4sy9jH+53UMSWnG4JQEB1MaY4JBfLVoHr6wMyNPaMHT367i6W9X8+5vG7nxlFZc1rcFNWL8//W1bf8R3pqzgQkLNlNYpFzetwVjzmxHPRt7xW9ENbj7xvfu3VsXLlzodAzjlrHnEOeNm0PnhNpMuLEfkRFCTm4B542fgyp8M/oUalWLdjqmKUFEFqlqb6dzhAo7J/nGvA17eOXHtczbsJfa1aMZ0TuRy/smkdygpk+Pq6oszNjHB3Mz+HbZdhQYnNKM0We0Jam+b48dro53TvJrUVVEBgHjgEjgbVV9ptTrscAHQC9gD3Cpqm7yZ0ZTNUn1a/LY4C6MnbyUf/2ynltPb8PD01ewdd8RJt/c3wosJuiUd94y/tGvVX0m3tSfRRl7effXTbz72ybemrORXkl1uah7M87q1JhmXhpVW1VZsS2bH1bu5IvUrWTsOUx8tSiu6p/MtScl2+1tB/mt0CIikcDrwFlAJvCHiExX1ZItN68H9qlqGxEZCTwLXOqvjMY7Lu6ZwMw1Wbw8Yy05uYVMXbyV0We0pVeSdf8zwaWC5y3jR72S6tErqR47s3OZungrny/J5OHpK3h4+graNY6jf6v69GhRly4JtUiqX5PoCgzqdiS/iLU7c1i29QCLMvYxd/0edmTnIgL9W9XnttPbcH63po7ckjL/y5+/gT5AuqpuABCRicBgoOR//sHAI+7HnwGviYhosN/DCjMiwpNDurI4Yx//+mU9PVvU4W8DrXuzCUoVOW8ZBzSuVY1bBrTmlgGtWb/rIDNW7uS39N18ujCT9+dmABAZISTUqU7D+Fjq1oihekwkURFCflExh/MK2Xsonx3ZuezMzvvvfhvExdC3VX1ObduAMzo2tq7LAcafhZYEYEuJ55lA32Oto6qFInIAqA/sLrmSiNwE3ATQokULX+U1VVC7RjSvXtaDF75fy7PDutkQ1iZYVeS8Zeckh7VuGEfr0+K4+bTWFBYVk77rIMu3ZrNp9yEy9h5md04emfsOk1dYTGFxMdGREVSPjqR+XCxtG8eTXL8GrRvG0SWhNol1qyNivYAClT8LLWX9FZSuQanIOqjqm8Cb4Gr0VvVoxhd6JdVjwk39nI5hTFXYOSnIREVG0KFJLTo0qeV0FOMD/rz8zQSal3ieCJSek/y/64hIFFAbCN6JKIwxwa4i5y1jjJ/4s9DyB9BWRFqKSAwwEpheap3pwNXux5cAP1t7FmOMgypy3jLG+Infbg+526jcBnyPq+vgu6q6QkQeAxaq6nTgHeBDEUnHVcMy0l/5jDGmtGOdtxyOZUzY8mv/LVX9Bvim1M8eKvE4Fxjuz0zGGHM8ZZ23jDHOsC4dxhhjjAkKQT+Mv4jsAjLKWa0BpbpNB5lgzh/M2SE88iepakN/hAkHdk4KeMGcHYI7f0WzH/OcFPSFlooQkYXBPLdKMOcP5uxg+Y1vBPvvJZjzB3N2CO783shut4eMMcYYExSs0GKMMcaYoBAuhZY3nQ5QRcGcP5izg+U3vhHsv5dgzh/M2SG481c5e1i0aTHGGGNM8AuXmhZjjDHGBDkrtBhjjDEmKIRNoUVEnheR1SKSJiKfi0gdpzN5QkSGi8gKESkWkaDo7iYig0RkjYiki8i9TufxhIi8KyJZIrLc6SyeEpHmIjJTRFa5/2ZGO53J/Jmdk/zPzknO8OY5KWwKLcAMoIuqdgPWAvc5nMdTy4GLgdlOB6kIEYkEXgfOBToBo0Skk7OpPPIfYJDTISqpEBirqh2BfsCtQfbZhws7J/mRnZMc5bVzUtgUWlT1B1UtdD+dh2uK+aChqqtUdY3TOTzQB0hX1Q2qmg9MBAY7nKnCVHU2rkk7g46qblfVxe7HOcAqIMHZVKY0Oyf5TpGZpQAABAtJREFUnZ2THOLNc1LYFFpKuQ741ukQIS4B2FLieSb2xel3IpIM9ADmO5vElMPOSb5n56QAUNVzkl9nefY1EfkRaFLGSw+o6hfudR7AVVX1sT+zVURF8gcRKeNn1r/ej0QkDpgCjFHVbKfzhCM7JwUUOyc5zBvnpJAqtKjqmcd7XUSuBi4AztAAHKCmvPxBJhNoXuJ5IrDNoSxhR0SicZ0cPlbVqU7nCVd2Tgoodk5ykLfOSWFze0hEBgH3ABep6mGn84SBP4C2ItJSRGKAkcB0hzOFBRER4B1glaq+5HQeUzY7J/mdnZMc4s1zUtgUWoDXgHhghoikisi/nA7kCREZKiKZQH/gaxH53ulMx+NuYHgb8D2uRlefquoKZ1NVnIhMAOYC7UUkU0SudzqTB04CrgQGuv/WU0XkPKdDmT+xc5If2TnJUV47J9kw/sYYY4wJCuFU02KMMcaYIGaFFmOMMcYEBSu0GGOMMSYoWKHFGGOMMUHBCi3GGGOMCQpWaDHGGGNMULBCizHGGGOCghVajF+IyIsislJE3hKRX9zTxFdkuxgRmS0iITXlhDHGWXZOCk5WaDE+JyKtgJNUtROQCkxV1aKKbOueQv4n4FIfRjTGhBE7JwUvK7SYMolIVxH5rcTzniLycyX20x74BUgSkSXADcAXJV6fKSJnuR8/ISLjy9jNNOByT49tjAkddk4yYMP4m2MQkQhcM6AmqGqRiMwExqrq4krs6wlgE/ABsFlVm5R47VTgMeAt4DJck8cVldo+Etihqg0r+36MMcHNzkkGrKbFHIOqFgMrgM4iMgzXf+z/OTmIyI8isryMZXCp3XUFlgINgP2ljjMbEOBOYOTRk4OIPF5inSIgX/6vvftniSOKwjD+HIT4B1LZaJ3SBGKfJiBol09gFWws06ULJE36tBYp0ppUFja2Qoo0guYDWClYiBKxeFPsCIuSweg2d/f5dbN39nAHlpczdy6zVU9HfJmSGmEmCcCNROqzz+DfOTeBtduDSVbuWWeJQdhMAzPDA1X1AlgETpOcd58tcPe3OQ38+Z/JSxo7ZtKEc6VFffaBT8D3JMcPKdDdiVwnuUxyBkxV1Uw3tgh8A94AF1W12n1tmcHmuJsa88BJkuuHX4qkMWAmTTibFvU5Aq6Az4+o8Rw4GDreBV5V1RywzeCZ9CHwEfjQnfOSoYAAXgM7j5iDpPFgJk04N+Lqn6rqC/AzydcR1lwG3iVZ7zlnC9jonmFTVdvA+yS/RzUPSe0xk+RKi+6oqmdVdQTMjjIcAJL8Avb6XuSU5O1QODwBfhgO0uQyk3TDlRZJktQEV1okSVITbFokSVITbFokSVITbFokSVITbFokSVITbFokSVITbFokSVIT/gLBqJo0q8gj7wAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "x = np.linspace(-2, 2, 101)\n", + "plot_loss_functions(\n", + " suptitle = 'Common loss functions for regression',\n", + " functions = [np.abs(x), np.power(x, 2)],\n", + " ylabels = ['$\\mathcal{L}_{abs}}$ (absolute loss)',\n", + " '$\\mathcal{L}_{sq}$ (squared loss)'],\n", + " xlabel = '$y - f(x_i)$')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Функции ошибки для классификации\n", + "\n", + "**0-1 loss**\n", + "\n", + "Предполагается решающая функция вида $\\hat{y} = \\mathrm{sign}(f_{\\theta}(x))$:\n", + "\n", + "$\\mathcal{L}_{0-1}(\\theta) = \\sum_{i=1}^n l_i \\quad l_i = \\begin{cases}\n", + " 0 & y_i f_{\\theta}(x) > 0 \\\\\n", + " 1 & в\\ противном\\ случае\n", + " \\end{cases} \\\\\n", + "$\n", + "\n", + "**логистическая функция ошибки**\n", + "\n", + "$\\mathcal{L}_{log}(\\theta) = \\sum_{i=1}^n \\frac{1}{\\log(2)} \\log(1 + e^{-y_i f_{\\theta}(x)})$" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# define and vectorize zero-one loss\n", + "def zero_one(d):\n", + " if d < 0:\n", + " return 1\n", + " return 0\n", + "\n", + "def logistic_loss(fx):\n", + " # assumes y == 1\n", + " y = 1\n", + " return 1 / np.log(2) * np.log(1 + np.exp(-y * fx))\n", + "\n", + "zero_one_v = np.vectorize(zero_one)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAADZCAYAAADsdQBUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd5iU5dXH8e9vZpfeewcFRAGRXtQYG7GLxobYUJRYo69Go4lRYolJTOxYQBGxERULMRg1EcWGAlKUoiKgNBUFKQLSzvvH8ywZ12V3dndmnpnd87muuZiZp53ZGe45c1eZGc4555xz2S4WdQDOOeecc8nwpMU555xzOcGTFuecc87lBE9anHPOOZcTPGlxzjnnXE7wpMU555xzOcGTFueyiCST1CFD12oqaYqk9ZL+nolrJlx7g6TdM3g9SXpY0hpJ76f5WmMl3ZTG8+/820mqLumfktZKelrSaZJeScM1fybp41Sf17nSyos6AOeSJWkIcDmwJ7AemAXcbGZvRRpY7hoOfAPUsTRO2CTpdeAxM3uw4Dkzq5Wu6+3C/sBAoJWZfZ/ha6dUob/diUBToKGZbQufe7y815BkQEczWxhe802gU3nP61x5eU2LywmSLgfuAP5EUEi3Ae4FBkUZV45rC8xLZ8KSRdoCS8qSsEjK5h93bYFPEhIW5yo2M/Ob37L6BtQFNgAnFbNPVYKkZkV4uwOoGm47EFgGXAV8DawEjgOOBD4BVgO/SzjXCOBp4DGCGp0PgT2Aa8LjlwK/SNi/BTAxPM9C4LxC53oKGBeeay7Qu5jXYUCHhNc9DlgFfA5cC8TCbR2AN4C1BLUl/wifF3B7GOdaYA7QtYjrjAW2AlvCv+2h4XM3JexzILAs4fES4DfhOdcC/wCqJWwfRFD7tQ74DDgcuBnYDmwOr3NPKV/nUOAt4G/AGmAxcETCNYcCi8K/7WLgtCJe67Dw+tvDGP4YPn9e+H6tDt+/FoXeh4uAT4HFu3iv9gfeAb4LPxNDE/62N4X36wMvhq9tTXi/VUnx7+r9TfzbAX8M37+t4esaVvD3Sti3C/Bq+Bq/IvycA32Bd8PYVwL3AFXCbVPCa3wfnveUIj4LewGvh8fPBY4t9NkaCfwrfF3vAe2jLkf8VjFukQfgN7+VdAu//LYBecXscwMwFWgCNA6/TG4Mtx0YHn8dkB9+Wa0CngBqhwX7ZmD3cP8R4ePDCJpQx4VfKL9POH5xwrXfIKj1qQZ0D899SKFzHQnEgVuAqcW8jsQv83HAC2GM7QgSrGHhtifDeGLhdfcPnz8MmAHUI0hg9gKa7+JaY/lxklL4ceEvqiXA+wRJWgNgPnB+uK0vwRfswDCmlsCe4bbXgXPL+DqHEnwpnxf+/S4gSEoF1CRIkDqF+zYHuuzitQ7lx1/mBxMkAz0JEt67gSmF4ns1fJ3VizhfG4Iv5FPDz0RDoHvhv2P4/AlAjfD1PQ08H27bZfy7en+L+NuNIGh6+8nrDK+3ErgiPEdtoF+4rRfQn+Dz3S58Ly8r6hqFPwvh610I/A6oEv4t1ye8jrEESVLf8PyPA+OjLkf8VjFu3jzkckFD4Bsrvgr8NOAGM/vazFYR/Ao9I2H7VoL+L1uB8UAj4E4zW29mcwl+LXZL2P9NM3s5vObTBInQnxOObyepnqTWBL+4f2tmm81sFvBgoWu/ZWaTzGw78CiwT0kvWFKc4BfuNWGMS4C/J5x3K0HTQIvwum8lPF+boN+PzGy+ma0s6XqlcJeZrTCz1cA/CZI0CH7ljzGzV81sh5ktN7MFJZ0sidcJ8LmZjQ7/fo8QfLk3DbftALpKqm5mK8P3MhmnhfF+YGY/ENSiDZDULmGfW8xstZlt2sXx/zGzJ81sq5l9G773PxI+P8HMNprZeoKap58n7LKr+Hf1/pbG0cCXZvb38Bzrzey9MK4ZZjbVzLaFf/MHCsVVnP5ALYL/D1vM7DWCGqRTE/Z51szeD///PM7/PifOlYsnLS4XfAs0KqFvQQuCpoUCn4fP7TxH+KUHUPAl9FXC9k0EBTG72PZNEcfXCq+xOvxCSrx2y4THXybc3whUS6KfRCOCX7GFX1PBea8iqG14X9JcSecAhF8g9xBUz38laZSkOiVcqzQKv5aCv1lrgiah0irpdf7omma2Mbxby4L+KacA5wMrJf1L0p5JXvdHnxcz20DwOUu87tJijk/q9UqqIekBSZ9LWkfQ9FJPUryE+It8f0tplzFK2kPSi5K+DOP6E8F7kYwWwFIz25HwXEmf+Ux3vHYVlCctLhe8S9DEclwx+6wg+GVaoE34XLqtABpIql3o2svLed5v+N+v7Z+c18y+NLPzzKwF8Cvg3oKh0mZ2l5n1Imj22gO4Mslrfk/QjFGgWSniXQq038W24jr6Fvs6SxLWhg0kqH1ZAIxO5jgKfV4k1SSo0Uu8bnFxF/d6E11BMOqmn5nVAQ4ouGRx8Rf3/pZCcTHeF16vYxjX7wpiSsIKoLWkxO+PVHzmnSuRJy0u65nZWoL+KCMlHRf+es2XdISkv4a7PQlcK6mxpEbh/o9lILalBP1nbpFUTVI3gqaScg07DWt1ngJullRbUluC4d6PAUg6SVKrcPc1BF+w2yX1kdRPUj5BElLQATUZs4AjJTWQ1Ay4rBQhPwScLekQSTFJLRNqDb4CipyTpaTXWZxwnpljw4TjB4JOo8m+1ifCeLtLqkpQ0/Be2FSSjMeBQyWdLClPUkNJRTWB1CaomftOUgPg+mTi39X7m2RsBV4Emkm6TFLV8O/bLyGudcCG8H26oNCxu3zPCDrWfg9cFf4/PBA4hqDZ1Lm08qTF5QQzu43gy+xago6uS4GLgefDXW4CphOMbPkQ+CB8LhNOJejMuAJ4DrjezF5NwXkvIfhyWEQwguYJYEy4rQ/wnqQNBCNfLjWzxUAdgl/rawiq7L8lGHmTjEeB2QQdbl8hGB2UFDN7HzibYOTSWoLOyQU1GXcCJ4YTu91VytdZnBhBTcYKgo6fPwcuTDLe/wJ/ACYQdFZtDwxO5tjw+C8IOldfEV57FkX3VboDqE5QozQV+HeS8e/q/U1a2GQ5kCCh+JJgJNRB4ebfAEMIOtCO5qfv9QjgEUnfSTq50Hm3AMcCR4Sv617gzGT6MDlXXjKrDFM0OOeccy7XeU2Lc84553KCJy3OOeecywmetDjnnHMuJ3jS4pxzzrmc4EmLc84553KCJy3OOeecywmetDjnnHMuJ3jS4pxzzrmc4EmLc84553KCJy3OOeecywl5UQdQXo0aNbJ27dpFHYZzOWvGjBnfmFnjqOOoKLxMcq58iiuTcj5padeuHdOnT486DOdylqTPo46hIvEyybnyKa5M8uYh55xzzuWEjCUtksZI+lrSR7vYLkl3SVooaY6knpmKzTnnnHPZL5M1LWOBw4vZfgTQMbwNB+7LQEzOOeecyxEZ69NiZlMktStml0HAODMzYKqkepKam9nK8lz3mw0/sHbT1vKcwpVBnWr5NK5dNeownMs6G7ds44vVG9mzWZ2oQ3Eu52RTR9yWwNKEx8vC58qVtIycvJCH315SnlO4MojHxNRrDvHExblCfvP0bN5fvJqJF+9Pi3rVow7HuZySTUmLinjOitxRGk7QhESbNm2KPekve7Sie+t65Q7OJW/mF98x9p0lrN20xZMWl7UkVQOmAFUJysJnzOz6QvtUBcYBvYBvgVPMbEl5rnv5wE4cP/Jthj86nad/tS/Vq8TLczrnKpVsSlqWAa0THrcCVhS1o5mNAkYB9O7du8jEpsDereqyd6u6qYrRJaFKPMbYd5awbUexb41zUfsBONjMNkjKB96S9JKZTU3YZxiwxsw6SBoM/AU4pTwX7dCkFncM7s6546bz2wlzuHNwd6SifrM55wrLpiHPE4Ezw1FE/YG15e3P4qIRiwUF8HZPWlwWs8CG8GF+eCv8oR0EPBLefwY4RCnIMA7Zqym/+UUnJs5ewQNTFpX3dM5VGhmraZH0JHAg0EjSMuB6gkICM7sfmAQcCSwENgJnZyo2l1p5nrS4HCEpDswAOgAjzey9Qrvs7GtnZtskrQUaAt8UOk/STdYFLjywPfNWruMv/15Ap6a1OWjPJuV6Lc5VBpkcPXRqCdsNuChD4bg0Kqhp8eYhl+3MbDvQXVI94DlJXc0scS6ppPralabJeueJJW49sRuLV33Pr5+cyXMX7UeHJrXK8CqcqzyyqXnIVRAFNS07PGlxOcLMvgNe56dzSe3saycpD6gLrE7VdWtUyWPUmb2okhdj+LjpPj2DcyXwpMWlXNxrWlwOkNQ4rGFBUnXgUGBBod0mAmeF908EXgtrhVOmVf0a3Hd6L5au2cglT85k2/YdqTy9cxWKJy0u5eLyPi0uJzQHJkuaA0wDXjWzFyXdIOnYcJ+HgIaSFgKXA1enI5C+uzXgxkFdmfLJKm55qXDe5JwrkE1Dnl0FkRf3pMVlPzObA/Qo4vnrEu5vBk7KRDyD+7ZhwZfreeitxXRqWpuT+7Qu+SDnKhmvaXEpF48FHytPWpwrnWuP2ov9OzTi989/yLQlKes641yF4UmLSzlvHnKubPLiMUYO6Umr+jU4/9EZLF29MeqQnMsqnrS4lPOOuM6VXd0a+Tx4Vm+2bN/BeeOms+GHbVGH5FzW8KTFpZz3aXGufNo3rsXIIT359OsNXDZ+lk8f4FzIkxaXcrGC5qHUjgx1rlI5YI/G/OGovfjP/K/468sfRx2Oc1nBRw+5lPvfNP4+34Rz5XHWvu349OsN3P/GZ3RoUosTe7WKOiTnIuU1LS7ldvZp2e41Lc6VhyRGHNuF/To05Jpn5/D+Yh9R5Co3T1pcyhUkLTu8eci5csuPx7h3SC9a16/Brx6dzhff+ogiV3l50uJSLs9HDzmXUnVr5PPQ0D7sMDjnkWm+RpGrtDxpcSkXi/noIedSbbdGNbn/9F4s+eZ7Ln7iA7b6GkWuEvKkxaVcnictzqXFgPYN+dPxe/Pmp98wYuJcUrx2o3NZz0cPuZSLe9LiXNqc3Kc1n32zgQfeWMRujWpy7s92jzok5zLGkxaXcj4jrnPp9dvD9uTzbzZy86T5tG1Yk4Gdm0YdknMZ4c1DLuW8psW59IrFxO2ndKdby7r8+smZfLhsbdQhOZcRGU1aJB0u6WNJCyVdXcT2NpImS5opaY6kIzMZn0uNPF/l2WU5Sa3Dsma+pLmSLi1inwMlrZU0K7xdF0Wsu1K9SpzRZ/WmQc0qDHtkGiu+2xR1SM6lXcaSFklxYCRwBNAZOFVS50K7XQs8ZWY9gMHAvZmKz6VOWNHiSYtLO0k1w7KltLYBV5jZXkB/4KIiyiOAN82se3i7oVzBpkGT2tUYM7QPm7Zs55yx01i/2YdCu4otkzUtfYGFZrbIzLYA44FBhfYxoE54vy6wIoPxuRSRRDwmT1pcykmKSRoi6V+SvgYWACvD2pJbJXVM5jxmttLMPgjvrwfmAy3TF3n6dGpWm5GnBYsrXvTETB8K7Sq0TCYtLYGlCY+X8dNCYgRwuqRlwCTgkqJOJGm4pOmSpq9atSodsbpyisfkHXFdOkwG2gPXAM3MrLWZNQF+BkwF/izp9NKcUFI7oAfwXhGbB0iaLeklSV2KOUekZdIBezTm5uO6MuWTVVz3wkc+FNpVWJkcPaQiniv8P+tUYKyZ/V3SAOBRSV3N7Ec/HcxsFDAKoHfv3v6/MwvFJZ/G36XDoWb2kzYQM1sNTAAmSMpP9mSSaoXHXWZm6wpt/gBoa2Ybwv51zwNF1uRkQ5k0uG8blq7ZyMjJn9G6QQ0uPLBDFGE4l1aZrGlZBrROeNyKnzb/DAOeAjCzd4FqQKOMROdSKi8mXzDRpVxRCUtZ9gEIk5sJwONm9mwR51lnZhvC+5OAfElZXR795hedGNS9BX/998e8MGt51OE4l3KZTFqmAR0l7SapCkFH24mF9vkCOARA0l4ESYu3/+SgeFxs3+Ft6y49JJ0kqXZ4/1pJz0rqWYrjBTwEzDez23axT7NwPyT1JSgvvy1/9Okjib+e2I2+uzXgyqfnMHVRVofrXKllLGkxs23AxcDLBJ3enjKzuZJukHRsuNsVwHmSZgNPAkPNG2dzUlxiu791Ln3+YGbrJe0PHAY8AtxXiuP3A84ADk4Y0nykpPMlnR/ucyLwUVge3QUMzoXyqGpenNFn9KZNwxoMHzedT75aH3VIzqVMqfu0SKoJbDaz7aU9NqxinVTouesS7s8jKExcjvPRQy7NCsqfo4D7zOwFSSOSPdjM3qLofnaJ+9wD3FPmCCNUt0Y+Y8/uw/H3vsPQMe/z3EX70bROtajDcq7cSqxpSdUQQ1e5eJ8Wl2bLJT0AnAxMklQVn+H7R1rVr8HDQ/uwdtNWhj7sc7i4iiGZ/+QpH2LoKr5YzJuHXFqdTNDUfLiZfQfUB66MNqTs07VlXe47vReffrWeCx77gC3bvJ+Zy23JJC2HmtmNZjYnceixma02swlmdgLwj/SF6HJRnjcPufQ6CnjVzD6VdC3B7NnfRBxTVjpgj8b8+YRuvLXwG656ZjY7/P+ly2ElJi0FwwcL9db/Q2Jv/WSHGLrKI+aTy7n0Km9H3ErlxF6tuPKwTjw/awV//veCqMNxrsxK0wacWEj8Ai8kXDHyYvJfdC6dftIRF6gSYTxZ78ID23PmgLaMmrKIB99cFHU4zpVJaZIWLyRc0uKxmNe0uHTyjrilJInrj+nCEV2bcdO/5vvkcy4nleY/uRcSLmnxmK/y7NKqcEfcBnhH3BLFY+L2U7rTb7cG/Obp2bz5qc/d6XJLaZIO763vkhaPxTxpcWljZhuBz4DDJF0MNDGzVyIOKydUy48z+qzedGhSm189OoPZS7+LOiTnklaapMV767uk+eghl06SLgUeB5qEt8ckFbkqvPupOtXyeeTsPjSsVYWzx07js1Ubog7JuaSUtSOu99Z3xYrLkxaXVsOAfmZ2XTirdn/gvIhjyilN6lRj3Dn9iAnOfOh9Vq7dFHVIzpXIO+K6tPBp/F2aif+VSYT3i52W3/3Ubo1qMvbsvqzbtJUzH3qfNd9viTok54rlHXFdWuTFxTZf5dmlz8PAe5JGhGsOTSVYtdmVUteWdRl9Vm8+X72RoWOn8f0P26IOybldKk9HXO+t73YpJuFLD7l0MbPbgHOA1cAa4GwzuyPaqHJX/90bMnJITz5avpbhj05n89ZSr4frXEYknbR4b31XGkFHXK9pceljZjPM7C4zu9PMZkYdT64b2Lkpfz2hG28v/JZfPzmTbdv9/6/LPkknLd5b35VG3Fd5dmkgab2kdUXc1ktaV8pztZY0WdL8cNX6S4vYR5LukrRQ0pyCpUsqqhN6teL6YzrzyryvuGrCHJ/V2mWdvFLsW9Bb/3sASX8B3gXuTkdgLrfFY2KHr/LsUszMaqfwdNuAK8zsg3BdtRmSXjWzeQn7HAF0DG/9CEZM9kthDFnn7P12Y/3mbdz26ifUrprHiGO7IHkfZ5cdSpO0eG99l7S4L5jospyZrQRWhvfXS5oPtAQSk5ZBwDgzM2CqpHqSmofHVliXHNyB9Zu3MvrNxdSsmsdVh+8ZdUjOAaVLWgp66z8XPj4O763vdsGHPLtcIqkd0AN4r9CmlsDShMfLwud+lLRIGg4MB2jTpk26wswYSfzuyL34fst27n39M2pWzeOigzpEHZZzpeqIW67e+pIOl/Rx2DZ89S72OVnSvLB9+Ylkz+2yjyctLldIqgVMAC4zs8L9YoqqTf7JB9vMRplZbzPr3bhx43SEmXGSuGlQV47v0ZJbX/6YMW8tjjok50pV04KZzQBmlPYikuLASGAgwS+VaZImJrYdS+oIXAPsZ2ZrJDUp7XVc9vBp/F06SXoEuDScfgFJ9YG/m9k5pTxPPkHC8riZPVvELsuA1gmPWwEryhZ17onFxK0ndmPz1u3c8OI8quXHGdIv92uSXO4qsaYlRb31+wILzWyRmW0BxhO0FSc6DxhpZmsAzOzr0rwQl128T4tLs24FCQtAWG70KM0JFPQufQiYH9YkF2UicGY4iqg/sLai92cpLC8e487BPTioU2N+//yHPDNjWdQhuUqsxJqWFPXWL6pduHAP/D0AJL0NxIERZvbvok5W0dqPK6J4TD5c0qVTTFL9gh85khpQyppjYD/gDOBDSbPC534HtAEws/uBScCRwEJgI3B2CmLPOVXyYtx3ei/OfWQ6Vz0zm/y4GNS9ZdRhuUqotP/JyyqZduE8gmGFBxJUwb4pqWvir6mdB5qNAkYB9O7d278Zs1BeLOY1LS6d/g68I+kZgrLkZODm0pzAzN6ihBGQ4aihi8oaZEVSLT/O6DN7M/Th97n8qdnkx2McuXfzqMNylUym1g5Kpl14GfCCmW01s8XAxwRJjMtBMXlNi0sfMxsHnAh8BawCfmlmj0YbVcVXvUqcMUP70KN1PX795Exenvtl1CG5SiZTScs0oKOk3SRVAQYTtBUneh44CEBSI4LmokUZis+lWLBgoictLn3MbK6Z3WNmdxeaEM6lUc2qeTx8dh/2blWXix7/gFfnfRV1SK4SyUjSYmbbgIsJFlycDzxlZnMl3SDp2HC3l4FvJc0DJgNXmtm3mYjPpZ4PeXbpIOmt8N/CAwRKPY2/K7va1fJ55Jy+dGlZlwsfn8F/PHFxGVKupEVS0p3SzGySme1hZu3N7ObwuevMbGJ438zscjPrbGZ7m9n48sTmohWX2O7T+LsUM7P9w39rm1mdhFttM6sTdXyVSZ1q+Yw7py+dm9fhgsdneI2Ly4jy1rT8MSVRuAqnoKbFPHFxaRCufVbicy696lbPZ9ywfnRuEdS4eB8Xl27JzNMyZxe3D4GmGYjR5aC8WDAow5uIXJoMLOK5IzIehaNu9XweHdaXLi2CPi4vfVipprFxGZbMkOemwGEEU/cnEvBOyiNyFUKsIGkxy9i4elfxSboAuBBoL2lOwqbaeHkUmTrVgsTlrDHvc/GTM7ljh3HMPi2iDstVQMl8n7wI1DKzWYU3SHo95RG5CsFrWlyaPAG8BNwCJK5htt7MVkcTkoOgc+64Yf045+FpXDp+Jlu37+CXPVtFHZarYEpsHjKzYeEkTDtJahZuG5KuwFxui4dJiw97dqlkZmvNbAnwLLDazD4nmNX2QUmlmsbfpV6tqnmMPacP/XdvyBVPz2b8+19EHZKrYMraEXdSSqNwFU5B0uITzLk0+YOZrZe0P0Hz9SPA/RHH5IAaVfIYM7QPB3RszNXPfsjYt311aJc6ZU1aip362rk8r2lx6bU9/Pco4D4zewGoEmE8LkG1/DijzuzFLzo3ZcQ/5zFy8sKoQ3IVRFmTltEpjcJVODHv0+LSa7mkBwjWHJokqSqZm+HbJaFqXpyRp/VkUPcW3Pryx/z13wt8CgRXbmUa2GFm96Y6EFexeEdcl2YnA4cDfzOz7yQ1B66MOCZXSH48xm0nd6dGlTzuff0z1m/exh+P7bLzR41zpZVU0iJpT2AQ0JJgRdUVwEQzm5/G2FwOi8eCH72etLh0MLONBJ1xCx6vBHyCkCwUj4k/Hd+VOtXyeGDKItZv3sqtJ+1DftwrxlzpJTO53G+B8QT9WN4nWPxQwJOSri7uWFd5FZRH3qfFpVIq1x6SNEbS15I+2sX2AyWtlTQrvF2XitdQGUni6iP25MrDOvH8rBWc/+gMNm/dXvKBzhWSTE3LMKCLmW1NfFLSbcBc4M/pCMzlNq9pcemQuPZQCk43FrgHGFfMPm+a2dEpuFalJ4mLDupAner5XPfCR5zx0Hs8eFYf6lbPjzo0l0OSqZ/bARQ1tWHzcJtzP+F9Wly2M7MpgE9Il2Fn9G/L3af2YNbS7zjlgXf5at3mqENyOSSZmpbLgP9K+hRYGj7XBugAXJKuwFxui8mTFpc+ki4v4um1wIyiZu8uhwGSZhP04/uNmc3dRTzDgeEAbdq0SeHlK6aju7WgXvUq/OrR6fzy3ncYN6wv7RvXijoslwOSqWl5GdiDYEXnl4FXgBFAJzN7CUCSdwV3P+I1LS7NegPnEwwOaEmQMBwIjJZ0VYqu8QHQ1sz2Ae4Gnt/VjmY2ysx6m1nvxo0bp+jyFdv+HRsxfvgANm/dzon3vcMHXxRe3s65n0omaZkMXASsMLMJZvaMmU0F4pIOlvQIcFZao3Q5Jx4vmFzOWxBdWjQEeprZFWZ2BUES0xg4ABiaiguY2Toz2xDenwTkS2qUinO7wN6t6jLhgn2pUz2fIaOn8uq8r6IOyWW5ZJKWwwlmn3xS0gpJ8yQtAj4FTgVuN7OxaYzR5aB4WPm2wyeTcunRBtiS8HgrQa3IJuCHVFxAUrOCWmRJfQnKy29TcW73P+0a1WTCBfvSqWltfvXodB6d+nnUIbksVmKfFjPbDNwL3CspH2gEbDKz79IdnMtdO6fx3+5Ji0uLJ4Cpkl4gmILhaIIfVjWBecmcQNKTBE1KjSQtA64H8gHM7H7gROACSduATcBg8yld06JRrao8Obw/lzwxkz88/xHL1mzkt4ft6ZPQuZ8o1Yy44bDnMk/gJOlw4E4gDjxoZkUOl5Z0IvA00MfMppf1ei46ce/T4tLIzG6UNAnYnyBpOT+hrDgtyXOcWsL2ewiGRLsMqFEljwfO6MWIf87lgTcWsWz1Jv5+8j5Uy49HHZrLImWaxr8sJMWBkcBAYBkwTdJEM5tXaL/awK+B9zIVm0u9nUmL/zB16bONYNoFI2gecjkuLx7jxkFdadOgBre8tIAVazcx6ozeNK5dNerQXJbI5DzKfYGFZrbIzLYQzLI7qIj9bgT+Cvjg/RwW91WeXRpJuhR4nKC5ugnwmCSfgqECkMTwA9pz32k9mb9yHceNfJuPv1wfdVguS2QyaWnJ/+Z5gaC2pWXiDpJ6AK3N7MXiTiRpuKTpkqavWrUq9ZG6cttZ0+J9Wlx6DAP6mdn1ZnYd0B84L+KYXAod3rU5T/1qAFu37+CE+95h8oKvow7JZYFyJy3h2kRJ7VrEczu/0STFgNuBK0o6kc+JkP28ecilmQhGNRbYTtFljMth3VrV44WL96NtwxoMe2QaD765CO8LXbmVuk+LpKcSHwLdgb8kcegyoOZsYccAABTQSURBVHXC41YEs0wWqA10BV4PRxk2AyZKOtY74+aePF97yKXXw8B7kp4jKIeOA8ZEG5JLh+Z1q/P0+QO44qnZ3PSv+Sz4cj03H9+VqnneQbcyKktH3HVmdm7BA0n3JXncNKCjpN2A5cBgYEjBRjNbS9A+XXDe1wmmzfaEJQf5Ks8unczstrCM2I8gaTkrxdP3uyxSo0oeI4f05M7/fsqd//2URas2cP/pvWhSp1rUobkMK0vScnOhx79P5iAz2ybpYoKlAOLAGDObK+kGYLqZTSxDLC5LFazyvMOTFpdCktaT0KxMQpOQJDOzOpmPymVCLCb+b+AedGpWmyuems3Rd7/F/Wf0omeb+lGH5jKo1EmLmS0u9DjpVVLDqbAnFXruul3se2BpY3PZI89HD7k0MLPaUcfgonXk3s3ZrVFNhj86ncEPTGXEsV0Y0s8XqawsMjl6yFUiBTNZek2LS6VkFmf1BVwrvr2a1+GfF+9P//YN+d1zH/LbZ+aweev2kg90Oa/MSYsvHOaK4zUtLk0mS7pE0o9+Wkuq4gu4Vi71alTh4aF9uOig9vxj+lJOuv9dlq7eGHVYLs3KU9PiPfXdLv1vGn9f5dmllC/g6naKx8SVh+3J6DN7s+Tb7zn67rd4bYGvFF2RlSdp8SpYt0sFqzz7kGeXSma22czuNbP9gLbAIUBPM2trZuf5CKLKaWDnprx4yf60rFedc8ZO588vLWDbdv/BVBGVJ2nxbyO3S/G4Nw+59DKzrWa20lecdwBtG9bk2Qv35dS+bbj/jc84dfRUVq7dFHVYLsW8psWlRZ6v8uycy7Bq+XFu+eXe3HFKd+auWMcRd77Jf+Z5c1FFUp6k5ZqUReEqnJh8Gn+X3SSNkfS1pI92sV2S7pK0UNIcST0zHaMrm+N6tOTFS/anRd3qnDtuOiMmzvXRRRVEmZMWMyvyP7pzkFDT4gsmujSTVLeMh44l6Ni7K0cAHcPbcCDZ2b9dFti9cS2eu2hfzt6vHWPfWcJxI9/mk698tehcV+qkRdIQSeMlPSbpCUmnpiMwl9viPuTZZc4LkiZIGinpXElVkjnIzKYAxU2OOQgYZ4GpQD1JzVMRsMuMqnlxrj+mC2OG9mbV+h845u63GPv2Yl90MYeVpabl52Y22MxON7MhwP6pDsrlPknEBDu8cHDp97aZnQBcBfQEbkrReVsCSxMeLwuf+wlJwyVNlzR91apVKbq8S5WD92zKvy87gH3bN2TEP+dx5pj3+XLt5qjDcmVQlqSlqqSjJHWTdCRQPdVBuYohLxbzmhaXCfUl9Qa2AnVI3cjGogYbFHluMxtlZr3NrHfjxo1TdHmXSo1rV2XM0D7cdFxXpi1ZzWF3TOGFWcu91iXHlCVpuRCoDxwJNAAuTmlErsKIxXz0kMuIywhqfO8HXgRS1d9uGdA64XErYEWKzu0iIInT+7dl0q9/xu6Na3Lp+Flc9MQHfLPhh6hDc0kqy4KJG4HHJO1lZvPTEJOrIPJiMU9aXNqZ2RbgjjSceiJwsaTxQD9grZmtTMN1XIbt3rgWz5y/L6OmLOL2Vz9h6qIp3DCoC0ft3Rxfuiq7JVXTIulKSe9I6pDw9DJJ56cpLlcBxGPypMWlnaQbJT0taaykTqU47kngXaCTpGWShkk6P6FcmwQsAhYCowlqmV0FEY+JCw5sz4u/3p/W9atz8RMz+dWjM/h6nfd1yWbJ1rR0AP4P2DnzpJmtl3QMQZWscz/hSYvLkPpmdlI4auh24KJkDjKzYkc+WtDZIalzudy1R9PaTLhgXx56azG3vfoJh9z2Br8/ci9O6dPaa12yULJ9Wv4LDCTo6AbsXOV5v3QE5SqGeEzeEddlwg/hxG8G1Iw6GJd78uIxfvXz9vz7sgPo0qIOVz/7IaeMmsrCr31el2yTVNJiZk8Ba4GFkqZJuhnYF/g4ncG53JYXk6/y7NJK0sNAXeBQYDzwj2gjcrlst0Y1efK8/vzlhL35+Mv1HHHnm9z68gI2bfHZdLNF0qOHzOxuoA1wPRAHfgMknYZKOlzSx+GU2FcXsf3ycJn5OZL+K6ltsud22Skm4QutunQys7OBS4C3gTeB46ONyOU6SZzSpw3/veLnHNOtBSMnf8Yv7njD1zDKEqUa8mxmm8xskpldbWYHADcmc5ykODCSYFrszsCpkjoX2m0m0NvMugHPAH8tTWwu++TFvabFpZ+ZbQJWm9kdZjY86nhcxdCoVlVuO6U744f3p1penHPHTeecsdNY8s33UYdWqZVnwUTM7I0kd+0LLDSzReHwxPEEU2QnnmtyOJwaYCrBnAguh3mfFpcOPprRZVL/3Rsy6dKfce1Re/H+4tUMvP0NbnlpPus3by35YJdy5UpaSiHp6bBDw4CXdrXRp8zODXHJp/F36VDkaEbgmMgichVafjzGuT/bndd+83MGdW/JA28s4qC/vcE/pn3hIyQzLFNJS9LTYUs6HegN3Lqrk/mU2bkhHhPbfJVnl3o+mtFFokntavztpH144aL9aNewBr+d8CFH3fUmUz7xH8+ZkqmkJanpsCUdCvweONbMfF7lHBf0afGkxaWWj2Z0UdundT2ePn8AI4f05Pst2zhzzPuc/uB7fLR8bdShVXiZSlqmAR0l7RZOADWYYIrsnST1AB4gSFi+zlBcLo3iEtu9ecilQXlHMzpXXpI4qltz/nP5z/nD0Z2Zu2ItR9/9Fhc/8QGLvbNu2pR67aGyMLNtki4GXiYoYMaY2VxJNwDTzWwiQXNQLeDpcBbCL8zs2EzE59LDZ8R16RSOGpoU3pD082gjcpVR1bw4w/bfjZN6t2L0lEU8+OZiXvroS07o2ZJLDu5I6wY1og6xQslI0gJgZjsLl4Tnrku4f2imYnGZ4X1aXCaVYjSjcylXp1o+V/yiE2cOaMd9r3/GY+99znMzl3Nir9ZcfHAHWtarHnWIFUKmmodcJRSPefOQc65yaVy7Ktcd05k3rjyQwX3a8MyMpRx462SunjCHz7/1ZqPy8qTFpU1eLObNQ865Sql53erceFxX3rjyIE7t24ZnZy7noL+9zqXjZ7Lgy3VRh5ezMtY85CqfmE8u55yr5FrUq84Ng7py8UEdePCtxTw29XNemLWCgzo1ZvgB7em/ewNfTboUvKbFpU1eTOzwpMVlqSTWQxsqaZWkWeHt3CjidBVDkzrV+N2Re/HO1QdzxcA9mLNsLaeOnsqx97zNC7OWs9UXakuKJy0ubXwaf5etklwPDeAfZtY9vD2Y0SBdhVSvRhUuOaQjb199MDcf35Xvf9jGpeNnsf9fXuOe1z7l2w0+RVlxvHnIpU1cXtPistbO9dAAJBWshzYv0qhcpVEtP85p/dpyap82vP7J1zz89hL+9son3PXaQo7euzmnD2hLj9b1vOmoEE9aXNrE42Kbr/LsslNR66H1K2K/EyQdAHwC/J+ZLS1iHyQNB4YDtGnTJsWhuoosFhMH79mUg/dsysKv1/Pou58z4YPlPDtzOXs1r8OQfm0Y1L0FdarlRx1qVvDmIZc2eT65nMteyayH9k+gnZl1A/4DPLKrk/l6aC4VOjSpzR8HdWXq7w7h5uO7AvCH5z+i383/5YqnZvPeom+xSj6NhNe0uLTxafxdFitxPTQz+zbh4WjgLxmIyzlqVc3jtH5tGdK3DXOWrWX8tC/45+yVTPhgGW0b1uCXPVrxy54tK+Vsu560uLSJx8R2nxHXZaed66EBywnWQxuSuIOk5ma2Mnx4LDA/syG6yk4S+7Suxz6t6/GHozsz6cMvmTBjGbf/5xNu/88n9GlXn0HdW3Lk3s1pULNK1OFmhCctLm3y4j56yGWnJNdD+7WkY4FtwGpgaGQBu0qvRpU8TuzVihN7tWLZmo28MGsFz81czrXPf8SIiXPZr0Mjju7WnF90bkbdGhW3/4snLS5tYhI7vHnIZakk1kO7Brgm03E5V5JW9Wtw0UEduPDA9sxfuZ4XZi/nxdkrufKZOfwu/iH7dWjE4V2acWjnpjSqVTXqcFPKkxaXNnk+T4tzzqWNJDq3qEPnFnW4+vA9mbX0O/790ZdM+mglVz/7IXruQ3q1qc/Azk05ZK+mtG9cM+eHUHvS4tImHot5nxbnnMsASfRoU58ebepz9RF7Mm/lOl6Z+xWvzPuKW15awC0vLaBdwxoctGcTDurUhL67NaBafjzqsEvNkxaXNvEYPnrIOecyTBJdWtSlS4u6/N/APVi2ZiOvLfia1xZ8zRPvfcHDby+hWn6Mfrs15GcdG/Gzjo3Zo2mtnKiF8aTFpU08FvPmIeeci1ir+jU4c0A7zhzQjk1btjN18be88fEq3vx0FTf9az4wn0a1qjKgfUP2bd+Q/rs3pF3DGlmZxHjS4tImHsMnl3POuSxSvUqcgzoFTUQAy7/bxNsLv+GtT7/h3UXf8s/ZwXRFTetUpU+7BvTdrQG92zagU7PaxGPRJzGetLi0icdibN9hmFlWZuzOOVfZtaxXnZN7t+bk3q0xMz5btYGpi1bz3uLVTFu8mhfnBFMV1aqaR4829YJ+M63r0b11PepHMDdMRpMWSYcDdxLMi/Cgmf250PaqwDigF/AtcIqZLclkjC518sKsfIdB3HMW55zLapLo0KQ2HZrU5vT+bTEzlq3ZxPTPVzN9yRo++OI77nntUwoq0Ns0qEG3VnXp1qouXVsGfWjqVk/vHDEZS1oSloIfSDCF9jRJE80scVXVYcAaM+sgaTDBtNmnZCpGl1oFVYnbd1hWVCs655xLniRaN6hB6wY1OL5HKwC+/2Ebc5atZc6y75i19DtmfvHdztoYgNYNqtO5eR32Krg1q0Or+tWJpeg7IJM1LcksBT8IGBHefwa4R5Kssq8QlaMKEpWrnplNPOZrc0bhysM60axutajDcM5VEDWr5jGgfUMGtG+487lvN/zA3BXr+GjFWuauWMe8Fet4Zd5XFHxz16gSp2PT2uzRpBZXHtaJJnXKXiZlMmlJZin4nfuE02yvBRoC3yTu5MvA54Z9WtWjbcMaTFuyJupQKq1NW7dHHYJzroJrWKsqB+zRmAP2+N8K59//sI1PvlrPx1+uZ8GX6/nkq/VM/ngV1x7duVzXymTSksxS8Mnsg5mNAkYB9O7d22thstSA9g1548qDog7DOedchtWsmrdzsrtUymSdfYlLwSfuIykPqEuwUJlzzjnnKrlMJi07l4KXVIVgKfiJhfaZCJwV3j8ReM37szjnnHMOMtg8lORS8A8Bj0paSFDDMjhT8TnnnHMuu2V0npYkloLfDJyUyZicc845lxt8HKpzzjnncoJyvcuIpFXA5yXs1ohCw6ZzTC7Hn8uxQ27Hn2zsbc2sccm7uWR4mZT1cjl2yO34y10m5XzSkgxJ082sd9RxlFUux5/LsUNux5/LsVd0uf7e5HL8uRw75Hb8qYjdm4ecc845lxM8aXHOOedcTqgsScuoqAMop1yOP5djh9yOP5djr+hy/b3J5fhzOXbI7fjLHXul6NPinHPOudxXWWpanHPOOZfjPGlxzjnnXE6oNEmLpFslLZA0R9JzkupFHVOyJJ0kaa6kHZJyZqibpMMlfSxpoaSro46nNCSNkfS1pI+ijqW0JLWWNFnS/PBzc2nUMbmf8jIp87xMikYqy6RKk7QArwJdzawb8AlwTcTxlMZHwC+BKVEHkixJcWAkcATQGThVUudooyqVscDhUQdRRtuAK8xsL6A/cFGO/e0rCy+TMsjLpEilrEyqNEmLmb1iZtvCh1OBVlHGUxpmNt/MPo46jlLqCyw0s0VmtgUYDwyKOKakmdkUgkU7c46ZrTSzD8L764H5QMtoo3KFeZmUcV4mRSSVZVKlSVoKOQd4KeogKriWwNKEx8vwL86Mk9QO6AG8F20krgReJqWfl0lZoLxlUkZXeU43Sf8BmhWx6fdm9kK4z+8Jqqoez2RsJUkm9hyjIp7z8fUZJKkWMAG4zMzWRR1PZeRlUlbxMiliqSiTKlTSYmaHFrdd0lnA0cAhlmUT1JQUew5aBrROeNwKWBFRLJWOpHyCwuFxM3s26ngqKy+TsoqXSRFKVZlUaZqHJB0O/BY41sw2Rh1PJTAN6ChpN0lVgMHAxIhjqhQkCXgImG9mt0Udjyual0kZ52VSRFJZJlWapAW4B6gNvCpplqT7ow4oWZKOl7QMGAD8S9LLUcdUkrCD4cXAywSdrp4ys7nRRpU8SU8C7wKdJC2TNCzqmEphP+AM4ODwsz5L0pFRB+V+wsukDPIyKVIpK5N8Gn/nnHPO5YTKVNPinHPOuRzmSYtzzjnncoInLc4555zLCZ60OOeccy4neNLinHPOuZzgSYtzzjnncoInLc4555zLCZ60uJSS9HdJ8ySNlvRGuBx8ScdUkTRFUoVaVsI5Fz0vkyoWT1pcykjaHdjPzDoDs4BnzWx7SceFy8T/FzglzSE65yoRL5MqHk9aXKlI2lvS2wmPe0p6TVIn4A2graSZwLnACwn7TZY0MLx/k6S7Cp36eeC09L8C51xF4mVS5eLT+LtSkRQjWBm1pZltlzQZuMLMPpB0E7AEGAd8YWbNEo47ALgBGA0MIVgkbnvC9jjwpZk1ztyrcc7lOi+TKhevaXGlYmY7gLlAF0knEBQEH4Sb9wZmA42A7wodNwUQcDkwuKBwkHRjuH07sEVS7Yy8EOdcheBlUuXinYxcWUwlWLXzQuDwhOe7EBQeVYFqiQdI2htoDnxjZuvD55rx489gVWBz+sJ2zlVQXiZVEl7T4spiKnAT8JyZLQcIf41sNbONZrYGiEuqFm5rDjwODAK+l3RYeJ4eBJ3jkNQQWGVmWzP7UpxzFYCXSZWEJy2uLBYAPwB/SXiuK/BRwuNXgP0l1QCeJWhjng/cCIwI9+lOWEAABwGT0hizc67i8jKpkvCOuK7UJN0DTDOzR4rZpwdwuZmdUcw+DwHnmdkOSc8C15jZx6mP2DlXkXmZVHl4TYtLmqT2khYA1YsrHADMbCYwubiJnMxsWFg4VAGe98LBOVcaXiZVPl7T4pxzzrmc4DUtzjnnnMsJnrQ455xzLid40uKcc865nOBJi3POOedygictzjnnnMsJnrQ455xzLid40uKcc865nPD/YbIc/OI+cmIAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_loss_functions(suptitle = 'Common loss functions for classification',\n", + " functions = [zero_one_v(x), logistic_loss(x)],\n", + " ylabels = ['$\\mathcal{L}_{0-1}}$ (0-1 loss)',\n", + " '$\\mathcal{L}_{log}$ (logistic loss)'],\n", + " xlabel = '$y f(x_i)$')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Строим нейросеть\n", + "Рассмотрим решение нашей задачи при помощи простейшей однослойной нейросети такого вида:\n", + "\n", + "\n", + "При этом модель будет описываться как\n", + "$$\n", + "f_\\theta(x) = W\\times x + b\n", + "$$\n", + "где параметры $$\\theta = $$" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[ 1.77202116, -0.25384488],\n", + " [ 0.28370828, -0.39610552],\n", + " [-0.30097433, 0.30513182],\n", + " [-0.8120485 , 0.56079421],\n", + " [-1.23519653, 0.3394973 ]])" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class Linear:\n", + " def __init__(self,nin,nout):\n", + " self.W = np.random.normal(0, 1.0/np.sqrt(nin), (nout, nin))\n", + " self.b = np.zeros((1,nout))\n", + " \n", + " def forward(self, x):\n", + " return np.dot(x, self.W.T) + self.b\n", + " \n", + "net = Linear(2,2)\n", + "net.forward(train_x[0:5])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Переходим к вероятностям\n", + "Расширяем нейросетевую модель с помощью функции **softmax**: $\\sigma(\\mathbf{z}_c) = \\frac{e^{z_c}}{\\sum_{j \\in J} e^{z_j}}$ для $c \\in 1 .. |C|$\n", + "\n", + "\n", + "\n", + "Можем рассматривать $\\sigma(\\mathbf{z})$ как распределение вероятности на классах $C$: $q = \\sigma(\\mathbf{z}_c) = \\hat{p}(c | x)$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.88348621, 0.11651379],\n", + " [0.66369714, 0.33630286],\n", + " [0.35294795, 0.64705205],\n", + " [0.20216095, 0.79783905],\n", + " [0.17154828, 0.82845172],\n", + " [0.24279153, 0.75720847],\n", + " [0.18915732, 0.81084268],\n", + " [0.17282951, 0.82717049],\n", + " [0.13897531, 0.86102469],\n", + " [0.72746882, 0.27253118]])" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class Softmax:\n", + " def forward(self,z):\n", + " zmax = z.max(axis=1,keepdims=True)\n", + " expz = np.exp(z-zmax)\n", + " Z = expz.sum(axis=1,keepdims=True)\n", + " return expz / Z\n", + "\n", + "softmax = Softmax()\n", + "softmax.forward(net.forward(train_x[0:10]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Ещё один взгляд на архитектуру сети\n", + "\n", + "![Архитектура нейросети](https://raw.githubusercontent.com/shwars/NeuroWorkshop/master/images/Cross-Entropy-Loss.PNG)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Cross-Entropy Loss\n", + "\n", + "* Повсеместно применяется в глубоком обучении\n", + "* Основная идея:\n", + " - трактуем выход модели как распределение вероятностей появления того или иного класса\n", + " - минимизируем вероятность неправильной классификации\n", + "\n", + "Два подхода к пониманию Cross-Entropy Loss:\n", + " * Цена ошибки, которую мы платим за неправильную классификацию, т.е. $-\\log p_y$, где $y$ - правильный класс\n", + " * Разница между двумя распределениями вероятностей. Энтропия $p$ + KL-расстояние между $q$ и $p$:\n", + "$\\begin{align} H(p, q) = & ~\\color{red}{H(p)} + \\color{blue}{D_{KL}(p||q)} \\\\\n", + " = & ~\\color{red}{-\\sum_{c \\in C} p(c) \\log p(c)} + \\color{blue}{\\sum_{c \\in C} p(c) \\log \\frac{p(c)}{q(c)}} \\\\\n", + " = & ~-\\sum_{c \\in C} p(c) \\log q(c)\n", + "\\end{align}$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "def plot_cross_ent():\n", + " p = np.linspace(0.01, 0.99, 101) # estimated probability p(y|x)\n", + " cross_ent_v = np.vectorize(cross_ent)\n", + " f3, ax = plt.subplots(1,1, figsize=(8, 3))\n", + " l1, = plt.plot(p, cross_ent_v(p, 1), 'r--')\n", + " l2, = plt.plot(p, cross_ent_v(p, 0), 'r-')\n", + " plt.legend([l1, l2], ['$y = 1$', '$y = 0$'], loc = 'upper center', ncol = 2)\n", + " plt.xlabel('$\\hat{p}(y|x)$', size=18)\n", + " plt.ylabel('$\\mathcal{L}_{CE}$', size=18)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": true, + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAADfCAYAAADm4+qPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd5hU5dnH8e9NXbAhHXZZiqA0ARUVRVCxoGgswYbR2KImaiyxx+QlGqPGrolGsUQJisZIQIxgrLGCgrogKAIKsoBSVECalOf9455xtsIsO7tnyu9zXec6M3POzNx7dmbu8zznKRZCQERERDJPnagDEBERkW2jJC4iIpKhlMRFREQylJK4iIhIhlISFxERyVBK4iIiIhmqXtQBVFXz5s1Dhw4dog5DRESkVkydOnVZCKFFRdsyLol36NCBKVOmRB2GiIhIrTCz+ZVtU3W6iIhIhlISFxERyVBK4iIiIhkq466Ji9SWDRs2UFxczLp166IORdJMXl4eBQUF1K9fP+pQJMcpiYtUori4mB122IEOHTpgZlGHI2kihMDy5cspLi6mY8eOUYcj6SQEeOghOOgg2HXXWnnL3K5Ov/lmX0QqsG7dOpo1a6YELqWYGc2aNVMNjZS3aBGcfz68+GKtvWVuJ/F334VRo6KOQtKYErhURJ8LqVBRka979661t8ztJN67N8yaBWvXRh2JiIhkungS79Wr1t5SSXzTJpgxI+pIREQk0xUVQfv20KRJrb1lbifxPn18HT97EsliZ599Ni1btqRnz55Rh5LWJk6cyG677Ubnzp255ZZbog5HMklRUSKv1JLcTuKdOsEuu4AaqEgOOPPMM5k4cWLUYaS1TZs2ceGFFzJhwgRmzpzJ6NGjmTlzZtRhSSZYuxY++6xWr4dDrifxOnVgzhy48MKoIxGp0PTp0+nfv/+P9z/44AMGDRq0Ta81cOBAmjZtmqrQ0kqqjtN7771H586d6dSpEw0aNOCUU05h3LhxqQxVstXHH8PmzbWexNVPXCRZBx1U/rGTToILLoA1a2DIkPLbzzzTl2XL4IQTSm97/fWtvmWPHj2YO3cumzZtom7dulx++eXccccdpfYZMGAAq1atKvfc22+/nUMPPXSr75FSl14KH32U2tfs0wfuvnuLu6TqOC1cuJB27dr9uK2goIDJkyen4I+QrBdBy3RQEoeXX4aLL4YJE7xBgkgaqVOnDj169GDGjBnMnj2bwsJC9txzz1L7vPnmmxFFlz5SdZxCCOUeU3cyScpHH8H220MtDwCkJL7ddvDJJ4lWhSKV2VLJuXHjLW9v3jypkndF+vXrx9tvv839999f4TXttCqJb6XEXJNScZwKCgpYsGDBj9uKi4tp27ZtzQUt2aOoyLuW1andq9RK4rvvDmZ+FnXMMVFHI1JOv379OPPMM7nwwgvJz88vt10lcZeK47T33nsze/ZsvvjiC/Lz83nqqad48sknayJcySYhwLRp8LOf1fpb53bDNvDqj86d1c1M0lbXrl1p2LAhV199dbVeZ9iwYey3337MmjWLgoICHnnkkRRFmB5ScZzq1avHX//6VwYPHky3bt046aST6NGjRwqjlKw0bx6sXFnr3ctAJXHXuzd8+GHUUYhU6J577uHmm29mu+22q9brjB49OkURpadUHachQ4YwpKJGiiKViahRG6gk7o48EgYO9O4BImli7ty5dO3albVr13LGGWdEHU7a0nGSyBUV+WXZCAZSUkkc4OyzfRFJI7vssguffvpp1GGkPR0niVxREXTp4g2la5lK4nEheF9fERGRqigqiqQqHdIkiZtZXTP70MyejyyIzp3hsssie3sREclAK1fC55/ndhIHLgE+iTSCdu3UQl1ERKpm2jRfR9AyHdIgiZtZAXAU8HCkgfTuDdOn+9SkIjEVjeAlos+F/CjClumQBkkcuBu4Coi2aXifPn5NfO7cSMOQ9JGXl8fy5cv1gy2lhBBYvnw5eXl5UYci6aCoCJo2hQoGGKoNkbZON7OjgSUhhKlmdtAW9jsPOA+gsLCwZoKJn0UVFcGuu9bMe0hGKSgooLi4mKVLl0YdiqSZvLw8CgoKog5D0kG8UVtEY+xH3cWsP3CMmQ0B8oAdzWxUCOG0kjuFEEYAIwD69u1bM8Wi7t3h2mtht91q5OUl89SvX5+OtTyZgYhkkE2b/DLs+edHFkKkSTyEcC1wLUCsJH5F2QRea/Ly4KabInlrERHJQNOnw9q1UGbGvNqUDtfE08fGjT7T1IoVUUciIiLpbtw4n7XsiCMiCyFtkngI4fUQwtGRBjF1Khx8MIwfH2kYIiKSAcaOhf79oUWLyEJImySeFvbeG9q08X+MiIhIZb74wqewPu64SMNQEi+pTh049liYONGvc4iIiFRk3DhfH3tspGEoiZd1/PGwejW88krUkYiISLoaOxZ23x122SXSMJTEyzroINhxR5gwIepIREQkHS1bBm++GXlVOkTfTzz9NGgA776rAV9ERKRizz8Pmzcriaet7t2jjkBERNLV2LE+adYee0QdiarTK3XddXDbbVFHISIi6WTNGvjvf70UHtFQqyUpiVfmww/hgQdAk1+IiEjciy9676U0qEoHJfHKHXecT/T+8cdRRyIiIuli7FjYeWcYMCDqSAAl8codc4xXlWjgFxERAR+ae/x4OPpoqF8/6mgAJfHKtW4N++8Po0Z5K0QREcltEybAt9/6eCJpQkl8Sy6+2IdiXbUq6khERCRqd93lrdJ/8pOoI/mRuphtyUkn+SIiIrmtqAheew3+/Geolz6pUyXxZBQV+WD3IiKSm+6+Gxo3hnPPjTqSUpTEt2blSthvP7jppqgjERGRKHz9NTz5JJx1lrdMTyNK4luz447w85/DP/4BS5dGHY2IiNS2v/0NfvjB20mlGSXxZFxyCaxfDw8+GHUkIiJSm9at8yR+9NFpOaeGkngyunWDI46A++7zszEREckNo0fDkiVw6aVRR1IhJfFkXXYZfP89TJsWdSQiIlIbQvAGbb16waBBUUdTofRpJ5/uDjsMiothp52ijkRERGrDhAlecHvkkbSY7KQiKokny8wTeAg+prqIiGSvDRvgiiugSxc47bSoo6mUknhVXXyxdzlbuTLqSEREpKaMGAGffOJTUjdoEHU0lVISr6ozzvBGDuo3LiKSnb79FoYPh4MP9smw0likSdzM8szsPTMrMrMZZnZ9lPEkpW9f7zd+110axU1EJBvdeCN88w3ceWfaXguPi7okvh4YFELoDfQBjjCzfhHHtHU33eRj5159ddSRiIhIKs2eDX/5C5x9NvTpE3U0WxVpEg/u+9jd+rElRBhScvLz4aqrYMoU+O67qKMREZFUueoqaNjQS+MZIOqSOGZW18w+ApYAL4UQJkcdU1KuugpmzoQmTaKOREREUuG//4WxY+Haa6F166ijSUrkSTyEsCmE0AcoAPYxs55l9zGz88xsiplNWZou45c3agR5ebB2rfclFBGRzPXtt16F3q2bD+6VISJP4nEhhO+A14EjKtg2IoTQN4TQt0WLFrUe2xZdf71PEP/ee1FHIiIi2+rCC322slGjvJCWIaJund7CzJrEbjcCDgU+jTKmKrv2WmjbFk4/HdasiToaERGpqqee8jHShw+HPfeMOpoqibok3gZ4zcymAe/j18SfjzimqtlpJ3j8cfjsM79OLiIimWPhQvjVr6BfP7jmmqijqbJIx04PIUwD9ogyhpQ4+GC/hnLXXV61Pnhw1BGJiMjWbN4MZ53ls1OOHOldhzNM5kWcrm66CZYtgw4doo5ERESScfvt8NJLPl94ly5RR7NNlMRTJS/Pz+TAz+42bkzr8XZFRHLaCy949fmJJ8L550cdzTaL+pp49gkBhg3zrgoh/cetERHJOZ9+6r/TvXvD3/+e9kOrbomSeKqZ+QTyTzzhs9+IiEj6+PZbn9SkYUMYNw622y7qiKplm5O4mWXGyGpR+O1v4eSTvarmhReijkZERMAvcw4bBvPmwZgxUFgYdUTVVp2SePeSd8xsVpn7J1fjtTObGTz6qA+ef8opPsa6iIhEJwQf0OXFF+H+++GAA6KOKCWqk8TLXvBtWeb+Q9V47czXuDGMH+8tHjdujDoaEZHcFQJceSWMGOEDdP3iF1FHlDKpbJ1eNqlnbkuBVMnP91J4vNHEqlWwww7RxiQikmtuvBHuuAMuugj+9Keoo0mp6pTEzcy2lJHUNBsSCfyee7zBW3FxtPGIiOSSu++G//s/OOMM/x3O4JboFalOEq8LfGdmC8xsIpBnZseZWX6KYssuBxwAy5fDoYf6MH8iIlKz/vY3H01z6FB4+GGok30dsqrzF20P9AauA2YBU4F/AF+a2SIgc6aBqQ177eUt1Rct8oQ+Z07UEYmIZK9bboELLoCjj/Yuvxk4pGoytvmvCiFsBj6OLSPB69eBrsBeQGZNBVMbDjgAXnvNx1YfOBBmzdI1chGRVArBG6/9+c9w6qnw2GNQv37UUdWYbU7iZjYIODyE8OO0LyGEAHwSW0ZVP7wstNde8OabMGmSEriISCpt2uTdyB580Gcm++tfs7IKvaTq/HVXADMr2mBmw8ws83vR15Ru3XzmHICJE+Hf/442HhGRTLd6tY+D/uCDXhK/776sT+BQvSS+B/BMJdtaA3+sxmvnhhB8+tKf/hRuvlljrYuIbIsFC/xy5bhx3hr9ppuyrhV6ZaqTxBsB6yrZNgYYWI3Xzg1mMHasDwP429/Cz38O6yo7pCIiUs5778E++8DcufD883DJJVFHVKuqk8Q/xxuwlRNCmA80q8Zr545Gjbzl5I03wqhRMGiQVwuJiMiWjRoFBx7ov6PvvgtHHhl1RLWuOkl8FPAXM8sru8HMWgLfVeO1c4sZXHcdPPOMT43XuHHUEYmIpK+1a+G88+D002Hffb003qNH1FFFojpJ/C/AGmCqmf3UzOoCmFkD4DbgtRTEl1tOOMEHJzCDmTPhhhu8taWIiLjZs2G//eChh7wB28svQ/PmUUcVmW1O4iGEDcAQYCLwBLDazL4AvgX2BX6bkghz1TPPwPDhcPjhPkCMiEiuGz3au+kuWAD/+Y83YMvSQVySVa329yGE9SGEy4EC4AzgbuBUYI8QgsYWrY7hw30603ffhZ494emno45IRCQa33zj0zqfeqr/Hn70EQwZEnVUaSFVneh2A14PIdwTQhgXQlibotfNbWedBUVFsOuu/gF+8smoIxIRqV0vvgi77w7PPusNgN94A9q1izqqtLHVJG5mO5vZr8zsTDOrbP/3gN/Hhl2VVOrSBd56ywcuGDrUH1u2LNqYRERq2ooV8MtfwhFHQJMm3njtuutyvvq8rGRK4hOAvwKPAOdWtEMIYSPwQmXbpZrq1fOB/Bs2hO+/92tCJ54IixdHHZmISOqNGeMjWz70EFx+OUydCnvsEXVUaSmZJL4DsAvwDt43vDKTgfOr8uZm1s7MXjOzT8xshpnlVi/9bdGwIZx/Powf7x/yESNg8+aooxIRqb7iYjj+eK91bNUKJk+G22+HvHI9mSUmmSQ+M4QwL4QwIITw0hb26wz0MbOqTBezEbg8hNAN6AdcaGbdq/D83FO/vo/uNm2an5mefz7sv783/BARyUTr1/usY127+nwSf/6zV5/37Rt1ZGkvmSQ+3sxuSGK/y2Lr7ZJ98xDC4hDCB7Hbq/DZz/KTfX5O23VXePVVePxxaN8edt7ZH9+wIdq4RESq4oUXvOHaNdfAoYfCjBlw1VVZPX1oKm01iYcQRgJ7m9ksM7vBzA43s4rm0BwArAohbNNIbWbWAZ9UZXIF284zsylmNmXp0qXb8vLZyczHW3/6ab+9aBF06uTVTxqDXUTS2YwZcNRRvpjBhAk+l0SnTlFHllGS7WI2FJgK/A5v6PaNmU0ysz+aWXz89PXArG0Jwsy2B54FLg0hrCy7PYQwIoTQN4TQt0WLFtvyFrnhhx/8jPbKK71a6okndL1cRNLLokVw7rnQqxe8/TbcdhtMn+6t0KXKkkriIYQ1IYRTgQOAR4FFwD7AdcB7ZvYGsAxYUNUAYtfQnwWeCCGMqerzpYQOHbxq6uWXoVkzOO00H1d4/fqoIxORXPfdd/D733u32ccfh4sv9pnHrrgCGjSIOrqMVaUOdyGEd/BW6vHq7wNLLB3xIVeTFutX/gjwSQjhzqo8V7bgkEPg/fd9iMKiIm/RDl59laOTBIhIRFatgnvugTvu8ER+8sk+XKqqzVOiOmOnzwshPB5CODuEsAt+Pbuqrar6A6cDg8zso9iisfRSoU4d+NnP4NZb/f60aT5c4aBBPuKRiEhNWrXKW5l37Ogl8IED4cMP4amnlMBTKFXDrhJCKAKureJz3gohWAihVwihT2x5IVUxSQldusDdd/vsaAceCAMG+HCGIUQdmYhkk2++geuv914z11wDe+/t3cXGjYM+faKOLuukLIkDhBCmp/L1JIUaNYJLLoHPP4d774V58+CnP4Vvq3QFRESkYgsXwtVXe/L+wx+85P3ee97qfO+9o44ua6U0iUsGaNwYfv1rb1Dy6qvQtKmXxk86ycdnX7066ghFJJNMnw5nnunV5rffDkcf7Zfvxo5V8q4FSuK5qkEDb7kO3thkwQK46CIoLPRJBoqLo41PRNLX5s1ewj7iCO8q9swzPlnJ7NneoHb33aOOMGcoiYuP9vbOOz5b2sCBcMst3l3t1VejjkxE0snKlX45rmtXn8+7qAj+9CcvBNx7rxqsRUBzuokzg/79fZk3Dx5+GPbbz7c9+aS3ND31VNihosH6RCSrTZsGDzwA//iHz6TYr583Xhs6VH28I6aSuJTXoQPceKM3hgN49lmvKmvbFn71K/joo0jDE5FasGYNjBzpEyz17g2PPuozjE2eDO++C8OGKYGnASVx2bp//cu/tEOHwmOP+expv/511FGJSKqF4HN3X3CBn7SfcQYsXw533unDpY4cCfvsE3WUUoKq02XrzLz6rF8//zI/8YRfEwNvAHfttT4Ry6BBULdutLGKSNUtWeKXzR5/3Gva8vLghBPgnHN8XAmzqCOUSiiJS9U0bVq6FD59Ojz/PIwaBfn5Pl77z38O3TUtvEhaW7fOv7sjR3pL840bYa+9vKvpqadCkyZRRyhJsJBhI3b17ds3TJkyJeowpKR162D8eD+LnzjRH/vqK2je3Oc317zAIulh0yZ47TUvdT/7rLc2b9s2cfKtuRXSkplNDSH0rWibSuJSfXl5cOKJvnz9tU8v2Ly5bxsyxGdRO/lkr55r1SraWEVyzebN3qbl6ae9P/dXX3kvk6FDvXHaIYfoMlgGU8M2Sa1WrXw4V/BGMgcf7A1jLrrIz/gPOQSeey7aGEWyXTxx/+Y3PgzqAQfAiBHebfSZZ/xk++9/h8MPVwLPcEriUnPM4Le/9SlQP/44MRLcnDm+fcUKHyBCo8OJVN/GjV5VftFF0K6ddw277z7Yc09vs7J0KYwZ4zVi8e6jkvF0TVxqVwh+Xa5ePR9b+fjj/fG+feG443zp3l2tYUWSsWYNvPSSf5fGj/dar0aNfDjUoUN9HPOddoo6SqmmLV0TVxKXaM2aBf/+t09TOGmSP/bZZz516tKl3kJWDeNEEhYv9lbl48fDyy/D2rX+PTnqKD8JPvJI2G67qKOUFFISl8yweDG88oq3lAXv5vKf/3ip4qijfN2yZbQxitS2zZt9AJYXXvDkHf/9a98efvITT9wDB+pkN4updbpkhjZtEgkcfLSoxo09kf/zn/7YySfDU0/57RBU7S7ZadkyryafONH7cC9dmhh06aabPHn36KHPvyiJSxobPNiXzZt9tqQXXkgMQLFhA+y2m19LHzzYW9m2axdtvCLbasMGH5P8xRd9mTLFT1KbNfMaqCFD/DMe77opEqMkLumvTh0fr32PPRKPrVrl3dcmTvQuM+BJ/dZb4ZhjoolTJFkhwCef+DXtl16C11/32cHq1EnMEDZ4sI+gpi5gsgVK4pKZmjaFRx7xH8MZM/yH8KWXfG508K4211zj/dIHDfIpVtWtRqI0f763+Xj1VV8WL/bHO3f2y0iHHeYnpvHPsEgSlMQls5lBz56+XHZZ4vF4N7bbboObb/YpE/fd10etatMmungldyxY4CeTr7/u63nz/PFWrfzEctAgP8ns2DHKKCXDKYlLdjr0UF9WrYI33/Qf0ffegxYtfPtVV3mXtoEDfZam/faD7bePNmbJXCHA55/DG2/A//7nSzxpN23qn7Hf/MZL2mqQJimkLmaSm+6910ex+uADL7XXrev9a8eP9+2rV6uvrVRu0yafwe/NN315661E9Xjz5n5yGD9B7NXLr3WLbKO07WJmZo8CRwNLQgg9o4xFcszFF/uyahW8847/CDdokNjevbvf79/fl/33h27d9GOcq1au9Nbjb7/ty6RJ3hANvFfEwQfDgAGeuLt1U0lbak2kJXEzGwh8D4xMNomrJC41btMmuPPOxA/2smX++MUXwz33+PZXXoF99tGcy9lo82YfNXDSJJ9E5N13fez/EPwkbvfdEyd2AwZAYWHUEUuWS9uSeAjhDTPrEGUMIuXUrQtXXulLCDB7tv+Q77abb58xw7v/AHTt6l2C9t3XB+DIz48ubtk2S5fC++97STu+fPedb9tpJ///nnCCt5vYd1/Yccdo4xUpQQ3bRLbEDHbd1Ze4zp29f++kSb48/zw89pgPg5mf7wnhiSdg77196dxZ1fDp4vvvvR3E++/7gCqTJ8MXX/i2OnW8l8OJJyZOzHQJRdJc5A3bYiXx57dUnW5m5wHnARQWFu41f/782glOJBkheEvkVq18mNjHHoMLLvCJKcBLc337ekO61q398bw8XTetaWvW+Eh/U6b42ONTpvgAK5s3+/Z27TxR77OPL3vuCTvsEG3MIhVI6wlQkkniJemauGSEjRu92v39932ZNs27HTVoAJdeCiNHetIouXTposS+rVat8oT9wQeesD/4wBP2pk2+vWVLP5GK14707esnXSIZIG2viYtkrXr1oHdvX37xi9LbBg3y0vjUqd5Q7ocffACaRYt8+6OP+nqPPbyVfMOGtRt7uvv6a/joI/jww8QyZ47XiIAn57328tm9+vb12/n5OkGSrBR1F7PRwEFAczMrBoaHEB6JMiaRGnfMMYnx3X/4AWbO9MQUd++9XqoEPxno1g2GDoXhw/2x777LjVbxGzZ4K/GiotLLV18l9unQwU92Tj/d13vtpRH5JKdE3Tp9WJTvLxK5Bg2gT5/Sj33wgZcsi4q8xFlU5Nd3wa/nFhb6tfdevRJL//6wyy61H38qhOAnMdOn+2WHadP89syZsH6979OggddKHHGEH68+ffzv1jjjkuMivyZeVbomLjlt/Xq4/35P7NOn+3X39evhd7+DP/4RVqyAc89NjCffs6cn93SZCWvFCo/54499mT7d1/G++OAl6fjJSe/evu7aFerXjy5ukQjpmrhItmjYsPRELxs3eqk9PkTs4sVekv/XvxLXiPPy4PHH4aSTYMkS7xbXo4dXRddUcl+1ykvSM2aUXoqLE/tst52fZBx3nA+g0rOnJ2zNmS2SNCVxkUxWr56XUuO6dvWkvnq1t86Ol3i7d/ft//ufJ3Pw5N61q2+7/nrvz75mjZd4ky31Ll/u7xNfZs70ZcGCxD7x9znwwNI1BIWF6oMtUk1K4iLZaLvtvGV23zI1cEce6aPPzZiRSLpvv50okT/4oM/w1rmzN6jr2tVHquvXz/vCf/IJfPqpL5984iX7uEaN/DkHHugnBt26eYm/U6f0qc4XyTJK4iK5ZPvtPSH361d+26pV3up9yBCYNctHpfv3v8vvV6+eV3l36uSt7Pv1825z7durZC1Sy5TERXLJ+vU+7/Xs2d59q+QSn0oTPBl37AgHHOCjzMWHIP3f/+A///EkHx929rnnEl3kbr/dxyKPD1XbpYv321YfbZEaoSQukm1++MGrvmfP9uvj8fVnn8H8+YlhR8FL1LvuCocf7tXm8aVz54oHmRk4EH7/e280t2yZJ/Nvv01sf+cdH0t+w4bEYwMGwBtv+O0HHvBq986dPcG3aKEEL1INSuIimWjNGi9Rz53rCTq+njOnfKLecUdPmPvuC6edlighd+kCTZtu2/ubeQJu0aL042PGeIv5+fP95GH27NLjkQ8fXvo6+g47+Ih2d97p9594AgoKvFtc27aqnhfZCvUTF0lHIXi19OefJ5L13LmJ2/EhWuOaNEmUbjt39iQYT9TNm6dPabdkLUH8xKN3bzjnHD8xiXeVA2/V3rGjjzV/3nleun/xRf/bOnTwEr1IDlA/cZF0tHq1J7QvvvDl889Lr7//vvT++fmewAYP9kZlu+ySSNjbWqKubQ0alJ/aNS4vz//2kjULc+d6YzzwY/WTnyT2z8/3JH/ttd4Yb+VKHwSnY0eV4iVnKImL1JR16+DLLxOJuuy6ZLUy+FCqnTr5MmiQJ+f4/VwoecYb03XsCIcdVn57u3bePS5eIxE/2Ykn6ylT4JBD/HaDBt5avkMHuOkm72q3ZInv36GDz2qWLrUTItWgJC6yrVav9iQ9f74n5rLrkq29wQdQKSz0JHXMMYmEFV+UWLYsL6/y7nHgE6BMnJio2YifMMX7qE+YAGeemXiteJL/29/8+M+b55cp2rf3oV9VkpcMoCQuUpF46+v58xOJuuy65HjfkEjS7dv7RB0dO3qSiC9t22rQk5q0885+qaEygwfD+PGerOPL/PmJGo5Ro7zlPfj/sl07/1+OGeNtDoqKvJ1CYaFvy/aaEckISuKSm1av9qFBFyzwpFx2/eWXXh1eUuPG/qNeWOhTXsZLcu3bJ0pvStLpq3VrOProyrefdZb/X0vWqBQXJ1rX338/jBiR2L9lS7/U8c47XoPy8sve3a5dO19at9bnQWqcWqdL9lm7FhYu9IRcXJxI1iWXkn2bwX+E27RJlLIKC0sv7dt74zFVd+euRYu8VX38JG/+fD/RGznStw8Z4lX2cfXqebe+t97y+48+6iePBQWJRN+ihartZavUOl2yx6pVnpgXLvR1Rcvy5eWf17RpIjn375/4EY0n7fx8bziTFtEAAA50SURBVAwlUpm2bX2pzJNPlq/VKVnlft99PsNcSQMH+ih4AFde6ev8fE/0BQV+SaZVq9T+HZJVlMQlPWzc6K2HFy4svSxalEjYCxd6Ei+rWbNEUt5/f/8RbNcu8UNYUOBV4SI1qUkTX3r1qnj7++97O4qStUMluwa+9ppPTFPyMs4pp8Do0X570CDYaSf/fLdt6+u99vIZ4SRnKYlLzdq82UvGixZ5a+1FixJLPEkvWgRffVV6lDHw6sg2bfzHqkcPHxq0oKB0SaVtW29pLJLu6tTx6+gtW8Kee5bfPmWKN6j85ptErVKzZr5twwavKZozx0vu8ctBV1wBt93mYwq0bp1I8PHl2GN9/PsNG7x2oE0bndBmGSVx2TabN3tL3cWLSy8lk3X8sZLjaMc1a5YoTey+u69LljDy8/3HTtcLJZeY+XejWTMfyS6ufn3vPhe3Zo1/x+LV9Rs3wrnn+onx4sU+Mc3ChV4jdcABnvzjc8o3aeLfszZt4Jpr4NBD/bv86quJx1u3TgyyI2lNSVxKW7fOS8WLF/s6fjt+P377669h06byz995Z/8RaNvWJ9Jo0yZRmo6XDlq3VulZpDoaN/bR+uKaNIG77iq9TwiJ72jLlvD3vyeSfPxEO779o4+86r6k7bf37nWHHebbH3ss8X1u3drXXbrouxwxJfFcsHGjn2nHk/LXX1d8e/FiWLGi/PPj1YDxL26vXokvc3xRchZJL2Z+SQq8ZB8f6KYiBxwA06eXr1nr0MG3z5kDjzxSfijgKVP8uvzo0T4NbevWiaVVKzjjDL+Ov3Kln1TsuKN6eKSYknim2rDBG4J9/fXWl2XL/AtU1g47+BetTRuv0j7ssMT9+Nl269beDaaePioiWatRI28gV1kjuRNO8OX770uf9MfHwN9+e/+tWLzYS+3xmrqTT/YkfscdcMMNfpLfqpUvrVt78m/c2Kv/v/wysa1VK69dUMLfKvUTTxch+BcknpiXLEks8fsl1998U/HrNGpU+kw4/mWpaF1yxigRkVSJN2ht1sxr8iZP9v7y8dq/eOHi/fd9+y9+4SX9knbayRvwmflUtdOne41gq1a+zs+Hgw9OvF8Wt59J637iZnYEcA9QF3g4hHBLxCGlztq1Xo29ZEn5dUXL+vUVv06TJokPbvfu/sGNJ+j4hzq+qDGKiEStTp3Sc83vu68vlbn1VrjkktI1iOvXJ0riX3wBr7zij//wgz/WubMPvgM+pO7UqYnW/y1bQp8+8Lvf+fbXXvPXatnS42raNGtG04u0JG5mdYHPgMOAYuB9YFgIYWZlz4msJB6Cj7a0dGnyS9nrR3ENGyY+aC1aJJJx2futWvl9DUIiIuK/wytWeKFn7dpEC/4HH/SSerymculSr+ofO9a3d+0Ks2YlXqdOHTj+ePjXv/z+hRf6ukWLxLLbbonX37Qp0qSfziXxfYA5IYTPAczsKeBYoNIknlLxkvGyZYklfr/kOn677FjacQ0blv7nd+ni63hSLpmgW7Twa9G61iMiUjVmiUF1Sjr//C0/b8wYv15fska0Y8fE9smTvbRf8jLlGWd4i/wQvIazUSP//W7e3NdDh8Lpp3uCf+IJf7x580QX2VoSdRLPBxaUuF8MbKHOJcX69Ck/XSR4C8r4P6ttWz8bi9+P/wNLLkrKIiLpq3v3RD/5isRrdzdu9Gv5S5cm+uBv2uT96UsW7L74IpE7vv3WE37cMcfAuHE183dUIOokXlHmK1e/b2bnAecBFBYWpu7d77gjce0mnqCbN1f1tYhILqpXL9G+qORjw4dX/pydd/YuePEkX7aWoIZFncSLgXYl7hcAi8ruFEIYAYwAvyaesncfNixlLyUiIjmobl3YZRdfIhB1m/z3gS5m1tHMGgCnAM9FHJOIiEhGiLQkHkLYaGYXAS/iXcweDSHMiDImERGRTBF1dTohhBeAF6KOQ0REJNNEXZ0uIiIi20hJXEREJENl3NjpZrYUmF+Nl2gOLEtROLlOxzI1dBxTR8cydXQsU6e6x7J9CKFFRRsyLolXl5lNqWz4OqkaHcvU0HFMHR3L1NGxTJ2aPJaqThcREclQSuIiIiIZKheT+IioA8giOpapoeOYOjqWqaNjmTo1dixz7pq4iIhItsjFkriIiEhWyMokbmZHmNksM5tjZtdUsL2hmT0d2z7ZzDrUfpSZIYlj+Rszm2lm08zsFTNrH0WcmWBrx7LEfieYWTAztQyuRDLH0sxOin02Z5jZk7UdY6ZI4jteaGavmdmHse/5kCjiTHdm9qiZLTGzjyvZbmZ2b+w4TzOzPVPyxiGErFrwMdjnAp2ABkAR0L3MPhcAD8RunwI8HXXc6bgkeSwPBhrHbv9Kx3Lbj2Vsvx2AN4BJQN+o407HJcnPZRfgQ2Dn2P2WUcedjkuSx3IE8KvY7e7AvKjjTscFGAjsCXxcyfYhwAR8Cu5+wORUvG82lsT3AeaEED4PIfwAPAUcW2afY4HHY7f/BRxiZhXNbZ7rtnosQwivhRDWxO5OwqeTlfKS+VwC/BG4FVhXm8FlmGSO5bnAfSGEbwFCCEtqOcZMkcyxDMCOsds7UcF00QIhhDeAb7awy7HAyOAmAU3MrE113zcbk3g+sKDE/eLYYxXuE0LYCKwAmtVKdJklmWNZ0jn4maaUt9VjaWZ7AO1CCM/XZmAZKJnP5a7Armb2tplNMrMjai26zJLMsfwDcJqZFeOTVf26dkLLOlX9PU1K5LOY1YCKStRlm+Ans49U4TiZ2WlAX+DAGo0oc23xWJpZHeAu4MzaCiiDJfO5rIdXqR+E1w69aWY9Qwjf1XBsmSaZYzkMeCyEcIeZ7Qf8I3YsN9d8eFmlRvJONpbEi4F2Je4XUL7658d9zKweXkW0pWqQXJXMscTMDgWuA44JIayvpdgyzdaO5Q5AT+B1M5uHXzN7To3bKpTsd3xcCGFDCOELYBae1KW0ZI7lOcA/AUII7wJ5+FjgUjVJ/Z5WVTYm8feBLmbW0cwa4A3Xniuzz3PAGbHbJwCvhljLAyllq8cyVgX8IJ7Add2xcls8liGEFSGE5iGEDiGEDnj7gmNCCFOiCTetJfMdH4s3usTMmuPV65/XapSZIZlj+SVwCICZdcOT+NJajTI7PAf8PNZKvR+wIoSwuLovmnXV6SGEjWZ2EfAi3vLy0RDCDDO7AZgSQngOeASvEpqDl8BPiS7i9JXksbwN2B54JtY28MsQwjGRBZ2mkjyWkoQkj+WLwOFmNhPYBFwZQlgeXdTpKcljeTnwkJldhlf/nqlCT3lmNhq/fNM81n5gOFAfIITwAN6eYAgwB1gDnJWS99X/QkREJDNlY3W6iIhITlASFxERyVBK4iIiIhlKSVxERCRDKYmLiIhkKCVxERGRDKUkLpKFzGw3M/uDme0WdSwiUnPUT1wky8Rm5HsT6IVPLTlQg3OIZCeVxEWyzzlAB6AP0JHEEMMikmWUxEWySGyc8JuBs0IInwNnA7eaWdMK9u1pZhvN7LBqvN/rZvb6Nj73ODP7wcw0MYnINlJ1ukiOMrP/Ag1DCNs8fWw8gYcQDtrG508F5ocQfrqtMYjkMpXERXJQbF7ow4A7Iw7lHuB4M+sRcRwiGUlJXCQ3XQAsx2dWitIYfEanX0Ych0hGUhIXyRJmdr2ZBTMbZGajzexrM1tjZu+Z2cAS+9UDjgNeCiFsKPMaD8Reo20Fr79b7Br2PVuJo5GZFZvZl2bWsMy2h81sk5mdAhBC+B5vSX/itv/lIrlLSVwke/TB585+CmgE/B6vLu8OTDCzNrH99sLngH+vgtd4N7bep4JtdwErgT9sKYgQwlp8LuV2eIkfADO7GW85/+sQwlNl3rOVmXXd0uuKSHlK4iLZow9QF7glhHBcCGFECOF3wEVAY+Ck2H7dY+u5FbzGpNi6VBI3s6OAI4H/CyF8m0QsjwEzgGvNbHszuxS4BhgeQri/zL7xOHRdXKSKlMRFsoCZ7QwUAm+FEMo2Vnsltu4QW7eIrb8p+zohhFmxx39M4mZWHy/Rfww8mEw8IYRNeNJuAYyNPf8vIYQbKth9eWzdMpnXFpGEelEHICIpsUds/XAF2+In69/H1vF+pVbJa00C+puZxUZ6uwTYFTg0lpyTEkJ43sw+AA7Bq/gvqWTXeBzq7ypSRSqJi2SHPrH1lAq27RtbfxhbL42tyw0AEzMJ2AnYzcxa4tfWx4YQXqlk/wqZ2Ukl4lq1haFf43EsrWS7iFRCJXGR7BBPlhsr2PYbvIr8v7H7H8fWlY2UVrJx20CgIXB5VYIxs8OBfwD/BjYAZ5vZXSGETyrYvXOZuEQkSSqJi2SHeHV6qdHXzOwcvCR+Y6w7F3iJfCXQr5LXmgxsxluSnwXcHRvCNSlmti/e//tt4GfA72Kvd3MlT+kHfB27Hi8iVaCSuEiGi/XF7oon57vMrD0wDzgIGAb8E7g7vn8IYZOZjQGONbOGIYT1JV8vhLDKzGbipfCvgD9VIZZuwH+Az4DjYq8918weAX5pZv1DCG+X2H97YADwaJX/cBFRSVwkC/TET8jvBK4GTgX+ipfOLwOGVXA9+m/AzsDRlbxmvA/5tSGEVckEYWaFeJX9CuDIEMLKEptvANYCt5Z52lC8+1tSrd5FpDRNgCKS4WJV5g8DvUMI06rwvInAdiGEAWUerw98Sqyr2ZbmItcEKCLRUklcJPPtgTce+7SKz7sc2C/WCK2kK/B5yH+9pQReXWZ2HLA7XnsgIttA18RFMl8fYFYI4YeqPCmEMIPYb0BsvvHBQC/gSuDOEMKkLTy92kIIY4EGNfkeItlOSVwkg5mZ4Yn3+Wq+1GDgSWAJPkb6NdV8PRGpBbomLiIikqF0TVxERCRDKYmLiIhkKCVxERGRDKUkLiIikqGUxEVERDKUkriIiEiGUhIXERHJUEriIiIiGUpJXEREJEP9P5VmBCAoNxpqAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "def cross_ent(prediction, ground_truth):\n", + " t = 1 if ground_truth > 0.5 else 0\n", + " return -t * np.log(prediction) - (1 - t) * np.log(1 - prediction)\n", + "plot_cross_ent()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1.429664938969559" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class CrossEntropyLoss:\n", + " def forward(self,p,y):\n", + " self.p = p\n", + " self.y = y\n", + " p_of_y = p[np.arange(len(y)), y]\n", + " log_prob = np.log(p_of_y)\n", + " return -log_prob.mean()\n", + "\n", + "cross_ent_loss = CrossEntropyLoss()\n", + "p = softmax.forward(net.forward(train_x[0:10]))\n", + "cross_ent_loss.forward(p,train_labels[0:10])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Задача минимизации\n", + "Описав нейронную сеть как модель $f_\\theta$ и функцию ошибки $\\mathcal{L}(Y,f_\\theta(X))$, можем рассмотреть $\\mathcal{L}$ как функцию $\\theta$ на всем множестве обучающей выборки $\\mathcal{L}(\\theta) = \\mathcal{L}(Y,f_\\theta(X))$\n", + "\n", + "В этом случае задача обучения сети будет формулироваться как задача минимизации $\\mathcal{L}$ по $\\theta$:\n", + "$$\n", + "\\theta = \\mathrm{argmin}_{\\theta} \\mathcal{L}(Y,f_\\theta(X))\n", + "$$\n", + "\n", + "Минимизацию можно осуществлять разными методами, например, стохастическим градиентным спуском (stochastic gradient descent, SGD)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Реализация нейронных сетей\n", + "\n", + " * Вручную\n", + " * С использованием готовых фреймворков\n", + " - PyTorch\n", + " - Tensorflow\n", + " - Chainer\n", + " - [Microsoft Cognitive Toolkit](http://cntk.ai)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Вычислительный граф\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.429664938969559\n" + ] + } + ], + "source": [ + "z = net.forward(train_x[0:10])\n", + "p = softmax.forward(z)\n", + "loss = cross_ent_loss.forward(p,train_labels[0:10])\n", + "print(loss)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Обучение сети\n", + "\n", + " * Для обучения сети необходимо предъявлять ей примеры, считать ошибку и подстраивать коэффициенты\n", + " * В соответствии с принципом градиентного спуска, необходимо расчитывать изменение коэффициентов в соответствии с градиентом функции $\\nabla f_\\theta$\n", + " * Итерация обучения выглядит так:\n", + " $$\\begin{align}\\def\\L{\\mathcal{L}}\n", + " W^{i+1}&=W^i-\\eta\\frac{\\partial\\L}{\\partial W}\\cr\n", + " b^{i+1}&=b^i-\\eta\\frac{\\partial\\L}{\\partial b}\n", + " \\end{align}\n", + " $$" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Обратное распространение ошибки\n", + "\n", + "\n", + "\n", + "$$\\def\\zz#1#2{\\frac{\\partial#1}{\\partial#2}}\n", + "\\begin{align}\n", + "\\zz{\\L}{W} =& \\zz{\\L}{p}\\zz{p}{z}\\zz{z}{W}\\cr\n", + "\\zz{\\L}{b} =& \\zz{\\L}{p}\\zz{p}{z}\\zz{z}{b}\n", + "\\end{align}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Обратное распространение ошибки\n", + "\n", + "\n", + "\n", + " * Не повторяем одинаковые вычисления\n", + " * Вычисляем ошибку на каждом узле начиная с конца\n", + " * Обратное распространение ошибки\n", + " * Все вычисления фреймворк берёт на себя" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Реализация обратного распространения\n", + "\n", + "* К каждому узлу добавляем функцию `backward`, которая вычисляет производную и значение ошибки\n", + "* После вычисления производных, реализуем обновление весов в соответствии с формулой выше\n", + "\n", + "Например, для линейного узла $z = x\\times W+b$:\n", + "$$\\begin{align}\n", + "\\frac{\\partial z}{\\partial W} &= x \\\\\n", + "\\frac{\\partial z}{\\partial b} &= 1 \\\\\n", + "\\end{align}$$\n", + "\n", + "Соответственно, если на вход пришла ошибка $\\Delta z$, то изменения весов вычисляются так:\n", + "$$\\begin{align}\n", + "\\Delta x &= \\Delta z \\times W \\\\\n", + "\\Delta W &= \\frac{\\partial z}{\\partial W} \\Delta z = \\Delta z \\times x \\\\\n", + "\\Delta b &= \\frac{\\partial z}{\\partial b} \\Delta z = \\Delta z \\\\\n", + "\\end{align}$$\n", + "\n", + "**ВАЖНО:** Вычисления производятся не для одного элемента обучающей выборки, а сразу для целой последовательности, называемой **minibatch**. Необходимые значения градиентов $\\Delta W$ и $\\Delta b$ вычисляются по всей выборке, а вектора имеют соответствующую размерность: $x\\in\\mathbb{R}^{\\mathrm{minibatch}\\, \\times\\, \\mathrm{nclass}}$" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "class Linear:\n", + " def __init__(self,nin,nout):\n", + " self.W = np.random.normal(0, 1.0/np.sqrt(nin), (nout, nin))\n", + " self.b = np.zeros((1,nout))\n", + " self.dW = np.zeros_like(self.W)\n", + " self.db = np.zeros_like(self.b)\n", + " \n", + " def forward(self, x):\n", + " self.x=x\n", + " return np.dot(x, self.W.T) + self.b\n", + " \n", + " def backward(self, dz):\n", + " dx = np.dot(dz, self.W)\n", + " dW = np.dot(dz.T, self.x)\n", + " db = dz.sum(axis=0)\n", + " self.dW = dW\n", + " self.db = db\n", + " return dx\n", + " \n", + " def update(self,lr):\n", + " self.W -= lr*self.dW\n", + " self.b -= lr*self.db" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Аналогичный образом функции обратного распространения `backward` добавляются к другим составляющим вычислительного графа:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "class Softmax:\n", + " def forward(self,z):\n", + " self.z = z\n", + " zmax = z.max(axis=1,keepdims=True)\n", + " expz = np.exp(z-zmax)\n", + " Z = expz.sum(axis=1,keepdims=True)\n", + " return expz / Z\n", + " def backward(self,dp):\n", + " p = self.forward(self.z)\n", + " pdp = p * dp\n", + " return pdp - p * pdp.sum(axis=1, keepdims=True)\n", + " \n", + "class CrossEntropyLoss:\n", + " def forward(self,p,y):\n", + " self.p = p\n", + " self.y = y\n", + " p_of_y = p[np.arange(len(y)), y]\n", + " log_prob = np.log(p_of_y)\n", + " return -log_prob.mean()\n", + " def backward(self,loss):\n", + " dlog_softmax = np.zeros_like(self.p)\n", + " dlog_softmax[np.arange(len(self.y)), self.y] -= 1.0/len(self.y)\n", + " return dlog_softmax / self.p" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Теперь напишем цикл обучения модели на нашем датасете. Будем рассматривать один проход по модели - т.н. **эпоху**" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Initial accuracy: 0.725\n", + "Final accuracy: 0.825\n" + ] + } + ], + "source": [ + "lin = Linear(2,2)\n", + "softmax = Softmax()\n", + "cross_ent_loss = CrossEntropyLoss()\n", + "\n", + "pred = np.argmax(lin.forward(train_x),axis=1)\n", + "acc = (pred==train_labels).mean()\n", + "print(\"Initial accuracy: \",acc)\n", + "\n", + "batch_size=4\n", + "for i in range(0,len(train_x),batch_size):\n", + " xb = train_x[i:i+batch_size]\n", + " yb = train_labels[i:i+batch_size]\n", + " \n", + " # forward pass\n", + " z = lin.forward(xb)\n", + " p = softmax.forward(z)\n", + " loss = cross_ent_loss.forward(p,yb)\n", + " \n", + " # backward pass\n", + " dp = cross_ent_loss.backward(loss)\n", + " dz = softmax.backward(dp)\n", + " dx = lin.backward(dz)\n", + " lin.update(0.1)\n", + " \n", + "pred = np.argmax(lin.forward(train_x),axis=1)\n", + "acc = (pred==train_labels).mean()\n", + "print(\"Final accuracy: \",acc)\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Для удобства опишем класс, который позволяет объединять узлы вычислительного графа в единую сеть, и применять функции `forward` и `backward` сразу ко всей сети последовательно:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "scrolled": true, + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "class Net:\n", + " def __init__(self):\n", + " self.layers = []\n", + " \n", + " def add(self,l):\n", + " self.layers.append(l)\n", + " \n", + " def forward(self,x):\n", + " for l in self.layers:\n", + " x = l.forward(x)\n", + " return x\n", + " \n", + " def backward(self,z):\n", + " for l in self.layers[::-1]:\n", + " z = l.backward(z)\n", + " return z\n", + " \n", + " def update(self,lr):\n", + " for l in self.layers:\n", + " if 'update' in l.__dir__():\n", + " l.update(lr)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ещё раз пробуем создать и обучить нашу нейросеть:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Initial loss=0.6212072429381601, accuracy=0.6875: \n", + "Final loss=0.44369925927417986, accuracy=0.8: \n", + "Test loss=0.4767711377257787, accuracy=0.85: \n" + ] + } + ], + "source": [ + "net = Net()\n", + "net.add(Linear(2,2))\n", + "net.add(Softmax())\n", + "loss = CrossEntropyLoss()\n", + "\n", + "def get_loss_acc(x,y,loss=CrossEntropyLoss()):\n", + " p = net.forward(x)\n", + " l = loss.forward(p,y)\n", + " pred = np.argmax(p,axis=1)\n", + " acc = (pred==y).mean()\n", + " return l,acc\n", + "\n", + "print(\"Initial loss={}, accuracy={}: \".format(*get_loss_acc(train_x,train_labels)))\n", + "\n", + "def train_epoch(net, train_x, train_labels, loss=CrossEntropyLoss(), batch_size=4, lr=0.1):\n", + " for i in range(0,len(train_x),batch_size):\n", + " xb = train_x[i:i+batch_size]\n", + " yb = train_labels[i:i+batch_size]\n", + "\n", + " p = net.forward(xb)\n", + " l = loss.forward(p,yb)\n", + " dp = loss.backward(l)\n", + " dx = net.backward(dp)\n", + " net.update(lr)\n", + " \n", + "train_epoch(net,train_x,train_labels)\n", + " \n", + "print(\"Final loss={}, accuracy={}: \".format(*get_loss_acc(train_x,train_labels)))\n", + "print(\"Test loss={}, accuracy={}: \".format(*get_loss_acc(test_x,test_labels)))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "def train_and_plot(n_epoch, net, loss=CrossEntropyLoss(), batch_size=4, lr=0.1):\n", + " fig, ax = plt.subplots(2, 1)\n", + " ax[0].set_xlim(0, n_epoch + 1)\n", + " ax[0].set_ylim(0,1)\n", + "\n", + " train_acc = np.empty((n_epoch, 3))\n", + " train_acc[:] = np.NAN\n", + " valid_acc = np.empty((n_epoch, 3))\n", + " valid_acc[:] = np.NAN\n", + "\n", + " for epoch in range(1, n_epoch + 1):\n", + "\n", + " train_epoch(net,train_x,train_labels,loss,batch_size,lr)\n", + " tloss, taccuracy = get_loss_acc(train_x,train_labels,loss)\n", + " train_acc[epoch-1, :] = [epoch, tloss, taccuracy]\n", + " vloss, vaccuracy = get_loss_acc(test_x,test_labels,loss)\n", + " valid_acc[epoch-1, :] = [epoch, vloss, vaccuracy]\n", + " \n", + " ax[0].set_ylim(0, max(max(train_acc[:, 2]), max(valid_acc[:, 2])) * 1.1)\n", + "\n", + " plot_training_progress(train_acc[:, 0], (train_acc[:, 2],\n", + " valid_acc[:, 2]), fig, ax[0])\n", + " plot_decision_boundary(net, fig, ax[1])\n", + " fig.canvas.draw()\n", + " fig.canvas.flush_events()\n", + "\n", + " return train_acc, valid_acc" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "import matplotlib.cm as cm\n", + "\n", + "def plot_decision_boundary(net, fig, ax):\n", + " draw_colorbar = True\n", + " # remove previous plot\n", + " while ax.collections:\n", + " ax.collections.pop()\n", + " draw_colorbar = False\n", + "\n", + " # generate countour grid\n", + " x_min, x_max = train_x[:, 0].min() - 1, train_x[:, 0].max() + 1\n", + " y_min, y_max = train_x[:, 1].min() - 1, train_x[:, 1].max() + 1\n", + " xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),\n", + " np.arange(y_min, y_max, 0.1))\n", + " grid_points = np.c_[xx.ravel().astype('float32'), yy.ravel().astype('float32')]\n", + " n_classes = max(train_labels)+1\n", + " while train_x.shape[1] > grid_points.shape[1]:\n", + " # pad dimensions (plot only the first two)\n", + " grid_points = np.c_[grid_points,\n", + " np.empty(len(xx.ravel())).astype('float32')]\n", + " grid_points[:, -1].fill(train_x[:, grid_points.shape[1]-1].mean())\n", + "\n", + " # evaluate predictions\n", + " prediction = np.array(net.forward(grid_points))\n", + " # for two classes: prediction difference\n", + " if (n_classes == 2):\n", + " Z = np.array([0.5+(p[0]-p[1])/2.0 for p in prediction]).reshape(xx.shape)\n", + " else:\n", + " Z = np.array([p.argsort()[-1]/float(n_classes-1) for p in prediction]).reshape(xx.shape)\n", + " \n", + " # draw contour\n", + " levels = np.linspace(0, 1, 40)\n", + " cs = ax.contourf(xx, yy, Z, alpha=0.4, levels = levels)\n", + " if draw_colorbar:\n", + " fig.colorbar(cs, ax=ax, ticks = [0, 0.5, 1])\n", + " c_map = [cm.jet(x) for x in np.linspace(0.0, 1.0, n_classes) ]\n", + " colors = [c_map[l] for l in train_labels]\n", + " ax.scatter(train_x[:, 0], train_x[:, 1], marker='o', c=colors, s=60, alpha = 0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "outputs": [], + "source": [ + "def plot_training_progress(x, y_data, fig, ax):\n", + " styles = ['k--', 'g-']\n", + " # remove previous plot\n", + " while ax.lines:\n", + " ax.lines.pop()\n", + " # draw updated lines\n", + " for i in range(len(y_data)):\n", + " ax.plot(x, y_data[i], styles[i])\n", + " ax.legend(ax.lines, ['training accuracy', 'validation accuracy'],\n", + " loc='upper center', ncol = 2)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "application/javascript": [ + "/* Put everything inside the global mpl namespace */\n", + "window.mpl = {};\n", + "\n", + "\n", + "mpl.get_websocket_type = function() {\n", + " if (typeof(WebSocket) !== 'undefined') {\n", + " return WebSocket;\n", + " } else if (typeof(MozWebSocket) !== 'undefined') {\n", + " return MozWebSocket;\n", + " } else {\n", + " alert('Your browser does not have WebSocket support. ' +\n", + " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", + " 'Firefox 4 and 5 are also supported but you ' +\n", + " 'have to enable WebSockets in about:config.');\n", + " };\n", + "}\n", + "\n", + "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", + " this.id = figure_id;\n", + "\n", + " this.ws = websocket;\n", + "\n", + " this.supports_binary = (this.ws.binaryType != undefined);\n", + "\n", + " if (!this.supports_binary) {\n", + " var warnings = document.getElementById(\"mpl-warnings\");\n", + " if (warnings) {\n", + " warnings.style.display = 'block';\n", + " warnings.textContent = (\n", + " \"This browser does not support binary websocket messages. \" +\n", + " \"Performance may be slow.\");\n", + " }\n", + " }\n", + "\n", + " this.imageObj = new Image();\n", + "\n", + " this.context = undefined;\n", + " this.message = undefined;\n", + " this.canvas = undefined;\n", + " this.rubberband_canvas = undefined;\n", + " this.rubberband_context = undefined;\n", + " this.format_dropdown = undefined;\n", + "\n", + " this.image_mode = 'full';\n", + "\n", + " this.root = $('
');\n", + " this._root_extra_style(this.root)\n", + " this.root.attr('style', 'display: inline-block');\n", + "\n", + " $(parent_element).append(this.root);\n", + "\n", + " this._init_header(this);\n", + " this._init_canvas(this);\n", + " this._init_toolbar(this);\n", + "\n", + " var fig = this;\n", + "\n", + " this.waiting = false;\n", + "\n", + " this.ws.onopen = function () {\n", + " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", + " fig.send_message(\"send_image_mode\", {});\n", + " if (mpl.ratio != 1) {\n", + " fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n", + " }\n", + " fig.send_message(\"refresh\", {});\n", + " }\n", + "\n", + " this.imageObj.onload = function() {\n", + " if (fig.image_mode == 'full') {\n", + " // Full images could contain transparency (where diff images\n", + " // almost always do), so we need to clear the canvas so that\n", + " // there is no ghosting.\n", + " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", + " }\n", + " fig.context.drawImage(fig.imageObj, 0, 0);\n", + " };\n", + "\n", + " this.imageObj.onunload = function() {\n", + " fig.ws.close();\n", + " }\n", + "\n", + " this.ws.onmessage = this._make_on_message_function(this);\n", + "\n", + " this.ondownload = ondownload;\n", + "}\n", + "\n", + "mpl.figure.prototype._init_header = function() {\n", + " var titlebar = $(\n", + " '
');\n", + " var titletext = $(\n", + " '
');\n", + " titlebar.append(titletext)\n", + " this.root.append(titlebar);\n", + " this.header = titletext[0];\n", + "}\n", + "\n", + "\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", + "\n", + "}\n", + "\n", + "\n", + "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", + "\n", + "}\n", + "\n", + "mpl.figure.prototype._init_canvas = function() {\n", + " var fig = this;\n", + "\n", + " var canvas_div = $('
');\n", + "\n", + " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", + "\n", + " function canvas_keyboard_event(event) {\n", + " return fig.key_event(event, event['data']);\n", + " }\n", + "\n", + " canvas_div.keydown('key_press', canvas_keyboard_event);\n", + " canvas_div.keyup('key_release', canvas_keyboard_event);\n", + " this.canvas_div = canvas_div\n", + " this._canvas_extra_style(canvas_div)\n", + " this.root.append(canvas_div);\n", + "\n", + " var canvas = $('');\n", + " canvas.addClass('mpl-canvas');\n", + " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", + "\n", + " this.canvas = canvas[0];\n", + " this.context = canvas[0].getContext(\"2d\");\n", + "\n", + " var backingStore = this.context.backingStorePixelRatio ||\n", + "\tthis.context.webkitBackingStorePixelRatio ||\n", + "\tthis.context.mozBackingStorePixelRatio ||\n", + "\tthis.context.msBackingStorePixelRatio ||\n", + "\tthis.context.oBackingStorePixelRatio ||\n", + "\tthis.context.backingStorePixelRatio || 1;\n", + "\n", + " mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n", + "\n", + " var rubberband = $('');\n", + " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", + "\n", + " var pass_mouse_events = true;\n", + "\n", + " canvas_div.resizable({\n", + " start: function(event, ui) {\n", + " pass_mouse_events = false;\n", + " },\n", + " resize: function(event, ui) {\n", + " fig.request_resize(ui.size.width, ui.size.height);\n", + " },\n", + " stop: function(event, ui) {\n", + " pass_mouse_events = true;\n", + " fig.request_resize(ui.size.width, ui.size.height);\n", + " },\n", + " });\n", + "\n", + " function mouse_event_fn(event) {\n", + " if (pass_mouse_events)\n", + " return fig.mouse_event(event, event['data']);\n", + " }\n", + "\n", + " rubberband.mousedown('button_press', mouse_event_fn);\n", + " rubberband.mouseup('button_release', mouse_event_fn);\n", + " // Throttle sequential mouse events to 1 every 20ms.\n", + " rubberband.mousemove('motion_notify', mouse_event_fn);\n", + "\n", + " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", + " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", + "\n", + " canvas_div.on(\"wheel\", function (event) {\n", + " event = event.originalEvent;\n", + " event['data'] = 'scroll'\n", + " if (event.deltaY < 0) {\n", + " event.step = 1;\n", + " } else {\n", + " event.step = -1;\n", + " }\n", + " mouse_event_fn(event);\n", + " });\n", + "\n", + " canvas_div.append(canvas);\n", + " canvas_div.append(rubberband);\n", + "\n", + " this.rubberband = rubberband;\n", + " this.rubberband_canvas = rubberband[0];\n", + " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", + " this.rubberband_context.strokeStyle = \"#000000\";\n", + "\n", + " this._resize_canvas = function(width, height) {\n", + " // Keep the size of the canvas, canvas container, and rubber band\n", + " // canvas in synch.\n", + " canvas_div.css('width', width)\n", + " canvas_div.css('height', height)\n", + "\n", + " canvas.attr('width', width * mpl.ratio);\n", + " canvas.attr('height', height * mpl.ratio);\n", + " canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n", + "\n", + " rubberband.attr('width', width);\n", + " rubberband.attr('height', height);\n", + " }\n", + "\n", + " // Set the figure to an initial 600x600px, this will subsequently be updated\n", + " // upon first draw.\n", + " this._resize_canvas(600, 600);\n", + "\n", + " // Disable right mouse context menu.\n", + " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", + " return false;\n", + " });\n", + "\n", + " function set_focus () {\n", + " canvas.focus();\n", + " canvas_div.focus();\n", + " }\n", + "\n", + " window.setTimeout(set_focus, 100);\n", + "}\n", + "\n", + "mpl.figure.prototype._init_toolbar = function() {\n", + " var fig = this;\n", + "\n", + " var nav_element = $('
');\n", + " nav_element.attr('style', 'width: 100%');\n", + " this.root.append(nav_element);\n", + "\n", + " // Define a callback function for later on.\n", + " function toolbar_event(event) {\n", + " return fig.toolbar_button_onclick(event['data']);\n", + " }\n", + " function toolbar_mouse_event(event) {\n", + " return fig.toolbar_button_onmouseover(event['data']);\n", + " }\n", + "\n", + " for(var toolbar_ind in mpl.toolbar_items) {\n", + " var name = mpl.toolbar_items[toolbar_ind][0];\n", + " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", + " var image = mpl.toolbar_items[toolbar_ind][2];\n", + " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", + "\n", + " if (!name) {\n", + " // put a spacer in here.\n", + " continue;\n", + " }\n", + " var button = $('');\n", + " button.click(method_name, toolbar_event);\n", + " button.mouseover(tooltip, toolbar_mouse_event);\n", + " nav_element.append(button);\n", + " }\n", + "\n", + " // Add the status bar.\n", + " var status_bar = $('');\n", + " nav_element.append(status_bar);\n", + " this.message = status_bar[0];\n", + "\n", + " // Add the close button to the window.\n", + " var buttongrp = $('
');\n", + " var button = $('');\n", + " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", + " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", + " buttongrp.append(button);\n", + " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", + " titlebar.prepend(buttongrp);\n", + "}\n", + "\n", + "mpl.figure.prototype._root_extra_style = function(el){\n", + " var fig = this\n", + " el.on(\"remove\", function(){\n", + "\tfig.close_ws(fig, {});\n", + " });\n", + "}\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function(el){\n", + " // this is important to make the div 'focusable\n", + " el.attr('tabindex', 0)\n", + " // reach out to IPython and tell the keyboard manager to turn it's self\n", + " // off when our div gets focus\n", + "\n", + " // location in version 3\n", + " if (IPython.notebook.keyboard_manager) {\n", + " IPython.notebook.keyboard_manager.register_events(el);\n", + " }\n", + " else {\n", + " // location in version 2\n", + " IPython.keyboard_manager.register_events(el);\n", + " }\n", + "\n", + "}\n", + "\n", + "mpl.figure.prototype._key_event_extra = function(event, name) {\n", + " var manager = IPython.notebook.keyboard_manager;\n", + " if (!manager)\n", + " manager = IPython.keyboard_manager;\n", + "\n", + " // Check for shift+enter\n", + " if (event.shiftKey && event.which == 13) {\n", + " this.canvas_div.blur();\n", + " event.shiftKey = false;\n", + " // Send a \"J\" for go to next cell\n", + " event.which = 74;\n", + " event.keyCode = 74;\n", + " manager.command_mode();\n", + " manager.handle_keydown(event);\n", + " }\n", + "}\n", + "\n", + "mpl.figure.prototype.handle_save = function(fig, msg) {\n", + " fig.ondownload(fig, null);\n", + "}\n", + "\n", + "\n", + "mpl.find_output_cell = function(html_output) {\n", + " // Return the cell and output element which can be found *uniquely* in the notebook.\n", + " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", + " // IPython event is triggered only after the cells have been serialised, which for\n", + " // our purposes (turning an active figure into a static one), is too late.\n", + " var cells = IPython.notebook.get_cells();\n", + " var ncells = cells.length;\n", + " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", + " data = data.data;\n", + " }\n", + " if (data['text/html'] == html_output) {\n", + " return [cell, data, j];\n", + " }\n", + " }\n", + " }\n", + " }\n", + "}\n", + "\n", + "// Register the function which deals with the matplotlib target/channel.\n", + "// The kernel may be null if the page has been refreshed.\n", + "if (IPython.notebook.kernel != null) {\n", + " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", + "}\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "res = train_and_plot(30,net,lr=0.01)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Важное замечание\n", + "\n", + "Простая линейная модель \n", + "* высокий training loss - \"недообучение\", выразительности модели не хватает, чтобы разделить данные\n", + "* valiadation loss и training loss примерно совпадают - модель хорошо обобщается\n", + "\n", + "Сложная многослойная модель\n", + "* низкий training loss - почти идеально приближает обучающую выборку (но может переобучиться)\n", + "* validation loss >> training loss и может возрастать - плохо обобщает данные" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## Выводы\n", + "\n", + "* Простые модели с небольшим числом параметров (\"low capacity\") менее склонные к переобучению\n", + "* Более сложные модели (high capacity) могут переобучиться (надо следить за validation error)\n", + "* Для более сложных моделей необходимо иметь больше данных\n", + "* \"bias-variance trade-off\" - необходимо достичь компромисса между недообучением и переобучением (обучением на распознавание нерелевантного шума во входных данных)" + ] + } + ], + "metadata": { + "celltoolbar": "Slideshow", + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + }, + "livereveal": { + "start_slideshow_at": "selected" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/3-NeuralNetworks/05-Frameworks/IntroKerasTF.ipynb b/3-NeuralNetworks/05-Frameworks/IntroKerasTF.ipynb new file mode 100644 index 00000000..94dd075c --- /dev/null +++ b/3-NeuralNetworks/05-Frameworks/IntroKerasTF.ipynb @@ -0,0 +1,1433 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "celltoolbar": "Slideshow", + "kernelspec": { + "name": "python3", + "display_name": "Python 3", + "language": "python" + }, + "language_info": { + "mimetype": "text/x-python", + "nbconvert_exporter": "python", + "name": "python", + "file_extension": ".py", + "version": "3.7.4-final", + "pygments_lexer": "ipython3", + "codemirror_mode": { + "version": 3, + "name": "ipython" + } + }, + "livereveal": { + "start_slideshow_at": "selected" + }, + "colab": { + "name": "IntroKerasTF.ipynb", + "provenance": [], + "collapsed_sections": [] + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "En2vX4FuwHlu" + }, + "source": [ + "# Введение в нейронные сети\n", + "\n", + "## Эпизод 2а: Многослойный персептрон на TensorFlow и Keras\n", + "\n", + "Дмитрий Сошников | dmitri@soshnikov.com\n", + "\n", + "http://github.com/shwars/NeuroWorkshop\n", + "-> Notebooks -> IntroKerasTF.ipynb" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fVjlmWBwwHl2" + }, + "source": [ + "## Нейросетевые фреймворки\n", + "\n", + "Мы видели, что для обучения нейросетей нужно:\n", + "* Быстро умножать матрицы (тензоры)\n", + "* Считать производные для вычисления градиента для метода обратного распространения ошибки\n", + "\n", + "Что позволяют делать нейросетевые фреймворки:\n", + "* Оперировать с тензорами, как на CPU, так и на GPU\n", + "* Автоматически вычислять производные (они вручную прописаны для всех элементарных функций)\n", + "\n", + "Опционально:\n", + "* Конструктор для нейросетей (описание сети как набора слоёв)\n", + "* Простые функции для обучения (`fit`, как в Scikit Learn)\n", + "* Набор алгоритмов оптимизации\n", + "* Набор абстракций для работы с данными" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8cACQoFMwHl3" + }, + "source": [ + "## Основные фреймворки\n", + "\n", + "* Tensorflow 1.0 - первый, получивший широкое распространение (Google). Позволял определять статический computation graph, и затем в явном виде выполнять вычисления\n", + "* PyTorch - Facebook\n", + "* Keras - надстройка над Tensorflow/PyTorch для унификации (Francois Chollet)\n", + "* Tensorflow 2.0 + Keras - динамический вычислительный граф, код получается похожим на обычные вычисления в numpy\n", + "\n", + "Мы рассмотрим Tensorflow 2.0 и Keras. Вам необходимо убедиться, что у вас установлена версия 2.x.x Tensorflow:\n", + "```\n", + "pip install tensorflow\n", + "```\n", + "или\n", + "```\n", + "conda install tensorflow\n", + "```\n", + "или выполняйте код в [Google Colab](https://colab.research.google.com/)" + ] + }, + { + "cell_type": "code", + "metadata": { + "tags": [], + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "xwqVx9-bwHl3", + "outputId": "2aa591b4-b647-441f-9c8e-4e0da2d517a0" + }, + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "print(tf.__version__)" + ], + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "text": [ + "2.4.1\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6tp2xGV7wHl4" + }, + "source": [ + "## Основные понятия в TensorFlow\n", + "\n", + "**Тензор** - это многомерный массив произвольной размерности. Удобно использовать при обучении нейросетей, например:\n", + "* 400x400 - чёрно-белая картинка\n", + "* 400x400x3 - цветная картинка\n", + "* 16x400x400x3 - minibatch из 16 картинок, используемый для одного шага обучения\n", + "* 25x400x400x3 - секунда видео\n", + "* 8x25x400x400x3 - minibatch из 8 1-секундных видео" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qG2bsaR7wHl4" + }, + "source": [ + "### Простые тензоры" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ybpnk08HwHl4", + "outputId": "fad9ed4a-df82-44a0-84ea-324bc71ea46f" + }, + "source": [ + "a = tf.constant([[1,2],[3,4]])\n", + "print(a)\n", + "a = tf.random.normal(shape=(10,3))\n", + "print(a)" + ], + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tf.Tensor(\n", + "[[1 2]\n", + " [3 4]], shape=(2, 2), dtype=int32)\n", + "tf.Tensor(\n", + "[[ 1.1846514 0.7508411 -1.2235912 ]\n", + " [ 0.9710432 -0.15184928 -0.10070106]\n", + " [ 0.01476252 -0.19518584 0.43259642]\n", + " [ 1.8912938 0.6803918 -0.41853315]\n", + " [-1.0234195 -0.46338254 -1.6521171 ]\n", + " [-2.6472843 0.72314113 -0.8514363 ]\n", + " [-0.2650735 -0.22901714 -0.4702833 ]\n", + " [ 0.4691955 -0.6513927 -0.22865944]\n", + " [-1.0068002 2.3035717 -0.8701199 ]\n", + " [ 0.35074237 0.192767 -1.013126 ]], shape=(10, 3), dtype=float32)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AXFMsV3r09Ux" + }, + "source": [ + "С тензорами можно производить обычные вычисления, которые производятся поэлементно (как в numpy). При этом тензоры автоматически дополняются до нужной размерности. Можно извлечь numpy-массив из тензора при помощи `.numpy()`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "e5Nu5Xgj1DnQ", + "outputId": "0dfc8758-4ffd-4968-c7bf-6ba8d435df2e" + }, + "source": [ + "print(a-a[0])\r\n", + "print(tf.exp(a)[0].numpy())" + ], + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tf.Tensor(\n", + "[[ 0. 0. 0. ]\n", + " [-0.21360815 -0.90269035 1.1228901 ]\n", + " [-1.1698889 -0.9460269 1.6561877 ]\n", + " [ 0.7066424 -0.07044929 0.80505806]\n", + " [-2.2080708 -1.2142236 -0.42852592]\n", + " [-3.8319356 -0.02769995 0.3721549 ]\n", + " [-1.4497249 -0.9798582 0.75330794]\n", + " [-0.7154559 -1.4022338 0.99493176]\n", + " [-2.1914515 1.5527306 0.35347128]\n", + " [-0.83390903 -0.5580741 0.2104652 ]], shape=(10, 3), dtype=float32)\n", + "[3.2695467 2.1187813 0.29417184]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uQ5zN6cVyrG7" + }, + "source": [ + "## Переменные\r\n", + "\r\n", + "Переменные могут содержать какие-то значения, которые мы затем можем модифицировать с помощью методов `assign` и `assign_add`. \r\n", + "\r\n", + "Например, вот глупый способ посчитать сумму всех строк тензора `a`" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7pu0UZ-_yqfB", + "outputId": "6708c83e-02e6-4442-8757-45918eb1fbc2" + }, + "source": [ + "s = tf.Variable(tf.zeros_like(a[0]))\r\n", + "for i in a:\r\n", + " s.assign_add(i)\r\n", + "\r\n", + "print(s)" + ], + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rIh1EHcezlNo" + }, + "source": [ + "Умный способ:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aQIdWZ1kzn6P", + "outputId": "1c123d9a-ecd2-4f2e-828e-5ade85ac8f63" + }, + "source": [ + "tf.reduce_sum(a,axis=0)" + ], + "execution_count": 8, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 8 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U-auwezDwHl6" + }, + "source": [ + "## Вычисляем производные\r\n", + "\r\n", + "Для обратного распространения ошибки, нам нужно уметь вычислять градиенты. Это делается с помощью `tf.GradientTape()`:\r\n", + " * Оборачиваем интересующие нас вычисления в `with tf.GradientTape`\r\n", + " * Помечаем интересующие нас тензоры вызовом `tape.watch` (переменные отслеживаются автоматически)\r\n", + " * Проводим вычисления\r\n", + " * Получаем градиенты через `tape.gradient` " + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "m8vFOXr7wHl6", + "outputId": "860ac72e-50c7-4ff2-f258-747f27194f90" + }, + "source": [ + "a = tf.random.normal(shape=(2, 2))\n", + "b = tf.random.normal(shape=(2, 2))\n", + "\n", + "with tf.GradientTape() as tape:\n", + " tape.watch(a) # Start recording the history of operations applied to `a`\n", + " c = tf.sqrt(tf.square(a) + tf.square(b)) # Do some math using `a`\n", + " # What's the gradient of `c` with respect to `a`?\n", + " dc_da = tape.gradient(c, a)\n", + " print(dc_da)" + ], + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tf.Tensor(\n", + "[[0.9086961 0.954213 ]\n", + " [0.85484374 0.9623889 ]], shape=(2, 2), dtype=float32)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8sfjBMBu59B5" + }, + "source": [ + "## Пример 1: Линейная регрессия\r\n", + "\r\n", + "Попробуем с помощью полученных знаний решить классическую задачу линейной регрессии. Для этого сгенерируем небольшой синтетический датасет:" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "j723455WwHl7" + }, + "source": [ + "import matplotlib.pyplot as plt\n", + "from sklearn.datasets import make_classification, make_regression\n", + "from sklearn.model_selection import train_test_split\n", + "import random" + ], + "execution_count": 10, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "WJNK_J6v6I-Z", + "outputId": "eb4a66a6-6b9a-4c8a-bc24-d81eeb2d3f27" + }, + "source": [ + "np.random.seed(13) # pick the seed for reproducability - change it to explore the effects of random variations\r\n", + "\r\n", + "train_x = np.linspace(0, 3, 120)\r\n", + "train_labels = 2 * train_x + 0.9 + np.random.randn(*train_x.shape) * 0.5\r\n", + "\r\n", + "plt.scatter(train_x,train_labels)" + ], + "execution_count": 11, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 11 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAb2ElEQVR4nO3df4xlZXkH8O+zwwUuP8pVmVoYXZemzZIKZZedEJo1RtZYbFHcgFEb0xbbZpM2bYVQ2tE/ivYfJjFWbW1qJpQWW6pLXdhQUJFkN6FuFDvDLgKyNChuZbBllB0QZsrODE//uPcsd86c97zvuec957zvvd9PsnF25+yZ9+yV5773eZ/3eUVVQURE4drU9ACIiCgfAzURUeAYqImIAsdATUQUOAZqIqLAnVLFTc8991zdsmVLFbcmIhpKc3NzP1HV8azvVRKot2zZgtnZ2SpuTUQ0lETkmOl7TH0QEQWOgZqIKHAM1EREgWOgJiIKHAM1EVHgKqn6ICIaJfsPz+NT9z+JZxeXcX6njZuu3Ird2ye83Z+BmoiohP2H5/Gxux7F8soaAGB+cRkfu+tRAPAWrJn6ICIq4VP3P3kySCeWV9bwqfuf9PYzGKiJiEp4dnG50J8PgoGaiKiE8zvtQn8+CAZqIiJ0c807pw/ggqn7sHP6APYfnnf6ezdduRXt1ti6P2u3xnDTlVu9jY2LiUQ08sosCCbfZ9UHEVGF8hYEXQLu7u0TXgNzGlMfRDTy6lgQLMMaqEVkq4gc6fv1oohcX8fgiIjqUMeCYBnWQK2qT6rqNlXdBmAHgCUAd1c+MiKimtSxIFhG0Rz1OwF8X1WNDa6JiGJTZEGwf7v4Oe0WRIDFpZVKFhEToqruF4vcBuBhVf18xvf2ANgDAJs3b95x7BhjORENl3R1SFq7NYZbrrl4oGAtInOqOpn1PefFRBE5FcDVAP4t6/uqOqOqk6o6OT6eeewXEVHw8uqps6pD+vneOp4okvr4DXRn0//rfRRERAGw1VO7VIFUUSlSpDzvtwB8yfsIiIgCYWuw5FIFUkWliFOgFpEzAbwLwF3eR0BEFAhbPXVWdUi/qipFnFIfqvoygDd4/+lERAE5v9PGfEawTmbJ6eqQuqo+uIWciKjnpiu3bqjqSM+Sq94unoWBmoiop44GS4NgoCYi6tPEjNmGTZmIiALHQE1EFDgGaiKiwDFHTUQjob+ZUiiLhK4YqIloKOQF4kGP2kruOb+4jDERrKliooEgz0BNRIWEODPNC8QAcOOdj2At1SnUdtRW+p7J3y9ynqIvhdqcupqcnNTZ2Vnv9yWiZmW1+SzT2tOXndMHMncUdtotvLL6am7Hu4lOO/NNx3TP/r93aGpX+cH3eGlzSkRka1rUFFOPjsXlldwgLejOkBWvzZSTtqa2Lnh1nqfIQE1EzkI9BHaQjnUCIJ1PKNIpr87zFBmoiciZKTid024Zm+3XwXTm4evOaGVePyayIUgnXDrl1X2eIgM1ETnLCl6tTYKXT6waUwhVSJ/CAgC3XHMxJjptCLr541uuuRg3v/etmQH80x+4BBOWk8d3b584eU+gG9zRd29WfRBRkLKaFi2dWMXxpZV119kqKsowVXjccs3FxsU9U5VKiJ3ysrDqg4hKuWDqvsw0ggB4evoq7z/PVI0xSBVGSKWGeVUfnFETUSm2Zvu++VzQDGXGbMMcNRGVYlrIq2qxzfQGUGcVRt0YqImolP5Ft/6FvKpmqnW/MYSAqQ8iKq3OFEKop7BUySlQi0gHwK0ALkK3Rvz3VPVbVQ6MiMgkltyyL64z6s8B+Lqqvl9ETgVwRoVjIiLKFFKVRp2sgVpEzgHwdgDXAYCqngBwotphERGtN2ir0mHgsph4AYAFAP8oIodF5FYROTN9kYjsEZFZEZldWFjwPlAialZ6N2Dd28RDbQhVB5dAfQqASwH8vapuB/AygKn0Rao6o6qTqjo5Pj7ueZhE1KRkNlvnNvH+n53XcrTphlB1cMlRPwPgGVV9qPf7ryAjUBPR8DLNZj9xz+O5p6qUzSdn9b9OG+b66YQ1UKvq/4jIj0Rkq6o+CeCdAL5X/dCIKBR5/Z4Xl7t9PtKnqvjIJ2e9QfQb9vrphGvVx58AuKNX8fEDAB+pbkhEFBrTNvG0/pyxKZ9cJFDnpTWaOLuwKU6BWlWPAMhsFkJEw++mK7daUxCJvOBaNJ9seoPwfQxW6LiFnIissraJm5ryn99pO/XjcKkiGcXt4lm4hZyInKR3A5oOuk2CaN73XGqik8XI5ZU1jIlgTXWk0h39GKiJRliZygyXnhum7+XVRO/ePrEhkK+pngz0oxakAR4cQDSyTDPiOo6ZMh02AHTTKqaFy6zc9LBsK887OIA5aqIR1eROP1MOW4Dc6pL0YmSTG3HqxEBNNKJ8npRSVNYioQDGWXYiHeBHZVs5AzXRkDNVV5hmtQpU3ssjXUXSabesQTqr2qPJN5s6cTGRaIjlVVfk1Ub77kxnyiP3LxzmMVV71H1eY1MYqImGUBIYs4JYkhpIFuVs15UN1LZSvLxt4rbFzaw3m3ZrDFdcOI6d0weiX2BMMPVB5EHTLUDTY0kW2EyS1MDu7RM4NLULYrmuDFseOe9n2CpQsjbiXLtjAvvm5odqgZEzaqKSQmtob2tkBGxMDfhMIaTTHLb2pHnbxF3+/dIbcXZOH/DSZyQknFETldRE5UHeDN42C85alPO1VTurXM40W0/eBHxvE3dZYAzpE5ALzqiJSqq78sA2g8+bxY6JrHsTSWaYvk72znrTUmSX3i2dWMX+w/PeTxW3fToI7ROQCwZqopLqrjywbb/OWmBrbRJAgJW1brjMCk4+TvY2vTkpuiV4Se9qADi+tLJuDL6CpGmBMZmh2/79QsTUB1FJdXd4s83gsxbYzjr9lJNBOmFKz5RJC5jenCY6bZx52sZ5YRUpoqzn71+UjLH2mjNqopJ8f3S3cZnBp2eoF0zdl3kv05bsQdMCebPZG/YecRqDD3kz9BhrrzmjJvIgKXN7evoqHJraVelH6EFm8C79oYHyC6N5s1nXMVQtxh7XnFETRWaQGbwtb5vwkRYwzWZdx1C1uj8B+cBATRShootvrsGpyrRASAHS5+JlHdiPmihQTfRZbrJH9ajL60fNGTVRgJqq9Q1p1kuvcQrUIvJDAD8DsAZg1RT1iciPJmt9Y0sLjIIiM+orVPUnlY2EiE6KsdaXqsPUB1EA0vnozhktHF9a2XBdyLW+VB3XOmoF8A0RmRORPVkXiMgeEZkVkdmFhQV/IyQaclmNjF76v1W0xta3Mwq91peq4zqjfpuqzovIzwN4QESOquqD/Reo6gyAGaBb9eF5nERDIz17XjqxuiEfvfKqotNu4czTTuGiHrkFalWd7/3vcyJyN4DLADyY/7eIKC2rmsPkheUVHLn51+saGgXMmvoQkTNF5OzkawC/DuCxqgdGNIxcmvonmI+mhMuM+o0A7haR5Pp/VdWvVzoqoiHlWrXBfDT1swZqVf0BgEtqGAvR0DNt0WY+mvKwPI+i1MT2ah9MjYk+cfVbjePvf9Zz2i2IAItLK1E9N5XDQE3RqXN7te0NoegbRtEt2uln7T8hJeu5Y30Do3xsykTR2Tl9wHhq9aGpXd5+jq1BUR0NjEzP2i95bjZUilteUyYeHEDRqWt7ta2Jfh2nj7s8U3JNE6ehUz0YqCk6dZ0UYntDqOMNw+WZkmvYH2R4MVBTdOo6Ssn2hlDHG0bWs/brf+5Qjroi/xioKTq2U6Z9sb0h1PGGkX7WTruF153RynzuKsZT5kRy8oeLiUQ5fFd9ND3eovfi4mR98hYTGaiJIlfVm0Vd1TXUxaO4iCJRNOhWWVPOxclwMEdNlNJUXjarL/XH7no09+dXWZLHxclwMFBTVKoOooMES9f72sY9SNCtctZbV3UN2TFQUzSqCqL9qpihuo57kKBb5ay3ruoasmOOmqJRx8ncVcxQXcdt6qyXF3RNTZ58zXp5InkYGKgpGnXtBCwaLG1M45tfXMbO6QMnFw6vuHAc++bmCwXdok2eKE4M1BSNKoJoWhUzVNO4Ba8dxTW/uIx9c/O4dscEDh5dKBR0OesdfgzUFI2qP+YD+TPUQeuVs8YtANI7GJZX1nDw6EItNcqhbdShfNzwQlFpKsBk7dIDgNed0cLN781u+p/X8N/UulQAPD19VRWPsG5c3HEYHm54oaFh+5hfVSA3HUp7fGklc4NJVsP/dmsMn/ngNuzePmHc9VdHjXIdi7LkF8vzyIsQmvdUWb6Xt2CZVb5nK/NrskaZOw7j4xyoRWRMRA6LyL1VDojiU0d9swtTcLzxzkdKj8U2000HOVswbLJGmTsO41Mk9fFRAE8A+LmKxkKRCuWjtCk4rqmW7n+RtSDYrz/I7T88j00iWMtY/+m/rqlqjToWZckvpxm1iLwJwFUAbq12OBSjUD5K580Iy+4uTGbAnXZrw/f6g1zy6SIrSIcSDLnjMD6uM+rPAvhzAGebLhCRPQD2AMDmzZvLj4yiUUd9swvbrLfsG0cyA85bsDQtOo6JBBUMWXsdF2ugFpH3AHhOVedE5B2m61R1BsAM0C3P8zZCCl4oH6WTwHPjnY9Y0w5lf44pyJneDF5VZWCkgbmkPnYCuFpEfgjgywB2ici/VDoqikpIH6V3b5/Apz9wSWMVFVyooyoU2vDSm1H/maq+J+86bnihpoW0MYabScgFN7zQyGkqB8smSVQFbiGnxjTZb4K9Lig0nFFTcHye9efrnMHZY88X7lxHVAfOqKkRpl4XQHcx0jVIFskJJwE9ryFS/38NzC1TnfJm1Oz1QY3Iq2kusgXd5eis/Yfnse2T38D1e48YgzSQ3XbUxyGxRGUxUFMjbOVqrkHStisymXEvLq8UH2TO/YnqxEBNznx2yMvqHpfmEiRtdcumnYJpUvD+RHVioCYnvjvk9W+SMXEJkrZ2oS7BfqLTxocv39zYJhkiG1Z9kJMqOuT1984YdAu6rW457zSV9GLh5Ftez5I9ChIDNTmpskNe2U0ieZtbTI2aso7QYqMiChUD9RCoY/NG1R3yqgqS3ClIw4CBOnI+N47kCaVD3iA4U6bYcTExci51xD6E1CGPaNRwRh25Ok9XqWNm2p/GOafdgkj3pO+x3tFWRXYtEg0LBurIhXK6io1LHj2dxunfpJIcBFBVaocoZEx9RM5WRxwC1xps180p3NpNo4aBOnIx5I5NefTr9x5Zt8OxSLqGW7tplDD1MQRCr2pwacAE5G9OSQsttUNUJQZq8i6dj+6c0cLxJXNTpCSVYTtFPBFaaoeoagzU5FVWXXdrk6A1JlhZM/c+f3ZxecPmFFZ9EHUxUJNXWfnolVcVnXYLZ552ijG1kaQy6k7j8EguigEXE8krUz76heUVHJrahc9+cFswVSq+OwISVcUaqEXkdBH5jog8IiKPi8gn6xgYvcZnH+iq2fpDh1SlUteuTqKyXFIfrwDYpaoviUgLwDdF5Guq+u2Kx0aor5eHLy49QUKpUqlzVydRGdZArd3Tb1/q/bbV++X/RFzK5LMPdB352Ji61cWyq5PI6RRyERkDMAfglwD8nar+RcY1ewDsAYDNmzfvOHbsmOehjqYLpu7LfFcUAE9PX+V8n6zm/Mmp21mVFKOwyFbkBHOiquWdQu5U9aGqawC2iUgHwN0icpGqPpa6ZgbADABMTk5yxu2Jr1lf1sw8eZHS6RRbuiWrcdLi0kp0AT2m2T+NtkLleaq6KCIHAbwbwGO264dVnbNNX32gbXnX/nSKbZHN1Dgp9Px5llDy5UR5XKo+xnszaYhIG8C7ABytemChqruky1eVhMsMPAnmeYtstsZJrJog8s9lRn0egNt7eepNAO5U1XurHVa4qjjk1cbHrM9le3YSzE3plk0iTr04WDVB5JdL1cd3AWyvYSxRiLGkK0nVLK+sndyKnSwkJvrTKaagnvX3siQBfxQWJInqwJ2JBdk2dISmP1UDdINtuzWGD1++2ZhOSdItYyIb7qfoVouYJAGfu/6I/GGvj4JCOuTVZcZqStUcPLqAQ1O7jPfevX0CN+w9kvm9pKQvr+pj5/SB2lNERMOKgbqgUEq6XHcslknVmHLVE512bpAv+3OJaD0G6gGEUNLluqhZpg67zKcH7voj8oc56opV1VDJdcZa5kzFMqWBMZzlSBQLzqgrVGVDJdcZa9lUzaCfHkJJERENA6deH0VNTk7q7Oys9/vGZuf0gcxgOiaCT3/gklJBi30qiIZL6V4fNBhTemJNtfTMmjNWotHBQF2hvFO1fZSq+V7U5AYVojBxMbFCWQtq/UIqVeMGFaJwMVBXKG+HHxBWqRqPpSIKF1MfFUtSB3XvZiyaxuAGFaJwMVDXwGXhz2d+eJCyQG5QIQoXA3VN8hb+fNdbD9KKNaQeJkS0HnPUAfCZH95/eN5YaZKXxvB1QAER+ccZdQB85YeTmbmJLY0RQg8TItqIM+oA+OpxnXdMFtMYRPHijLokH4uApvzwFReOY+f0gdKVGwCYxiCKGAN1Cb4WAbOqQq64cBz75ua9VG5MdNoM0kQRG9pAXXamW+b0FNet4Xk/Y5ATUli5QTScrIFaRN4M4IsA3ojuKUwzqvq5qgdWRtmZbh2np9h+xiD3HqRRE/t7EIXPZUa9CuBGVX1YRM4GMCciD6jq9yoe28DKznTrOD3F9jMGvXeRyo0q+2UTkT/Wqg9V/bGqPtz7+mcAngAQ9H/FZcvd6jg9xfYz6jghhf09iOJQqDxPRLYA2A7goYzv7RGRWRGZXVhY8DO6AZUtd3P9+2U2idh+Rh0bUNjfgygOzouJInIWgH0ArlfVF9PfV9UZADNA94QXbyN0kM6zpismgGKz0SKLcoNuEnH5GVVvQGF/D6I4OM2oRaSFbpC+Q1XvqnZIxWT1Ud43N49rd0wMPBsdZDZb9BDbELZs8wBaojhYz0wUEQFwO4DnVfV6l5vWeWai6VzCiU4bh6Z2lb6/S1VEzOcXsuqDKAx5Zya6BOq3AfgPAI8CeLX3xx9X1a+a/k6VgTodWEwNiATA09NXFb5ff6DKCsCCbo3iRN+1Vb9ZENHwK3W4rap+E9341LiscrIkcKa55Flt5WlZVRHJz+q/lotyRFSlqJoymQJn+l3ENc9qK0+zBdrkWl9NlYiIskQVqE2BM0lFCIBOu4XTW5tww94j1kU920zYJdA+u7jMRTkiqlRUgdoUOJNc8Gc+uA2vrL6K40srTidp22bCtlPEk2tDqOAgouEVVVMmW+1x0a3jtvv1987Iyoenr2VgJqIqRBWobU2Hii7quTQx6g/AIZSyhTAGIqqXtTxvEHXWUfcb9jK5mOu1iShfXnleVDlqm1gW9YruYkywiRLRaIoq9WEzSD/mupVpLcp6baLRFHWgNuVrBw3MvvK/efcp0yubTZSIRlO0gdp303tf9yt7cktekOdRW0SjKdocte98ra/72e6TV7ud1Qmwvw6c9dpEoym4GbVr+sF3vtbX/VxObjHNil3SIqzXJho9QQVqU9pg9tjzOHh0YV3w9p2v9XU/233yFjxv2Hsk855cLCQabUEFatOM8o5v//eGrnXX7pgodYpLmq/8b5mTW7hYSERZgspR5zVd6re8soaDRxe85mt95X/L3CeWOnAiqldQOxNNOwuzuB4MEBtuEScaTaUODqhTVtqgzMEAMeJiIRGlBZX6yEobfPjyzUwHENFIC2pGDWTPKCff8nqmA4hoZAUXqLMwHUBEo8waqEXkNgDvAfCcql5U/ZCqw4U6IoqRy4z6nwB8HsAXqx3KRj4Dq+/eIEREdbEuJqrqgwCer2Es69j6XhTFXs5EFKugqj76+Q6s7OVMRLHyFqhFZI+IzIrI7MLCQun7+Q6sthPHiYhC5S1Qq+qMqk6q6uT4+Hjp+/kOrNyeTUSxCjb14TOwJouSyytrGBMBAHTaLZze2oQb9h4pdG4hEVHdrIFaRL4E4FsAtorIMyLy+9UPy1+TpP5FSQBYU0Vrk+DlE6s4vrTiZaGSiKhKQTVlqkKRRk8TnTYOTe2qeERERBvlNWUKNvXhS5HFR1aAEFGIhj5QF1l8ZAUIEYVo6AN11qJka5OgNSbr/owVIEQUqiiaMpVhOqMw68+4lZyIQjT0i4lERDGI4oQXdrYjIsoWRKBmZzsiIrMgFhPZ2Y6IyCyIQM3OdkREZkEEana2IyIyCyJQs7MdEZFZEIuJplpnLiQSEQUSqAGeNE5EZBJE6oOIiMwYqImIAsdATUQUOAZqIqLAMVATEQWuku55IrIA4NiAf/1cAD/xOJwmDcuzDMtzAHyWEA3LcwDlnuUtqjqe9Y1KAnUZIjJravUXm2F5lmF5DoDPEqJheQ6gumdh6oOIKHAM1EREgQsxUM80PQCPhuVZhuU5AD5LiIblOYCKniW4HDUREa0X4oyaiIj6MFATEQWusUAtIu8WkSdF5CkRmcr4/mkisrf3/YdEZEv9o7RzeI7rRGRBRI70fv1BE+O0EZHbROQ5EXnM8H0Rkb/pPed3ReTSusfoyuFZ3iEiL/S9Jn9Z9xhdicibReSgiHxPRB4XkY9mXBP8a+P4HFG8LiJyuoh8R0Qe6T3LJzOu8Ru/VLX2XwDGAHwfwC8COBXAIwB+JXXNHwH4Qu/rDwHY28RYPTzHdQA+3/RYHZ7l7QAuBfCY4fu/CeBrAATA5QAeanrMJZ7lHQDubXqcjs9yHoBLe1+fDeC/Mv4/Fvxr4/gcUbwuvX/ns3pftwA8BODy1DVe41dTM+rLADylqj9Q1RMAvgzgfalr3gfg9t7XXwHwThGRGsfowuU5oqCqDwJ4PueS9wH4onZ9G0BHRM6rZ3TFODxLNFT1x6r6cO/rnwF4AkC6cXvwr43jc0Sh9+/8Uu+3rd6vdFWG1/jVVKCeAPCjvt8/g40v2slrVHUVwAsA3lDL6Ny5PAcAXNv7SPoVEXlzPUPzzvVZY/FrvY+uXxORtzY9GBe9j8/b0Z3B9Yvqtcl5DiCS10VExkTkCIDnADygqsbXxEf84mJi9f4dwBZV/VUAD+C1d1lqzsPo9lW4BMDfAtjf8HisROQsAPsAXK+qLzY9nkFZniOa10VV11R1G4A3AbhMRC6q8uc1FajnAfTPLN/U+7PMa0TkFADnAPhpLaNzZ30OVf2pqr7S++2tAHbUNDbfXF6zKKjqi8lHV1X9KoCWiJzb8LCMRKSFbnC7Q1XvyrgkitfG9hyxvS4AoKqLAA4CeHfqW17jV1OB+j8B/LKIXCAip6KbbL8ndc09AH639/X7ARzQXmY+INbnSOUKr0Y3NxejewD8Tq/C4HIAL6jqj5se1CBE5BeSfKGIXIbufwehTQIAdCs6APwDgCdU9a8NlwX/2rg8Ryyvi4iMi0in93UbwLsAHE1d5jV+NXK4raquisgfA7gf3cqJ21T1cRH5KwCzqnoPui/qP4vIU+guDH2oibHmcXyOPxWRqwGsovsc1zU24Bwi8iV0V93PFZFnANyM7iIJVPULAL6KbnXBUwCWAHykmZHaOTzL+wH8oYisAlgG8KEAJwGJnQB+G8CjvZwoAHwcwGYgqtfG5TlieV3OA3C7iIyh+2Zyp6reW2X84hZyIqLAcTGRiChwDNRERIFjoCYiChwDNRFR4BioiYgCx0BNRBQ4BmoiosD9P3fFq1WlCzEwAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ng4rZmGc6oxk" + }, + "source": [ + "Линейная регрессия вычисляется как $f_{W,b}(x) = Wx+b$, где $W, b$ - параметры модели, которые необходимо найти. Функция ошибки на наборе данных $\\{x_i,y_u\\}_{i=1}^N$ может быть определена как среднеевадратичное отклонение\r\n", + "$$\r\n", + "\\mathcal{L}(W,b) = {1\\over N}\\sum_{i=1}^N (f_{W,b}(x_i)-y_i)^2\r\n", + "$$\r\n", + "\r\n", + "Опишем модель и функцию ошибки:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "QxhI4GlB6aiH" + }, + "source": [ + "input_dim = 1\r\n", + "output_dim = 1\r\n", + "learning_rate = 0.1\r\n", + "\r\n", + "# This is our weight matrix\r\n", + "w = tf.Variable([[100.0]])\r\n", + "# This is our bias vector\r\n", + "b = tf.Variable(tf.zeros(shape=(output_dim,)))\r\n", + "\r\n", + "def f(x):\r\n", + " return tf.matmul(x,w) + b\r\n", + "\r\n", + "def compute_loss(labels, predictions):\r\n", + " return tf.reduce_mean(tf.square(labels - predictions))" + ], + "execution_count": 14, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JUxwj3367gD2" + }, + "source": [ + "Обучать модель будем на сериях примеров - minibatches. Для обучения используем градиентный спуск, подстраивая парметры в соответствии с формулой:\r\n", + "$$\r\n", + "\\begin{array}{l}\r\n", + "W^{(n+1)}=W^{(n)}-\\eta\\frac{\\partial\\mathcal{L}}{\\partial W} \\\\\r\n", + "b^{(n+1)}=b^{(n)}-\\eta\\frac{\\partial\\mathcal{L}}{\\partial b} \\\\\r\n", + "\\end{array}\r\n", + "$$" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "-991PErM7fJU" + }, + "source": [ + "def train_on_batch(x, y):\r\n", + " with tf.GradientTape() as tape:\r\n", + " predictions = f(x)\r\n", + " loss = compute_loss(y, predictions)\r\n", + " # Note that `tape.gradient` works with a list as well (w, b).\r\n", + " dloss_dw, dloss_db = tape.gradient(loss, [w, b])\r\n", + " w.assign_sub(learning_rate * dloss_dw)\r\n", + " b.assign_sub(learning_rate * dloss_db)\r\n", + " return loss" + ], + "execution_count": 13, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "idr2VEWb9rr0" + }, + "source": [ + "Теперь приступаем к обучению: делаем несколько проходов по всему датасету (эпох), разбиваем его на minibatches, и вызываем функцию обучения:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nOuu0qpx-wAp" + }, + "source": [ + "# Shuffle the data.\r\n", + "indices = np.random.permutation(len(train_x))\r\n", + "features = tf.constant(train_x[indices],dtype=tf.float32)\r\n", + "labels = tf.constant(train_labels[indices],dtype=tf.float32)" + ], + "execution_count": 12, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3zdIf6c_85Ht", + "outputId": "43b04684-8b90-4c65-d5ff-20ebac61c73c" + }, + "source": [ + "batch_size = 4\r\n", + "for epoch in range(10):\r\n", + " for i in range(0,len(features),batch_size):\r\n", + " loss = train_on_batch(tf.reshape(features[i:i+batch_size],(-1,1)),tf.reshape(labels[i:i+batch_size],(-1,1)))\r\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 15, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 94.5247\n", + "Epoch 1: last batch loss = 9.3428\n", + "Epoch 2: last batch loss = 1.4166\n", + "Epoch 3: last batch loss = 0.5224\n", + "Epoch 4: last batch loss = 0.3807\n", + "Epoch 5: last batch loss = 0.3495\n", + "Epoch 6: last batch loss = 0.3413\n", + "Epoch 7: last batch loss = 0.3390\n", + "Epoch 8: last batch loss = 0.3384\n", + "Epoch 9: last batch loss = 0.3382\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "US6q0nCBD-LL", + "outputId": "65a79620-a3eb-445b-aafb-60a60575ab0e" + }, + "source": [ + "w,b" + ], + "execution_count": 16, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(,\n", + " )" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 16 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "_e6xRMZFDnyI", + "outputId": "d202b7fe-4383-4d82-b98e-a20f3180093e" + }, + "source": [ + "plt.scatter(train_x,train_labels)\r\n", + "x = np.array([min(train_x),max(train_x)])\r\n", + "y = w.numpy()[0,0]*x+b.numpy()[0]\r\n", + "plt.plot(x,y)" + ], + "execution_count": 17, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 17 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhU1Z3/8ffp7uqFrRulQWhoQFZFZOsxGowKiBhN3KNJRo3JTJiJGbcYlGQWZ/L8ZgTcMyYTiZplYhKNInFiZFFQXFFWUeluEGRp9qWbpffq8/uju9qmum7Vra5b1beaz+t5eNJ03br3XMp876lzvud7jLUWERHxr4zOboCIiESnQC0i4nMK1CIiPqdALSLicwrUIiI+l5WMk/bp08cOGTIkGacWEemSVq9efcBaWxjptaQE6iFDhrBq1apknFpEpEsyxmxzek1DHyIiPqdALSLicwrUIiI+p0AtIuJzCtQiIj6XlKwPEZGTycK1FTywuIxdlTUMKMhj1oxRXDWhyLPzK1CLiCRg4doKfrRgAzUNQQAqKmv40YINAJ4Faw19iIgk4IHFZa1BOqSmIcgDi8s8u4YCtYhIAnZV1sT1+45QoBYRScCAgry4ft8RCtQiIjSPNU+es4yhs19m8pxlLFxb4ep9s2aMIi+QecLv8gKZzJoxyrO2aTJRRE56iUwIhl5X1oeISBJFmxB0E3CvmlDkaWAOp6EPETnppWJCMBExA7UxZpQxZl2bP0eMMXemonEiIqmQignBRMQM1NbaMmvteGvteGASUA28mPSWiYikSComBBMR7xj1NOBTa61jgWsRkXQTz4Rg2+Xi+XkBjIHK6oakTCKGGGut+4ONeRpYY619PMJrM4GZAMXFxZO2bVMsF5GuJTw7JFxeIJP7rxnboWBtjFltrS2J9JrryURjTDZwBfCnSK9ba+dba0ustSWFhRG3/RIR8b1o+dSRskPa8nrpeEg8Qx9fprk3vdfzVoiI+ECsfGo3WSDJyBSJJz3vG8AfPG+BiIhPxCqw5CYLJBmZIq4CtTGmOzAdWOB5C0REfCJWPnWk7JC2kpUp4mrow1p7HDjV86uLiPjIgII8KiIE61AvOTw7JFVZH1pCLiLSYtaMUe2yOsJ7ycleLh6JArWISItUFFjqCAVqEZE2OqPHHIuKMomIeOCjiiqeWZmchX7qUYuIJOCzA8d5cEkZf/lwN3175nDNhIHkZTtnhnSEArWISAfsO1rLf7+2mT+8v51AZga3TR3Ody843fMgDQrUInKSaFtMKZFJwqO1DcxfsYWn3tpKfWMTXz9nELdPG0HfnrlJaHUzBWoR6RKiBeKObrUVOmdFZQ0ZQBOQYaDJwlfO7s8PLxnFkD7dk31rCtQiEh+veqZet8kpEAPc/dx6gmGVQmNttRV+zqaW3zdZyMnK4OIz+qUkSEOcZU7dKikpsatWrfL8vCLSuSKV+UyktKdXJs9ZFnFFYUFegLrGpqgV74oK8iI+dL54/2vsqqqN+r63Z09NvPEtopU5VY9aRFxLdBPYZHGq0VFZ0xD1fQZaA3zbXnjxqd2iBulo10wGBWoRcc2vm8A61eiIxgDh4wk1DUFmL/iQ2oam1rHoaNdMFS14ERHXnIJTfl7Asdh+Kjjtedi7WyDi8ZnGtAvSIbUNTdw9fST3Xz3WsVJeqvdTVKAWEdciBcRAhuF4fSMVlTVYPh9CSGawDt+FBeD+a8ZSVJCHoXn8+P5rxnLfV8dEDOAPXT+OIoeHzmm9crlt2ghuOKe49ZzQHNxpc+5UDvVo6ENEXItUtKi6vpHD1SeOBSdz3Nopw+P+a8Y6Tu6FZ6nMGHMar27c2264JC+Qyewvj279u1/qfijrQ0QSMnT2yxGHEQywdc7lnl/PKcPDTRZGY7CJP63eyaOvlrP3SB1jBvRi/9E69h+t6/RUQ2V9iEjSxCq277WOTGhaa1n88R7mLS5jy/7jTCwu4L+/MZFzhp6SlDZ6TYFaRBLipti+l+J9MLz76UHmLipl3Y5KRvTtwfybJjH9zH6YljHndKBALSIJSXWxfbcPhk92HWHuolLeKN9P//xc5l13NtdOHEhmRvoE6BAFahFJWCon3WI9GHYcquahJWX8ef0ueuUG+PFlo7n5vCHkRtmU1u9cBWpjTAHwJHAWzTni37HWvpvMhomIOIn0YDhwrI7Hl23mmZXbyMww/OOFw/jHC4eRnxc5lzqduO1RPwYsstZeZ4zJBrolsU0iIhFFKgh18Zn9ePLNLfxyxRZqG5u4vmQQd148gn69kld2NNViBmpjTD5wAXALgLW2HqhPbrNERE4UKX961p/WkxPI5FhdI18+6zR+OGMUwwp7dHJLveemRz0U2A/8yhgzDlgN3GGtPd72IGPMTGAmQHFxsdftFJFO1tnlTSMVhGposphgEwu/P5nxgwpS1pZUc7OEPAuYCPyPtXYCcByYHX6QtXa+tbbEWltSWFjocTNFpDOFerOpXCbe9tpOi1wA6hubunSQBnc96p3ATmvtypa/P0+EQC0iXZdTedN/f+njqLuqJNoDj1T/OpxTzY6uJGagttbuMcbsMMaMstaWAdOAT5LfNBHxi2j1nkM1n8N3VenI1lfh/uuvG6MG6VRXsessbrM+bgOeacn42AJ8O3lNEhG/cVvvOVSMKfRzpNfcBOq9R2p59NVN7Dta53hMkU+2AUsFV4HaWrsOiFgsRES6vkirAZ1Eq7kRa4OBqpoGnnjjU55+eyvBJkv37EyO17e/ptfbYPmdViaKSExuy5vC5zU3YtXjaDuG3T8/l5Ihp/BG+X6O1DZw5bgB/GD6KNZsP5zSOiJ+pUAtIq6ErwZ02ug2FESjvRb+3l1Vtby0fhejT+vJ77/7BcYMyG8N5DUNQTKNIWjtSTXc0ZYCtchJLJHMDDfFmJxem7eoNOIwytHaxtYg3TaQB61tDfQnW5AGbRwgctJy6hEne5up97ce4vonnEsFFUWZuIw0Nt3ZC3G8oo0DRKQdp9zoZG2hVbrnCA8sKuO10n2OO3wbIo9th4RPRjptywXxpQH6nQK1yEmqIzuldMTOw9U8vLScF9dW0CMni3svHc2p3bO576WPT3hQGHDcGTwkfHOAVD9sOosCtUgX5zQ04JQbbWnelzDRIYRDx+v52fLN/O+728DAzC+dzvcuGkZBt2wAsrMyWtuVnxdoXTjjJFK2R6oeNp1NgVqkC4s2NBAtNzqRIYTq+kaeenMr81ds4Xh9I9dNGsgZ/XvxZMvv2j4srppQ1NrGaJyyPVK9X2NnUaAW6YJCvehIQSw0NBCalIt1nNtA3RBs4o8f7OCnr21i/9E6LjmzH7NmjOLjXUeijiNHGr4IiTW56bQt15TRhUyesyztJxhDFKhFPOCnzAM3hYxCQwOhXu3Q2S9HHB92M4TQ1GR5ecNuHlpSxmcHqzlnyCn84sZJTBrcG4BbfvVB1HHkaNeIlYESKUVwyuhCXlhd0aUmGBWoRRLkt8yDaD3UkPChgY4OIby16QBzF5WyoaKK0af15OlbSqiqbuD2P6xtDZxOWRyhAO10TFFBnqt/v/CFOJPnLOtyE4xu6lGLSBTRMg+SJVSjeejsl5k8Z9kJdaFj9YIjTcrNmjGKvLDNX6Mt1d6ws4obn1zJjU+t5NDxeh6+fhwv3/4ljtQ08uMXPzqhbrXTnt+hh0C8147FzQRjtH8/P1KPWiRBqc48iNWDj9aLzTTmhIdIqIfpZpUhwGcHjvPgkjL+8uFuencL8K9fOZMbzy0mJyuz9f3hDy1L5NS76vpGFq6tcH1tt2J9O/DbNyA3FKhFEpTqzINYucORJtgCGQYMNASbw2Wk4BRpZ++QfUdr+elrm/jj+zsIZGZw29ThfPeC0+mVe+IO304PJwsUhKXgHa5uOKENXgVJpwnGUA89HXOvFahFEhQrMHgtVg/ebaU7p+DUdmL0tPxcxhbl8+amAzQEm/jGOcXcNm04fXtG3uE72ngz0C5XOhkBMlYPPR1zrxWoRRLk9Vf3WNz04MN7qENnvxzxXLGWZO+uqmV3VS0TBhXwyA3jGdKne9S2RXto3fXsOldt8EK0Hno65l5rMlHEA1dNKOLt2VPZOudy3p49NalfoTsy+eYUhMJ/71TVbt/RuphBGpr/He6/ZixFBXkYmnvSoRQ7t21INq8nL1NBPWqRNNORHnys4RlrLcvL9rGrqjbi++Pp9Tr1ZlM9ROQk1d+AvKBALZKG4p18ixacVm87zNxXSnn/s0NkZhiCEcraedHr9VOA9HLyMhVUj1rEp5K92nHT3qPMW1zG0k/2UtgzhzumjSAvkMm/LPwo5TWqRfWoRdJOMnN9d1XW8Oir5Ty/eifds7P44SUj+c75Q+mW3RwOMjOML3q98jlXgdoY8xlwFAgCjU5RX0S8kYxc38rqen7++qf8+p3PwMJ3Jg/l1inDOaV79gnHpduwwMkgnh71FGvtgaS1RERaeZnrW1Mf5FfvbOV/Xv+UY3WNXDNhIHdNH8HA3t0SbaakiIY+RHwgfDy6oFug3QIViG9SrzHYxHOrdvLYa+XsPVLHtNF9mXXpKEaf1svLpksKuA3UFlhijLHAE9ba+eEHGGNmAjMBiouLvWuhSBcXaTw6kGEIZJrWJd/gPpXNWsuij/bwwJIytuw/zqTBvXn8mxP5myGnJO0eJLncBurzrbUVxpi+wFJjTKm1dkXbA1qC93xozvrwuJ0iXUZ477m6vrHdeHRDk6UgL0D3nKy4JvXe+fQAcxeVsX5HJSP69uCXN5dw8Rl9Mcaphp2kA1eB2lpb0fK/+4wxLwLnACuiv0tEwkXqPTupqmlg3X2XuDrvx7uqmLeojDfK9zMgP5d5153NtRMHkpmhAN0VxAzUxpjuQIa19mjLz5cAP0l6y0S6IDdF/UPcjEdvP1jNQ0vL+PO6XeTnBfjny87gpvMGkxu2RFrSm5sedT/gxZavTlnA7621i5LaKpEuym3WRqzx6APH6nh82WaeWbmNzAzDrRcN4x8uHEZ+XsDxPZK+YgZqa+0WYFwK2iLS5TlVbnM7Hn2srpFfrtjCk29uobaxietLBnHnxSPo1yty2VHpGpSeJ2nJT5vJxsOpMNG/XzHGsf0L11Ywb1Epu6pqT9gppW/PHL4w9BQF6ZOAArWknVRupRTrgRDvAyPewkQL1uzk3hc+bE3Ta5tOte9oXbv7TtcHmESnokySdibPWea4i8jbs6d6dp3wBwKcWKAo1uuJsNbyRvl+vvvbVSfkUkcSuu9ktkeSL1pRJm0cIGknVVspxdpdPFm7j6/bUck3fvket/zqg5hBGj6/787YDV1SQ4Fa0k6qdgqJ9UDw+oHx6f5jfO93q7nqZ2+zed8x/uOKMQzIjz3+HLrvdNwLUNxRoJa0k6qtlGI9ELx6YOypquVHCz7kkkdWsKJ8P3ddPJLXZ03hW18cwj2Xjm53r221vW+/bHUl3lOglrQTbV8+L8V6ICT6wKiqaWDuolIuenA5z6/eyU3nDuaNe6Zwx8Uj6JHTPM8ffq8FeQF6dwtEvO9kPMAWrq1g8pxlDJ39MpPnLGPh2ooOn0s6TpOJIlF4nfUBUNsQ5DfvfMbPX/+UI7UNXDW+iB9MH8mgUxIvO+pl1ocmJ1Mr2mSiArVIijQGm1iwpoJHXi1nd1UtF40q5J4ZozlzQGJlR5OVkpeq7Bpppq24RDqRtZYln+zlgcVlbN53jHGDCnj4+vGcN+zUdsfGG3STvWVXPL+X5FGgFgnjZQ/1/a2HmPPKRtZsr+T0wu784saJzBhzWsSyox0JusnYsivEabm7JidTT5OJklaSPbkVCpYVlTVYPg+W8V6ndM8RvvPrD7j+iXepqKzhhpJB1NYH+d7v1nD+3OURz9eRPOhk9npTlV0jsalHLWkjFUvHE+2h7jxczcNLy3lxbQU9c7K499LRnNo9m/te+jhmuzsSdJPZ6413ubskjwK1pI1kfs0P6WgP9dDxeh5ftpnfvbcNY2DmBadz64XDye8WYPKcZa7a3ZGg61Tkyater3Yk9wcFakkbqZjcijdYVtc38tSbW3lixRaq6xv52qRB3Dl9BP3zPz/eqX0VlTVMnrOstbc6ZXQhL6yuiCvoqtd7clCglrSRiskttz3UhmATf3x/O4+9tpkDx+q45Mx+3HPpKIb37em63YbPt+KqqKzhhdUVXDupiOWl++MKuur1dn0K1JI2kv01H6L3UNvWhc7MMASbLOcMPYUnbprEpMG942p327rSITUNQZaX7k9JjrLKoaYXBWpJG6n6mh+ph7pwbQX3PP8h9cEmAIJNzWG2fM8Rdhyqjhio2wbD/LwAuYEMKqsbHHvYkJoc5VTW8xZvKFBLWon1NT8ZPcUPd1Zy7wufB+m2KmsaIwa58GBYWdNAXiCTR24Yz1UTihxX/aUiRzkVk7LiLeVRiyf8ULzHqxzokK0HjvP936/hisffpq6xfZAOiZTrHCsnujNzlLXiMP247lEbYzKBVUCFtfYryWuSpBu/fJV2Co53P7c+rrbsO1rLT1/bxB/f30F2Vga3Tx3On1bvZHdVreN7woNcrGDYmdkaWnGYfuIZ+rgD2AgkVkFGuhy/fJV2Co5Ba109OI7UNjD/jS089dZWGoJNfPMLxdw2dQSFPXM4vbBHuwnBttoGuYVrK8gwhmCEgmdtj+usbI1UTMqKt1wFamPMQOBy4D+BHyS1RZJ2/PJVOtokXbQHR21DkN+9t42fLd/M4eoGvjpuAHdPH8mQPt1bjwm9799f+pjKmoYT3t82yIW+XUQK0n4Jhsq9Tj9ue9SPAvcA7ZNEWxhjZgIzAYqLixNvmaQNv3yVjtRTbCv8wRFssry4toJHlpZTUVnDl0b04d5LR3NWUX7E94d6wNEmLCN9uwDINMZXdZyVe51eYgZqY8xXgH3W2tXGmIucjrPWzgfmQ3M9as9aKL7nl6/SocBz93Prow47WGtZVrqPeYvKKNt7lLMH5jPvurOZPLyP6+s4BTmnbxFN1iowSoe56VFPBq4wxlwG5AK9jDG/s9bemNymSbrw01fp0DWdHhyrtx1iziulfPDZYYac2o2ffXMil42NXHa0I/zy7UK6lrh2eGnpUf8wVtaHdniRzhY+PHHjucWs3lbJqxv3UtgzhzumjeCGvxlEINPbDFVtXyUdpR1e5KQTGp7YVVnDI0vLeWBxGd2zs5g1YxTfnjyEbtnJ+U/fT98upOvQnonSaZJZb6Kyup6fv/4pv37nM7Bw83mD+f6U4fTunp30a4t0hHrU4jteLpJpG3RPy89lYnFvVmzaz7G6Rq6dOJC7po+kKCzPOdK1V207FHflOpFUUI9aOoVTrQto3uXabZCMNCYMMGZALx6+fjyjTut5wrEPLC5zvG54RTuNLUsqqUctvhNtMUw8vet5i0oj5i1XVje0BumFaysiLlQJF6nsqAoViR+oKJN0iljparE2dQV459MD7HKovxF6EIR63LGCtBMVKhI/UI9aXPNyAi7WKkJwDpIfVVQxb3EZK8r3kxmjpobTSsFwkQr5tz2PSGdSoBZXvK6Q1zaNzWnMODxIbj9YzYNLynhp/S4KugX458vOoKBbgH/788eOqyLd9IiLOrhfoUiqKFCLK8mokNe2dka0Jej7j9bx+LJN/P797WRmGL4/ZRgzLxhGfl4AgEBmhmNPP1qhpvDJwpLBpyhlT3xJgVpcSWaFPKdFIhef2Y+Hl5bz5JtbqGts4oa/GcQd00bQr1duu/c7BVSnIZbe3QLc99UxJ7xPhYrErxSou4BULN5Idg2LtkGyrjHI71du58J5yzl4vJ7Lx/bn7ktGcnphjw6dF7RSUNKbAnWaS9XuKqmokNfUZHlp/S4eXFLGzsM1fHHYqdx76WjGDSpI6LzqKUu6U6BOc6naXSWZPVNrLa+X72feojI27j7Cmf178dvvjOVLI/p4VtVOJJ0pUKe5VO6ukoye6drth5nzSikrtx6i+JRu3HTuYF7buJdvPf0++XkBjIHD1Q2taXjxrFoU6SoUqNNcutQ/Dh9Hv/m8wazdXsmij/fQp0c2P7lyDN0Cmfxrm1S7totUQrnSnbVxrkhn0srENDdrxijyApkn/M5v+b+hcfSKyhoszcH2/ldKWV62j7suHskbs6Zw83lDeOTVTa4Wp7hZtSjSlShQp7mrJhRx/zVjKSrIw9C8eMNvhYScVgfWNTbx3KodLP1kLxDfcI2WdsvJREMfXYCfsxpqG4KOC07gxKGMaItTwvltaEckmRSoxXML11Ywb1Epu6pqyXCRtBEaynBT/wP8N7QjkmwK1OKpF9fs5N4XNlAfbAKgyUKmgYwMQ0PQufb5rsqadimAyvoQaaZALZ5ZueXgCUE6JGihZ3YW3XOyYhZgSvUwjrbkknSgQC0J27j7CPMWlbK8bL/jMVU1Day775KYBZhSKVWrOkUSFTNQG2NygRVATsvxz1tr70t2w+Rzfu317ThUzSNLy3lxXQU9c7KY/eXR/PadzyIW82/bYwZ/1N5I1apOkUS56VHXAVOttceMMQHgLWPMK9ba95LcNsGfvb6Dx+p4fPlmnnlvO8bAzAtO59YLh5PfLcBpvXJj9pj9kqWSylWdIomIGaht8+63x1r+Gmj54/2OuBKRl72+RHvmx+saeeqtrcxfsYXq+kauLxnEHRePoH/+56lyfuoxx5IuqzpFXO1CbozJBFYDw4GfWWvvjXDMTGAmQHFx8aRt27Z53NST09DZL0d8Khpg65zLXZ8n0thwaPupSJkUbYN6//xczht2Km+UH+DAsTpmjOnHrBmjGN63Z/sLpRGn8XK/LRiSk0PCu5Bba4PAeGNMAfCiMeYsa+1HYcfMB+YDlJSUqMftEa96fZF65qEPKXw4JTyA7aqq5YU1FQwr7M78myex/WA133r6gxNS6CqrG3zde44knXr/cnKLK+vDWltpjFkOXAp8FOv4riqVk3te1YGONe7adjjFacl3TX2Q7QerT2hP28JJfhg/j5dfxstFoolZ68MYU9jSk8YYkwdMB0qT3TC/ilRg6EcLNrBwbUVSrudVLQ83PfBdlTV8uLPSMdd5d1VtzF29VTBJxHsxx6iNMWcDvwEyaQ7sz1lrfxLtPSUlJXbVqlWeNdJPJs9ZFjGQFRXk8fbsqZ3QIncijceGywtkUtMQJMM0rygMF1odGEu84+cikuAYtbX2Q2CC561KU+mY0hUaqqlpCLYG29BEYltBa7l92ggG5OfyH//3Sbug7vS+cKHeu1/zv0XSjVYmxindUrrCe9JBa8kLZPLVcf3564Y9HKtrBOD84X145IbxFPbMASA3kMndz61v14O2EDVYh8bP/Zj/LZKuVI86Tn4q1L9wbQWT5yxj6OyXmTxnWcRxcqc87D+t3smxukauGDeAN2ZdxO/+/gutQRqag2mTwzBHKKXPAAV5AXp3C7QbP4+W/y0i8VGPOk5+Sely22N1GpKxFv5y2/mcVZTveA2nbw9uxuPTcYhIxK8UqDvADyldblcs9s/PjVh7o6ggL2qQhsRSA9NtiEjEzzT0kWRuhic6wk2PddVnh8gJG6YB98E2kdRAPw0RiaQ79aiTKJkTatF6rOV7jzJvURmvbtxL3545fG3SQN7efIDdVbVxD9V09NuDX4aIRLoCV7U+4tWV86jj4ZRznWkMD10/LqGgFSkvOicrg3EDC1i17RDds7P4x4uG8e3JQ+iWreexiN8lXOtDOsZpeCJobcI967Y91orKGnrkZFHbEGTdzkr+7vyh3HrRcHp3z+5Yw0XEVxSokyjartpeFKi/ZEw/Kipr+MXrn3K8vpFrJg7krukjKerghJ0WqIj4kwJ1EsXaVbujqWoNwSaeW7WDx17dxL6jdVx8Rj/uuXQUI/t1vOyoFqiI+JcCdRKFAlykFX4Qf6qatZa/btjDg0vK2HrgOCWDe/Pzv51IyZBTEm6rtqUS8S8F6iQLBblES5W+s/kAcxeVsn5nFSP79eDJm0uYdkZfjDERj493GEMLVET8S4E6BdykqjkF1o8qqpi7qJQ3Nx1gQH4uD35tHFdPKCIzI3KADp0r3mEMLVAR8S+l5/mAU6rdmAG9WLO9koJuAf5pynBuPHcwuREWsITrSClWbUsl0rmUnudzkcaH6xqbWLu9ku9PGcY/XDiMXrkBV+dauLbCMdMk2jCGFqiI+JcCtQ84Fk4CZs0Y7fo8oV6xk1jDGH6oYSIi7anWRyerawzSKy9ybznefOho22SpzoZI+lKPOkEdXSTS1GT58/oKHlpSTlVNQ7vtr/ICmUwZXcjkOcsSztwANNYsksYUqBPQkewKay2vl+1n7qJSSvccZcyAXvzX1WM5eKyOB5eUtwblKaMLeWF1hSeZG0UFeQrSImmsywbqRJdDu3l/vItE1mw/zNxXSlm59RCDT+3GzecN5tVP9vKtp99vd43Jc5bFvQAlkfrRIuJfMQO1MWYQ8FugH83zW/OttY8lu2GJSHQ5dKK7p4T/fvO+YzywuJTFH++lT49sfnLlGLoFMvnXP3/seI2OLEDpSOaG6nuI+J+bHnUjcLe1do0xpiew2hiz1Fr7SZLb1mGJLod2+/5Yi0T2VNXy6KvlPLdqB92ys/jB9JH83flD6Z6TFbPH3NEFKPFkbqi+h0h6iJn1Ya3dba1d0/LzUWAj4Ov/Fye6HNrt+512Mfn+lGHc/8pGLnxgOS+s2cm3vjiEN2ZdxO3TRtA9J8vVNVKxQ4o2oBVJD3GNURtjhgATgJURXpsJzAQoLi72oGkdl+hyaLfvDx9q6J+fy6TBvZnzSilH6xq5enwRd00fyaBTusV9jVQsQFF9D5H04DpQG2N6AC8Ad1prj4S/bq2dD8yH5iXknrXQhfBx1vCMCYivNxrPpNxVE4r4ytn9eX71Th59dRP/9+Fupowq5J5LR3NG/14JXSPZC1BU30MkPbha8GKMCdAcpJ+x1i5IbpPiExpnraiswdI8zvrC6gqunVTUoU1Zwf2mrtZaFn20hxmPrmD2gg0cPF4HQPneY5TtOerJNZJJG9CKpIeYRZlMcx3N3wCHrLV3ujlpKosydaQAUTycsiLe23KQuYtKWbu9kr49c6isbjgHR4kAAAn7SURBVKA+2NT6vnQpaKSsDxF/iFaUyU2gPh94E9gAhCLRj621f3V6TzIDdXhgcSpAZICtcy6P+3xtA1WkinJt5ecF+PFlo3ns1U3sqqpt97pXDwsR6foSqp5nrX2L5rjX6SKlkxmak7vDuRlnjZWeFq12BkBdQ5CcrEx2RwjSoEk5EfFGWhVlihQ4Le2fIm7HWWOlp8UKtLWNTTywuMzxoaBJORHxQloF6mjlQEOTcgV5AXIDGdz17Domz1nGwrUVcZ9vV2UNx+sa6ZEbOylmV2WNJuVEJKnSKlA79VBDY8GP3DCeusYmDlc3tGaA/GjBBsdg7XS+XnkBLnxgOUdrG4my41XrOfyQwSEiXVdaBepYPdd4V9pFOp8BqmoaGFbYgwW3fpGHrx/fWhc62hDLVROKeHv2VLbOuZy3Z09VkBYRz6RV9bxYq/XiXWl31YQirLX8v5c3cvB4PQD983P5z2vGctHIQowxTCzufUIWSGensvmhDSKSWmkVqCH6ar14V9qt31HJn1bv5ODxegb2zuPuS0Zy5bgiMhzGOzp7qyoVURI5OaVdoI7G7dLvLfuP8dCScl7esJtTumdz31fP5JtfKCYnK/YO317oaK840aqAIpKeulSgjjU0su9ILY++tolnP9hBTlYGt08bwXe/NJSeLnf49kIivWIVURI5OaV1oHbqmYYHvCO1DTzxxqc89dZWgk2WG79QzD9NHUFhzxxX5/OqXZBYr1hFlEROTmkbqN30TGsbgvzvu9v42eubqaxu4IpxA7j7kpEMPrV7h87nRbti9YqjBXlttSVyckrbQB2tZ/rVcQNYsGYnjywtZ1dVLReMLOSeGaM4qyi/Q+eLJ1DHOk+0XnGsIJ+KGtUi4j++C9Ruhx+ceqYVlTV8+bEVlO89xriB+Tz4tXF8cXifmNf1avzXzc4tTr1iNw+Lzs48EZHU81WgdupRrtp2iOWl+08I3tEq5zUGLT//24l8+azTaK7SGptX47+J7Nxy17PrIp5Tk4UiJzdfBWqnHuUz721vrZAXCt7XTipqt4sLwPUlA/nPq8cSyIxv0aVX47+J7NyiyUIRicRXS8ijFV1qq6YhyNJP9jK2zZhzr9ws5l17NvOuGxd3kAbvdlxJ5Dwq7iQikcTcOKAjOrpxgNNuLU6yszK45YtD+N6Fw+jdPTvu6/mRloiLnJwS2jgglSINGzhtDNAtO5OlP7iwtWBSV6HJQhEJ56uhj0jDBt84ZxCBsNobOVkZ/NfVY7tckBYRicRXPWr4vEdpreXlDbt5aEk5DU2W7MwM6oNNFGk4QEROMr4L1ABvbz7A3EWlfLizipH9evDkzSVMO6Ov61Q7EZGuJGagNsY8DXwF2GetPSuZjfmoooq5i0p5c9MBigryePBr47h6QhGZsbZZcUkTdSKSjtz0qH8NPA78NpkNqapp4Gu/eJecQAb/cvkZ3HjuYBZ9tIcL5i33JLCqlrOIpKuYgdpau8IYMyTZDcnPC/DETZMYX1xAr9yA54FVtZxFJF35KuvjgpGF9GqpDR3v/oexqJaziKQrzwK1MWamMWaVMWbV/v37Ez6f14HVaRm2lmeLiN95FqittfOttSXW2pLCwsKEz+d1YNXybBFJV74a+mjLy8AayvaoaQiS2ZLiV5AXIDeQwV3PrmPynGUsXFvhSbtFRLwWM1AbY/4AvAuMMsbsNMb8XfKb5V2RpNCkZKiGSNBaAhmG4/WNHK5uwPL5RKWCtYj4ka+KMiVDPIWeigryeHv21CS3SESkvWhFmXw79OGVeCYflQEiIn7U5QN1PJOPygARET/q8oE60qRkIMMQyDxxWboyQETEr3xZlMlLTnsURvqdViiKiB91+clEEZF0kBY7vKiynYhIZL4I1KpsJyLizBeTiV4XYBIR6Up8EahV2U5ExJkvArUq24mIOPNFoFZlOxERZ76YTHTKddZEooiITwI1NAdrBWYRkfZ8MfQhIiLOFKhFRHxOgVpExOcUqEVEfE6BWkTE55JSPc8Ysx/Y1sG39wEOeNicztRV7qWr3AfoXvyoq9wHJHYvg621hZFeSEqgToQxZpVTqb9001XupavcB+he/Kir3Ack71409CEi4nMK1CIiPufHQD2/sxvgoa5yL13lPkD34kdd5T4gSffiuzFqERE5kR971CIi0oYCtYiIz3VaoDbGXGqMKTPGbDbGzI7weo4x5tmW11caY4akvpWxubiPW4wx+40x61r+/H1ntDMWY8zTxph9xpiPHF43xpifttznh8aYialuo1su7uUiY0xVm8/k31LdRreMMYOMMcuNMZ8YYz42xtwR4RjffzYu7yMtPhdjTK4x5n1jzPqWe/mPCMd4G7+stSn/A2QCnwKnA9nAeuDMsGNuBX7R8vPXgWc7o60e3MctwOOd3VYX93IBMBH4yOH1y4BXAAOcC6zs7DYncC8XAX/p7Ha6vJf+wMSWn3sC5RH+G/P9Z+PyPtLic2n5d+7R8nMAWAmcG3aMp/Grs3rU5wCbrbVbrLX1wB+BK8OOuRL4TcvPzwPTjDEmhW10w819pAVr7QrgUJRDrgR+a5u9BxQYY/qnpnXxcXEvacNau9tau6bl56PARiC8cLvvPxuX95EWWv6dj7X8NdDyJzwrw9P41VmBugjY0ebvO2n/obUeY61tBKqAU1PSOvfc3AfAtS1fSZ83xgxKTdM85/Ze08V5LV9dXzHGjOnsxrjR8vV5As09uLbS6rOJch+QJp+LMSbTGLMO2AcstdY6fiZexC9NJibf/wFDrLVnA0v5/CkrnWcNzXUVxgH/DSzs5PbEZIzpAbwA3GmtPdLZ7emoGPeRNp+LtTZorR0PDATOMcaclczrdVagrgDa9iwHtvwu4jHGmCwgHziYkta5F/M+rLUHrbV1LX99EpiUorZ5zc1nlhastUdCX12ttX8FAsaYPp3cLEfGmADNwe0Za+2CCIekxWcT6z7S7XMBsNZWAsuBS8Ne8jR+dVag/gAYYYwZaozJpnmw/aWwY14CvtXy83XAMtsyMu8jMe8jbKzwCprH5tLRS8DNLRkG5wJV1trdnd2ojjDGnBYaLzTGnEPz/w/81gkAmjM6gKeAjdbahx0O8/1n4+Y+0uVzMcYUGmMKWn7OA6YDpWGHeRq/OmVzW2ttozHmn4DFNGdOPG2t/dgY8xNglbX2JZo/1P81xmymeWLo653R1mhc3sftxpgrgEaa7+OWTmtwFMaYP9A8697HGLMTuI/mSRKstb8A/kpzdsFmoBr4due0NDYX93Id8D1jTCNQA3zdh52AkMnATcCGljFRgB8DxZBWn42b+0iXz6U/8BtjTCbND5PnrLV/SWb80hJyERGf02SiiIjPKVCLiPicArWIiM8pUIuI+JwCtYiIzylQi4j4nAK1iIjP/X8W5zmilIONHQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0giuwC9GHzi8" + }, + "source": [ + "## Вычислительный граф\r\n", + "\r\n", + "Для проведения вычислений Tensorflow строит внутри себя вычислительный граф, который, в т.ч., может вычисляться на GPU. Однако в нашем случае, поскольку мы использовали пользовательские Python-функции, они не включались в вычислительный граф, и при вычислениях на GPU производилась бы передача данных между GPU и CPU и обратно.\r\n", + "\r\n", + "Для ускорения высчислений и построения единого статического графа, необходимо отметить все функции соответствующим декоратором:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "HK7HPLz3Hyrl" + }, + "source": [ + "@tf.function\r\n", + "def train_on_batch(x, y):\r\n", + " with tf.GradientTape() as tape:\r\n", + " predictions = f(x)\r\n", + " loss = compute_loss(y, predictions)\r\n", + " # Note that `tape.gradient` works with a list as well (w, b).\r\n", + " dloss_dw, dloss_db = tape.gradient(loss, [w, b])\r\n", + " w.assign_sub(learning_rate * dloss_dw)\r\n", + " b.assign_sub(learning_rate * dloss_db)\r\n", + " return loss" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "J7HusxWkGjLX" + }, + "source": [ + "## Dataset API\r\n", + "\r\n", + "Для работы с данными в Tensorflow присутствует удобное API, которым мы в данном случае можем воспользоваться:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "oYro9Lbr8q0M", + "outputId": "78c0a6de-71bd-4eef-8819-439495b28672" + }, + "source": [ + "w.assign([[10.0]])\r\n", + "b.assign([0.0])\r\n", + "\r\n", + "# Create a tf.data.Dataset object for easy batched iteration\r\n", + "dataset = tf.data.Dataset.from_tensor_slices((train_x.astype(np.float32), train_labels.astype(np.float32)))\r\n", + "dataset = dataset.shuffle(buffer_size=1024).batch(256)\r\n", + "\r\n", + "for epoch in range(10):\r\n", + " for step, (x, y) in enumerate(dataset):\r\n", + " loss = train_on_batch(tf.reshape(x,(-1,1)), tf.reshape(y,(-1,1)))\r\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 18, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 173.4585\n", + "Epoch 1: last batch loss = 13.8459\n", + "Epoch 2: last batch loss = 4.5407\n", + "Epoch 3: last batch loss = 3.7364\n", + "Epoch 4: last batch loss = 3.4334\n", + "Epoch 5: last batch loss = 3.1790\n", + "Epoch 6: last batch loss = 2.9458\n", + "Epoch 7: last batch loss = 2.7311\n", + "Epoch 8: last batch loss = 2.5332\n", + "Epoch 9: last batch loss = 2.3508\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A10prCPowHl7" + }, + "source": [ + "## Пример\n", + "Рассмотрим пример двухмерной задачи классификации на 2 класса. Примером такой задачи может быть классификация опухоли на 2 типа - доброкачественная и злокачественная, в зависимости от её размера и возраста.\n", + "\n", + "Сгенерируем тестовые данные случайным образом:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "id": "j0OTPkGpwHl7" + }, + "source": [ + "np.random.seed(0) # pick the seed for reproducability - change it to explore the effects of random variations\n", + "\n", + "n = 100\n", + "X, Y = make_classification(n_samples = n, n_features=2,\n", + " n_redundant=0, n_informative=2, flip_y=0.2,class_sep=1)\n", + "X = X.astype(np.float32)\n", + "Y = Y.astype(np.int32)\n", + "\n", + "split = [ 70*n//100, (15+70)*n//100 ]\n", + "train_x, valid_x, test_x = np.split(X, split)\n", + "train_labels, valid_labels, test_labels = np.split(Y, split)" + ], + "execution_count": 19, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "id": "c-_BjSHPwHl8" + }, + "source": [ + "def plot_dataset(features, labels, W=None, b=None):\n", + " # prepare the plot\n", + " fig, ax = plt.subplots(1, 1)\n", + " ax.set_xlabel('$x_i[0]$ -- (feature 1)')\n", + " ax.set_ylabel('$x_i[1]$ -- (feature 2)')\n", + " colors = ['r' if l else 'b' for l in labels]\n", + " ax.scatter(features[:, 0], features[:, 1], marker='o', c=colors, s=100, alpha = 0.5)\n", + " if W is not None:\n", + " min_x = min(features[:,0])\n", + " max_x = max(features[:,1])\n", + " min_y = min(features[:,1])*(1-.1)\n", + " max_y = max(features[:,1])*(1+.1)\n", + " cx = np.array([min_x,max_x],dtype=np.float32)\n", + " cy = (0.5-W[0]*cx-b)/W[1]\n", + " ax.plot(cx,cy,'g')\n", + " ax.set_ylim(min_y,max_y)\n", + " fig.show()" + ], + "execution_count": 20, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 283 + }, + "id": "tq0vFchQwHl8", + "outputId": "9a5aa6a0-c92f-4d72-9e78-c0f615804bff" + }, + "source": [ + "plot_dataset(train_x, train_labels)" + ], + "execution_count": 21, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd5iU5dX/v2dmtrINdpfOsoBKV9AFsQRFNIgx1thiNJZojDH6mkSNKSbRn0l8TaKmGDUxRo2vFbEiYgsIEaWIIFUEgaXsLmzvOzP374/vPM6WqbvT93yua67dnXn2mTPzzJxz36eKMQaKoiiKYou3AIqiKEpioAZBURRFAaAGQVEURfGgBkFRFEUBoAZBURRF8aAGQVEURQEAOOItQF8oKioypaWl8RZDURQlqVizZs1BY0xx9/uT2iCUlpZi9erV8RZDURQlqRCRXb7uV5eRoiiKAkANgqIoqYgxQHs70NERb0mSiqR2GSmK0gecTmDjRuDNN4GdOwG7HZg+HZgzBxgzBhCJt4Th09YGrFoFLFoE7NvH+w47DDjjDOCoo/gaFb+oQVCiijFAYyN/5uQANt2TJgatrcCDDwKffALk5gKFhYDbDXz4IfD++8BZZwHnnptcRqGpCbj/fmDrVqCoCBg9mvdXVgL33QccfzxwzTWAQ9WeP/SdUaJCRwfwwQdcqFVUUK/k5wPz5wNf+QqQlRVvCfs5TzwBbNgAlJZ2VfrDh3PnsHAhlers2XETMWwefxz4/POeu5tBg4CBA4GVK4Fhw4BzzomfjAlOwqzXRGSUiLwnIptEZKOI3BRvmZTe0d4O/PnPwN//zt9LSnhzOIB//xv43e+AhoZ4S9mPqayktR41yvcOwOEAhg4FXnoJcLliL19vqKqiq2jkSN+vSYTGbvFi7o4UnySMQQDgBPAjY8wkALMAfF9EJsVZJqUXLFwIrF8PjB0L5OV5v58DBvC+8nLgscfiK2O/5uOP+TOQ/27AAKCmhrGFZGDDBvolA72mjAyuUD77LHZyJRkJ4zIyxuwHsN/ze4OIbAYwAsCmuAqmhEVTE/D228CIEf7dzyNHUidVVABDhsRWPgVAbS2Qlhb8OJuNFzRW1NRwlb9vH5CZCUydCkyYEFoguKkp9ICx7hD8kjAGoTMiUgpgOoAP4yuJEi7bt9MFHUjfiHAxt3EjMHgwvRJ2e3LFL5OavLzQ0jGNiU2wx+Wie+r11/mcmZn8EC1ezA/ID35A91YgCgpCc29Z2Q2KTxLOIIhIDoAFAP7HGFPv4/FrAVwLACUlJTGWTglGW1toit3lApYtA155hQtWhwM49ljg1FMZE1SiyLRpwLPPUjn6u1gtLcw+Gjs2+vIsWAC8+iqzgrpnAB06xKDTHXcE3k5OncodjbW68IX1mg4/PHKypxiJFEOAiKSBxuApY8yLvo4xxjxijCkzxpQVF/doxaHEmbw86plANDQAa9YAH30EpKcz0WXoUHoLfv1r4N13YyJq/2XYMODooxnM8XWx3G66bc48M/opmlVVwBtv8EPg67kKC7nKeP11733G0O3TeUdQUADMnQvs2uX7NTmdfE3nnqtppwFImHdGRATAowA2G2P+GG95lN5x2GFML21uBrKzez7ucgHLl/P3adNoEAB+R4cPZ8zvX//i7xMmxEzs/sfVVzNnf9s2pmValvzgQRaOfPWrVLDR5oMPuLIP5P8fOhRYsQI47TQGn5YsYcxABJg5k/ePGwdceCFXGx98wA9fQQFf06FDdJGdcw5w8snRf01JjJhgy7kYISInAngfwAYAbs/dPzXGLPL3P2VlZUab2yUe778PPPIIU027xxLKy4H33gPKyoCJE33/f1UVPRU/+lH0Ze3XtLcDq1dzhV5eTgU7dSqNwaRJsQnqPPAAA0+FhYGP27yZsQW7nXGFzEyuLioruYO47DIaMLebRu6tt1iTIAIceSSrr0eP1kCVBxFZY4wp635/wuwQjDHLAejVSgFOPBGorwdeeIGLv4ED+T2srfWmo44f7///CwsZcG5ooMtXiRLp6azePf54KlKR2CtMh4PPHQinE/j0U+4GOscN7Xa6v9rbWZQ2ciQ/WBMm6PaylySMQVBSBxHga1+jm3r5choBY2goLAUfKF3cZuM5rBigEgPi1VNk+nQGjwLFA3ftotHwl2mUns7MoUWLAq80lKCoQVCixrBhwAUX8Gbxl78AmzbRZe0Pt5sGxFcMQkkxpk/nhW5s9J0Oagw/MIcfHthoFRVx5eHvPEpIJFSWkZL6nHQSA86BqKwEjjlGv9f9gqws4PrrGfg9eLBrhlBbG/DFF/Q5jhsX+DzWtrKtLaripjpqEJSYMnEid/779/t+vKWFGYVnnBFbuZQ4MmUK8POf84OxaxewZw+wezeDTmefzQ9De3vgc7hcNAi6rewT6jJSYorDAdx8M/CHP3DxV1DAtjkdHVwg2mzADTdocVrCcegQW0s4HOxLEkrri3AYNw647Tb2M6mu5vOUlLD/0Cef0B0UKM5QWQnMmqVtdPuIGgQl5gwaBPziF8C6dUwpr6hgFuGZZzLwPHhwvCVUvmTXLuDFF6mQbTZvcOf004F58yJvGIYM6VmRPHmyd1s5bFjP/2lq4opi3rzIytIPSZg6hN6gdQiKEkW2bgXuvZdKv7jYG9RtaaFynjaNfYYibRR8UV3NbWV5eddt5aFD3E18//uciKaERMLXISiKkkC0tXGoRV5ez5SwrCz69NatA955h7uFaDNoEPsZffIJi84qK7mtPO881lEMGhT6uZqaaNDcbm5HCwqiJ3eSoQZBiQvxqoNSQmTdOirOoiLfj4uwpcQbb7AjYSz6A2VksDht5sze/X9DA/Dyy8DSpV37IM2cyR5H2otdDYISO+rr2ZLmrbfoAcjMZMxgzhzGKZUE4uOPgwdos7OZCXDgAKuEE5n6enZN3b+fjbIsN5fLxU6Ln34K/PSnfKwfo2mnSkzYu5eB5Oee42Jy9Gi2qFi6lPevXBlvCZUutLeHNnBGJDnGbD7zDLMXRo/uGvOw27kaMQZ46KHgrXpTHDUIStRpbWU80OXi9zE7m3okLY3fxcGDgYcfZhqqkiCUlASfluZyUYEOHBgbmXpLbS1XHIFW/0VFrH9IlpGhUUINghJ11q6li8ifOzozk+1oliyJrVxh4XJxbu8//gHcdx/w5JPsppmqK8rjjuNrC7T6r6hg7n+gPiSJgLXSCLTjsYJZ27ZFXZxERmMIStRZtowzEgIxeDDw4YfAt7/N2GFCUVHBNs3WrN/0dPbXeecd5sh/73up12djyBDOGVi8mNu67sq0poZK9Otfj4984RCqS8tmC220aAqjBkGJOrW1wZW83c4FaUtLghmE+nrgnnvo9yot7fqYMezT/6c/AbfemnqTuC66iEryzTep/LOyqFxbWxkAuv1234ViiUZhobdjYqC0Nrc7OV5PFEmxT7CSiAwcyKByoDYzTqdX5yQUy5ZxNTx6dM/HRFhBu3UrdwxHHhl7+aKJ3Q5cfDHTSleu5EXMyGCH0smTk8cAjh7N61Rb6z/e0drK1zZ1amxlSzCS5IoqycycOcCDD/qvHWpsZNq7zcaF9ogRHNo1ZUqcdY7bzcBGoF4aInQXvfVW6hkEi6Ii9hVJVkSAb30L+O1vmcnQ3b3X1kZj953vJNj2NPZoUFmJOkcdRZ1aWdnzsV276JHYvZuT1LKzmexx333A//4vjUXcaG1lMVOwbUtuLuMLSuIyfjzwwx/SJ7lrF2snKioYcD54ELjySmD27HhLGXd0h6BEnYwMfhfvvZffv8JCKv59+zh/ecAA7iKGDvUeP2gQR+0+9BBnK8elotnanrjdgYezuFwMNCuJzZFHAn/8I9tfbNrE6zp2LAd8p1pSQC9Rg6DEhKFDgTvv5LTEt97i4mzzZg7CmjKFRqEzlnv+009pROLSDjs9ncLt2BG49fKhQ+zb3wusrFVt4REj+tr+IsVRg6DEjAEDgJNP5q2qivGCkhL/ytAqXvvgg14ahMZGYONGZgoNGABMmhR+I7PTT2eW0cCBvgMaLS3cPZxwQsindDpZ0vDGG9wFiXBw0Lx5jNXGa7yxoqhBUOJCYyOTWIKtjDMz6eINC6eTPfyXLOHvFjYbZ3hefHHowcNJk7j6f+kl+rry8ym0MRSssZF1CP6q7rrR1gb87W8s1svP97YA2rmTLrXZs4ErrkieBB4ltdCPnRIXMjO9nQ8CGYX29jALYd1u4LHHGJwoKemqWV0u4L336OK58cbQtK4IWyyXlACvvMLot83G57GMxRFHhCzes88yo2rMmK6vu7iY9mbZMv4899wwXrOiRAg1CEpcGDqUrWXq6wNXMbe3A8ceG8aJt24Fli9nEVl334vdzpz0devYzXPGjNDOKcJjy8qYKtXWxiBkOD34wTT4//yHsRFfRtBm445h8WJg/nwaTUWJJeqtVOKCCHDWWYwldPbqdMbqqjx+fBgnfvttpjD5c8SLMB6waFHYMkOELR1KSsI2BoA3sSVQS530dBrBLVvCF09R+ooaBCVuzJxJb8zu3Vx4u928v6mJmUW5ucBNN4UZZN26NXj3zYICOu1j3La5oSH0bKKWlujKoii+SCiXkYj8E8CZACqNMVPiLY8SXUToK584kcVpn3zC+wsKgG9+kxk3tbVAXR3dSyG1tQhV48YhzzMvL/TmqIHafChKtEgogwDgXwD+AuCJOMuhxAgr5XLiRC7YnU4agZdf5kwTK6HH4QBOOYUdFALWEE2ezBiBVeXmi+pqFkCEMgAmgkyezKd0Ov3Hs9vamAAVlptMUSJEQrmMjDHLAFTHWw4lPtjtTAC66y4WsI0YwQBsSQmzOhcvZklAQ0OAk8ydS3+LP3eQMdxyzJ8fldcQiLw8GrU9e3zvFNxuoLwc+NrXNKCsxIeEMghK/8YY5ugbQ2PQeQGfns7Eob17geefD3CSww5jH/8vvuByuzMdHYwdzJoVt0Z0F17I2MnOnd6AutPJyu1du2jPzjgjLqIpSsK5jIIiItcCuBYASkpK4iyNEkk+/5wr5ECXdfhwYMUK4Bvf8FOfIMIAxMCBwKuvdjUKaWlMbTr77Ji7izqLcN11LEB7803gs88o8qRJtGMTJmgbCyV+JJ1BMMY8AuARACgrK0vR+YX9EyvVMpBCdDi4g/jiiwCLfJuNfpdTT+VJGxsZpR0/PiGitXY72+7389b7SgKSdAZBSV06OkJfuPurXehCRgZ7byuKEhIJFUMQkacBfABgvIiUi8jV8ZZJiR3DhwdX9MYw+FpYGBuZFKU/kVA7BGPMJfGWQYkM9fXeRqO5uUy5DNSiAqALKCPDm3oJAHZXO4bsX4fDPn8TefXlaOlw4OCYGShxnwygNMqvQlH6FwllEJTkp6ODWUDvvMPMT6uOwGYDvlpWjfOG/Rfp2zfz4IkTgeOP/7INRFYW57o/9hjTTXPQiGM/uB+F1Z+hNbMANWmD0druwpyMFZBfvceD58/XKKyiRAg1CErEcLmAv/8d+PBDKvQvi6+MQen2tzD8vmewbSAw6dhctqPYvJltqi+6iEOURTBnDt1Gzz1rcOT6h5HZuBPl2aVwtgnS0oCZJ9pRMGQELc/TT3M2Z1lZPF+2oqQMahCUiLFpE7ByZc/WziW738f0DU+idsgobKtPR2E7MGwYmBra3g78+9/cHsyeDRHahuNH7EbDDzdg/9DRyLYJiotZfPylkUlLY7XaSy8BxxyjuwRFiQAJFVRWkpslSxgv6Kybba4OTN74PBpyh8M40pGZySlhX5Kezmjyc89x1e8hZ/MqDBtpx9HHCKZPZ9fTHu0e8vI4mFkH3CtKRFCDoESMbdt6NhotOrgF6e0NcKaxM11WFlsJdWndkJXFWoHNm733VVcH798gwuBEU1PfhW9sZN+M1ta+nyuWVFez7HnfPm+7WEXpJeoyUiKGzdazR09WSzXS25swuGID0tsb4LKlQWwjANcQwNGt6KC21vt7fn7P1hPdMYaBi74Um23eDLz+Ov1dIrwdfzwHHI8Y0fvzRpvPPwcWLmQql/XGFxUBX/865zvrYGalF6hBUCLGpEnUq0OG8G+HsxVHbH0FQw98jLaMPLhtaXB2uDHJsRfydiZ7CnUeep+W5v195kxOoQ80Y7OhgU/WW8W9eDHwf/9HP5c1xszpZCDkv/8FfvhDvqhEY+1a4M9/piEcNcqr/BsagEceAXbsAC67TI2CEjb6iVHC4uBB4LXX2ITu0Uepm9rb+dhpp9F743YDMAbT1/wduQ170Z6eg/b0XHQ4stAiA5A9vICKfsWKTv8AtqS2KC1lYx9/8QGnk93hzjmndwHlrVuZpTRqFAcaW+dwOBjTyM8H/vQndkZNJOrqgIceoszFxV2Vfm4uI/rvvAOsWRM/GZWkRXcISki4XMALL3BRLcLFqcvFWfb5+cAPfsBWQaecArz7LjA1eweG71+D6sLxyGyrx4D6/ahHPvLzgQEDAEgWq9a2b2fZ8YwZdHlYiADf+x7wxz/SR15YSIXndtMQtLRw3FpYA5c78eabfBGddyUADdWhQ3yOigrm0V53XZAhDDHkww9pDP25yWw274jQsjLNvlLCQg1CClBVxWZvbjfT8ktLI68Hnn+eOmb06J79hurqgP/9X+AXv6CnorAQqP7jMtS1pKMWghr7UTjc2YiitFoMyM6B9bFz2jNg1m1Ey/zzkP3Ny3t+GPPzgdtvB1avpvto924+eVkZ+0QfdljvXmhbG/Dxx9wddH8hq1YxwGyzMevpySfZIO+882JfBFdfT3k++ojbsJISYMMGP21eO1FQwF7aDQ3Bj1WUTqhBSGIOHmQKvzV6UoRGYeRI4PLLgSOOiNzzLF7s2xgA1NstLSwJ+MEPGNfsWF2OA5tysP4LoL4lA58POxGNTTswuPxzHNrTBIcDaHc50Jxegservov2X+Vi/nzq+S6L9sxM4MQTebMi1n1Vyu3t3gwli8ZGYPly3m/FNTo6+JxDhtC95HZzZFssWLeOfrn2dr7BdjvwwQeMbwwezPiLw0GlX1/P/8nJoQGwguMhdQBUFC9qEJKUQ4eA3/yGeqxzXNEYJuv89rfALbdEJia6ciX1S6BOpIMHc9FdU0OPRYdkYOd2F9w2FpSJpKO1aAK2NR2OAztb4GwDikZmYnTaXhSNyYVUb8MX/+9DvP9wDWafPRCOE2b13AFEanWelcX6h/Z2/gS4C3C5uq6oOzqojNPSuDp/8UUaps6B8GiwfTvwwAN0oQ0Y4L0/N5dWeetW73tRVcWfVo+QgQN50dPSeLwSHk4nd2GLF3t3pEcfDcyZE52td4KhQeUk5bnnuDgcPrzrQleEOmHgQODhhyOzSNyzJ3hmp83GW7VnAOp/mmfC1ljfpVDN7QbK99vhzMyBLS8HTZWNaEgvQtlHf8Npq3+Dsub34dz8GfY9swy4+25avGgEdR0ObkUOHODfra0cxdY9TtDeDowdy9/T0qhwP/oo8vJ058UX+YZ3NgYWpaWUf+1aoLKSBquggD/z8xmkf/ttDlvoHh9RAtPcDPzhD8D99zOZobCQC4SVK4Ff/5oDl3zNPk0h1CAkITU1dC0HmiOfl0ddunFj358vPT20mie32+vFePXADKTnZSK9vfHLx5uavAPm7eJGfkcV9h1MQ379btQVlKIxbziksBAba0bAPWo0AyN/+EN0isXmzKE7qqaGiqC7C6m+ngp28GDvfdnZDHBHk4MHWRvROcDemUGDuJOxZm/62kFlZHAUW4orr4jzz39yp1haSmPgcHgr6UeOZCBt5cp4SxlVwjYIIjJAROIzf1ABwMUsEHyYjMPRrU1EL5k+nTGCQLS0cEE7fDiTc9ocA7Dq2BuQ2VKDnMYDEOP+Mlab7azHsNad2JAxA1vaxqAxz1tHkJbGhXlDo7C+YNcuroYjTVERcOut/H3vXhqd9nZardpaWtTjjuv6JlsWL5rU1fFN8ueaaGykIcvP57H19TRojY2UOy2Nk+JqayNz8fsL+/YxVbekxPd773BwcbBwYUpXhAc1CCJiE5FvisjrIlIJYAuA/SKySUTuFZHDoi+m0plAtVqdsYLMfWXqVK/+8SfP/v0s7rW8FCJA1eDJWHrSHagYPAV59eUY2FSOYR274LSl45Vh38VHOBbt6b7TOb+Ue+BA+nOjQWkp8LvfATfc4A3GFhXREMye3bN1RktL9OdepqcHXtnX11PO4mLGCsaO5Wp2+HDKPXeuNw6ya1d0ZU0l1q71BuP9kZvLmM3u3bGTK8aEstx5D8DbAG4H8Kkxxg0AIjIIwBwA94jIQmPMv6MnptKZoUOpMN3uwMWoHR3UeX0lLQ248UbgnnuoEwcP9j5vSwuNwZFHskupJZ+V5FJXMBqrZt6AjNY6lG+qx9YdaXAVDoaBIG/3Cgwb2rUPkaULv9TFVgO7aJGVRffRLbfQR+zvDWts5BYo2iM5hw9nTKCpyXcMwVoNuFwMund2afk6VgmN6mpvgkEgItU7K0EJxWV0qjHmLmPMessYAIAxptoYs8AYcz6AZ6MnotKd4mIq4IoK/8c0N9Pl7XcQfZiMGwfccQcnn5WX87Z7N5/nwgtpMKzvU04OF9id9XhbZj6yx4/CQcdQuIwNHU6Bw+7C+OzyLs/T1ESDkpXlucPp9BkcdbkivHM/4wyutnft6tJ1FcbQr3/oEHD99d5RbtHCbmfe7oEDvl9gTg6tcHa2/ziDRSL3Yko08vO7Xnd/GNPpw5l6BN0hGGM6RGQCgBEAPjTGfBklFJHTjTGLjTEhvJNKJLnoIuCuu7iDLSrqutNtaqKx+MEPIqu/Ro7kOWtquKCyujz4SmY56yxm7+3dy9kHNhsXvOPGMWbqcABnT6jFoOZ9aEwfDoD1Ym43O1Z8SVUVm7V5Hl+1ijVqe/fynFOncmcyaVIfMwKzsoAf/5g+4v/8hxYHoECHHQZcfDF/xoLZs9m8bulSXlzLndXeTpdRXh6LTPxtD+vr6UYaPz428qYCRx/Nax/IH9vUxN3b6NGxlS2GiAmyrRSRGwF8H8BmANMA3GSMednz2FpjzNFRl9IPZWVlZvXq1fF6+rhTXs7OCpar2EpFz89nxXC8B4nV1ABPPMEaKwur84TdDkzK2olzN/waNVkj0OZOQ0YGe9oVFnoO7uig5v/1r9EwaDTuu496ctAg6kRr8d7YyPjFJZdEqJ9bUxO3P04nhRk2LPb55243M1oWLfJaP4eD7q0jjmDRWnZ2z37j9fV843/8Y27nlNAwhm1SNm/myqc7Lhe/aFddBZx8cszFizQissYY00NDhGIQNgA4zhjTKCKlAF4A8KQx5gER+dgYMz0aAodCfzcIAD/HO3eywaXLRd01aVL0k2HCobLSK19hIXvYHTzIwltZ/AYmfPw08scMRNG4AjjSxFtdV10NXHopzLzTcd99TKHt3m0C4Hm/+AK48kr2UkoprPfC5WJQ09ry7djB1cCBA11XtYMGUWmpMQifhgb4XHVUVdE3evrp3CmmQHFaXwzCRmPM5E5/54BGYROAU4wx0yItbKioQUgBjOEW4qWXuCq32bg6Li0Fzj4bmDYN5eXAz3/Onbq/72JzMz0qv/99YhnDqOJ2M7V0927+PmIE/W3B8pEV/7S1eXtn7d3LD9y0aWzlO2FCShgDwL9BCOWrUyEi04wx6wDAs1M4E8A/AUQ5B09JeURY6DBtGrcSVjR88OAvv3zr1gVOzQf4LwcPcqcQK1d/3LHZ6D6KVNMqhTuwE07gze0OnoqaYoRiEC4H0KUBgjHGCeByEXk4KlIp/Q8R72SdbtTVhdaFQSR4AZ2ihEw/HDAUSpZReYDHVkRWHEXpycCBoWUEut2+U/eTjpoaNrBra6Mfe+LE4POlFSUC9Bdvq5LEHH00m/kFyghsbGTAOhKFeHGjuZkjPf/73641CJmZnAw3b16/cl8osUcNgpKQGMPEGrudhWqzZnFYmK9WM04n6y6uuy6Jd/ltbcxw2b6daY+dA8NtbcBTT9HqnX++GgUlaoRsEEREAFwKYKwx5k4RKQEw1BgTsX7AInI6gAcA2AH8wxjzu0idW0kO6uuZjvrGG8y2dDhYmzB7NvXhhg3MvszP5yL64EFmF11wAXD88fGWvg+sWEE30ZgxPRV+RgZTrF57jZbRV568okSAcHYIDwJwAzgFwJ0AGgAsADAjEoJ4Oqj+FcBpAMoBrBKRV4wxmyJxfiXxOXCAozhratieo7SUu4Q1a6gvL7mE2X9vvslsIrud/dxOOSVwSmrC43azAK1TZlUPHA7eli4FLr00tvIp/YZwDMKxxpijReRjADDG1IhICN2gQmYmgO3GmB0AICLPADgbrHdQUpyODhaKtrV17Qxgt7PYrr2d7vWf/MTbtTplaGxkEV5JSeDjCgpYSasoUSIcj2uHZxVvAEBEisEdQ6QYAWBPp7/LPfd1QUSuFZHVIrK6yhofqCQcLhdX+jU13rZAgfj0U5YhFBf7fjw9nQk3r70WWTkVRfESzg7hTwAWAhgsIncD+AaAn0dFqgAYYx4B8AjASuVYP78SmNZW4P336QGx5ifk5wPz5/seMWCxfHnwlNHCQg60qqvjOVOGnBy2Smhs7DnGszO1tZx3kAg0NbGid8MGWvxx4+i/+7IRVQRoaGDDLrebNSrBursqfSYkg+AJKC8DsAbAXAAC4BxjTCT3r3sBdO5UM9Jzn5IkNDXR7fPZZ/z+Wh6Qpibg3/9mltAPf+hb8dfXB+/MahWNtrSkmEGw2YCvfQ147DG+Ob7iCE4nFe9XvhJ7+bqzZg3wyCP07+XkUP5PPgEWLGB67Fln9S2gU1fHc63oVObkdnMWxYUXssWuEhVCchkZNjxaZIzZYoz5qzHmLxE2BgCwCsDhIjLGE5u4GMArEX4OJYo8/TQb7Y0d21XpDxjA+3buZPakLwYNCl5l7HYzHbUvxWetrRxt0NDQ+3NEheOPZ6+c3bt7+tja2thp88wz459htHEj8Oc/0yKXlnLVPmgQrf+IEcALLzBFrLfU1gK/+Q23jMOGsZuhdduyBbjzTmDPnuDnUXpFOC6jtSIywxizKhqCGGOcInIDgDfBtNN/GmMiMAuADNAAACAASURBVCJeiQU1NaynCjSTZcQIppRecEHPrs2zZtHN1NLCZJqiop7upaoqtjzKzQ1fvv37mZ20YgX1rTEcF/C1rwFTpiRAhlJGBnDzzbSqy5d7C9NEOKvhW9/yjqSLF8ZQvvx8No/qjsNBw7BwIXDSSb2z3M8+y1zi7gF2m40FKYcOAQ89xGEgSVt0kriElWUE4FIR2QWgCXQbGWNMhGZyAcaYRQAWRep8SuzYupU6LFCjTbudOmXrVhoAgP+zZAl1yJ49fCw7m3qwpITKOi2NhqKlhQo8XLZtYxdUY+jKcjj4+969wL33starr16OsLDmjhrDKLo1Azkri62rzzuPb0R7O63fxInRn9QWCrt3800LlA2VlsaUsY8/Bk48Mbzz19TQrxhoVVFYyN3S9u3a1C8KhGMQ5kVNiiTDGCqvigqv4go02rY/0N4e/rHG0FX8yiv0hMybxx1EfT31ys6d1BFjx9KYfO97jF2GQ1MT8MADdHVbehfgdSss5GJ3wQKed8qUEE/qcvHEDofvlbI/GhuZJvXee4wJWMyaRd+7lWJVUAAce2zo540VNTXB284CvHgHDoR/fmvSU7D23SK08moQIk7IBsEYsyuagiQLO3dyCtjOnd4gpzGcXfytb/lPm0x1OivbQIh4jy0vB15/na5oSwfMnk1d8vnn3krkuXPZlsJPM9SArFrFFkH+rovDQXkWLQrBINTVsTBsyRKe1BhO+znjDAY8AynKhgZW3e3Zw6CoNYDa5eKqeP164Kc/pd88UbG2VsFwuXrXjM9qNx0Mmy20XGYlbMJpXXGHr/uNMXdGTpzEZvt24He/486+c2Ws2816obvvBn72s/5pFCZO5GK5tdW/Lmht5Xs3cSL/XrqUi8nOC0KHg7sFK3ba0MDFeG/f0xUrgmckFRby+gXM+qys5MWvqfGmQBpD188f/kD//qWX+vdrP/cc3S3du+/Z7XyxlZX0jf/qVwkQ0PDDmDGUt6PDfz9yY/iFsC5yOBQXewM8gd4Dp5Nb8i1b+OHIzuYQjFB6pCsBCcdl1NTp90wAZ4JzlvsFLhfw8MNUGAUFXR+z2bjo27eP1bQ33RQfGeNJRgZd348/TmPZfWqZ00mPwDHHMLvS7WbsNFhqeW4uXdfNzYFT9P3R0hJcT4jwGvp1e7nd9Du1tHQto7b8TgMHctcwejS3ON2pqwsecS8u5htkpWklIgMGMFj89tv+28pWVfF96M1rGDmSRufgQf/1DM3NDCw/8QSzryyys5mF9dWvarC5D4TjMvpD579F5PdgRlC/YOtWftYDtVceOpTTvQ4e7J81NHPncpX98sv8TlqGs7aWC2mnkxXJVvLJxo3UtdOn++7pFgmGDgU2bQqc8NLRwYWv32O2bvW9urew2bhife01BlK7K6SdO7nqDeQbt/yPW7dGxiBYw7bffZd1A04nUzdPP52pWum97Dpz/vlsJPXZZ3RvWdvBjg76+nJyGOzpzcUUod/1N7+hFe/uh2xpoWEdMICPZWV5H2ttZU5zZSVw2WWJu8tKcPrS/jobLB7rF+zYEXzhYcXb9uwJbBA6Orjbra/n9/KII1Kj0EqEsdFjj+Xqf5OnC9XgwXytY8d2/Q6PG8dYwbp11JWdF98WDQ18L8OJ3XZmzhwW1AbyQlRUACefHCCR56OPgitQayuzb1/PWoHqan4oNm3iqjYjg2/GqFFd3xC7PbzovD+MoXFasIByFxfzw3noEPDXvzLu8T//07stV1YW8KMfAW+9xTzeqiqvMTvpJKaB9aVaedw44JZbgAcfpOFJT+e529poEHJygLKynlvQzEyuKt5+m49PmtR7Gfox4cQQNsDTxwisEygGcFc0hEpEgrk1Lawgs79zLF3K2p2mJu85RViAevHFXfVDsjJsGGsNAO4AbruNmVjdX9vo0TS0AwZw59B9DIAx1DdXXtl7L8CECdR/O3bw/N2vYV0dn/O00wKcpLExtBW1zdbVjQFwJf3oo1RuhYV8UYcOccuUnc1WrVaAxOmMTFD5ww8Zs+juuxs4kNu2HTtYaXzzzb1bSWdlMU/39NO5Ine7+doiNa5uwgTGZTZu5I7J5eLu7KOPuJLqbgwsbDYajLfeUoPQS8LZIZzZ6XcngArPbOV+wahRwRMbrKEuQ4f6fvy11/g9HT686w7C5QKWLaOO+NGPEiPlPFJ8/jldaL5W/3l5VNZbt/K9q6z06kOXi1lIEyb0bc6B3Q7ceCPwpz9RN2dlUW91dHDXkpPDBWnADKYhQ5hXHwjr4neumquupmIrLqby//xzCmSz8fjmZuD55+n7HjKEF/7IPpb1uN3ASy9xW+ZLcYrww7x+Pd/gUaN6HhMq6enRq5xOS6Nra9o0731PPBF891FYyP5Koa7glC6Es+663hizy3Pb66ksvidqkiUYkyfzu97U5P+YqioqMF+tVg4c4A7e10rZbuf9W7bQ1ZJKHDrk/3spwoXckUfSU7JjBxfSX3xBl/1XvsJFbF8NZG6ut2321Kk0CMOGsQbsnnuYoBKQY4/l6j1QymV1Na1b54KU5cv5whoaqPxtNiq69HS+KGtF/e67XA1ffHHfZyeXl9MHFqicW4QfutWr+/ZcscbtDr5VFOk6flQJi3B2CKcBuK3bffN93JeSpKVRgdx/Pxd83d2v1dVcIH7zm77/f/lyrz7whQh1yaJF9HunSqJEsNR1ESpky202YQL15YQJPbO5fFFdzZvDwSQef++v3U7j0ytPwsiR9EuvXu17Ek9LC7cb3/9+1/vffZd9fpYu5cXNy+M2sKOj6wVuaOAH6qSTeiFcNyzDE4yMDL5xycS4cTR4gXYJdXXRy1DoBwQ1CCLyPQDXAxgrIus7PZQL4L/REiwRmT6dK9bHHuMq1mbzLkaGD2cnT39V/Z9+2rN/T3dycph5WF8fmjJMBqyEGWs+si+sBJwzzgjdA7F7N9tdfPKJ9zpkZ9Ot/dWv9j6JxiciwNVXU5GvW8cnys31Dn2w2ZhZM2FC1/+rr6el6ujgbiAtjT+t4grAuyNIT+fFHzOmb7JmZ4e2Qm5vD/6BTDTmzWM73UGDfCt8Y3g9Lrss9rKlCKHsEP4PwBsAfgvgJ53ubzDGJNkSo+9Mm0a38KZNTCgRodIbNy4yq/pUW9gMGsTODB995N9dXVFBXRooTb8z27ax6DctjQbEet9bWxmj2bKFcYOIGoWsLBaYbNvGlf+uXVS+c+b4nwOQl0ffV+eLarMxpcxKK+vooGGx2Rhs6atBGDmSW9iGBv9uI2PoAisr69tzxZopU7gq++QTrrw6f+HcbmZyTZ3KqnGlVwQ1CMaYOgB1AC4RkYEADgcL0yAiMMYsi66IiYfDQb93OPG/CROYERfItdvURB3Sm26eicwll1B/7tpF372lqDs66EEpKAC+853QjGFbG7sv5+X1TFO3Mg/Xr+d7fcYZEX4hNhsvZPedgD9OOYVpnoF8Zs3NPF+wOoUQaGgAVq2yocl+DiYu/xuyJmRhyHBHz9jy3r1Urn0JKMcDh4M7saee8gbb7HZvtsesWcDll2vFch8IJ+30OwBuAmsP1gGYBeADAKdER7TUYvZsYPFiLsz8Zc1VVLAup496IeHIywNuv53xkXff9fZ1E2H+/5lncicRCuvXU/H5qxETodF54w2mksZVN5xwAvDMM1y5+sp6aWvjxR4xgtH3QFWPATCGmZbPPUfdmJV5HMpzKzB12UvYkZGJ8ScWY/BQO/3rVrfA7343ObejmZl0351zDt13dXX8gB11VP/sGRNhwgkq3wRgBoCVxpg5IjIBwG+iI1bqMXw48PWvMyNw5MiumTNuN2NlY8cmxkCsaJCbC1x0EdPXKyu9nZ/DTV3/+OPgRWpZWcz4OnAgzovgwkLgl79kj6OKCv7tcNAiNjfTGBx3HEu5Z8wI3Sp24513mJFZUmLtvgQVg89BxxFTMHzzO3D/Zw0ypriQP2UUM5mmT0/+3ObCwsQZJ5pChGMQWo0xrSICEckwxmwRkfFRkywFOfdcKrOXXmJMz/IkiFAfXH554hemNTQwHvDppzRkhx/OOoFQdZnVGLC3dE/Q8YfN1rXDdNwYP54Nnm65hZFwh4OrXCs/uaaGRSn+0tOC0NzMncGoUd1iJiKoLjwc1ScejkOHgKVFBr+4Q5JyU6DEjnAMQrmIFAB4CcBbIlIDQFtih4HNxmHzc+Yw7byujgu18eOTo/fRihXAv/5FRZuTQ0O2YQPw4otscXPGGdH3QowaxdY8gXC5aKx6ueCOPJMnMzVtwQL247YmBdXV0Zd47rm97l3yySc0koEW/IMGATu/EOzdy6detYq7tJwcNhuMVEKEkvyE09zuXM+vvxKR9wDkA1gcFalSnMxMfhGTibVr2e11+PCutVODBlEhPfMMldKpp0ZXjuOOY7ppoDTWigruuBKqP9SQIcD113NHYE1WGj68zxkEe/f6j0lZWLVaDz/McIbNxmvY3s64VmkpcMMNybEoUaJLyOsCId8SkTuMMUvBwPK0YP+nJD9uN0fpFhf7LqS10j+ff56pn5GkqYktJ7Zt44K6uJh1Bl984buVSG0tFeBZZ0VWjogxcCDdRePHRySdLCMjtJYqW7dyNzF6tHfC38iRNAYVFazYrq/vszhKkhOOy+hBAG4wq+hOAA0AFoCBZiWFCdSPyCIjg8bg008jk95eX8/RmkuXUuFZTQNnzfLOP37zTe8MepeLzz9wIJvphVrTkOyMHx+8Dq2ykklMJ5/s26U3ZAgN7LJlzPhS+i/hGIRjjTFHi8jHAGCMqRGRSJb+KAlKTU1ox9ntzO7pK3V1HE5WUcEUUit11OWi/3vDBk6mO/VUYOVKuk0yMlg0OHlylFJNnU5WI65fz3TRUaPol4pzte9hh9H4BZrBsW4ddwOB2iQNGUIDO39+6qU9K6ETjkHoEBE7PC2wRaQY3DEoKY7DEVqw2O2OTHXwM8/QsHRvA2Kl7FdW0h9+xx0xWtHu3An85S/s/ZOeTkGWL6egZ57JnPg4RWWtrhm/+Q0r54cM8Sr09nYW/tlswYt3rVTd5ubUK4xUQiccg/AnAAsBDBaRuwF8A8DPoyKVklCMG0eDEKwfkTF0YfSFmhqu+gP1NCoupo7etavXtVyhs28ftyuZmT19Zk4nI9wA54fGiZEjaRwXLuQOyuqubbez/U9xcXC3knX9dHfQvwmlud2TxpjLABQBuBXAXAAC4BxjTL+Zqdyfyc9nrcGKFf6b9x04wLnqffXd79zJn8GmTQIMNEfdILz0En/6cg05HDQSr71GB30c81yHDuVO4ZJLuNK35nxnZfH28suBB6RZBcwh1cE0NzPt7N13GcXPy+PrnzEjckNylLgQyj73GBEZDuAqABUAngYb3lWISKJkeitR5uKL6TbfubNrJlF7O1fqubnANdf0vQ4hWMaMRUwKz2pr2fI60PQcK+dz5cooCxMaBQUsFhw3zqvcTziB18VfBpjbTYMwf34I12/vXuDnPwf+8Q+vC62ujsV3P/0pPwxK0hKKQXgIwDsAJgBYA2C152b9rvQDBgxg9s5557Faec8ettuoqmIg97jjqDu3bOnbfBIrMBqoHxzA5/A3mS5iWNN9gsUHsrISWhEWFQFXXEHvV01N1/e2qYlGfs4c4Oijg5yosRH4/e+5CigtpfXJzOQWcvRonvjee2lIlaQklG6nfwLwJxH5mzHme9EQQkQuAPArABMBzDTGqKFJQLKzgbPP5kqyqor1AS++yFTTjRt5jNWj6NpruVINl9JSZhbV1fmfCdHSQlmmTOn1SwmNUB3qbnfCd9j8ylf4fr74oneWhzHU5VdcEeJQpo8+orL3l388aBDbc6xYAXztaxF+BUosCCWGIIb4NQbWMX2Q41MA5wF4uA/nUGJEejoXi48/TiXT3Y9fW8s47O23hzCeshsi7PhqzTvo7pJua2PmzDXXRHjegS+ssuzW1sA5my0tXWf/JihTp9KI7t/PXV5GBt2AIQeSlywJHicpLmbr1VAMwr59dLXt2cP3t6yMQkb9wir+CCXL6D0RWQDgZWPMbutOTw3CiQC+DeA9AP/qrRBWcFq081ZS4HYDjz5KY+CrPURBAY/517+Au+4KP64waRIn0z38MPPrMzK8PnCHg6NMZ85kRs3771O5DRzICZSTJwdv5RAy6elM01mwgFbP1wupr2dQNZzhGHHE6pjRK2pqus6M9kVmJgtIAs0/7ugAnnySF89mo9V3Omkc8vM53cgatafElFC+OqeDAeWnRWQMgFpwQI4dwBIA9xtjPo6eiF0RkWsBXAsAJf5SXpSosn07XUaBKpcHDqT3YOfO3n23jzoKuO8+FlVt3kz3xtixXETW1TGuWVXFzJmMDOqgtWuZgnnzzYHH7obFvHksSNu8mZrU6iLndtNatbcDt97aP1a1Awbw9QZKRerooFHwZwyMYa/upUtpZLsfV1vLPhq/+hV9h0pMCSWG0Aq2rXhQRNLA9NMWY0xYkSMReRuArzDgz4wxL4d6HmPMIwAeAYCysrK+uKmUXrJ/f/Cgr7WY3r+/94u9jAzg2GN5s6ivpzupo6Orq8oqptq3j7rkqqt4n3dGQC/JyKCFWbSIrpDWVioxl4v+l298o2/9vJOJk05i/mqghVhlZeA5BXv3cmfgyxgA3F42NzOV95pr+iyyEh5hba6NMR0A9vfmiYwxUe6DqcSKcFxAkfYCrljBHYIvHVxVxdqE8nLuTAYP5mJ2/nwu9Hsd983IYIvqM87gyZ1ObkH6W3vQE04AXn+dASRfRQ3NzVwpnHyy/3OsWEGfXqAI9pAhdB9dckng4gkl4kTK25o0VFdz1QrQAxDnVjRJiTWFzNdUSAur8jWSE8uMYb8dX5MS9+5lEkxmJnV1YyPdS62tHCCzYwdw/QVVcHz0X7b9dLmYBjV7NoUMxXJlZDDBv79SVAT84AfAAw/QKg8eTCvrdNJn53IB3/9+YFfPnj3Bi9esKHdtrRqEGJMQBkFEzgXwZwDFAF4XkXXGmHmRfI6KCrZnXrvW+903hsWVF1zQ/xZ7faG0lCv0Q4f8v29VVdS3gVpQhIvLRR3RPauprY1Dc3JyqJ9cLubXAzQQY0oNml99F1VvPEVdVVDAD8HSpcDbb7NL3iWXaN+GUJg6FbjzTs7tXLaMxsBu5+7h1FODrwBC6dcNMEYTsewAJVR69Y6LyFBjzIFICWGMWQj2SYoKBw4Ad99NxTFypHe36nLRQGzdyu6ZOqM7NEQ45/zuu+kyLi72Gtn2dhantbQAJ57IwHJJSWRcR3a7dyRxZ11RXt61FKB7WcCIfR/h6L3/wua8URg6M90rS24uPwRvvknf0vnn913IRMUYrs63bOG2afBgZkYFG1Dti+HDgcsu49jP1lYq+VCV94wZtN6BVmCNjdzmBctoUiJOb03wIgDB6hoTAmOYIul09ky3s7pn7t/PnPof/zg+MiYjJSXM9HnySfrtAb6PO3dSzx5xBNPWlyxhLcK11/b9+y3CiugPP+x6Lffv7zpCsrmZzw8AYtyYvOl5tBcMQUNL+pdFbV9it3O7s2gRJ++kYqvPgweBRx7hhbIqr10uRtvPO48Blt5YbLs9/N5F06ZxK2el63bHGG7nr7pK53rGgd6+40lTMFBezoraQMpo6FBW2x6I2J6nfzBqFIvP7r6b33Oru+bXv86up5Zrqbwc+O1vGb/pK3PnchfS3u69z+326rOODv5uJcIMrNmB7OZDaE+nL9pndpTDQQX5ySd9FzDRqK3lm797Ny/G6NG8cKWlXKU/9RTw6quxkyczk3UG9fX8wnXuc9LUxGDP8ccztqPEnN4ahL9HVIoosnNn4OAnwMdEvJ02ldAR4Yp7/Xpg+vSe+f8iNLh1dUxQCURzM/XWnj1dFX5nSkuBb3+bRqaykvpk4EC6AxsbqVPKyrwL14zWOhgRuFxccPodRu9wRGa6T6KxeDEt8dChPb8E6ek0EAsXchcRK444AvjlLxmPKC/nBd+9m0b5yiu5ndR4TlzorcvoxYhKEUWs8Yuh0JembP0Zq9FnoLTOYcMYgzz//J5u6+pqpp2//773GmRkAKedxh1H9+PnzKHL6PXXOT0tLY3G4PDDeeucOeaypwPGoLGRriu/rm6Xq3f+9ESmtRV4773AWT9WCugHH3BrFytGjWJGUl0ddzEOB42WGoK4kvIxhECdiztjjMawestnnwV3JaeleYt7O9c1VVZy2ldDA/WBZVRaWzmKYP16xna6n3/8eN4aG7mzWLiQKe7d3dLVgw5Dc3sasgZ0YOxYPxbLypGNere8GHPoEINnwSrzcnJ4EeNBfr7v/idKXEj5GML48VwxNjb6P6aujoajP6eY9wW7PXjlskXn3ZoxwEMPUfmPGtV1h5GZCYwZw67Szz/v/3w5OTTkV18NnH466xF276Z7urwc2L43C+UTT8VXxpQjK9OPkPv2sQlSr5v8JCjhbI21j5iCfhBDsNuByy/3zovtTmMje3ZdfrkmNfSWyZO9ef/+aGujG6jzLmzXLsYQA+3ihg/3NrALhMPBLMh77wUuugiYNYvupp/+FLjwqXOQNesoBok6n6ilhb2gi4oiM90n0SguZjqtv8k4Fo2NSdOcT4kuvXIZGWMejLQg0WT6dCY2PPooXRTWDrq9nSvMm2+mUlN6x4wZwP/9H/Wrv75n+/dzlkLnoO6WLd6Avj8cDu4kduwIPigeYFB7Xo+SxnRW2P73v0wv3e1p2jtgAHsRnXxyalbEpqVx2/T889xu+cJqITtzZmxlUxKSflMKeMwxdBGvX0/lAjAAOWVK/2hUGU0GDAC+8x3gr39lu/zOLmGnkx6Z0aN7Kuq2ttB3ZX0el5mWxuZsX/kKUx6NYc1BqlfDzp3L6sudO1l00/n11tczqHPddalZf6GETYp/G7qSkcHV7IwZ8ZYk9Zg5k37/p5+mK8ha9YuwYvmii3om8QwZEryLgRXvjdj8epvN/yi2VCQri1H5F15gmpeVdmcML8DNN3MLrSgIbWJaKF9Fd7jtsJXQsDoObN7MFXVxMV0niZgheeSRTC3fuZOppA4HW1/7Kki1jk9Pp+vO3y6tvp5Zk937FylhkJ3NINm553J73NHBTIsxYzRwpnQhlB3CPs8tUMTNDkCn1USYgwc5Neyzz7p2HEhLY8eB009PvDioCI1AKDMQsrPpwn/ySaaidq9jaGmhYfnxjxPvdSYlubmhBWJiSGOjN5anrtv4E4pB2GyMCbinFJGYTUzrL1gdB5qa6H/vrBDb2xnEbWsDzjknfjJGgtNO4+t50VPqOGAAd0WNjVQQN9zAXYeSOhjDWN7rr3OxY7MxG/Ckk/h50Hqg+CEmSAK5iGR6pqb16ZhoUFZWZlavXh3rp40JzzzDJpz+hnE5ncy5v/fe1GjdXV3Niuft26kcpkzp2oJCSQ2MofF/+WUmHwwaxMVORwd72jkcwG23+U+KUiKDiKwxxpR1vz/UEZr+TnqlMeaxeBiDVCbUjgMizKQ866zYyRYtBg3iQDIltfn4Y1agl5Z27VKRlsbW9LW1wP33c6xyZmbcxOy39DWi9OuISKF0IdSOA7m58es4oCjhYgx7Vg0c6L9lUUEBOwesWxdb2RQSSpbRen8PAQixU5ASDqEmfhijSSJK8lBbywy0kiDpJ7m53PnOmhUbuZKKxkbOtWhvp8/t8MMjWksTypmGAJgHoKbb/QLgvxGTRPmSoiJm4ASq/AXYhUE7DijJQns7FzDBMsbS0oK3Qul3tLay4nzp0q61JHl5TNU78cSIpOKFYhBeA5BjjOmxiROR//RZAqUHaWms6tWOA0oqYRVDu1yBu1w3NQETJ8ZGpqSgvR144AH2eulebd7cDPz971wdRiAIF9ThYIy52hiz3M9j3+yzBIpPTj2Vu8Fdu3q2baivZzuIK6/UjgNK8pCdTTdQRYX/Y4xhOrUOTOvEihXAxo30tXV3D2Vns1Xw888HfmNDRD3QCUpmJvDDH3IYzIED7Me2ezcNRHo6Ow4cf3y8pVSU8Jg/n922fXWvtaryJ07kYkgB36zXX2dxhj+XUFoafXErVvT56UIJKq81xgQchhPKMUr4WB0HzjsP+Pxz7hQKCrTjgJK8jBzJyvMHHmA2XUEBF71NTfR+TJzIQWr6+fZQX88inWCR+IICjg8877w+PV0oMYSJATKNAAaXdeRRFMnJSbiOA0osaWqiYkhLY3/vJO/jMWECCypXrWI2UWsrW53MmcNxyzpFsxOhTp4CIjIDOBSDMCGEY4L0rFQUJWz27wfeeINa0xh+4YcNA848k874JF5G5+TQAMyZE29JEpy8PAYKm5sDd7Ssq2Npfx8JpVJ5V+e/ReROz/+tA7DOGLOtz1IoitKVHTtYrut2c9i0NSmovh7429+YcXLFFUltFJQQsNvZxfKZZ/ynHLpc9CdHIBIf9qfJGHMHgAcA1AE4V0SSZpymoiQFbW3s35CV1TXNUITFSGPGsLfJBx/EV04lNpx0EmMI5eU93UIdHRwDO39+RGaCh2wQROQBETovjTEVxpg3jTH3GGOu6asQInKviGwRkfUislBE+tEEE0XpxiefcCfgb5CPzcbBGK++Gp6PWUlOsrOBW29lFeru3TQA1s+qKuDCC4ELLohZYZpFA4BXRORiY0yTiMwDcIcx5oQ+SwG8BeB2Y4xTRO4BcDuA2yJwXkVJPj78MPiM59xcKoWqKu0X3R/IzeVg+IoKTstqbWVHyClTIjotK2SDYIz5uYh8E8B/RKQdQCOAn0RCCGPMkk5/rgTwjUicV1GSEqsMPRDWxKT29tjIpCQGQ4bwFiXCcRnNBXANgCYARQBuNMa8HwWZrgLwRgA5rhWR1SKyuqqqKgpPryhxZsQINjELhDWMOl8zvpXIEU5Q+WcAfmGMORlcwT8rIqeE+s8i8raIfOrjdnanY34GWx6JlAAADjtJREFUwAngKX/nMcY8YowpM8aUFRcXhyG+oiQJJ5zArJFA8YGKCuDYY7V3iRJRwnEZndLp9w0iMh/AAgAhNVAwxpwa6HERuQLAmQDmmmBj3BQllSkpYdfCDz/kJJnuwcL6ehoLnSikRJheN9I2xuz3uJH6jIicDuBWACcZY5ojcU5FSVpEgKuvptJftYoVyjk53DU0NHCu6C23sA+EokSQPk1WMMa0REiOvwDIAPCWJ7N1pTHmugidW1GSj4wM4Prr2c3w/fc5QDsjgzuHo48OPChDUXpJ5Ebt9AFjzGHxlkFREg4RuoxKS+MtidJP0Lp3RVEUBYAaBEVRFMVDQriMFCVROXCALvyNGxnjnTCBPcSGD0/6LtSK0gM1CIriA2M4qGrBAir+gQN5/zvvAG++yYzPb3xDm40qqYUaBEXxwbJlwLPPsiQgLc17f24ui4RffZWZoFoKoKQSur5RlG50dAAvvEC3UGdjYGG3c675yy8DLZFKvFaUBEANgqJ0Y9s2thIKlOqfns6xBRs3xk4uRYk26jJSlG7U1YV2nAhQUxNdWRKGtjZg3ToGUaqr6S876SRgxozgrbqVpEENghJzqqqArVupYwoKgMmTgczMeEvlJT09tOOM6ScFw5WVwO9/z5/5+bxYdXXA448z6v6jH/kf76gkFWoQlJjR0AA88QSwejUnAYrwlp4OnHsuMG9eYqRyHnEEs4ecTv9jCSz5J0yIrWwxp6WFxqChoWvFdFYWjUNtLXDvvcBddwGFhXETU4kMGkNQYkJzM/XK2rUMyI4ZQ/0yejQHPz31FLBwYbylJHl5rDUoL/ffgXrvXqCsDCgqiq1sMWftWu4M/A1lKSjgQJ9ly2IrlxIV1CAoMWHpUo6AHTWqZ+5+RgYNwyuvsBAsEbjoIq7+d+7sOqumuRnYsYOv49vfjp98MeOdd/zPdrYYMgR46y2d75wCqEFQoo7TCSxeDAwd6v8Yh4PpnO9HYwZfL8jMBG6+GbjiCv69ezdv7e3At74F3HZbP4mlVlUFn9mbkUHXko7zTHo0hqBEnbo6uqCtal9/5OcDW7bERqZQyMgATjkFOPlkzqQBWJhmt8dVrNiSk0NFHyjS7nTyTfFVtKEkFbpDUKJOIgSK+4LNRq9JQUE/MwYAgymHDgU+prISOP547eORAugVVKJOfj5vTU2Bj6utZQqqkkAcdxy3Sg0Nvh9vbWVp96kBJ+QqSYIaBCXq2O3A/PmcC+8v7tjRwcdOPDG2silBKCgAbrqJPrPycl4ogA2d9u1jFsB3vsOmT0rSozEEJSbMns3xwNu3cxRwZ9dLSwuwfz8zewYPjp+Mih8mTgTuvBN4912mizmd9AMeeyxw2mlalJZCiEniVLGysjKzevXqeIuhhEhzM/DMM8CKFSzsssjJYSvp2bOTP96Q8jiddBNlZGgQOYkRkTXGmLLu9+sOQYkZ2dnAVVcB55/PBnLt7YwtjB+vuiVpcDj6Sb5t/0QNghJz8vPZE01RlMRCg8qKoigKADUIiqIoigc1CIqiKAoANQiKoiiKBzUIiqIoCoAEMQgicpeIrBeRdSKyRESGx1smRVGU/kZCGAQA9xpjjjTGTAPwGoA74i2QoihKfyMhDIIxpr7TnwMAJG/5tKIoSpKSMIVpInI3gMsB1AGYE+C4awFcCwAl2lBLURQlYsSsl5GIvA3A18ysnxljXu503O0AMo0xvwx2Tu1lpCiKEj5x72VkjAm1YfpTABYBCGoQFEVRlMiREDEEETm8059nA0igQYqKoij9g0SJIfxORMYDcAPYBeC6OMujKIrS70gIg2CMOT/eMiiKovR3EsJlpCiKosQfNQiKoigKADUIiqIoigc1CIqiKAoANQiKoiiKBzUIiqIoCgA1CIqiKIoHNQiKoigKADUIiqIoioeEqFRWoo/TCezYATQ1AVlZwLhxQFpavKVSFCWRUIOQ4hgDLF0KvPQSUFcHiPC+7GzgzDOBr34VsNvjLaWiKImAGoQUxhjgxRdpDIYNA0aP9j7W2go8/TRw4ADw7W8DNnUeKkq/R9VACrNzJ/DqqzQE2dldH8vMBEpLgffeAzZujIt4iqIkGGoQUpj33gPS0wGHn32gzQbk5gJvvhlbuRRFSUzUIKQw69YBhYWBjykq4g7B7Y6NTIqiJC5qEFIYtzu02IAxvCmK0r9Rg5DCjBsH1NYGPqa+Hhg1SjONFEVRg5DSnHYa0NDgf/VvDHDoEDB/fmzlUhQlMVGDkMJMmgSUlQG7dvWMERgD7NkDjB/PYxRFUbQOIYWx24Hvfhd46ing/fd5n8PBqmVjaAiuvJKZSIqiKGoQUpyMDOCqq4Czz2bWUU0NkJcHHHUUMGRIvKVTFCWRUIPQTygsBObOjbcUiqIkMhpDUBRFUQCoQVAURVE8qEFQFEVRAABikrhEVUSqAOyK4lMUATgYxfNHk2SVXeWOPckqe7LKDcRf9tHGmOLudya1QYg2IrLaGJOUWfrJKrvKHXuSVfZklRtIXNnVZaQoiqIAUIOgKIqieFCDEJhH4i1AH0hW2VXu2JOssier3ECCyq4xBEVRFAWA7hAURVEUD2oQFEVRFABqEIIiIneJyHoRWSciS0RkeLxlCgURuVdEtnhkXygiBfGWKVRE5AIR2SgibhFJuNS87ojI6SKyVUS2i8hP4i1PqIjIP0WkUkQ+jbcs4SAio0TkPRHZ5Pmc3BRvmUJBRDJF5CMR+cQj96/jLVN3NIYQBBHJM8bUe36/EcAkY8x1cRYrKCLyVQDvGmOcInIPABhjbouzWCEhIhMBuAE8DODHxpjVcRbJLyJiB7ANwGkAygGsAnCJMWZTXAULARGZDaARwBPGmCnxlidURGQYgGHGmLUikgtgDYBzEv09FxEBMMAY0ygiaQCWA7jJGLMyzqJ9ie4QgmAZAw8DACSFBTXGLDHGOD1/rgQwMp7yhIMxZrMxZmu85QiRmQC2G2N2GGPaATwD4Ow4yxQSxphlAKrjLUe4GGP2G2PWen5vALAZwIj4ShUcQxo9f6Z5bgmlT9QghICI3C0iewBcCuCOeMvTC64C8Ea8hUhRRgDY0+nvciSBckoVRKQUwHQAH8ZXktAQEbuIrANQCeAtY0xCya0GAYCIvC0in/q4nQ0AxpifGWNGAXgKwA3xldZLMLk9x/wMgBOUPWEIRXZFCYSI5ABYAOB/uu3kExZjjMsYMw3csc8UkYRy1emAHADGmFNDPPQpAIsA/DKK4oRMMLlF5AoAZwKYaxIsWBTGe57o7AUwqtPfIz33KVHE44NfAOApY8yL8ZYnXIwxtSLyHoDTASRMUF93CEEQkcM7/Xk2gC3xkiUcROR0ALcCOMsY0xxveVKYVQAOF5ExIpIO4GIAr8RZppTGE5x9FMBmY8wf4y1PqIhIsZXtJyJZYCJCQukTzTIKgogsADAezHrZBeA6Y0zCrwBFZDuADACHPHetTIbsKAAQkXMB/BlAMYBaAOuMMfPiK5V/ROQMAPcDsAP4pzHm7jiLFBIi8jSAk8FWzBUAfmmMeTSuQoWAiJwI4H0AG8DvJQD81BizKH5SBUdEjgTwOPg5sQF4zhhzZ3yl6ooaBEVRFAWAuowURVEUD2oQFEVRFABqEBRFURQPahAURVEUAGoQFEVRFA9qEBRFURQAahCUFEFESkWkxdMnxrqvR1tqEcnytDJvF5GiPj5nlogs9XQ8hYjcKCKbRSSsNiEiUiAi1/dFlhCeo0eraxFJF5FlIqIdCxQAahCU1OJzT58Yqy31XwHMBzAJwCUiMskY0+I5Zl8Enu8qAC8aY1yev68HcJox5tIwz1Pg+d+wEBLqd/hfYJuEL/F0Z30HwEXhPreSmqhBUJIGz1CU0zy//z8R+XOAw2PRlvpSAC975HkIwFgAb4jIzSLyLc8wlHUi8nCnXcRLIrLGMyDlWs95fgdgnOfYez27nc4r+R+LyK88v5d6dj1PgD1wRvl7rs4EaHX9kud1KIoaBCWp+CWAn4nIpWDL4/8JcGxU21J7+haNNcZ8AQCetiD7AMwBsBhcdZ/g2Y244FW6VxljjgFQBuBGESkE8BN4djfGmFtCePrDATxojJkMIDvAc4XCpwBmhHG8ksKo71BJGowxyzyNzX4I4GTLVSMid4HNzmJJEdhnyRdzARwDYBXFRRbY/x6gETjX8/soULkfCPO5d3WashXouYJijHF54im5nmEzSj9GDYKSNIjIVADDAByylJeIDAUnT3Un7LbUIvJ9ANd4/jwDwLmd/zbGdI47tADI9HcqAI8bY27vdv6TAZwK4DhjTLOI/MfPOZzounvvfkxTsOcKkwwArX34fyVFUJeRkhQI5+g+BcYBGj3tvQFgGoB1Pv4l7LbUxpi/etw204wx+7r/3e3YGgB2EfGl0N8B8A0RGeyRfZCIjAaQD6DGYwwmAJjlOb4BQG6n/68AMFhECkUkA5xp4Q9/zxUSHpfVQWNMR6j/o6QuahCUhEdEsgG8COBHxpjNAO6Cd0iRT4PgmSd9A4A3wZm7zxljNkZYtCUATvTx3JsA/BzAEhFZD+AtcGezGIBDRDaDgeSVnuMPAVjhmRh3r0c53wngI8//+u2ZH+C5uuBpdf0BgPEiUi4iV3semgPg9d68eCX10PbXSlIjIo+Cbp0SAK8ZY0IaSSgiXwAoM8Yc7MNzHw3gZmPMZb09R7wRkRcB/MQYsy3esijxR3cISlJjjLnaGOMGs2vyOxem+cIqTAPjDu5Ax4bw3GsBvOcrzTMZ8LjSXlJjoFjoDkFRFEUBoDsERVEUxYMaBEVRFAWAGgRFURTFgxoERVEUBYAaBEVRFMWDGgRFURQFgBoERVEUxYMaBEVRFAUA8P8BBYGhJSJXix0AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SjPlpf2-wHl8" + }, + "source": [ + "## Обучение простейшего одноуровневого персептрона вручную\r\n", + "\r\n", + "Используем возможности tensorflow по вычислению градиента для обучения одноуровневого персептрона.\r\n", + "\r\n", + "Для начала, задаём архитектуру сети, в которой будет 2 входа и один выход. Соответственно, матрица весов $W$ будет иметь размерность $2\\times1$, а вектор сдвига $b$ -- $1$.\r\n", + "\r\n", + "Функция обучение будет такая же, как в прошлом примере, но функция ошибки будет представлять собой логистическую функцию ошибки. Для этого нам нужно получить на выходе сети значение **вероятности** класса 1, т.е. необходимо привести выход сети $z$ к диапазону [0,1] с помощью передаточной функции `sigmoid`: $p=\\sigma(z)$.\r\n", + "Далее, если для примера с номером класса $y_i\\in\\{0,1\\}$ был получен выход сети $p_i$, то ошибка вычисляется как $\\mathcal{L_i}=-(y_i\\log p_i + (1-y_i)log(1-p_i))$. \r\n", + "\r\n", + "В Tensorflow оба эти этапа (применение сигмоиды и взятие логистической функции ошибки) делается одним вызовом `sigmoid_cross_entropy_with_logits`. Поскольку мы делаем обучение по минибатчам, то необходимо усреднить ошибку по всем компонентам минибатча с помощью `reduce_mean`. " + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "kdDxWeCqwHl8" + }, + "source": [ + "W = tf.Variable(tf.random.normal(shape=(2,1)))\n", + "b = tf.Variable(tf.zeros(shape=(1,),dtype=tf.float32))\n", + "\n", + "learning_rate = 0.1\n", + "\n", + "@tf.function\n", + "def train_on_batch(x, y):\n", + " with tf.GradientTape() as tape:\n", + " z = tf.matmul(x, W) + b\n", + " loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=y,logits=z))\n", + " dloss_dw, dloss_db = tape.gradient(loss, [W, b])\n", + " W.assign_sub(learning_rate * dloss_dw)\n", + " b.assign_sub(learning_rate * dloss_db)\n", + " return loss" + ], + "execution_count": 22, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zAAgw0h6KzUd" + }, + "source": [ + "Далее, разбиваем входные данные на минибатчи по 16 элементов, и по-очереди проводим обучение, подстраивая веса $W$ и $b$" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PfyqjVb2wHl8", + "outputId": "308850b8-fe17-4cda-ac27-8bcda210f113" + }, + "source": [ + "# Create a tf.data.Dataset object for easy batched iteration\n", + "dataset = tf.data.Dataset.from_tensor_slices((train_x, train_labels.astype(np.float32)))\n", + "dataset = dataset.batch(16)\n", + "\n", + "for epoch in range(15):\n", + " for step, (x, y) in enumerate(dataset):\n", + " loss = train_on_batch(tf.reshape(x,(-1,2)), tf.reshape(y,(-1,1)))\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 25, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 0.5500\n", + "Epoch 1: last batch loss = 0.5502\n", + "Epoch 2: last batch loss = 0.5504\n", + "Epoch 3: last batch loss = 0.5506\n", + "Epoch 4: last batch loss = 0.5508\n", + "Epoch 5: last batch loss = 0.5510\n", + "Epoch 6: last batch loss = 0.5513\n", + "Epoch 7: last batch loss = 0.5515\n", + "Epoch 8: last batch loss = 0.5516\n", + "Epoch 9: last batch loss = 0.5518\n", + "Epoch 10: last batch loss = 0.5520\n", + "Epoch 11: last batch loss = 0.5522\n", + "Epoch 12: last batch loss = 0.5524\n", + "Epoch 13: last batch loss = 0.5525\n", + "Epoch 14: last batch loss = 0.5527\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s4_Atvn5K4K9" + }, + "source": [ + "Для демонстрации того, как сработало обучение, построим граничную прямую $W\\times x + b = 0.5$" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 283 + }, + "id": "PgRTHttLwHl9", + "outputId": "e4407e1b-edf5-48e5-fdc2-da28120a3c6b" + }, + "source": [ + "plot_dataset(train_x,train_labels,W.numpy(),b.numpy())" + ], + "execution_count": 26, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd3hU1dbG3z2T3gtJgJCQQhFpAlFERJqCYPeq2CufBa9YsNBROna59nLtF70qliuo9A7SkY4ISYA0SkhPJpPZ3x8v46RMz7Qk+/c884hzzpyzZjKz115dSCmhUCgUCoXG2wIoFAqFwjdQCkGhUCgUAJRCUCgUCsU5lEJQKBQKBQClEBQKhUJxDqUQFAqFQgEA8PO2AI2hVatWMiUlxdtiKDzIieITyCvNw/lx5yPYP9jb4igUTZJt27adklLG1X++SSuElJQUbN261dtiKDxIYUUh0uanITUpFT/f/rO3xVEomiRCiCxzzzdphaBoeUQHR2N8//EYv3w81matxYD2A7wtUtPm1Clg40bgyBFAqwW6dwcyMoDQUG9L5hxSAllZwOrVwMGDgEYD9OwJDBgAtG7tbel8HtGUK5UzMjKkshBaHuXV5egwvwPSotOw9r61EEJ4W6Smh8EA/Pgj8NNPgBBAWBifKysDAgKAhx4Cevf2tpSOUVMDfPEFsHIl4O8PREVRQRQW8r3dfDMwYgTfbwtHCLFNSplR/3kVVFa4jaoq4PffgU8+AT7+mJu2srLGXzfEPwTTBk7D+mPrsejPRY2/YEtk0SJg4UIgMRFITgZiYoBWrYD27YHISGD+fGD/fm9L6Rg//AAsX873k5hIKycsDEhKAtq2BRYsANav97aUPo2yEBRuYdcu4L33gPJyICiIm7KKCm7c7rwTuOyyxm3Uqmuqcf7b5yPILwg7H9oJrUbrOuGbOyUlwJNPAgkJ/IOYo7AQiI4Gpk1rGjvqkhLgiSeANm0APwue8LIyQK8HXn6Z7rEWjM9bCEKIJCHESiHEPiHEXiHE496WSeEc+/YBr73GDVpKCl23CQn8d6tWwEcfNX6j5q/1x8zBM7GnYA8W7FngCrFbDjt20L1iSRkAdLdkZgLHj3tMrEbxxx98T5aUAcAv5NmzwOHDnpOrieEzCgGAHsA4KeX5AC4G8KgQ4nwvy6RwECmBL7/kemIuLhkYyE3cf/4D6HSNu9fNXW9Gr9a9MGXlFOhqGnmxlkRurnVlANAq0GhoKTQFzpyxb9cvBFBc7H55mig+oxCklLlSyu3n/l0CYD+ARO9KpXCUzEwgJ4duaEsEB9N9tGdP4+6lERrMGToHmWcz8f629xt3sZZEUBB30/ZgbcftanJygG++oXn51lsMQFVV2ffakBD73pOU3JUozOKTaadCiBQAvQD8bubYgwAeBIDk5GSPyqWwzalT3ITZcjsLAeTlNf5+w9KHYVDKIMxYMwP3XnAvwgLCGn/R5k7XrsD333NxtPSHqq6mMvBE4Wd1NfD558CaNbyncXHfupVB4bFjgY4drV/j/HPOBFvvyd8f6NDBtfI3I3zGQjAihAgD8B2AJ6SUDWw7KeX7UsoMKWVGXFyDQjuFl9Fq+Zu0hV4P/PUXMHs28PjjwHPPMQPy9GnH7ieEwJyhc1BQVoDXN73unNAtjfR0ZuKcOmX+uJTAiRPAkCFcnN3NF18Aq1ZRpnbtmPEUF8eMJz8/4MUXbccy2rQBevWi3OaQkte4/HLPvKcmik8pBCGEP6gMvpRSLvS2PArHMW4orVnv5eXMQlq1ilZCSIgpLX78eGD7dsfueXG7i3H9edfjpQ0v4VS5hUVOYUIIYMwYau9jx6idjVRUAEePAp06Addf735ZcnJoGaSkMGZRn8hIPv/TT7avdd99VAxHj5rym6VkzODIEeCCCzzznpowPqMQBKuLPgKwX0r5qrflUThHTAxw0UX8nZvDYGDdkFYLdOvG8wMDGYBOSmKm45tvMhbhCDMHz0SprhRz181t9HtoEbRuDUydCgwcSK187BiQnU1tfcstwFNPMdbgbjZs4IJvThkYiY+n+6ioiLL+9BPw/vvMXti/n18qAIiIACZMAO64g0ouO5uPoCAW2v3znyy6U1jEZ+oQhBCXAlgLYDeAc39hTJRSLrb0GlWH4JsUFQFz5gAFBdywGRNaamr4+927Fxg5EggPN//6vDx2G3jkEcfue9+P92HB7gX487E/kRSZ1Lg30ZIoK2OWjkbDxddWBpIref117t5jYqyfl5UFdOnCnGatlrsIvZ5B53btgMceq9uawmAASktNVdhNoZbCg/h8HYKUcp2UUkgpe0gpLzj3sKgMFL5LZCQwcSIwaBCVwrFjdN+eOMG1pm9fy8oAoPt461bHq5qfH/g8JCReWP1Co+RvcRjNs8REzyoDgLv32i4rc0gJHDgAbN7MOENSEhVX27ZAaipTY+fOrZsiq9HQYggPV8rAAXxGISiaFxERwN13cwM4fjzw7LPAq69ykxcba/21xnRyRxVC+6j2GJMxBh/v/BgHTh1wTnCFZ+nTh24qa+TkMA7QpYt511J8PI+vWOEeGVsQSiEo3EpoKDMGO3VisVp4uO3Ucilp8Tvjwp44YCJC/EMwecVk5wRWeJaePbl7KCoyf1xK+hhTU60XniUkAMuWMbVU4TRKISg8yqWX0rVrjdOnmVYeEeH49eNC4/B0v6fx3f7vsOXEFueEVHiOgADmHZeV0RKonZ5WWsr4QlQUdxTWCAzkTkNVITcKpRAUHqVHD7qMLKXAV1ezT9lVVzl/j6f6PYVWIa0wYfkE5y+i8Bzp6Wyil5FBpWDMeAKYSnrxxbaLW6Tkw5OV1c0Q9ekpPEpgIDMa583jbz4+3tRJoaCAm7w77jAVnjpDeGA4Jg+YjCd+ewLLjizD5WmXu+4NKNxDYiLw4IPAbbexAZ2fH78cWi13CV99Zb0fSnExs42cMSsVf6MsBIXHSUwEpk8HbryRXoGsLGYgXXABMHkyMGxY4+/xcMbDSI5Mxvhl4+ErqdUKOwgPZxZRmzammEHfvvx3RYX510hJk/Oqq1RGUSNRFoLCK0RFAVdfzd9wVRWzHV3Zoj7QLxDTB03HvT/ei+/2f4ebzr/JdRdvSRiLR5YtY+5wQAAX6P792cvcE0RFsbDszTdpAcTEmBb+8nJ2bx0wgBWRikbhM4VpzqAK0xTWqDHUoMe7PaA36LF3zF74adT+xyHKyrgI79vH4q6wMCoIYxHb6NH073uKAweA777jPAOtlqloERHcVQwZ0uKH3jiCpcI09QtRNFu0Gi1mD5mN67++Hp/s/ASje4/2tkhNBynZHuLgQfYZqu2KCQuj++add9hrpHNnz8h03nmseCwoYJpqQADjBiqQ7DJUDEHRrLm287W4uN3FeH7V86iotuCDVjQkK4sdCJOSzPvlg4Pp7//xR8/KJQRrDjp1oqJyVBlIyeZ3q1ezqVbtXkgKZSEoPI+Upnb71nqauQIhBOYOnYtBnw7CW1vewtOXPO3eGzYXNm7kH8hakDY2lgvq6dO2y899gWPHOL81M7Pu3IRWrZje2rWrV8XzBZRCUHiMwkJg7VpgyRK6p7Va4JJLgKFD2freXQxMGYgrO1yJ2WtnY3Tv0YgKinLfzZoLBQW0AqxhHLNZUuL7CuHECWDWLH7p2revq+iKi4GXXwbGjWML3haMchkpPEJ2NjBlCgd1hYXxN9m6NbBpE2uS1q517/1nD5mNwspCvLzhZffeqLkQHm67DYSxx0hTGEn55ZdUAnFxDa0eY+bShx/abrTXzFEKQeF2KiqAV17hZrJ9e1OPIj8/Nqxs04aW/F9/uU+GXm164dZut+K1Ta8hr9QFszubOxddBFRWWj+nuJh/vNptp32R3FxmSsXHWz4nPJwFcfv3e04uH0QpBIXb2baNa4ellveBgVQSv/3mXjlmDJ4BXY0OM9fMdO4Cej1THvfsYdC1Cads26RLF2rrggLzx2tqWAx2zTW+XwyWm8vdiC05tVr+XVswKoagcDtr1tjuKGCcgVBZ6b5BXR1iOmB0r9F4b9t7eKrfU0iLTrPvhQYDC7MWLaK/XAg+17o18I9/sAdPc0OrZdO5efO4SMbHM6YgJYPIxcXM//dkHYK7ac4K3k6UhaBwO8XFtt3MxpoiW62xG8uUgVPgr/HH1JVT7XuBwQB88gnw+ed8E8YBLcnJFHb+fEbJmyMJCQzw/OMfrAjOyuKjfXvgmWeAUaN83zoA2CvFGO+whsHANtstGGUhKNxObCyDyiEhls/R62nV20psaSxtw9vi8b6PY976eXjmkmfQs3VP6y/YtQtYtYoLRe0cWSFo9gQFAQsWAN2705/e3IiMZI+RkSMZDPLzaxpB5NokJDB76M8/Lcc7ior4RT3vPM/K5mMoC0HhdgYNoqfFGvn5bI/jiaLTZ/s/i8igSExaMcn2yb/8wkXRUsFEQACPrV7tWiF9DY2G046amjIwcscd/HLl59d1DUnJfOiiInZbbeHtL5RCULidnj25McvPN3/8+HEmgSxZAtx/P/Dcc9yU25qs6CzRwdEY3388Fv25CGuzrOS7VlUBhw7ZHgAfGwvs2OFaIRWupXVrYNIkuruys1mcZnSBhYUBEyZ4rgWHD6Oa2yk8wsmTrP3Jz2fzytBQrre7dvE32aOHaWRuaSnjlm3a0FVtaz12hvLqcnSY3wFp0WlYe99aCHO+8IoKYMwY21VzlZXM2X/lFdcLqnA9J05QKUhJd1JaWtOIhbgQS83tlIWg8AhxccDzzwP/93/sh1ZUxIxGvZ4u6m7daK0LwZTwlBQqhX/9yz2tZkL8QzBt4DSsP7Yei/5cZP6koCAKW1Zm/WJFRRRY0TRITAT69WOZfHp6i1MG1lAKQeExgoMZJ5g2jQt9aqppxro52rZlH7I//3SPPPf3uh8dYjpgwvIJqDHUNDxBCGDECMu5+AB3meXlwOVqKpui6aMUgsIr5OfTarc2FRFgzPb33xtxIykBnc5sjrm/1h8zB8/EnoI9WLBngfnX9+/PFssnTjS8hsFAf1dGhsP+5+pqhh2+/x744Qdg9+4W3zVB4QOotFOFV6ioMLmIrBEYyI4CDnPyJDN/li2jj9/fn+lOgwfT9DjHzV1vxrz18zBl5RTc0vUWBGgD6l4nJISBjHff5YAWjYZayqhkLr0UuOsuh9q2bt8O/PvfjJX4+fEyNTWMrTz8cIvPfFR4EaUQFF4hNJSLYO0uxOaorHSikeahQwzw6vWssA0M5JZ85Uo+Hn0U6NULAKARGswZOgdXfnkl3t/2Pv550T8bXi8qiqlP2dnc1hu7e2ZkWO+PY4adO4E33uDL6k+gLC4GXnyRCS8dOzr4nhUKF6BcRgqvEBfH5I7CQsvnGOcmONQd4exZ4LXXqHGSkkx58/7+dP3ExABvvcX+NucYlj4Mg1IGYcaaGSjVlZq/rhDMNrr+eloEI0c6rAz0ehY9x8VRvPpERDCg/umnqouCwjsohaDwCkIA111HhaDTmT/nxAm6T9LsbDkEANiwgfmsliLVxnLpVatqySIwZ+gcFJQV4PVNrztwM8c4cIAJSWFhls+JimJdRmam28RQKCyiFILCa/ToAdx7Lzfrx49zHdfrqSSOHuWGfswYB7MCV62y7WNKSOB5tbbhF7e7GNefdz1e2vASTpWfcuLd2CYvz/bO3zhzxlIRn0LhTpRCUHiVoUOBGTMY7y0tZSw4NpZu/okTbXdJbUBJCYO+1vD3p/apqZtqOnPwTJTqSjF33VwHb2ofGo19riAp3T9aVKEwh08FlYUQ/wZwNYACKWXLnmXXgmjXDrjzTj7qY1xA7bYSYmJMWUWWqKqi66he35qu8V1xd8+78ebmN/F438eRFJlk503tIyWF78NaIF1KPpKTXXprhcIufG0f8gmAK70thMK7VFdzNsKMGcADD7C6+Y032O/I5g576FDgzBnr5xQUAFdcYXZVfn7g85CQeGH1C86/AQukpjIufcqKRyovj41TfX0ImaJ54lMKQUq5BoCNX7OiOVNRAbz+OscMnDpF66FNG+DgQc5q+fxzG60sLrqI1W6nT5s/XlRE6+Gyy8webh/VHmMyxuDjnR/jwKkDjX9DtRACGD2a8ufl1X0fBgOD6EFBTGJSKLyBTykEheLzz2kJpKayjZBGQ89OfDx318uWAUuXWrlAWBjw9NN8UWYmk/urqxmgyMqiu+jpp60GnicOmIgQ/xBMXjHZ5e8vKQmYPBno0IFlDcbHsWO0DCZPdjibVaFwGT7X7VQIkQLgZ0sxBCHEgwAeBIDk5OQ+WS18Bmpz4uRJ4NlnuWhaCqpWVLB10KuvWg8ToLQU2LIFWL6caUsREaxS7tvXdr8MAC+segHPr34em0dvxoWJFzr3hmyQn09LQQj2W3O4AE+hcBJL3U6bnEKojWp/3bxYtgz48kvb3aazsqg4unRxnywlVSVIm5+Gngk9sezuZe67kULhBVT7a4XPU1xs38Q0Idw3PMdIeGA4Jg+YjOVHl2PZEaUQFC0Dn1IIQogFADYC6CyEOC6EeMDbMik8R2Qk3f32YK71g6t5OONhJEcmY/yy8fA1S1qhcAc+pRCklLdJKdtIKf2llO2klB95WyaF5+jZk7t/a1lEFRVUBunp7pcn0C8Q0wdNx7bcbfhu/3fuv6FC4WV8SiEomheFhWxJYasswEirVhw/YJxuaCSk/BQ6Hfwf+mx6C51Wv487O2+Bv6HKPULX484ed+L8uPMxecVk6A1qYIGieeNTlcqK5sGBA8BPPwH79zP7s6aG7Zyvuw7o2uYMkJPDE9u2bTAw+c47GUvYuROICq/BJbnfoeNfv6CySqBchmBAkgEdtmwADoQBjz3m9sHoWo0Ws4bMwg1f34BPd36KB3orL6ai+eJzWUaOoLKMfI/Vq4GPPmKWZ2ysqVWD7sRJdP7jG1wRvRWJ7c5VCBsMQJ8+wKhR7Al9Dr0e2LMHOPLiN2iz5SecCU9BfBst0tKoP4QAtUZxMTBliu20pEYipUS/j/rhRMkJHPrnIQT7B7v1fgqFu1FZRgq3c+IE+/0nJtL9Y+wMEVaWj+t2z8D5uh3YmtcOZ8OTWGyQlATs2sUeFbXae/r5ARckn8GNQb+g360puOpaLS680KRgAFDj+PsDCxe6/X0JITD38rk4Xnwcb2952+33Uyi8hVIICpexahVdRMaZNAAAKdFr58fQ1uhQHpUIjZ8GR4+eO6bRUHvodMDHH9cNHBgHKddrQFeHuDjgjz8st6lwIYNSBmF4+nDMXjcbRZVFbr+fQuENlEJQuIyNG+t4fgAA4SU5iD11AGWhCQCYIXT8eL0XJiQw8GCMLQDs5RAUZP2GGg0f9katG8nsobNxpuIMXt7wskfup1B4GqUQFC5Dp2tYWBZ1NpP/OOfrEcI0S/lvjH6g2mPCAgMbzCswi5T2VbPZorycaVFVlrOXerfpjVFdR+HVTa8iv9RHJticOcPPLSfHRtc/hcI2KstI4TJat+Z8mtqtgoQ0wE9fiZhThxCgK0WV9EdgeGsItAJQr/107QXtgguAlSut37CyEggOptvJWQ4eBH75hbEMIeiiGjCA7bHbtGlw+ozBM/Dtvm8xc81M/Gvkv5y/b2P56y/ghx8Yfddq+dnFxQFXX83cXTVhR+EE6lujcBnDh9f13mhqqpGUvQ5tc7Yi7tR+hJfmIvJMJroXrQdWrKD2MCJEXX9T165MKSosNH8zKTl7c/hw2xPSLLF0KTBrFnDoEAPcycl0X61ZA0ydSjdWPTrGdsTo3qPx3rb3cKTwiHP3bSw7dwIzZ1IpJCWxR3hyMtOz3n8f+OIL+0azKRT1UApB4RBSMpto9262qa7dU6hPH65NOTk8sefOTxF3ch/KQ+Kg9wtCGUJRExKOkLZR3N2vW8fS4+JiphB16mS6mJ8f6wwqK3nB2u6jigoOXe7WDbjSyXlKf/7JhbNdO/abNu6o/fz4XEQEp/IUFzd46dSBU6HVaDFt1TTn7t0YioqAt99mGldtuQEgPJx9w5cuBbZt87xsiiaPchkp7ObwYeCrr7gxFcI0EH7QIOCGGziV8umnOdymeHc24g6tQ15UKooiwpCStxGBIQJtkwOYOBQaysVt714qg6efbujmSE0Fpk1jldvmzaahxCEhwC23AMOGOW8d/Porr2Pu9VLy+dxcRsqHD69zuG14Wzze93G8uP5FPHPJM+iR0MM5GZzh999pCVhq5qTR0LJatIga2u7ZowqFE4VpQohQAJVSSjsifu5FFaZ5jj17OIMgNLRWcRjYjC4nh2v3M8/QpW8wALkvfY7in9fgdGAigoOB1KBcRGXugKZaZ1r4q6t58scfc9KZNYqK6I/SaunbtzoMwQZVVcDDDzccvGA0fw4fBs6e5cIbFARMnAgMGQJERf19amFFIdLmp+HS5Evxv9v+57wstqip4aAIvZ4Tg157jWm21mY6SMn+H2+8QUtHoaiHpcI0mxaCEEID4FYAdwC4EEAVgEAhxCkAiwC8J6U87GJ5FXZy9iywaRPdyno9J3ENGMCuEK7aHFZWAu+8Q0UQFlb3mL8/C4WPHGFs9sYbz5UX6DKReGE4yrRcm3bktIFBG482/vlICi9ESLDEaUM0KoqrcehgCtJj2LDOosyRkXYNtrELnc5k3hiRkh/i0aPUapGR/ECrq4Gff2ZcYfz4v4cdRwdH47n+z2HC8glYl70OlyZf6hrZjOj1vOeiRYyjGGXNz6dLy9pnYXxvetV7SeEY9riMVgJYBmACgD1SSgMACCFiAAwGME8I8b2U8gv3iakwx/r13FzX1HB90Gg4ZOa33zhr/rbbrNd12cuuXUBZWcMag9q0aQMsWQJcddW5wjR/f5zIrsHWv3g8OBgQWi0yq9pi+59todMBMdES8VXH8PMvWpQs54Z9zBizyT2uJTiYLiGdzuQyysykMoiOrmv+REYyYFtQQF/YjBl/f6hj+47F/N/nY/yy8Vh731oIV2lgvZ4aePNmBrmTk03PHzxIn92wYdTOubm0GKSkxm7blv/292+ovRX2U1PDqXtaLc3iFuJ6s0chXC6lbNClXkp5BsB3AL4TQjTCflc4wx9/AO+9x99/7fqtiAh+l3/7jQvzzTc3/l67d3MNtUZgINfX3FwgJQU4GpuBYzu+RGib6DplAsXF/J0JAaC0BDIxHgmpIeietwlxy1Zj9+KzCB8ZhbCrB7Eftq3iNGfw8+M4zSVLqIUMBmYahYXV/eHrdEBaGv8dH89RbYcO/T2qLcQ/BFMHTsUjix7B4j8X46pOV7lGvuXLqQzS0urK4+fHdNxly/gw1moYP+CsLH4xEhKA++5zPr7SkikpYUOu335jxoTBQIU8ciSQkeGaHZYPYzPLSEpZLYQ4TwgxVAhRZ8shhLjSeI67BFQ0RErg66+5ITS3Xmq1dOP88gtd741Fr7cvrd04y0BK4KsjF0Eb7I8gaUpD0uvpDg8KAoICJfzPnsTB6H4YsnIyLtz2LtoYTsBQXYPMDSeAd99l47rc3Ma/AXMMGcJd9NmzXAQqK+suoCUl1K61J977+wP1YlYP9HoA6dHpmLB8AgzSBYVhej2weDHNJHO70pgYPrKyaMFERVGRhYXx31LSf6diB45z+jQtwG+/ZcKBMRW5qAh4803gww/tK5Zswtj8mQshxgL4EcBjAPYIIa6rdXi2uwRTWCY7m4Fca25kPz8uztu3N/5+qam2R1Yaq49bteLg+IO5Edjd7yGElhUguJwuDWMGZ5CsQNuqo9gTciFOHzgJv+pKFEWloDI4BgFRIfjrTAyqWrfnTV96iSaFq4mL42Dmmhp+oDodH+XlVBJhYUC/fnV3hP7+9J3Vwl/rj5lDZmJ3wW4s2L2g8XLl5FAZhYRYPkdKppgWFtLkqqjgZ3T2LE25wYOB//3P/vFzCn6mb7/NxT8lxWQSC0FFm5ZGH+2SJV4V093YU4fwfwD6SCmvBzAIwBQhxOPnjrUMx5qPUVRkSvu0RkAAXd+NpW9f3sva+pKXB1x8MTemJSVcR/MS+2DdgIkoCW+DyKJshBYeQ9uabATKSiyLvx0rQq5GSYUW5aGm4ITREikvBxftwkK6T9xBWhowbx5wzz10v2g0jCFcfDEwcGBDP1llZV2L4Ry3dL0FF7S+AFNWToGuRtc4mfR663/YwkJ+OImJHDKRlkaZ27alAhs6lC6j0lKm9Crs4+hRxmbOJQ00QAhabYsXN2tFa08MQSOlLAUAKWWmEGIQgG+FEO2hFIJXsDfjUq+3vtG0l6goZg99/TUt6Pr3P3WKyue6c7ZjYKCpUPZ0bCesGzARoaX5OLytCHmn/VEanYQa4Yfw/ZvgF2nez/33mhgbS3/ukCGNfyPmCAkBrr2WebW5ubyfOaSkNXHxxQ0OaYQGc4bOwYgvR+CDbR/g0YsedV6eqCiadgaDeT+d0VTT66mczjvP/HWEcM1uoKWwcyd3MdaUcXAwfZ5ZWUzna4bYYyHkCyEuMP7POeVwNYBWALq7SzCFZVJTuSjrrGxGpeSa0q2ba+551VXA7bcz69HYS+3YMf47PByYMIEbU4BZkVFRdT09ZWEJ0Kd1QpYmFTXCjwJWVyM9sm7rar2ev8u/E2RCQjzS3hr/+AdNm4qKhsek5CJw8cXciZthePpwDGw/EDPWzECprhEurpgYoEcPLjzmMC5YUvKDtoQx00hhH6Wl9n1eQlhtgNjUsUch3A0gr/YTUkq9lPJuAJe5RSqFVYKCWDx74oTlljX5+czrT0lxzT2FAEaMAF5/HXjgAW7Yr7qKimDWrLprk1bLTXd+ft1+da1bM7ah0wGlZQIJQcVIDqjz1UJpKeX+OzNJp3NPplF9OncGHn2UxW9ZWfTNl5XRasjKYtXvffdZfLkQAnOGzkF+WT7e2PRG42S54Qa+79q9noxERvL59u0tp5UavxTp6Y2ToyURF2d9hwWYdlnNOGBv02Ukpazfvb72sfWuFUdhL1dfzd35jh3cmRvXhqoq+vNjYoBHHnF9+nRYGAvfbHHZZYzVLlvGNSwmhhuwbt3YwigyEujXqxJhOQUoiWwHg4FrcFRUPWv85MkGrSPOnqWbSqvlhr3OQHCVzBwAACAASURBVJ7GcOGF9Mtv3Ahs2cIFIiODvTk6dLD5YfZL6ofrOl+HFze8iIczHkZsiAX3ky1SUlj2PX8+raOwML7ZsjKaUP36WX/TBQWUNynJufu3RC68EPjvfy276gAG79q1s26ZNXHUTOUmTHU1sHYt41xnznC98vdn5+bLL3ddYa+zSMmitsWL2UsO4OaqSxcqs8qjubhxx2SUBraC3j8Y7dvz2N/ZnxUVXBBnzgRat0ZuLjs+b9liWpuN73fkSNu1Ep5gb8FedH+nO8b1G4eXhr3UuItVVFDjb91K5dS+PVtbBwcDc+ZQK7ZtazKnampo0YSGst2G0YensI9PPmHL9ZSUhsq/qoom+bhxrI9p4lhqXaEUQjPAYGDyicFAJeCL9UiVlaYgt7FH3fHjQPnqzYj79h2ExwTAPzGeO+GaGu5ydTq6cTIykJ3NNbCmhuucMRu0qorxjPR09sdzRRC9sdz7w734as9XODz2MNpFuGk3WVQE/Pgj21sYf8NS0nq44Qbm/yocQ6cD/v1vWohBQdy9GH9cQgB3383ss2ZAoxWCYF3+HQDSpJTThRDJAFpLKd2UE2gbpRCaCZmZzO/+/XcuakIwgDtsGNC+PQwGthGqqLCcBJSZSc/Sbbd5UnDzZJ7NROc3O+PuHnfjg2s/cO/NSktpFUhJTelts7CpIyXTT1etYiqqnx/Quzcts2akZF2hEN4BYAAwRErZRQgRDWCJlPJC14pqP0ohNDOqqrjqBwfX8ZHv389yAWsB8upqhhtef903rIQnfn0C/9r8L+wdsxfntbKQGqpQeAlLCsGRATl9pZSPAqgEACllIQAfdE4omiyBgYwq1wuY7t1re2yyvz9dUllZbpTPASYOmIgQ/xBMWTnF26IoFHbjiEKoFkJoAUgAEELEgRaDQuFWdDr7eooJ4TutZuJD4zGu3zh8u+9bbDmxxdviKBR24YhCmA/gewDxQohZANZB9TJSeIDERPtTxC3FGLzBU/2eQquQVpi4YqK3RVEo7MKuEZrnAsprAGwDMBRsWXG9lHK/G2VTKAAwpvf554wTWComPX2aJQRun6XgABGBEZg0YBKe/O1JLDuyDJenXW77RUVFnHi0di3bVMTHsz9Rz56+mT6maFY4ElTeLaX0qVYVKqjccli8GFiwwHwvpZISZgZOmuR7xbmV+kp0frMz4kPjsXn0ZutDdPbtYzFaVZWpkq+8nBV7iYnAU0/5lgmkaLK4Iqi8XQjh1owiIcSVQoiDQojDQojx7ryXwrfR602dnQG2zRg1ytRJ4uRJU1+lqioW9vqaMgCAIL8gvDDoBWzN2YqF+xdaPjEnh/OSQ0NZgBYezlz4mBimV506xePNuNOmwvs4YiEcANABQBaAMtBtJKWUPVwiCAPWhwBcAeA4gC0AbpNS7rP0GmUhND9OnwZWrODQMJ2OsYGuXakQzj+fHpXNm00p4t2705visvYVbqDGUIMe7/ZAjaEGe8bsgZ/GjKf200/pJrLWFiEzE3j8caBXL7fJqmgZWLIQ7IohnGO47VMaxUUADkspjwCAEOIrANcBsKgQFM2L7GzgxRfpJWndmi5zKbn4z5vHAtzrr2e9WlNCq9Fi1pBZuOHrG/Dpzk/xQO8H6p6g01EZWOrFbyQsjK0VlEJQuAm7XUZSyixzDxfKkgjgWK3/P37uuToIIR4UQmwVQmw9aalFsKLJUVXFojKNhnECY/xUCDaiTE4Gvv+evZGaItd1vg59E/vi+dXPo6K6Xott4+xeW8UWISGW22IrFC7AboUghJhq7uFO4cwhpXxfSpkhpcyIi4uz/QJFk2DXLgaGY2LMH/fzY83azz9bbvntywghMPfyuThefBxvb3m77kHjRCGDjbIenc5yy2uFwgU4ElQuq/WoATACQIoLZTkBoHa/3nbnnlM0MUpKGAd4+20+Vqww39q/Nhs2MJ5qjehotpk5e9Z1snqSQSmDMDx9OGavm42iyiLTgeBguoFs7f6LithX3FfQ6TglKTu7waxpRdPE7hiClPKV2v8vhHgZwG8ulGULgI5CiFRQEdwK4HYXXl/hAdasAT77jBXDxp5CW7YA//kPcNddlptFlpXZTrMXgi6lpjywavbQ2ejzfh+8vOFlzBgyw3TgyiuBbdu4yJr7IM6epXXQp4/nhLVERQWwdClHm1ZUmAZ8DxjAQR2uSI09c4a7hD/+4JepUyde38LEOoVrcCSoXJ8QcBfvEqSUeiHEP0ElowXwbymlmhLehNi0CfjgAybK1M/6qaoCPvyQz5sZS4z4eG42rQ2jMralCA93ncyepneb3hjVdRRe3fQq/nnRP5EQdm5mQadOwD33UJsGBvID0Wr5wRUUUEk8+6z3O/eVlwOvvsoBF23aMMADME94zRpg+3aO0bMVILfGqlX8HAB2bxWCCujXX9nS9pZbLA+xUTQKR2IIu4UQf5x77AVwEEAjZwXWRUq5WErZSUqZLqWc5cprK9yLXs/CsdatzaeABgby2IIFPLc+l13GmQnW4gMFBRxsZcu1ZAkpOeNk3z66nryV0j9j8AxU6aswc83MugeGDAGmTAEuuICCZmXRMhg5Epgxg8O0vc3ChfzwUlPrjjb18+OEtupq4J13nA/0bN3KmQQJCcwkiIzkLsE4qWzRIuCnn1zzXhQNcMRCuLrWv/UA8qWUZn7aipbIwYMsJGvf3vI5oaF0kx84wFGatUlIoNJYt44bz4SEupZAaSkVyciRzsm3Zw/w7bdcY40DekJDeb1hw+xrnucqOsZ2xOjeo/HetvfwZL8nkRadZjqYns7Zp6NHc3ENDPSscNYoKwNWr7butomL44d85IjjlYIGA8dYxsWZ31VotVQSP//MkYAqwO5yHLG7xtRKNz1xzsUzz22SKZoUp0/bvyk8c8b0b72ea8C4cVz0ja18Fi3i2nPyJGOWpaXAE09wPXCUDRtY31BYyNcnJfG/ISG0WN5/38NdUvV6TO3yMLRCi2nLJ5s/x9+fAvqKMgBYEFJTY7mhFGAK9OxzonzoyBH+wa35BI19zptq/rGP44iFcAWA5+o9N8LMcy2C8nIugkLQ3dvS+47ZSqE3d66UwBdfMAspOZnPp6ezS8ORI2xTsXs3R2MOHOjcMLAzZ+iBaNu2rocDYHJPaioVRq9e5mMbLqWyEli2DPjtN7QtK8PjUR3x4p4FeKa0J3pc/xDzan0Zvb7hrGFzGGMfjlJUZN/1/fzoP1S4HJs/YyHEIwDGAEgTQvxR61A4gA3uEsxXOXOGjdZWrzaljQcGMtY1bJhvDHr3BkbvgMFgOd5n/LyM5x49yvhhSorpNRoNFWx8vOmcoCDnJ0Nu2MD71lcGRoRgUsyiRUDfvjbWo5oa+ruWL6fZEhDAF11yiSm4aomKCgZjDx6kTyw2Fs8hDu/hT0za/yb+t7uAwVhfHtMYHc0P0zjm1BLV1c4Fle3dVdVOYVO4FHtcRv8BcA2An8791/joI6W8w42y+RwFBYztrVjBBSspiY+ICMbajG0XWiIJCdxl5+ZaPic3l32HEs4l1qxaxTXAWsJIfDzwyy+2a7YssX277Y13ZCRw/LiNWomKCjaXe/FFKgWtlimiP/7Igc8bbOyNvv0WOHyYJsm5XUM0gvEc+uPnkONYpztM35UvV90lJzOwa60QxNij3Jn2Gunp/FytRfulNDW4UrgcmwpBSlkkpcyUUt4GoBhAAoD2ALoJIXyoSsa9SAm89x7XhfotmIOC+DvPzAS++cZrInqdu+/mJjI7u+5vurqaz0VHsxbh9Gm6hQ4etL3zDw2lJ8HZuie93naGotHtbTGOICXzaffupTkTH89FPSyMX4a4OODddzn82RylpTQpExMb7KzHoi/ayDCMj90Beeggc299FSGAW2/lH8Tczkev5x/6hhucSwULCeHshxMnLCvG3Fz+2Pbto0X1yCPAc8+xJqKoyPxrFHbjSNrpaHBIzm8AXjj33+fdI5bvkZXFbDvj7tYciYlMxbZVldtciY7mTIIrrmBs8NgxPk6eZAygXz9g1iy2qn72WXpeDh+2Pg3NuCG0x7VsjpQUZj9Zo7KSSt1iLPP4cZoaycnmBTH6tBYuNL+QHTlCbWMm0BICf0zFQKwXx7A45IRlpeIs5eW0aPbuZb/wxtK1K/DYY1x8s7Ko2U+fpiLIyQFuvpmtaZ3lhht4jyNHqEiNVFZyx6XR8O+xYAF3GrGx/Gy/+gqYOJGvUziNI0HlxwFcCGCTlHKwEOI8tKARmgcPmgoyLeHnx/XgyBG6RloikZHAbbexK6kx7hcezg308uWm9HKA68iePfxv//7mMw2Ns2GcrT0YNMgU77FkKeTnU16LgfFNm3jQ2h8/JobFWidPmgIgRqqr+dqKCpNLpVaw6QH0wstyAyZEb8eIinKHUv8sUl5Od9aKFSZ/m8EAdOnCwq6UFOev3acP3WfbtpkqidPSGEux1IzKXgID2eJ77Vr6CrOz+dkFBTFIt2oVLYnoaNNrwsL4OHsWePll7jpqH1fYjSMKoVJKWSmEgBAiUEp5QAjR2W2S+RiVlfYXR5orvKrNmTP8LRUU8LvdsyetYGd3wb5IcLCpJuHTT7lW1n+PHTpwk1lcDOzY0TDLR0puQG+6yfnPJjWV69T69XWD18br5+dzDRs0yMpFCgosR6WNCEGlUVLSUCGcPMmF0+i/MhgYPO7cGYiLgz+0mIkhuC3gOyww7MIduMG5N2ukvBx46SXuqNu2Nfk3peRzM2fSzdKxo/P3CAlhK4kBAxonqzkCAug6GjyYucJScqexbBkVqqU6iKgoKpD169lCQ+EwjiiE40KIKAA/AFgqhCgEh+W0CBISbOeqGxtWWtqc6PXA11/zew3we6/XcyNnrEfy5SQTZyguphvNjPsc4eH0DvzxB70ApaWmWiOdjq7kjAwm8jiLEMD993PjuXo1/z8wkOuKXk8l8eijNmIZERHW/VpGzKUzrVgBfPklj/n78zrl5dzNZmbyzXXsiFt0HTFPxGJK9qe4uWYiArSNyGP+3/947fpVgsYc6aIi4M03uZu2VlPgbTSaun2Rli+3/QOJi+MPTCkEp3CkuZ1x2/K8EGIlgEgAv7pFKh+kRw8u4JZ6jwFc/Nq0Md9hQErulFev5u+0dr2RsaXCvHnsXGCtn09T488/rbf679CBC/Tvv9N9ZNz8+fsD117LhyM1Dubw9wfuvRe46io22isooAuqVy8qYpvWR0YGFyNrlJRwsWrTxvRcdjbw+efMzDlzBti4kW/G+IbKy9mfR6+HJiAAc0aOxYhD0/DBtg/w6EWPOvdmKyqohGrLUZ/ISCqMPXua1rCds2etvy+ACrmgoHGBpxaM3T81wengdwBIk1JOF0IkA7gAwGZ3CedLhIQAN97IzZ65Qe8VFfzNjxtn/nuYlcWdcn23BcDzW7fmb3TVKi6CzQVb/YKE4OdpMND7cMEFVLgdOri+piMuzsnWF507M784P998VkFNDRehBx+s+8dduZKLf24uo+tJSTzPYOCOwFh1u3IlMHEiho8ai4GfrcCMNTNwzwX3ICzAidYMx4/zQ7eV0x8YyEBzU1II4eHckVn7Yuh01PZKGTiFI/GrtwH0A3Dbuf8vAfCWyyXyYYyNFnNyuPkzpk9mZdHV+eijtCTMYU/OfevWzJ6zFYNoSjhSUNajB+OV3bv7WIGfRgOMHctdQVYWtT9gCnJkZTGz5tJL675u0yb6D/fsodkXHU2TJCGBO9mgIP67VSsgPx9CCMwZOgf5Zfl4Y5OTfSOtRc/rvydvdfdzlsGDbc+MKChgk0CFUzhijPeVUvYWQuwAAClloRCiRTVsEAK45hpmxGzcyDRUjYaN2jIyrPfaOnrU9uJotHZLSppPkkTHjnzfZWWWM4V0Om5Y6ze8s4Zez1T0Zcu4+TYWDV96acOYrkuIiwOmTWP2y2+/mVKozjuPyqB794a70qoqU7DC6Cby8+Mf1/gHlpK+xtxc4MgR9Evvh+s6X4cXN7yIhzMeRmyIg7MF4uKoFGwphspK650IfZH+/VlSXlJiPke4tJSWlzsC3S0ERxRCtRBCC0ACgBAiDoCT9aNNm5gY+qMdQau1XW1rzLlvTq3e/fyYJfTee/SY1Pdk6PVc0G+/3X6roKyMMdF9+6iEw8PptVm0iI/77+fa4XIiIviHHzGCVoJWaz37qHVrCmnNfWEciykEfY7p6Zg5ZCZ6vNMDc9fNxUvDXnJMxpgYpq3t32+5fUR1Nb9kGRmOXdvbxMayw+FrrzGeEB/PnYROR3eeRsOUVTVa12kcWXrmA/geQLwQYhaAdWhBdQiNpXdv26MfjUHp5hRUBrg433UXkJdHV1tREd/rsWN0eV9/PQeG2YOU7PBw8CDjMXFxXJNDQ6lw4uKofFxd31UHjYY3tJWKOnw4A8fW2lGUlzNgIsTfmQbd4rvhrp534c0tb+J48XHH5bvpJt6zsLDhMZ2Of4R//KNpftG6dGHa7MiR/EFlZVGRDh/O5x0xMxUNsKe53edSyrsAtALwLIChAASA66WU7vzZNSv69WMhq7Eqtj5Gd/RDDzW/eJgQrCnq3Zstf3bvprXUrx+te0f6oGVlATt3UhlYKxr+/nuuHV7lootMxRbGyV+1KSnhopyQQBdUmmkuwguDXsCC3QswffV0vH/N+zZvVVPDGPGSJcDRo4mIqxyPGzPfQkpYJsKj/SG0GioDrZbmmL0a2BeJj6dCu/FGvnGttvn9aLyEkDaaaQkh9gG4HMAvAAaByuBvpJRnzLzMI2RkZMitW7d66/YOs3YtW+LExNRdHyor6ULOyADGjPGtFvi+xldfMW7QzsrwVim5CX7xRR/wHpw6xXYOOTn0bfn50U+m1/NLcPHFPKdvX2Yp1eLxXx7HW1vewt4xe9G5leUa0KoqDinbsYPep6gofganC2rQ6uR+XJO+F317VUMktbMd7FK0CIQQ26SUDXyG9sQQ3gWwHEAagG21rwnGE9LMvUjRkAEDuCZ8+y0XLOPkrqAgWvkjRihlYIuTJ23HGozN6kpLfUAhtGoF/Oc/bN505Ah93mFh9G+Fh9OP1q4d+33UY9Jlk/DRjo8wZeUU/Pfm/1q8xRdfmLea2rTTQt+6G97O7oaqZPaTUiisYVMhSCnnA5gvhHhHSvmIB2Rq1lxwAWN+x47Rjx4QwB9yUxmwU13NLMoDB+j2SU/ne7LlTncV9hQNGyvGzfVG8gpt2gCvv84K4nXr+FxlpWkm6MiRZnft8aHxGNdvHKavmY6tOVuR0bZhEPjUKV7SUt89Pz+65L7/nhlYasOhsIY9LiMhbZxkzznuoKm5jJo6e/YwYFtaypoqIeiuCAwE7rmHPYPczf79rOi21putuJgyzZ7tg67l0lKaOcZqRBuatLiqGGlvpKF3m95YcteSBseXLWOxpK0M0uxsdotOTWWH2ZISfkYdOqhZMy2RxriMVgohvgPwo5Qyu9YFAwBcCuAeACsBfOIiWRU+yMGDwCuvMP5Rf/GpqGA3U43G/WMojUXDeXnmg9E1NVxvx4zxQWUAmDpz2klEYAQmDZiEp5Y8heVHlmNo2tA6xwsL7WvtISUr5d96yzSpUkoq9iuuAK67rulYqQr3YU/a6ZUAagAsEELkCCH2CSGOAPgTrFp+XUr5iRtlVHgZKdmSJyLCfD1QcDAX5y++cH/xq7FoOCyMrT6Mc1oMBiqCrCwWD7p9PrIHeeTCR5AUkYQJyyegviEeEWG76SJAF+WPP1IBtG9PF1P79gxx/Pwz8PbbzatCXuEc9kxMq5RSvi2l7A9OShsKoLeUsr2U8v+klDvcLqXCq2Rmsl7A2ijKkBB6Q/budd19S0rYp23+fOCNN1h0duaMqWh41Ci6rLKyuOClpTF2e/PNPmodOEmQXxBeGPQCtuRswfcHvq9zrGdP05hjSxQUsHni+ec3rBY3xrC2bWODQUXLxmYMwZdRMQTPsHkz0xrt8VM3dmCWkQ0bgI8/5q7VWMhbWsqF76abGIcVgothVRWDpc3Z5aE36NHjnR6QkNj9yG74aUx+ojff5EC3pKSGitBgYO+8kBDWfViiqIjnzJzZvJSpwjyNiSEoWjj2ZqYYm3g2lh07GJNo27ZuzDUmhi6pr77i4n/FFXQheaQRXkEBB69s3sw0p6Qk4PLLWf3mgdQdP40fZg2ZhRv/eyM+2/UZ7u91/9/H7ruPC/rBg6YaFynZfLGkhJZdZxujrCIiqNDLy52fTqdo+jSjrjkKd2HM6LHWi8loaHbo0Lh7GQwcl9uqlfkEHH9/pu1/+y0zNz3CunXA+PFMG9XrKcShQ5xK9tpr5gfOu4Hrz7seFyVehGmrpqFSb3rzoaGcU/3ww4zxGF1oqal0oXXtars/lnE8bBN2GChcgFMKQQjhQLMBRVMnNpZtJ3JzLZ9z6hRdSuaGAznCkSMMDltrsxMYSDfR7t2Nu5dd7N3L5knx8YzEGnsYxcdTU+7dC3z0kUdWUiEE5g6di+PFx/H2lrfrHAsIYNrvjBkU56OPgKefZmufTp1s99EqK6MloVJQWzbOWgiLXSqFwue58066Io4dq5vVYjCYFMWDDzbe/3zmjH3X0GhMHajdhpSs6IqKMm+uGKf7bNvGqLsHGJw6GMPSh2H22tkoqiwye45WW9ciGDyYCtSahVdQwEaudnXaNRhYmfjxxxzD+eGH7OpqT7qTwqdxViGosFMLIzoamDSJu9CcHCoG46NbN47+TExs/H38/e3bbHukErmggEMvrA2nEIKFAJs2uVkYE7OHzMbpitN4ZeMrdp2fksKZMZmZDVNLpaQuS0mxs7Dw7FlGnufO5Xs+dgzYupWNo55/noELRZPF2aDyBy6VQtEkiI4GHniAmUQnTnAxSUhgILOigq704ODGWQnp6dylGptYmsOoMGwFShtNSQmFsfWGgoJsT/JyIX3a9sEtXW/BqxtfxaMXPoqEMDNjPWshBC280FDgl19Mn60xXbV3bwambQbnq6pYnZiXZ75UPC+PimHaNOV7aqI4qxAWulIIIcTNAJ4H0AXARVJKlUvqw0REmHoKbdoELF7MtUAItu0ZOZLNO+vPnbb32v37m/rzmCM/n8rAWsdTlxAUZJ+5otOZr9hzIzMGz8B3+77DrLWzMH/EfJvna7VM1x0+HPjjD7rmQkJo3ZkbE22W7duZimQpUGQcDP777/RTKZocvhJD2APgRgBrXHxdhZuoqABefZXuY52OG8bkZD7/3nssJKuqcu7ao0bxWkeP1s0kqq7mehQaCvzf/3kgX75tW6Y7lZRYP0+n4+wDD9IpthMe6PUA3t36Lo4WHrX7deHhVLjXXAMMHeqAMgA4OtTWbNfYWODXXx24KOjHsjVOUOERfCKGIKXcL6U86MprKtzLf/7DvPfUVFNGkBAMPKelsRHefy13bLZKaCjTJW+4gY3qsrPpqj55krUHU6ZwnTbitgQfjQa49lrGEiwFTPPyqL06dnSTEJaZOnAqtBotpq2a5pkb5ubatoTCwmjC2VrgS0uBpUuBceOA0aP5ePdddt5TeI0mF0MQQjwI4EEASLbkU1C4lcJC1mi1a2d+ly4Ej61ezfGYznhTQkL42pEjqQik5ObT6Oc+eZIDh1as4NoSFcU6sf79bW9iHaJ/f2qkX3+l5ouN5RssL6eiiIsDHnvMK4OwEyMSMfaisXhpw0t45pJn0D2hu3tvGBjI3by1QryaGlMrXEucPMlYw8mT/PxSUvi6HTuAjRtpIo4YoUqmvYBT32Ip5du2z6qLEGKZEGKPmcd1Dt77fSllhpQyI87r009aJsZZCNbWBT8//sYPHGjcvQICmL3Urp1JGezZw4ynRYu4IW3fnmvVwoXA5Ml0NbkMITi85umnaQlkZ7Pyq7ISuOUWYOpUr07hee7S59gRdcUk99+sb1/bwfOCAp5naTGvqeFsiJISKgJjWbRWywBUu3YsRd+1y6WiK+zDY60rpJSXe+peCvdSUWHfeVI6H0ewRH4+4xORkXW7SAcHc70+e5YZkU8/zeynmBgXbDSFAHr04KOqiotaUJBXrIL6xATH4Ln+z2HiiolYn70e/ZP7u+9mgwZxAINxCEZ9qqt5bOjQhseM7N/PFDVLAy38/Wnu/fQTO/cpK8GjeP8brWhyWKsiro0Qrk++WbWK1om5kQIlJax0XrmSXpxnnmEG5ObNLowzBAbSn+UDysDI2L5j0TqsNcYvH9+gPbZLSUzkJKQTJ1iabryXlExbys6mNWWtXH3jRtv5rdHRNPNUTYPHsWkhCCFi7LiOQUppozje6j1uAPAvAHEAFgkhdkophzt7PYV76dqVrhydznKH0aoq/u7PO89195WSi725zJjTp9khFaBVUFrK/nNFRcC//gVceSVw+20S4vgxunyMRRQdO/rU4u4MoQGhmHrZVIxZPAa/HP4FIzuOdN/NBg1i246ffmJWgVZLiyk9nalf3W3EMYqKbLelNQ7FttcUVbgMe1xGOece1mw3LQCnI7xSyu8BfG/zxEZgrLZfupTfY4D94S+/nDntyjK1n+Bgpi3+97+0/OvHEmpqWP16xx2urSbW66lo6q8n1dWshwgI4P1qaujiF4Leh/BwYNPCHAxc+xHaVf5leqEQ9P/fey+/DE2Y0b1H45WNr2DC8gm4ssOV0Ag3Krnzz+fjzBk2QQoONgXbbREbC/z5p/VzjEOxHZgsp3AN9iiE/VLKXtZOEEL49JAcvR7497+ZGRMaaspC2bsX2LKFNTR33aUGkDvCVVcxJXTpUrp9Y2L4Oy4s5AJ99dXAsGGuvaefH1339S2T3Fze0xifrKmpq4giy3Mx7M9ZyMmSSBzZHkJTa+EqKmLX0nHjWKXVRPHX+mPG4Bm4feHt+GrPV7i9++3uv6kxSOMIl1zC9DNrnD7NtuIuTRdT2IM92wgrYzUcOsdrLFzIytf27Wnt+vvzU8U2mwAAG4NJREFUkZDA51asYMaKwn40GuD22+mjN/bAMRiYJBIfz/m9Tz8NfP219S6pjiAEe/Lk59d9/vjxugqgvJy1EEa67/4SQf41yNHHo7Kq3i42MpKL2ocfNvkZkqO6jULPhJ6YsnIKdDW6ugfz8oBvvqHie/RRYNYs7oZ0OvMXcxcdO/JhqRlgZSV3Gtc5lHyocBF2jdC0dEwIcZ+tc7xNaSmwZAkzUMy5ijUaLmKLFnmwv34zQQjGD++7j+meUVEM6lZWcp319+dnP2kSFYQrGDiQllzt4mG93vS3ragwpaoCQGhpPuIL9qAsLAFCWKgvCw+npbBvn2uE9BIaocGcoXNwpPAIPtz+oenAunXAxImspfDz4x8qPx946y1g9my+d48JqQH++U/+gTIzTSZlRQWrDwsK2DbX7Y2qFOZorKPxBZdI4UaMXXn9rDjHjAHS/fs9J1dzQkoOaT96lDGFmBh+psHBDOwmJJg6JDeW+HjgySep6LOzuY6EhtKVXVREWS65xGQxhJfkQAoNDFJACCsxDY2GweYmzpUdrsRl7S/D9NXTUaYrY8Dsgw/4wSUl8Y9i9PGlpHCn/uabnm0dERnJXcKYMYzhnD3LH+Dw4bRcLr3Uc7Io6mBPltEflg4BcKQTilcwzuG1h7Iy98rSXDlyhMq0fXvzccXAQK4BP/xgPXar03Ft0GjoPrYU0+nShRvb9euZdRQcTKXfvTvXPHOjC4xZR8403GtKCCEwZ+gc9P93f7zx+xuYuMmfwVlzHwrAnfqff7JlRKdOnhM0IID9nzzcA0phHXuCygkAhgMorPe8ALDB5RK5GEe68HpkNm8zZONG/r6tJZnExHDdOXWqbh8igO6fZcsYoK6qogKPimL3goEDzWcpxsayzdC11/L8119ncWt9C6AkIhH6KgNEkAEdOlgxiA0Gy8VSTYxLki7BtZ2vxbx1c/HQ4asQ287KQm80m9av96xCUPgk9riMfgYQJqXMqvfIBLDKrdK5gPPP507TWrywupoupS5dPCdXc+L0acsbUCNC8O9QWlr3+TNnOG/lxx9pRSQlMd6j1QKff86RxbZiO0JwnvAFF9Drk5ND99Hp08Dek/E4HtMDl3UusFwkV1xMDdSMvgCzhsxCia4U86L22jfP4dQpzwim8GnsCSo/IKVcZ+GYB3LbGkdEBGtpjh0z7zqSkseuuELN9HCWyEjbLSqkpFunthUmJd3bhYXcnNfe3YeGMmC9fz+nWNoiOBh4/HF2Qr3kErqmU1OpKEZ+cTsiY/0YsKz/JTh7lgKMHm090NTE6BbfDXd1vhn/CtuH49JG0Fin4x9R0eJpPr8AK4waxZ3o1q30TRvTm8+c4U6yXz+2WlY4R9++bClhjaIi7vzj403PHT9uij2YQwi6uFesoGvIWGdgCSFYMJueXv9IawYx//1vU1GUEFQOrVtTa7iypNpHeGH4XCw4+B2mVy7F+4E3WT6xstLO+ZmK5o49QeXtUsrejT3HmwQEMPX6jz84QtDYcr1zZ46E7N5dFaU1hs6dudjn5XF9rY9eT/fNXXfV9V7s38//t+bR8Pfn6//6i73lnKZNG6ZeHj9Ok9BgYPqTcWZnMyQlOhWPpN2Ct/76CuOq89DZ38wfJz+fn00zcpcpnMceC6GLlUwjgMFln7c3/fw4O7Z3b64FthYihf1oNMDYsSz4zcykuyY0lJ9zQQFTQ2+6iZ99bSoq7FPEQjDO02iEYJAiKckFF2saTLrhNXz02veYUvYr/nv6Sv5xtFqm1Bkj/E88oXZECgD2KQR7bGkL46R8k2a6IfQqrVrRf79pE60wY0p/Rgb7RXXq1FABx8XZVxwspXJxO0t8WALGXfospq+Zjq3J0cjYcYIfaHQ0O5NeconqGaT4G5sKQUpZp1pHCDH93Ot2AtgppTzkJtkUTYywMC7+Q4eaBmtZU749etByq662XB9QUkLFUbsVhcIxxl0yDm9teQsTo7ZhyYe/mio1lYmsqIfDe2Up5VQAbwAoAnCDEMJr4zRbCidPsrXz6tWMg7jEfeJGhOACb8sSCwtjsDg723xLiaoqupxuuUVZdY0hIjACkwZMwtIjS7E8c6XtEZeKFouwd6CGEOINAE9It07gcIyMjAy5detWb4vhNoqKgM8+A7Zvr5stGRbGRXLAgKb/uzYYgG+/BRYvpkUREcH3WlzM93b33SxOUzSOSn0lOv2rE1qHtcbvo3+HaOpfHEWjEEJsk1Jm1H/ekbTTEgA/CSFulVKWCSGGA5gqpXTjzL6WS2kpMG8ed8hJSXV3yBUV7A1UWen6FtOeRqMxKbd165gVqtHwffXrpzogu4ogvyC8MOgF3P/T/fj+wPe4scuNXpXn7FlWuG/YwO9x69Z0NXbr1qzKQZocdlsIACCEuB3AkwB0AEoBzJRSrnWTbDZpzhbCwoUcSmWpm4JOxzTPV15Ri6bCPvQGPXq80wMSErsf2Q0/jXdW3j/+YKNVnY4tTfz8mPRUUsIs4CeesH9Mq8I5LFkIdntmhRBDAfwfgDIArQCM9aYyaM7odOzr06aN5XMCAuha2bTJc3IpmjZ+Gj/MGjILB04dwGe7PvOKDFlZ7DsVEcGCxPBwVpm3asXK8uxsYP58zzZfVZhwJFQ3CcAUKeUgADcB+FoIMcQtUrVwTp2iUrA1fjI8XLXsVjjG9eddj4sSL8K0VdNQqff8AJBffmFM21LVeWIiC0eNY24VnsVuhSClHGLsaSSl3A1gBICZ7hJMoVC4HiEE5g6di+PFx/H2lrc9eu+yMmDz5rrtS+ojBHvtuWqgksIxnE7mk1LmAhjqQlkU52jVitaBrYZxJSWq44DCcQanDsaw9GGYvXY2iio9Ny2ttNTU9dYaISFMplCYITcXWL4c+PlnRuTrtw9uJI2KKkkpK1wliMJEQAC7r/74o/WgskbDTByFwlFmD5mNjA8y8MrGVzB98HSP3DMwkLEBKa2nS+t0qni6AUVFbM64axf/X6s1FRhedRULelzQfkSV+/goV1xBf+rx4w0DbOXlfP7229nGX6FwlD5t++CWrrfg1Y2vIr803yP3jIoCOnZkl2FrlJYC/VUyu4myMjYK27ePkfiUFOaip6SwQePChcDXX9s/GtIKSiH4KGFhwHPPAb16MfMiM9P0qKjgHPIhKqSvaAQzBs9Apb4Ss9bO8tg9r7mGm11LPazOnGEqas+eHhPJ91m9mh16ExMbmlb+/lQMS5YAJ040+laqBMSHiYhg2+5Tp5h5UV3NmoPOnZv/bGCF++kU2wkP9HoA7259F09e/CRSo1Pdfs9u3ViI+M03zDRq1Yquz6oqduIOCQGefNJ2hl2LoaYG+O03833ljWi1dB2tXg3ccUejbqcUQhOgVauGc4gVClcwdeBUfPbHZ5i2aho+u8H9tQlC0OWdns51btcu01jnq6/mdMOYGLeL0XQoLqYPzVb1aVQUcKjxfUaVQlAoWjCJEYkYe9FYvLThJTxzyTPontDd7fcUgtlxXbrQMqiuZnGaGslgBuNkP1tI6ZIOkCqGoFD4MtXV3EavXs00QzfkYz536XPsiLpiksuvbYvAQMbLlDKwQEQE3QMlJdbPO3uWox8bibIQFApfREpg7Vpmj5SV1c3V7N0buOcel00NigmOwbP9n8WkFZOwPns9+ierFB+fQaMBRo5kymlYmPl8XZ2O349LL2387Rp9BYVC4XqWLgU++ICLQEoKG/2kpHB49e7dbIXrwqKkx/s+joTQBExYPgE+1OFeAXCqXffubARVfxhKWRkzkEaNsl4CbidKISgUvkZhIfDVV8w1Dw6ue0yjYfphTg5TDV1EaEAopg6cirXZa/Hr4V9ddl2FCwgI4NDy4cPpMszKMj30euDhh13WB9+h9tfuQgjxEoBrwLbafwG4T0p51tbrmnP7a0ULZvFiTg1KTrZ8TlUVE/pff50LhgvQ1ejQ5a0uCA8Ix/aHtkMj1H7R5ygrYw66Tsf4QocOTgVgGt3+2s0sBdBNStkDwCEAE7wsj0LhPQ4cYCtbaxibXZ065bLbBmgDMGPwDOzK34Wv93ztsusqXEhoKKv2LryQBUkujsb7hEKQUi6RUhprFzcBaOdNeRQKr+Mly/3WbreiR0IPTF45GboanVdkUHgPn1AI9bgfwC+WDgohHhRCbBVCbD158qQHxVIoPESXLrbTDKuqaCW4uGJRIzSYM3QOjhQewUfbP3LptRW+j8cUghBimRBij5nHdbXOmQRAD+BLS9eRUr4vpcyQUmbExcV5QnSFwrP068fgsc7KDj0nhx0QXRQ/qM2IDiMwIHkApq+ZjjJdmcuvr/BdPKYQpJSXSym7mXn8CABCiHsBXA3gDukLkW6FwltERbGV7bFjbG1bG4OBzycluSyzpD5CCMwZOgd5pXmY//t8t9xD4Zv4RGGaEOJKAM8CGCilLLd1vkLR7BkyhC6hb74BTp6sW5h24YXAXXdZnkPpAvon98c1na7BvPXz8FDGQ4gJVg2GWgK+knZ6GEAggNPnntokpXzY1utU2qmi2aPXM+vo7Fl2tOzYEYiN9citd+fvRs93e+KZS57BvCvmeeSeCs9gKe3UJywEKWUHb8ugUPgkfn7sGe0Fuid0x5097sT8zfMxtu9YJEYkekUOhefwxSwjhULhI7ww6AXUGGowfbVnxmwqvItSCAqFwiKp0al4OONhfLTjIxw63fh++wrfRikEhUJhlUkDJiHILwhTVk7xtigKN6MUgkJhg8pKZnpmZ3OedUsjISwBT/V7Cv/d+19sy9nmbXEUbsQnsoycRWUZKdxJaSnwyy/AsmWmrsN+fsDgwcCIESwXaCkUVRYhfX46+rTtg9/u/M3b4igaia83t1MofIriYmDOHGDRIs74TU7mIy6OXadnzQLOnPG2lJ4jMigSEwdMxJK/lmDF0RXeFkfhJpRCUCjM8OWXQF4eZ9IEBpqe9/cH2rdn5+mPWlirnzEXjkG7iHZqiE4zRikEhaIeZ84AW7ZwDo0l2rQB9u1jS6GWQpBfEF4Y9AI2n9iMHw784G1xFG5AKQSFoh5//cVOERorvw4heM7hw56Tyxe4u+fdOK/VeZi0YhL0Br3tFyiaFEohKBT1qD+21hrWGpI2O/Ly4Lf4V8wSQ7H/1H58vuI1r81tULgHn2hdoWh5VFVxMQ0OZuaOLxHjQB83F48j8E0qK4FPPgE2bQI0GtwQ4I8LY1ph2uoXcNvWKgQ98hgQGeltKRUuwMd+iormzoEDwG+/ATt30u0SEAAMHcpUTl9ZXDt2BKKjmXYaFmb+nIoKICQEOP98z8rmcWpqgHfeAXbtYjRdCAgAczESQ8VneCfvZzz5chUwcSK1u6JJo1xGCo+xeDFTOQ8eZDv/5GTuxn/9FZg2DcjK8raERKvlOIKCAm6O66PTMZh8661umU/jW+zfT+19ThkYGYJUXCHTMCvqDxQfO0zrQdHkUQpB4RH27gW++gpo1w6IjzcFbAMCqBz8/IDXXjO/AHuDjAzgoYeA06eBzEzOsj99mkorPx+45x5gwABvS+kBli6lmVRLGRiZjaE4LSrwSttMVvCpeEKTR7mMFB5h0SIgIoJ5/OaIjubCu3MncPHFHhXNIv37Az16MAV13z6ud507A337tiCXeWYm/3BmyEBb3CzPxysB2/BoXjLidbq6RRuKJodSCAq3U1JCz0NysvXzIiKAtWt9RyEAQHg4h5cNGeJtSbyEnx/HdlpgBgZjIfZjVvRuvKHVelAwhTtQLiOF26mspIvIjNehDv7+QJma6e5b9O5NX5kFOqMV7q/sgnciDiGz9LgHBVO4A6UQFG7HmKlTU2P9vIoK9gpS+BADB/IPZ6k4w2DA1PzzoNX4YdqqaZ6VTeFylEJQuJ3gYLqB8vOtn1dezvVH4UO0aweMGsXe38XFdQPHZWXA0aNoN/g6PNb3MXy+63PsKdjjPVkVjUYpBIVHGDGCG83SUvPHT5wA0tKALl08K5fCDoYPB8aOZcA4O5vDIbKyaDXcd9//t3fvMVaUdxjHvw8gLKKgiHdAhKLWC6UCpoq0KErF1FrSNtVAjVFrDBov1FpwVQqUWELSNBCbaoKpGtJKIqWNRcQKYtECCgWLogStN6wX8ApCKfDrH/OuLutezpHdnTPL80k2ObPMOe+zu+z89p2Z83th7FgmDJtI105dqV5cnXda2we+qGytomdPGD8eZs3KTkl3755dr9y2LSsSffrADTdk7wGwCiPBkCHZvbivv5790KqqsvcmpB9Y987duWXoLVQvrubpN57mrF5n5RzavgwvkGOt6qOPYMUKWLYsu2Zw1FHZO5VPOaXhW1KtGLbt3Ea/mf044bATWHr5UtTUXQSWm4YWyPEMwVpVt24wcmT2YW1Ll45duONbd3DtgmtZuHEho/qPyjuSlcnXEMys2Vx1+lX0PbQvEx+fyJ5o+P0LVplcEMys2XRs35Gp50xl7TtreXDdg3nHsTK5IJhZs7rk1EsYcOQAbltyGzt3708LRhSfC4KZNat2asedI+7klQ9eYfbq/Wzh6YJzQTCzZjfqK6MY1nsYU56cwrad7kdSFC4IZtbsJHHniDt5e+vbzFwxM+84ViIXBDNrEUN7D+WiEy5i+lPTeX/7+3nHsRK4IJhZi5l27jQ+/u/HTF82Pe8oVoKKKAiSpkp6TtIaSYskHZN3JjPbd6cdeRpjB4xl5sqZbPp4U95xrAkVURCAGRExICIGAg8Dd+QdyMyax+Thk9m9ZzdTlk7JO4o1oSIKQkR8XGuzC1DcBktmtpfjDz2eawZfw+x/zmbDlg15x7FGVERBAJA0TdIbwBgamSFIulrSs5Kefe+991ovoJl9adXDqqnqUMXtS27PO4o1otUKgqS/SVpXz8fFABFRHRG9gDnAdQ29TkTcExGDI2Lw4V5ey6wQjjzoSMafOZ65z89l1Vur8o5jDai49teSegMLIuLUpvZ1+2uz4vhox0f0m9mPQccM4tGxj+YdZ7/WUPvrijhlJKl/rc2LgRfzymJmLaNbVTduHXYri15exOJ/L847jtWjIgoC8Kt0+ug5YCRwQ96BzKz5jRsyjp5dezLx8YlU2tkJq5CCEBHfj4hT062nF0WEb1g2a4OqOlQxefhkVm5ayfwX5+cdx+qoiIJgZvuPy752GSf1OInqxdXs2rMr7zhWiwuCmbWqDu06MO3caazfvJ4H1j6QdxyrxQXBzFrd6JNGM+SYIUx6YhI7du3IO44lFXfbaTkkvQe81oJD9AA2t+Drt6SiZnfu1lfU7EXNDflnPy4ivvBGrkIXhJYm6dn67tUtgqJmd+7WV9TsRc0NlZvdp4zMzAxwQTAzs8QFoXH35B1gHxQ1u3O3vqJmL2puqNDsvoZgZmaAZwhmZpa4IJiZGeCC0KSirvcsaYakF1P2P0k6JO9MpZL0Q0nPS9ojqeJuzatL0gWSXpK0UdKEvPOUStK9kt6VtC7vLOWQ1EvSEkkvpP8nhWiGKalK0kpJa1PuyXlnqsvXEJogqWvNEp+SrgdOjohrco7VJEkjgcURsUvSdICI+HnOsUoi6avAHuBu4OaIqNhFLyS1BzYA5wNvAs8Al0bEC7kGK4GkbwJbgftLWX+kUkg6Gjg6IlZLOhhYBXyv0r/nkgR0iYitkg4AlgE3RMTynKN9xjOEJhR1veeIWBQRNZ3DlgM988xTjohYHxEv5Z2jRGcAGyPilYjYCfyRbE2PihcRTwLv552jXBHxn4hYnR5/AqwHjs03VdMiszVtHpA+Kup44oJQglLXe65gVwCP5B2ijToWeKPW9psU4ODUVkjqA3wdWJFvktJIai9pDfAu8FhEVFRuFwSab73n1tZU7rRPNbCLLHvFKCW7WWMkHQQ8BNxYZyZfsSJid0QMJJuxnyGpok7Vdcg7QCWIiPNK3HUOsACY1IJxStZUbkmXA98BRkSFXSwq43te6TYBvWpt90yfsxaUzsE/BMyJiHl55ylXRHwoaQlwAVAxF/U9Q2hCUdd7lnQBcAvw3Yj4NO88bdgzQH9Jx0vqCFwC/CXnTG1aujg7G1gfEb/OO0+pJB1ec7efpM5kNyJU1PHEdxk1QdJDwIlkd728BlxThCU+JW0EOgFb0qeWF+HuKABJo4FZwOHAh8CaiPh2vqkaJulC4DdAe+DeiJiWc6SSSPoDMJysFfM7wKSImJ1rqBJIOhv4O/Avst9LgFsjYkF+qZomaQBwH9n/k3bA3IiYkm+qvbkgmJkZ4FNGZmaWuCCYmRnggmBmZokLgpmZAS4IZmaWuCCYmRnggmBthKQ+kranPjE1n/tCW2pJnVMr852SeuzjmJ0lLU0dT5F0vaT1kspqEyLpEEnj9iVLCWN8odW1pI6SnpTkjgUGuCBY2/Jy6hNT05b6LmAUcDJwqaSTI2J72uetZhjvCmBeROxO2+OA8yNiTJmvc0h6blmUKfV3+PdkbRI+k7qzPg78qNyxrW1yQbDCSIuinJ8e/1LSrEZ2b4221GOAP6c8vwP6Ao9IuknS2LQYyhpJd9eaRcyXtCotkHJ1ep1fAf3SvjPSbKf2X/I3S/pFetwnzXruJ+uB06uhsWprpNX1/PR1mLkgWKFMAqoljSFreXxjI/u2aFvq1Leob0S8CpDagrwFnAMsJPure2iajezm84PuFRExCBgMXC/pMGACaXYTET8rYfj+wG8j4hTgwEbGKsU6YEgZ+1sb5nOHVhgR8WRqbDYeGF5zqkbSVLJmZ62pB1mfpfqMAAYBz2Rx6UzW/x6yIjA6Pe5FdnB/u8yxX6u1ylZjYzUpInan6ykHp8VmbD/mgmCFIek04GhgS83BS9JRZCtP1VV2W2pJ1wI/SZsXAqNrb0dE7esO24Gqhl4KuC8iJtZ5/eHAecCZEfGppCcaeI1d7D17r7vPtqbGKlMnYMc+PN/aCJ8yskJQto7uHLLrAFtTe2+AgcCaep5SdlvqiLgrnbYZGBFv1d2us+8HQHtJ9R3QHwd+IOmIlL27pOOAbsAHqRicBHwj7f8JcHCt578DHCHpMEmdyNa0aEhDY5UknbLaHBH/K/U51na5IFjFk3QgMA/4aUSsB6by+SJF9RaEtJ70dcCjZGvuzo2I55s52iLg7HrGfgG4DVgk6TngMbKZzUKgg6T1ZBeSl6f9twBPpRXjZqSD8xRgZXpugz3zGxlrL6nV9T+AEyW9KenK9E/nAH/9Ml+8tT1uf22FJmk22Wmd3sDDEVHSkoSSXgUGR8TmfRj7dOCmiPjxl32NvEmaB0yIiA15Z7H8eYZghRYRV0bEHrK7a7rVfmNafWremEZ23WFPY/uWMPZqYEl9t3kWQTqVNt/FwGp4hmBmZoBnCGZmlrggmJkZ4IJgZmaJC4KZmQEuCGZmlrggmJkZ4IJgZmaJC4KZmQHwfyDHwTuSEudxAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "oEQswfCGrmHw", + "outputId": "3cf61882-60e1-4baa-8e51-0c31ea80875c" + }, + "source": [ + "pred = tf.matmul(valid_x,W)+b\r\n", + "fig,ax = plt.subplots(1,2)\r\n", + "ax[0].scatter(valid_x[:,0],valid_x[:,1],c=pred[:,0]>0.5)\r\n", + "ax[1].scatter(valid_x[:,0],valid_x[:,1],c=valid_labels)" + ], + "execution_count": 27, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 27 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5xU9b3/8ddnZnZnC0WaIqKCXSxE3UiIGsSKFTXq1dxr7MRekvvLtXujN15bOpZwNZY8jDUxoqLYg8aoLIoCIoqFACIsSN02Mzuf3x+7moWdhZWZnTMz5/18PHiw8z2H832rn/l4OHPmfM3dERGR0hcJOoCIiOSHGr6ISEio4YuIhIQavohISKjhi4iERCzoAOvTv39/HzJkSNAxpERNmzZtqbsPyPe8qmvpTuur64Ju+EOGDKG2tjboGFKizGxeEPOqrqU7ra+udUlHRCQk1PBFREJCDV9EJCTU8EVEQkINH2hY3ciMV2czf87CoKOI5NSiTxYz49XZrF6+JugoUgAK+i6dfHj0FxO558oHcRxPO4N33IKbn7+avgP7BB1NZKOtWVHP1UffyOw3PsIihqed4y4+nLNvPgUzCzqeBCTUZ/hTJ0/nrsseIJlIkUq00JJKM2/WfC7e56qgo4lk5boTfsHM1z6gJdVCKpGiJdXCo794kifGPxt0NAlQqBv+PVc+SLol3WH8i0+X8NE7nwSQSCR7q75czfSXZmbcdu81D+U5jRSSUDf8RZ8u7nRb7eTpeUwikjtrVtTT2ToX9Ssb8pxGCkmoG/5mW3f+rfpB2w7MYxKR3Nl0q/6dbovGQv2WD71Q/9c//fqTIMPnV2XxMkYcsVf+A4nkQCwWY5d9dsq4bd/vfyfPaaSQhLrhjzhiL75/6ZFEYxHMjEg0QllFGddP/C8qquJBxxPZaNf99acMHLppW21DrCzKVjsP5pI7xgUdTQIU+tsyz7n1VMaeN4baydOpqK7gu2NrqO5dHXQskaz06teT+z76HW+/MIMFcz5nq2GD+dboXYhEQn2OF3qhb/gAm2+zGUede2jQMURyKhKJUHPIcGoOGR50FCkQ+t+9iEhIqOGLiISEGr6ISEio4YuIhIQavohISKjhi4iERE4avpn9wcyWmFnGJzaZ2f5mttLMprf9uiYX84p0J9W1lJpc3Yd/LzAeuH89+7zq7kfmaD6RfLgX1bWUkJyc4bv7FODLXBxLpFCorqXU5PMa/kgze9fMnjGzXTrbyczGmVmtmdXW1dXlMZ7IRlFdS9HIV8N/G9ja3YcDvwP+2tmO7j7B3WvcvWbAgM4fXyxSAFTXUlTy0vDdfZW7r2n7eRJQZmadP7RbpAiorqXY5KXhm9lAa1s52cz2bpt3WT7mFukuqmspNjm5S8fMHgT2B/qb2QLgWqAMwN3vBI4HzjWzFNAInOSdrcEmUiBU11JqctLw3f3kDWwfT+vtbSJFQ3UtpUbftBURCYmSafirvlzN3OmfUr+yPugoIjnj7iz4aBGfzZpPOp0OOo4UuaJf8SqVTPHrcyfw0gOvURaPkUqkOPr8MZx9039oOTcpav/8YCH/fdwtLJlXh0WMql5VXPGnixk+qtPb/UXWq+g74t1X/IlXHvw7yeYkDasaSTQlefKO53j8t5OCjiay0ZKJJD/Z/xoWzFlIc2OCpvpmvly0nKuO/F+WLVoedDwpUkXd8NPpNE/d+RzNjYm1xpsbmnn0F08GlEoke29NeodEY5J17/lJt6R5/v5XAskkxa+oG34q2UJinWb/ldXLVuc5jUjufLloOS2plg7jiaYkS+brVn/ZOEXd8MvjZQzabmDGbTt+e7s8pxHJnV333Qms43hljwr2GL1r/gNJSSjqhg9w4fiziFeV0/aFRyIRo6I6zjm/PDXgZCIbb+huW/OdI2uoqI5/PVZeWc7gHQcx8uiaAJNJMSv6u3T2PGh3fvm36/jTDX9h3vsL2GGvbfnBFcey9bAtg44mkpXLH7iIyfe8wtO/f45kIsUBP9iPYy48jFhZ0b9tJSBWyN8Er6mp8dra2qBjSIkys2nunvfTZdW1dKf11XXRX9IREZGuUcMXEQkJNXwRkZBQwxcRCQk1fBGRkMhJwzezP5jZEjOb2cl2M7PfmtlcM3vPzPbMxbwi3Ul1LaUmV2f49wJj1rP9MGD7tl/jgDtyNK9Id7oX1bWUkJw0fHefAny5nl3GAvd7qzeATcxs81zMLdJdVNdSavJ1DX8LYH671wvaxkSKmepaikrBfWhrZuPMrNbMauvq6oKOI5ITqmspBPlq+AuB9g+3Gdw21oG7T3D3GnevGTBgQF7CiWwk1bUUlXw1/InAD9vuavgOsNLdF3X3pJ9//AX/eLKW+XMyvgdFshVIXbe0tPDelPd5c9Lb1K9q6O7ppITk5LF7ZvYgsD/Q38wWANcCZQDuficwCTgcmAs0AKfnYt7OJJqT/PykX1E7eTqx8hipZAu77bcz//2X/0dFVXzDBxCh8Ooa4ON3P+Pyw35OU30TZkYq2cL5vzmdw886qLunlhKQk4bv7idvYLsD5+dirq64/9qHqX3uXRJNSRJNSQBmTHmfO39yH5fcMS5fMaTIFVpdt6RauOyQ61lRt2qt8dsvuYcdarZlu28NzVcUKVIF96FtLjz9fy90WPow0ZTk+fteoZAfBy2yPu+8NJNEc7LDeLI5xdMTng8gkRSbkmz4zQ3NGceTzSnS6XSe04jkRv3KBshwvpJuSbNq2Zr8B5KiU5INf/fvDcMyrAe604jtiEaj+Q8kkgO7jxpGKpnqMF5RHWffY0cEkEiKTUk2/PN+cwZVvaooi5cBECuPUdmjgotuPzvgZCIbr8+mvfmPa04gXhX/+oSmojrOtt8ayn7fV8OXDSvJxTG32mkL7n7/10y87VnmTJ3LNsOHcMwFY9h0K93/LMXt5MuOZdjIHXh6wgvUr2xg1AkjGX3yPlrnVrqkZKuk3+Z9OP1/1nuThUhRGj5qF4aP2iXoGFKESvKSjoiIdKSGLyISEmr4IiIhoYYvIhISavgiIiGhhi8iEhJq+CIiIaGGLyISEmr4IiIhoYYvIhISOWn4ZjbGzOaY2VwzuyzD9tPMrM7Mprf9OisX84p0N9W2lJKsn6VjZlHgNuBgYAEw1cwmuvv76+z6sLtfkO18Ivmi2pZSk4sz/L2Bue7+ibsngIeAsTk4rkjQVNtSUnLR8LcA5rd7vaBtbF3fN7P3zOwxM9uys4OZ2TgzqzWz2rq6uhzEE9loOatt1bUUgnx9aPskMMTddweeB+7rbEd3n+DuNe5eM2CAnl8vBa9Lta26lkKQi4a/EGh/VjO4bexr7r7M3b9aaPYuYK8czCvS3VTbUlJy0fCnAtub2VAzKwdOAia238HMNm/38mhgdg7mFeluqm0pKVnfpePuKTO7AJgMRIE/uPssM7sOqHX3icBFZnY0kAK+BE7Ldt5cSSaS/OXXT/PMXS+SSrUw+qR9OPny46jqWRl0NAlYsdf2go8Wcf+1DzPj1dn0G9SHky8/jn2O2TvoWBIgc/egM3SqpqbGa2tru+347s7lY/6Hma99QHNjAoCyeBlbbD+QO6bdrHVCS5yZTXP3mnzP2911DfD5x19w7l4/pWlNE+l063u8oirO6TeczHEXHdGtc0uw1lfXof6m7QdvzWXW63O+bvYAyeYkiz+r4/UnpgaYTCQ7f7zuUZrqm79u9gBNDc3ce9VDJJqTASaTIIW74b/5ES2pdIfxxjVNzPz7BwEkEsmNma99QLqlY21jsOiTxfkPJAUh1A1/wJb9iJVHO4zHK8sZOHTTABKJ5MZmW2e+9TOVbKHPZr3znEYKRagb/ogj9qSyRyUWsbXGo2VRDvr37wWUSiR7J19+LPGq+Fpj5RVl7DP22/Tq2zOgVBK0UDf8svIyfv3q9Wy/51DK4mWUV5QxeMdB3PLitfTqpzeFFK+9Dh7OhePPpEefaiqq45TFy9j3uBH85O7zgo4mAQr9bSibb7MZt711E8sXryCVbGHA4H5BRxLJiUNPG81B//E9lsxfSq9+PanuVRV0JAlY6Bv+V/pstknQEURyLhqLsvnQzYKOIQUi1Jd0RETCRA1fRCQk1PBFREJCDV9EJCTU8EVEQkINX0QkJHRbZh698dQ0nhj/DKtX1LPf90dw9LmHUtlDj2GW4ubp1XjDH6HpRYj0wapPxeL7BR1LMlDDz5N7rnmIv/zqKZrqWxdH+nTGP3nu3r9x29QbqVjnK/AixcLTa/Blx0DLEqC1tj0xFe9xPpEe44INJx3okk4eLF+8gkdvnfh1swdINCZYPK+O5+//W4DJRLLjDQ9DSx1fNftWjbDmd3h6VVCxpBM5afhmNsbM5pjZXDO7LMP2uJk93Lb9TTMbkot5i8X7//iQsvKOf5lqbmjmrUlvB5BIukq1vQHNLwFNHcetDJIz8h5H1i/rhm9mUeA24DBgGHCymQ1bZ7czgeXuvh3wK+CmbOdtz92ZU/sxbzw1jS+/WJ7LQ+dE7wG98HTHlcUi0Qh9N+8TQCLpikKo7cb6JqY++w7TX55JKpnK5aFzI7opYBk2tEBEtV1ocnENf29grrt/AmBmDwFjgffb7TMW+O+2nx8DxpuZeQ7WV1z6+Zdcdsj1LJ5XRyQaIdmcYuwFYxh38ymYZSrE/Bs2cgd69e9JU0PzWo2/LB7j6PMODTCZbECgtf23R17n1jNvJxJtPS+LlkW5/onL2OW7O2Z76Jyxqh/iTS+y9ll+BCJbQGznoGJJJ3JxSWcLYH671wvaxjLu4+4pYCWQ8bGUZjbOzGrNrLaurm6Dk193/K3Mn/M5TfXNNKxqJNmc5Kk7n+OVh1/fmH+WbhGJRLjlhWsZvMMgKqrjVPWqpKpnJZdOOIdthw8JOp50Lme1/U3retEni7n59Nu+ruuGVY2sXraGKw7/OY31GS6hBMTK94BeV4NVgvUAKiG2Pdb37oI54ZJ/Kbi7dNx9AjABWhd7Xt++S+Yv5ePpn3VYyq2pvpnHf/s0o0/ap/uCfkObb7MZd8/6FfPeX0DDqga223MbyuNlOTn26uVr+OCtuWwyoBfb7TFUb7QC9E3qGuC5+18hnWrJdBzefGoa+/9b4dR2pOoEvPIoSL4Pkd5YbNucHdtTc6Hlc4jtjEUzr+IlXZeLhr8Q2LLd68FtY5n2WWBmMaA3sCzbiRtWNRKNRYGOizKvWdGQ7eFzzswYssuWG97xG3jopsf5488epSxeRkuqhU236s//PnMlm26lN0cOBFbbq7+sJ5Xs2PDTLWnqVxZibVdA+Z45O56nV+LLx0FydusHwN6MV/0b1vMqndBkIReXdKYC25vZUDMrB04CJq6zz0Tg1LafjwdeysU1zi13HEQsw90vZeUx9jnm29kevuBNnTydB/7nzySaktSvbKCpvpkFHy7iqqNuDDpaqQistkccvgcVPSo6jHva2ePA3bI9fMHzFT+F5EygCXw1kICGx/CGR4KOVtSybvht1y0vACYDs4FH3H2WmV1nZke37XY30M/M5gI/Bjrc3rYxorEoP/6/c4hXlX/9wVa8spw+m2/CCf959Ab+dPF7/LdPr3VvP7SeAX7+8RfMm70goFSlI8ja3uuQ4ez+vZ2pqP7Xl/IqquMcdd6hDNp2YC6mKFieXgWJ1+j4N/dGaLgviEglIyfX8N19EjBpnbFr2v3cBJyQi7nWte+xI/jdP27gr+OfZfG8OmoOGc5hZx0YiuXcVi5dnXE8GouyZnl9ntOUpqBqOxKJcN0T/8WUR9/gpQdfpTxexpgzD6TmkOG5nqrweD2dnov6yrxGKTUF96Htxhi629Zc+vsfBR0j7/Y5Zm8+m/FPEk1rnwml0852ewzJ6VzuSWiZB9YHi2rd33yIRqOMPmmfgrr5IC8iAyGyCaQXr7MhCvH9cz6dp5dDehlEt6L1yl3p0qMVitjY88fQf3A/4pWtRWpmxKvinPfr04hX5u75POmGJ/AlI/Flx+N1o0h/eZa+Ni/dxsyw3jcAFUC0bbQcrDfW48KczePeRHrFJfiS/Vpre8kI0vX35+z4hagkzvDDqrpXFXdMu5lJ//cCbzw1jX6D+nDMhYez84jtczaHJ6bCqqtZ64s1iX/gKy7E+up6qnQPi+8H/f6MN9wLqXlQPgKr/ncs0jdnc/jKK1qf8EkCPNE6uPoXeHQQVnFQzuYpJGr4Ra6qZyXH//gojv/xUd1yfK+/m47PSklC4m28ZSEWXfd7SCK5YWXbY71/3i3H9vRqaHoOSKyzpRGvv7NkG35JXtJZvXwNX3y2hHQ6veGdZf1aPs88bmXQsjS/WULO3Vk8r46VS3U5LWvp5fzrctE6Wtb97KB0lNQZ/urla7jp1PG8/fx7RKJGVa8qLv39jxh5VE3Q0YpX+Xch9TEdbpHzFOTwG5WyftNfnsnNp41n1dLVpNNpho3ckSv+dDF9B+oBZRslOqjtC12N62yIQHnpfoenpM7wrz3mZqY99y7J5iTNDQmWf7GCn5/8K+a+82nQ0YqWVZ/R9oyU9ucGldDjAizSI6hYobLok8VcfdSN1M1fRnNjgmRzipmvfcBPD76OHHzHK5TMYtDzclo/GP5KBKwK63FxULG6Xck0/AUfLeLD2o9JJdZ+hGyiKcljv3wyoFTFz6KbYv2fgMoTILollO2BbXKLVjPKoyduf7bDo5FbUi0smbeU2W9+FFCq4hep+j7W53Yo2xuig6HiaKzf41hs66CjdZuiuqSzdOEyJt4+mbnvfMoOe23DUeeNoV/b8+Tr5i8lVh6juXHtD2E87Sz6pHSvyeWDRQdivX8WdIyS9u4rs5h014s01Tcx6sTvMuqEkW3PiYJFcxdnfK6OmVE3fxl8J99pS4fF98Xi+wYdI2+KpuF/OvOfXLLvVSSbkySbU0x/eRZ/Hf8sv3n952y982C2HT6EZHPHh6iVxWN8a/SuASQW6Zr7f/YIj9wykURjM+7w9gvvMfnel7lh0hVEo1G+dcAuTHvhXZob1j6ZSSVT7FCzTUCppRgVzSWd351/V9vz7lv/aptsTtKwqoHbLvoDAL369WTsBWPWevZINBahqmclx1x0eCCZRTakbsEyHr7przQ3tDZ7aH289/v/+JA3n2pd/vKQ00bTq19PYmX/uqskXhVn/5P2YfOhmwURW4pUUTR8d2fm3z/IMA7v/W3W16/PvukULrrtbIbuvhX9t+jLIaeN5o63b6HPpr3zGVeky6a/PJNIrOPtgU1rmnh94lSg9Qt2t9fexJHnHMKALfux1c6DGXfLKfzkrnPzHVeKXFFc0jEzyivKaW5o7rCt/SMEzIyDfziKg384Kp/xRDZada8qIhme7x6NRejZ9193QW0yoDfn/+YMzv/NGfmMJyWmKM7wAQ49bX/KK9ZeIaq8oowxZx4QUCKR7NUcOvzrR3u3Fy2LMeb00QEkklJWNA3/7JtPYbfvDSNeWU5Vr0rKK8sZPnpXzrzhB0FHE9lo5RXl/O+zV9Krf0+qelZS1auSeGU5F912FlsPy+3qaCJFcUkHoKIqzo3PXsW82QtYMOdzttp5C7bcUc9xkeK3097b8/DCCcx4dTbNDQl2HzWMqp6VQceSEpRVwzezvsDDwBDgM+BEd1+eYb8WYEbby3+6+0YvR7X1zoPZeufBG/vHRbok37UdK4uxxwGlv3ShBCvbSzqXAS+6+/bAi3S+vFuju3+r7Vfprz0opUC1LSUn24Y/Fvjqoej3AcdkeTyRQqHalpKTbcPfzN0Xtf38BdDZt0AqzKzWzN4ws/W+ccxsXNu+tXV1dVnGE9loOa1t1bUUgg1ewzezF4CBGTZd2f6Fu7uZdfbovq3dfaGZbQO8ZGYz3P3jTDu6+wRgAkBNTY0eBSjdaQczm5lhPOe1rbqWQrDBhu/unS79YmaLzWxzd19kZpsDSzo5xsK23z8xs1eAPYCMDV8kjz5094yLJai2pRRle0lnInBq28+nAk+su4OZ9TGzeNvP/YF9gPeznFeku6m2peRk2/BvBA42s4+Ag9peY2Y1ZnZX2z47A7Vm9i7wMnCju+tNIYVOtS0lJ6v78N19GXBghvFa4Ky2n18HdIOxFBXVtpSionm0goiIZEcNX0QkJNTwRURCQg1fRCQk1PBFREJCDV9EJCTU8EVEQkINX0QkJNTwRURCQg1fRCQk1PBFREJCDV9EJCTU8EVEQkINX7qdN79OetnxpBfvRXrpsXjzlKAjiWTNPUF6zXjSS/YjvfjbpFf8FG9ZHHSs9VLDl27lzVPw5edA8j3w1ZCahS+/gHTj80FHE8mKr7gY1kyA9GLwldD0JL7sODy9Ouhoncqq4ZvZCWY2y8zSZpZxqbi2/caY2Rwzm2tml2UzpxQXX3Uj0LTOaBOsuTGIOF2m2pb18dRcaP47a9d2C6RX441/DirWBmV7hj8TOA7o9O/oZhYFbgMOA4YBJ5vZsCznlWLR8mkn4/NxT+c3yzej2pbOJWeDRTNsaILE23mP01VZNXx3n+3uczaw297AXHf/xN0TwEPA2GzmlSISGZB53PpiVrhXFFXbsl7RLQHPsKEcYtvmO02X5eMdtwUwv93rBW1jGZnZODOrNbPaurq6bg8n3azH+WCV6wxWQo9zAomTY12ubdV1iSkbDtGt6LBKrMWwqn8LJFJXbLDhm9kLZjYzw69uOZNx9wnuXuPuNQMGdHJ2KEXDKk+EHpeC9QTKW3/vcT5WdWrQ0QB2yFdtq65Li5lhfe+D+CigDIhBbAes7x+x6MCg43Vqg4uYu/tBWc6xENiy3evBbWMSAmaGVZ+GV50CvgqsJ2YbLLt8+dDdO/1AtgtU2yFmkT5YnztwbwRPYZGeQUfaoHxc0pkKbG9mQ82sHDgJmJiHeaWAmEVb3yCF0+xzQbUtmFUWRbOH7G/LPNbMFgAjgafNbHLb+CAzmwTg7ingAmAyMBt4xN1nZRdbpHuptqUUZXW65e6PA49nGP8cOLzd60nApGzmEskn1baUosK9L05ERHJKDV9EJCRK6hM06bqlC5cx5bE3SDQlGXHEngzddaugI4lkzd2Z8epsZrw6mz6b9mbUiSOp7l0ddKyCoYYfQi8//HduPf12wGlJpfnjdY9y9HmH8qNbfhh0NJGNlkqmuGbsTcx4dTaJxgTlleX8/j/v56bnr2anvbcPOl5B0CWdkFmzop5fnHE7iaYEiaYkLakWEo0JnrrzOWa9vqEnCYgUrmfufon3psymqb6ZdNppqm+mYXUjPzv+VtwzPQYhfNTwQ+atZ94hEuv4n725IcGLD+g59VK8Jt/zMs0NzR3G16xo4LOZ/wwgUeFRww8bnelI6Kjmv6KGHzLfPmwPWlIdH0tcXlnOAT/YL4BEIrlx6OmjiVfFO4z36F3NEN2UAKjhh07PPj34yd3nUl5RRnlFGdFYlPLKco4852B23WenoOOJbLTDzjyA3fbdiYrqOBYxKqrjVPWs5JrH/hMzCzpeQdBdOiF0wEn7svt+OzPl0TdINCVab8vcbeugY4lkJVYW44ZnruS9Ke8z89UP6LOZbstclxp+SPXfoh/HXXJE0DFEcsrMGD5qF4aP2iXoKAVJl3REREJCDV9EJCTU8EVEQkINX0QkJNTwRURCItsVr04ws1lmljazTtcGNbPPzGyGmU03s9ps5hTJB9W2lKJsb8ucCRwH/L4L+45296VZzieSL6ptKTnZLnE4G9C32KTkqLalFOXrGr4Dz5nZNDMbt74dzWycmdWaWW1dXV2e4olstC7VtupaCsEGz/DN7AVgYIZNV7r7E12cZ193X2hmmwLPm9kH7p7xWbzuPgGYAFBTU6PH3El32sHMZmYYz3ltq66lEGyw4bv7QdlO4u4L235fYmaPA3sDevh6EXN3SH0AnoCyYZiVBR1pY3zo7p1+INsVqu3S4+k1kPoQIv2xWGk9ZbPbn6VjZtVAxN1Xt/18CHBdd88r3ceTH+LLfwS+HDAgBpvcisVHBR0tr1TbpSe95m5Y8xuwGHgSL9sF63M7FukbdLScyPa2zGPNbAEwEnjazCa3jQ8ys0ltu20GvGZm7wJvAU+7+7PZzCvBcU/gX54C6YXgDeD14Cvx5RfiLQuDjpczqu3w8aaXYc1vgSbwNUAzJN/Dl18YdLScyfYunceBxzOMfw4c3vbzJ8DwbOaRAtI8BUhk2NCCN/wZ63lRvhN1C9V2+HjDPUDjOqOp1qbfsgiLbh5ErJzSN23lm0kvA++4YhYkIa27T6SItXRSvxaD9PL8ZukmavjyzZTvDWRo+FaFxffNexyRnImPAjq5+SC2XV6jdBc1fPlGLDYUKscCle1GKyC2A8QPDCqWSNas+myI9AbK241WQM8rMSvv7I8VFa14Jd+Y9boeykfiDQ8BTVBxNFZ1ImYqJyleFu0H/Z/C6+9r/awqOhCrPgMr/3bQ0XJG71D5xswMKo/AKrVEopQWi/TFel4KPS8NOkq30CUdEZGQUMMXEQkJNXwRkZBQwxcRCQk1fBGRkFDDFxEJCTV8EZGQUMMXEQkJNXwRkZBQwxcRCQk1fBGRkMh2xatbzOwDM3vPzB43s0062W+Mmc0xs7lmdlk2c4rkg2pbSlG2Z/jPA7u6++7Ah8Dl6+5gZlHgNuAwYBhwspkNy3Jeke6m2paSk1XDd/fn3D3V9vINYHCG3fYG5rr7J+6eAB4CxmYzr0h3U21LKcrlNfwzgGcyjG8BzG/3ekHbWEZmNs7Mas2stq5OS+ZJQci6tlXXUgg22PDN7AUzm5nh19h2+1wJpIAHsg3k7hPcvcbdawYMGJDt4UTWZ4d81bbqWgrBBhdAcfeD1rfdzE4DjgQOdHfPsMtCYMt2rwe3jYkE7UN3r+lso2pbSk22d+mMAX4KHO3uDZ3sNhXY3syGWuvCkCcBE7OZV6S7qbalFGV7DX880BN43symm9mdAGY2yMwmAbR98HUBMBmYDTzi7rOynFeku6m2peRktaatu2/XyfjnwOHtXk8CJmUzl0g+qbalFGkRcyk57mlIvIY3vQSRXljlMVhsm6BjiWTN06vwxschNRdiu2CVR2GR6i7/eTV8KSnuLfiK8yHxBngDEMPr78V7XU+kSrfIS/Hy1N54wcoAAAOCSURBVKf4shPBE0AjUInXj4d+j2HRgV06hp6lI6Wl+YV2zR5a76hsglVX4+k1QSYTyYqvvAp8Fa3Nntbf08vwVTd2+Rhq+FJSvPGpds2+HYu2/o9ApAi5JyE5DVj37uAWSLzc5eOo4UtpsQrAMm0Ai+c7jUiOROi8XZd9o6OIlAyrPB6oyLAlAuUj8h1HJCfMohA/kI4fu5ZDRdc/m1LDl5Ji8RFQfRoQByrBqsGqsT530vrdKJHiZL1/BtEhrTVNBVgVlA3Dev64y8fQXTpSciI9L8UrT4DE661vjvhoLFIVdCyRrFikL/R/ChJvQss8iO0AZXtglukSZmZq+FKSLDYYYicGHUMkp8wiEB8JjNyoP69LOiIiIaGGLyISEmr4IiIhoYYvIhISavgiIiFhmRfyKQxmVgfM68Yp+gNLu/H4uVAMGaE4c27t7nlfb1B1/TXlzJ0u1XVBN/zuZma161virhAUQ0ZQzkJSLP+Mypk7Xc2oSzoiIiGhhi8iEhJhb/gTgg7QBcWQEZSzkBTLP6Ny5k6XMob6Gr6ISJiE/QxfRCQ01PBFREIi1A3fzE4ws1lmljazgrvtyszGmNkcM5trZpcFnScTM/uDmS0xs5lBZ+mMmW1pZi+b2ftt/70vDjpTdyvk2i6GuobSrO1QN3xgJnAcMCXoIOsysyhwG3AYMAw42cyGBZsqo3uBMUGH2IAU8BN3HwZ8Bzi/QP9d5lJB1nYR1TWUYG2HuuG7+2x3nxN0jk7sDcx190/cPQE8BHR9LbM8cfcpwJdB51gfd1/k7m+3/bwamA1sEWyq7lXAtV0UdQ2lWduhbvgFbgtgfrvXCyjxJpUPZjYE2AN4M9gkoaW67iZdqe2SX/HKzF4ABmbYdKW7P5HvPBIcM+sB/Bm4xN1XBZ0nW6pt+UpXa7vkG767HxR0ho20ENiy3evBbWOyEcysjNY3xAPu/peg8+RCkda26jrHvklt65JO4ZoKbG9mQ82sHDgJmBhwpqJkras83w3MdvdfBp0n5FTXOfRNazvUDd/MjjWzBbSuCPy0mU0OOtNX3D0FXABMpvWDmEfcfVawqToysweBfwA7mtkCMzsz6EwZ7AOcAhxgZtPbfh0edKjuVKi1XSx1DaVZ23q0gohISIT6DF9EJEzU8EVEQkINX0QkJNTwRURCQg1fRCQk1PBFREJCDV9EJCT+P2uLgQUAFhskAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HUjdeIefsIsg", + "outputId": "f267f505-8ba4-43ef-9ebe-df124c3c05a1" + }, + "source": [ + "tf.reduce_mean(tf.cast(((pred[0]>0.5)==valid_labels),tf.float32))" + ], + "execution_count": 28, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 28 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_95qF9lY2kHp" + }, + "source": [ + "## Используем оптимизаторы TensorFlow\r\n", + "\r\n", + "Tensorflow достаточно плотно интегрирован с библиотекой Keras, которая содержит в себе множество полезного. Например, мы можем использовать оптимизаторы, реализующие немного другие алгоритмы обучения, чем градиентный спуск.\r\n", + "\r\n", + "Также попробуем выводить точность на всех этапах обучения." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ups7nlV22ofp", + "outputId": "aa4dff06-82b9-4b2f-ca00-33970ea2b989" + }, + "source": [ + "optimizer = tf.keras.optimizers.Adam(0.01)\r\n", + "\r\n", + "learning_rate = 0.05\r\n", + "\r\n", + "W = tf.Variable(tf.random.normal(shape=(2,1)))\r\n", + "b = tf.Variable(tf.zeros(shape=(1,),dtype=tf.float32))\r\n", + "\r\n", + "@tf.function\r\n", + "def train_on_batch(x, y):\r\n", + " vars = [W, b]\r\n", + " with tf.GradientTape() as tape:\r\n", + " z = tf.sigmoid(tf.matmul(x, W) + b)\r\n", + " loss = tf.reduce_mean(tf.keras.losses.binary_crossentropy(z,y))\r\n", + " correct_prediction = tf.equal(tf.round(y), tf.round(z))\r\n", + " acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\r\n", + " grads = tape.gradient(loss, vars)\r\n", + " optimizer.apply_gradients(zip(grads,vars))\r\n", + " return loss,acc\r\n", + "\r\n", + "for epoch in range(40):\r\n", + " for step, (x, y) in enumerate(dataset):\r\n", + " loss,acc = train_on_batch(tf.reshape(x,(-1,2)), tf.reshape(y,(-1,1)))\r\n", + " print('Epoch %d: last batch loss = %.4f, acc = %.4f' % (epoch, float(loss),acc))" + ], + "execution_count": 30, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 6.1202, acc = 0.6667\n", + "Epoch 1: last batch loss = 5.9872, acc = 0.6667\n", + "Epoch 2: last batch loss = 5.8651, acc = 0.6667\n", + "Epoch 3: last batch loss = 5.7551, acc = 0.8333\n", + "Epoch 4: last batch loss = 5.6570, acc = 0.8333\n", + "Epoch 5: last batch loss = 5.5700, acc = 0.8333\n", + "Epoch 6: last batch loss = 5.4931, acc = 0.8333\n", + "Epoch 7: last batch loss = 5.4255, acc = 0.8333\n", + "Epoch 8: last batch loss = 5.3660, acc = 0.8333\n", + "Epoch 9: last batch loss = 5.3140, acc = 0.8333\n", + "Epoch 10: last batch loss = 5.2685, acc = 0.8333\n", + "Epoch 11: last batch loss = 5.2289, acc = 0.8333\n", + "Epoch 12: last batch loss = 5.1945, acc = 0.8333\n", + "Epoch 13: last batch loss = 5.1649, acc = 0.8333\n", + "Epoch 14: last batch loss = 5.1394, acc = 0.8333\n", + "Epoch 15: last batch loss = 5.1177, acc = 0.8333\n", + "Epoch 16: last batch loss = 5.0993, acc = 0.8333\n", + "Epoch 17: last batch loss = 5.0840, acc = 0.8333\n", + "Epoch 18: last batch loss = 5.0714, acc = 0.8333\n", + "Epoch 19: last batch loss = 5.0612, acc = 0.8333\n", + "Epoch 20: last batch loss = 5.0532, acc = 0.8333\n", + "Epoch 21: last batch loss = 5.0472, acc = 0.6667\n", + "Epoch 22: last batch loss = 5.0430, acc = 0.6667\n", + "Epoch 23: last batch loss = 5.0404, acc = 0.6667\n", + "Epoch 24: last batch loss = 5.0391, acc = 0.6667\n", + "Epoch 25: last batch loss = 5.0391, acc = 0.6667\n", + "Epoch 26: last batch loss = 5.0403, acc = 0.8333\n", + "Epoch 27: last batch loss = 5.0424, acc = 0.8333\n", + "Epoch 28: last batch loss = 5.0454, acc = 0.8333\n", + "Epoch 29: last batch loss = 5.0491, acc = 0.8333\n", + "Epoch 30: last batch loss = 5.0535, acc = 0.8333\n", + "Epoch 31: last batch loss = 5.0584, acc = 0.8333\n", + "Epoch 32: last batch loss = 5.0638, acc = 0.8333\n", + "Epoch 33: last batch loss = 5.0696, acc = 0.8333\n", + "Epoch 34: last batch loss = 5.0756, acc = 0.8333\n", + "Epoch 35: last batch loss = 5.0818, acc = 0.8333\n", + "Epoch 36: last batch loss = 5.0882, acc = 0.8333\n", + "Epoch 37: last batch loss = 5.0946, acc = 0.8333\n", + "Epoch 38: last batch loss = 5.1010, acc = 0.8333\n", + "Epoch 39: last batch loss = 5.1073, acc = 0.8333\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dvAiaj_JndyP" + }, + "source": [ + "**Задание 1**: Постройте графики ошибок на обучающей и тестовой выборке в процессе обучения\r\n", + "\r\n", + "**Задание 2**: Попробуйте решить задачу классификации на датасете MNIST с помощью этого кода. Подсказка: используйте `softmax_crossentropy_with_logits` или `sparse_softmax_cross_entropy_with_logits` в качестве функции ошибки. При этом в первом случае на выход сети необходимо подавать целевые значения в формате *one hot encoding*, а во втором - в виде целочисленного номера класса." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "995iCprDrgYQ" + }, + "source": [ + "## Keras\r\n", + "### Deep Learning for Humans\r\n", + "\r\n", + "* Раньше работал поверх Tensorflow, CNTK или Theano, сейчас включен в состав Tensorflow\r\n", + "* Оперирует нейросетями на уровне слоёв\r\n", + "* Включает упрощённый \"обучатель\", средства работы с типовыми данными (картинками, ...)\r\n", + "* Много готовых примеров\r\n", + "* Functional API vs. Sequential API\r\n", + "\r\n", + "Keras даёт более высокоуровневое API для реализации нейросетей, позволяя определять нейросети как комбинации слоёв и оперировать понятиями \"модель\", \"слой\", \"алгоритм обучения\".\r\n", + "\r\n", + "Книжка от создателя Keras: [Deep Learning with Python](https://www.manning.com/books/deep-learning-with-python)" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "QJWplVfy34Eo", + "outputId": "9be976f2-4f9a-495c-bddc-a7f9ec30989a" + }, + "source": [ + "inputs = tf.keras.Input(shape=(2,))\r\n", + "z = tf.keras.layers.Dense(1,kernel_initializer='glorot_uniform',activation='sigmoid')(inputs)\r\n", + "model = tf.keras.models.Model(inputs,z)\r\n", + "\r\n", + "train_x_norm = train_x-np.min(train_x) / (np.max(train_x)-np.min(train_x))\r\n", + "\r\n", + "model.compile(tf.keras.optimizers.Adam(0.1),'binary_crossentropy',['accuracy'])\r\n", + "model.summary()\r\n", + "h = model.fit(train_x_norm,train_labels,batch_size=8,epochs=15)" + ], + "execution_count": 32, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Model: \"model_1\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "input_2 (InputLayer) [(None, 2)] 0 \n", + "_________________________________________________________________\n", + "dense_1 (Dense) (None, 1) 3 \n", + "=================================================================\n", + "Total params: 3\n", + "Trainable params: 3\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n", + "Epoch 1/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 1.1443 - accuracy: 0.3283\n", + "Epoch 2/15\n", + "9/9 [==============================] - 0s 2ms/step - loss: 0.6759 - accuracy: 0.5823\n", + "Epoch 3/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4717 - accuracy: 0.7888\n", + "Epoch 4/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.3932 - accuracy: 0.9044\n", + "Epoch 5/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4254 - accuracy: 0.8513\n", + "Epoch 6/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.3866 - accuracy: 0.8534\n", + "Epoch 7/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4578 - accuracy: 0.8344\n", + "Epoch 8/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4432 - accuracy: 0.8192\n", + "Epoch 9/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4172 - accuracy: 0.8408\n", + "Epoch 10/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.3778 - accuracy: 0.8819\n", + "Epoch 11/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4008 - accuracy: 0.8800\n", + "Epoch 12/15\n", + "9/9 [==============================] - 0s 3ms/step - loss: 0.3769 - accuracy: 0.9019\n", + "Epoch 13/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.3995 - accuracy: 0.8336\n", + "Epoch 14/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.3816 - accuracy: 0.8719\n", + "Epoch 15/15\n", + "9/9 [==============================] - 0s 1ms/step - loss: 0.4211 - accuracy: 0.8244\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "K2Kf60IrZcqs", + "outputId": "b60b868d-3562-4715-f5d5-1f9764e45f09" + }, + "source": [ + "plt.plot(h.history['accuracy'])" + ], + "execution_count": 37, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 37 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAbKElEQVR4nO3dfXRU933n8fdXT0iAAAkE2EJIGAtsbKd+EPhpm+MmddZJeiDd7Db41E+JU7a7cZJts921d3u8OW67p/vUdM+ptz3eOMa1U7Neb9LQLq3tk6Yn3fgBCRs/AJYgWAIBGgQSYiTQ43z3jxnhsRBoECPdufd+Xudw0Ny5zHyA0Ud3fvd3f2PujoiIhF9R0AFERCQ/VOgiIhGhQhcRiQgVuohIRKjQRUQioiSoJ16yZIk3NDQE9fQiIqG0a9euE+5eM9l9gRV6Q0MDLS0tQT29iEgomVnHhe7TkIuISESo0EVEIkKFLiISESp0EZGIUKGLiESECl1EJCJU6CIiERHYPHSZvv6hUX74VidXLKzglvoqquaVBR1JRAqACj1kegeGeeiZnbzT2XduW+PS+TQ1VNFUX836hmrqqiswswBTikgQVOgh0tU3yP1Pv0lHzxn+9NdvpnpeGS0dvTS39/DX7x7jhZ2HAaipnMP6rIK/9opKSoo1uiYSdSr0kGg/McB9T79J78Awz355A7evXgzArVelf0+lnLbjSZrbe9nV3kNzey873usCYG5ZMTetXERTfTVNDVXctLKK+XP0Xy8SNRbUR9A1NTW51nLJzb5jp7n/6Z2MpVI8+5UNfGLFopz+3NFTZ2np+KjgP+g6TcqhyGDdlQvOFfz6hmqWLSif4b+FiOSDme1y96ZJ71OhF7ZdHb18+ZmdzC0r4bmHN9C4rHLaj5UcHOHtQ6doyRT87sOnODsyBkBddQVN9enhmeKi/A/PLK2cQ1NDFVcsrMj7Y+fL4MgY7x3pY+/R04ym9Fm7M6Gs2LhhxSKuu3IBpQU6DOjuHO45y65DPfQMjMzIc9yxejHXXrFgWn/2YoWu990F7B/2d7Plz3exbMEcnnv4Vuqq517W41WWl/LJNTV8ck165c2RsRR7j56mub2HXR29/MP+E/zw7SP5iH5BtYsq0idwG6pZ31DFmqWVFBUFcwK3d2CYXR29NHf00NLey3udfQyPpQLJEjflpUXcWLeI9Q3V3FJfxc31VSwoLw0ky+hYin3HkrRkXgfN7T0cTw7N6HP+/heun3ahX4yO0AvU37x3jG9se5vVNfP584c3sLRy5odE3J3k0Ch5f0k4HOo5c+4HR/Y3TGV5CbfUp4d9muqr+IW6RZSXFuc5QPrvdqjnDC3tvbR0pN+hHDjeD0BpsXFD7UKaZjiDwMDQKLsPn6K5PV2ee4+dZizlmME1yxewvqHq3OvhykUz825uYGiUtw+dOvd6fOtQL2eG0+9Usw84muqrZixDeWkRc0qm9xrTkEvIvNh8mEd/8C43raziew+tZ2FFMEcuM2X8Le14sba097A/q1yvr11IU/1H31SL58+55OcYHUux99jp9EnizPN0Z/0QyX58FXhwsgt+V0cvb3X0MpBVrulyr+KW+mrWLq+keBrv5hKnB88debd09LDvWPLcD5Frly+YlQLPJxV6iPzPnx7kD3bs45Nraviz+25mblk8RsVOnckMf2QK+J3DHw1/XLVkXtYwTTUNi+eeN8++f2iUtw/1nvsBsfvwqXNHXSuqKs4V+PqGahqXzg9smEcubnQsxQddyfR5no70/2XidOYH8ZwSbq6vOvd/eWPdIirKPv6DOJVyDnT3p39AtKeH0w73nAXSR8U31WV+QDRUc9PKRYEN81wOFXoIuDv/7ZU2/uQnB/j8DVfwnS/dSFlJYZ40mg2DI2O8f6SPlsw3dUtHL6fOpE9QLZ5XRlNDFTfWVZE4PUhzew/7jn00g+faKxacG5st9BOxcnHuTmfvR+/mdrX30ppIAlBSZFxXu5D1mauld3X0squjl76z6dfJkvlzMuWfHsJZV8AnYi+FCr3ApVLOf9i+h+fe6GDz+jr+4FdvmNZbyyhLpZyDJ/ppzrx13tXRS8fJM1SUZubYZ06yao599PWdGeGtQ73nxuF3d55ieDTF6pp56XMxmeGT+kneyUWBCr2AjYyl+Nf/+x1+tPso//yTV/HoZ6+J5ItwJvQODDO/vCQSR10yfUOjYwwOp1g4N3zDJ9OhaYsFanBkjK99/y1+/MFxfucfr+Vf3rVaZX4JtCiZAMwpKZ72jJGoUaEHJDk4wlefbWFnew+/94Xruf+2+qAjiUjIqdADcLJ/iAef2ckHx5L88ZduZNONtUFHEpEIUKHPsqOnznL/02/S2XuWpx64hU9dsyzoSCISESr0WfThiQHu++6bnD47wnMP38qGVdVBRxKRCFGhz5I9R/t48Hs7cYcXttzG9bULg44kIhGjQp8Fze09fGVrM5VzSnjuq7eyumZ+0JFEJIJU6DPs71uP85vP7+LKhRU899VbqQ3BWhEiEk4q9BnUOzDMv3j+La5akl4xcck0FpkSEcmVLrGbQduaD3N2ZIzvfOlGlbmIzLicCt3M7jGzVjM7YGaPTnL/SjP7iZm9bWbvmtnn8h81XEbHUjz3ejt3rF7M2uXT/5QhEZFcTVnoZlYMPAl8FlgH3Gtm6ybs9rvAi+5+E7AZ+B/5Dho2r+5NcLRvkIfuaAg6iojERC5H6BuAA+5+0N2HgW3Apgn7ODD+eUoLgaP5ixhOW19rZ0VVBZ++VhcOicjsyKXQa4HDWbc7M9uyfRu4z8w6gR3A1/OSLqT2Hj3Nmx/28MDt9VoGV0RmTb5Oit4LbHX3FcDngOfM7LzHNrMtZtZiZi3d3d15eurC8+xr7VSUFvOlppVBRxGRGMml0I8AdVm3V2S2ZXsYeBHA3V8HyoElEx/I3Z9y9yZ3b6qpqZle4gLXMzDMX+4+wq/eXBub9ZlFpDDkUujNQKOZrTKzMtInPbdP2OcQ8GkAM7uWdKFH9xD8IrY1H2JoNKWToSIy66YsdHcfBR4BXgb2kZ7NssfMnjCzjZndvgX8hpm9A7wAPORBfRRSgEbHUjz/egd3rF7MmmWaqigisyunK0XdfQfpk53Z2x7P+novcGd+o4XP+FTFb2+8LugoIhJDulI0j57RVEURCZAKPU/2HO1j54c9PHh7g6YqikggVOh5Mj5V8dea6qbeWURkBqjQ8yA9VfEo/0RTFUUkQCr0PNjWfIjh0RQPaqqiiARIhX6Z0qsqdnDn1ZqqKCLBUqFfplf2JjjWN8hDd6wKOoqIxJwK/TJt/Vk7ddUVfOqapUFHEZGYU6Ffhj1H+9jZrqmKIlIYVOiXYXyq4j/TVEURKQAq9Gn62FTFCk1VFJHgqdCn6YWd6amKWlVRRAqFCn0aRsZSPP9GB//o6iU0aqqiiBQIFfo0vLJnfKpiQ9BRRETOUaFPw9bXPqSuuoJf0lRFESkgKvRL9P6RPprbezVVUUQKjgr9EmmqoogUKhX6JTjZP8SP3jnKF2/RVEURKTwq9EuwrflwelXF2xuCjiIich4Veo5GMqsqaqqiiBQqFXqOXtmToOu0piqKSOFSoedo62sfsrJ6rqYqikjBUqHnYHyq4gO312uqoogULBV6Dra+1s7cMk1VFJHCpkKfwon+IbZrVUURCQEV+hS27TzE8JimKopI4VOhX0R6VcVD/GKjpiqKSOHLqdDN7B4zazWzA2b26CT3f8fMdmd+tZnZqfxHnX0v7+nSVEURCY2SqXYws2LgSeBuoBNoNrPt7r53fB93/62s/b8O3DQDWWfd1p+1U794Lr+0VlMVRaTw5XKEvgE44O4H3X0Y2AZsusj+9wIv5CNckN7r7KOlo5cHbm+gSFMVRSQEcin0WuBw1u3OzLbzmFk9sAr4uwvcv8XMWsyspbu7+1KzzqqPpiquCDqKiEhO8n1SdDPwkruPTXanuz/l7k3u3lRTU5Pnp86fE/1D/NU7R/nizStYUK6piiISDrkU+hEg+4qaFZltk9lMBIZbzk1VvKM+6CgiIjnLpdCbgUYzW2VmZaRLe/vEnczsGqAKeD2/EWfXyFiK597o4Bcbl3D1Uk1VFJHwmLLQ3X0UeAR4GdgHvOjue8zsCTPbmLXrZmCbu/vMRJ0df/t+F4nTQ3z5zoago4iIXJIppy0CuPsOYMeEbY9PuP3t/MUKztbX0lMV71qjqYoiEi66UjTLe5197NJURREJKRV6Fk1VFJEwU6FnjE9V/Ke3aKqiiISTCj3jhTfTUxUf0KqKIhJSKvSMH39wnKb6Kq5eOj/oKCIi06JCB1IpZ38iyfW1C4OOIiIybSp04MipswwMj7FGa56LSIip0IG2RBKAtcs13CIi4aVCB1ozha5PJRKRMFOhA21dSa5cWK7piiISaip0oDXRz5rlOjoXkXCLfaGPjqX4+fF+1mq4RURCLvaF3tFzhuGxlMbPRST0Yl/obV2ZGS4qdBEJudgXemsiiRm6QlREQi/2hd6WSFJfPZeKsuKgo4iIXJbYF3prV1JXiIpIJMS60AdHxmg/eYa1mrIoIhEQ60I/2D3AWMp1hC4ikRDrQv9oDRcVuoiEX6wLvTWRpLTYaFg8L+goIiKXLdaF3taV5Kol8ykrifU/g4hERKybrDWR1BouIhIZsS30/qFROnvPsnaZLigSkWiIbaHv1xroIhIxMS70fkBruIhIdORU6GZ2j5m1mtkBM3v0Avv8mpntNbM9ZvYX+Y2Zf62JJOWlRdRVzw06iohIXpRMtYOZFQNPAncDnUCzmW13971Z+zQCjwF3unuvmS2dqcD50pZI0ri0kuIiCzqKiEhe5HKEvgE44O4H3X0Y2AZsmrDPbwBPunsvgLsfz2/M/NMaLiISNbkUei1wOOt2Z2ZbtjXAGjP7mZm9YWb35CvgTOgdGOZ4coi1yzXDRUSiY8ohl0t4nEbgLmAF8FMzu8HdT2XvZGZbgC0AK1euzNNTX7rxS/51hC4iUZLLEfoRoC7r9orMtmydwHZ3H3H3D4E20gX/Me7+lLs3uXtTTU3NdDNfNq3hIiJRlEuhNwONZrbKzMqAzcD2Cfv8Jemjc8xsCekhmIN5zJlXrYkkleUlLF9QHnQUEZG8mbLQ3X0UeAR4GdgHvOjue8zsCTPbmNntZeCkme0FfgL8jrufnKnQl6utq5+1yyox0wwXEYmOnMbQ3X0HsGPCtsezvnbgtzO/Cpq705pI8vlPXBF0FBGRvIrdlaLHk0P0nR3RFaIiEjmxK/TWLs1wEZFoil2hfzRlUXPQRSRaYlnoS+aXsXj+nKCjiIjkVewKvTXRr+EWEYmkWBV6KuXsT2gNFxGJplgV+pFTZzkzPKYrREUkkmJV6JrhIiJRFq9C1wwXEYmwWBV6WyJJ7aIKKstLg44iIpJ3sSr09Ida6OhcRKIpNoU+MpbiYPcAa3RCVEQiKjaF3nFygOGxlNZwEZHIik2ht3b1A5rhIiLRFZ9CTyQpMrh6qcbQRSSaYlPobV1J6hfPo7y0OOgoIiIzIj6FflwzXEQk2mJR6IMjY7SfGNAJURGJtFgU+s+7+0k5mrIoIpEWi0If/1ALHaGLSJTFotBbu/opLTYalswLOoqIyIyJRaG3JZKsrplPaXEs/roiElOxaLj0Gi4abhGRaIt8oScHRzhy6qw+1EJEIi/yhb7/uC75F5F4iHyht3VphouIxEPkC701kaSitJgVVRVBRxERmVE5FbqZ3WNmrWZ2wMweneT+h8ys28x2Z359Nf9Rp6ctkb7kv6jIgo4iIjKjSqbawcyKgSeBu4FOoNnMtrv73gm7/i93f2QGMl6W1q5+7lpbE3QMEZEZl8sR+gbggLsfdPdhYBuwaWZj5UfPwDAn+oc0fi4isZBLodcCh7Nud2a2TfRFM3vXzF4ys7rJHsjMtphZi5m1dHd3TyPupRm/5F9ruIhIHOTrpOhfAQ3u/gngVeDZyXZy96fcvcndm2pqZn4YRGu4iEic5FLoR4DsI+4VmW3nuPtJdx/K3PwucEt+4l2e1q4kC8pLWLZgTtBRRERmXC6F3gw0mtkqMysDNgPbs3cwsyuybm4E9uUv4vS1JZKsXV6JmWa4iEj0TVno7j4KPAK8TLqoX3T3PWb2hJltzOz2DTPbY2bvAN8AHpqpwLlyd63hIiKxMuW0RQB33wHsmLDt8ayvHwMey2+0y5M4PcTpwVGt4SIisRHZK0Vbx2e46AhdRGIisoU+voaLCl1E4iKyhd6aSFJTOYfqeWVBRxERmRWRLfS2RFLzz0UkViJZ6KmU05ZI0rhsftBRRERmTSQL/XDvGQZHUjpCF5FYiWShtyUyn1KkKYsiEiMRLfT0DJfGpRpyEZH4iGSht3YlqV1UQWV5adBRRERmTSQLfXwNFxGROIlcoY+Mpfh5d78uKBKR2IlcobefGGBkzFm7XOPnIhIvkSt0reEiInEVuUJv60pSZLC6RkfoIhIvkSv01kSShiXzKC8tDjqKiMisilyhtyX6dYWoiMRSpAp9cGSM9pMDGj8XkViKVKEfON6Pu06Iikg8RarQWzMfaqEpiyISR5Eq9LbjScqKi6hfPC/oKCIisy5ahd6V5KqaeZQWR+qvJSKSk0g1X1uiX2u4iEhsRabQk4MjHDl1VidERSS2IlPo4x9qoTnoIhJXESr08RkuKnQRiafIFHprV5K5ZcXULqoIOoqISCByKnQzu8fMWs3sgJk9epH9vmhmbmZN+YuYm7ZEksZllRQV2Ww/tYhIQZiy0M2sGHgS+CywDrjXzNZNsl8l8E3gzXyHzEVbIsnaZbqgSETiK5cj9A3AAXc/6O7DwDZg0yT7/R7wn4DBPObLyYn+IU70D2uGi4jEWi6FXgsczrrdmdl2jpndDNS5+//NY7ac6YSoiEgeToqaWRHwR8C3cth3i5m1mFlLd3f35T71OW1d+pQiEZFcCv0IUJd1e0Vm27hK4Hrg782sHbgN2D7ZiVF3f8rdm9y9qaamZvqpJ2hN9LOwopSllXPy9pgiImGTS6E3A41mtsrMyoDNwPbxO929z92XuHuDuzcAbwAb3b1lRhJPYn8iydpllZhphouIxNeUhe7uo8AjwMvAPuBFd99jZk+Y2caZDjgVd6c1kWSNlswVkZgryWUnd98B7Jiw7fEL7HvX5cfKXdfpQZKDo7rkX0RiL/RXirbqhKiICBCBQh+fsqhCF5G4C32ht3b1s7RyDlXzyoKOIiISqNAXelsiqQuKREQIeaGPpZz9x5MabhERIeSFfrjnDIMjKc1wEREh5IXeOn5CVEMuIiLhLvTxNVwal+qiIhGRUBd6ayLJiqoK5s3J6fooEZFIC3Wht2XWcBERkRAX+vBoioPdAxo/FxHJCG2ht58cYDTlOkIXEckIbaFrDRcRkY8LbaG3JZIUFxlX1cwLOoqISEEIbaG3diVpWDyX8tLioKOIiBSE0Ba61nAREfm4UBb62eExOnrOaPxcRCRLKAv9wPF+3NEMFxGRLKEsdK3hIiJyvlAWelsiSVlJEfXVc4OOIiJSMEJZ6K1dSVbXzKekOJTxRURmRCgbMb2Gi1ZYFBHJFrpC7zs7wrG+QY2fi4hMELpCP3A8fUJUM1xERD4udIXe2tUPaA0XEZGJQlfoS+aXcfe6ZdQuqgg6iohIQQndR/185rrlfOa65UHHEBEpODkdoZvZPWbWamYHzOzRSe7/TTN7z8x2m9n/M7N1+Y8qIiIXM2Whm1kx8CTwWWAdcO8khf0X7n6Du98I/Gfgj/KeVERELiqXI/QNwAF3P+juw8A2YFP2Du5+OuvmPMDzF1FERHKRyxh6LXA463YncOvEnczsa8BvA2XApyZ7IDPbAmwBWLly5aVmFRGRi8jbLBd3f9LdVwP/FvjdC+zzlLs3uXtTTU1Nvp5aRETIrdCPAHVZt1dktl3INuALlxNKREQuXS6F3gw0mtkqMysDNgPbs3cws8asm58H9ucvooiI5GLKMXR3HzWzR4CXgWLge+6+x8yeAFrcfTvwiJn9MjAC9AIPzmRoERE5n7kHMyHFzLqBjmn+8SXAiTzGmWlhyhumrBCuvGHKCuHKG6ascHl569190pOQgRX65TCzFndvCjpHrsKUN0xZIVx5w5QVwpU3TFlh5vKGbi0XERGZnApdRCQiwlroTwUd4BKFKW+YskK48oYpK4Qrb5iywgzlDeUYuoiInC+sR+giIjKBCl1EJCJCV+hTrc1eKMyszsx+YmZ7zWyPmX0z6Ey5MLNiM3vbzP466CwXY2aLzOwlM/vAzPaZ2e1BZ7oYM/utzOvgfTN7wczKg86Uzcy+Z2bHzez9rG3VZvaqme3P/F4VZMZxF8j6XzKvhXfN7IdmtijIjOMmy5p137fMzM1sSb6eL1SFnuPa7IViFPiWu68DbgO+VsBZs30T2Bd0iBz8d+Bv3f0a4Bco4MxmVgt8A2hy9+tJX3G9OdhU59kK3DNh26PAj929Efhx5nYh2Mr5WV8Frnf3TwBtwGOzHeoCtnJ+VsysDvgMcCifTxaqQieHtdkLhbsfc/e3Ml8nSRdObbCpLs7MVpBei+e7QWe5GDNbCHwSeBrA3Yfd/VSwqaZUAlSYWQkwFzgacJ6PcfefAj0TNm8Cns18/SwFsujeZFnd/RV3H83cfIP0IoKBu8C/K8B3gH9Dnj87ImyFPtna7AVdkgBm1gDcBLwZbJIp/THpF1kq6CBTWAV0A89khoe+a2bzgg51Ie5+BPivpI/GjgF97v5KsKlysszdj2W+7gKWBRnmEnwF+JugQ1yImW0Cjrj7O/l+7LAVeuiY2Xzg/wD/asInOxUUM/sV4Li77wo6Sw5KgJuBP3X3m4ABCmc44DyZsedNpH8QXQnMM7P7gk11aTw9v7ng5zib2b8nPdz5/aCzTMbM5gL/Dnh8Jh4/bIV+qWuzB8rMSkmX+ffd/QdB55nCncBGM2snPZT1KTN7PthIF9QJdLr7+Duel0gXfKH6ZeBDd+929xHgB8AdAWfKRcLMrgDI/H484DwXZWYPAb8C/LoX7gU2q0n/YH8n8722AnjLzJbn48HDVuhTrs1eKMzMSI/x7nP3gv/QbHd/zN1XuHsD6X/Xv3P3gjyKdPcu4LCZrc1s+jSwN8BIUzkE3GZmczOvi09TwCdxs2zno6WwHwR+FGCWizKze0gPF2509zNB57kQd3/P3Ze6e0Pme60TuDnzmr5soSr0zEmP8bXZ9wEvuvueYFNd0J3A/aSPdHdnfn0u6FAR8nXg+2b2LnAj8B8DznNBmXcSLwFvAe+R/r4rqEvVzewF4HVgrZl1mtnDwB8Cd5vZftLvMv4wyIzjLpD1T4BK4NXM99qfBRoy4wJZZ+75CvediYiIXIpQHaGLiMiFqdBFRCJChS4iEhEqdBGRiFChi4hEhApdRCQiVOgiIhHx/wHO1VhtMoQ9YwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iJruFXmb_dur" + }, + "source": [ + "Выше мы использовали **функциональный** способ задания модели, когда мы сначала описываем входную переменную, затем - происходящие с ней преобразования, и потом определяем объект `Model`.\r\n", + "\r\n", + "Мы можем также задавать модель как последовательности слоёв с помощью `Sequential`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "iWc_kSr8_YXt", + "outputId": "345dbe65-629d-468f-ed75-1d412c966340" + }, + "source": [ + "model = tf.keras.models.Sequential()\r\n", + "model.add(tf.keras.layers.Dense(5,activation='sigmoid',input_shape=(2,)))\r\n", + "model.add(tf.keras.layers.Dense(1,activation='sigmoid'))\r\n", + "\r\n", + "test_x_norm = test_x-np.min(train_x) / (np.max(train_x)-np.min(train_x))\r\n", + "# это не ошибка, мы нормируем тестовые данные так же, как нормировали обучающие!\r\n", + "# (поэтому min и max считаем для обучающих данных)\r\n", + "\r\n", + "model.compile(tf.keras.optimizers.Adam(0.1),'binary_crossentropy',['accuracy'])\r\n", + "model.summary()\r\n", + "model.fit(train_x_norm,train_labels,validation_data=(test_x_norm,test_labels),batch_size=8,epochs=15)" + ], + "execution_count": 45, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Model: \"sequential_6\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "dense_14 (Dense) (None, 5) 15 \n", + "_________________________________________________________________\n", + "dense_15 (Dense) (None, 1) 6 \n", + "=================================================================\n", + "Total params: 21\n", + "Trainable params: 21\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n", + "Epoch 1/15\n", + "9/9 [==============================] - 1s 19ms/step - loss: 0.6962 - accuracy: 0.5596 - val_loss: 0.5533 - val_accuracy: 0.7333\n", + "Epoch 2/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.5057 - accuracy: 0.8035 - val_loss: 0.4924 - val_accuracy: 0.8000\n", + "Epoch 3/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.4173 - accuracy: 0.8726 - val_loss: 0.5273 - val_accuracy: 0.6667\n", + "Epoch 4/15\n", + "9/9 [==============================] - 0s 4ms/step - loss: 0.4405 - accuracy: 0.8145 - val_loss: 0.5641 - val_accuracy: 0.6667\n", + "Epoch 5/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3523 - accuracy: 0.9039 - val_loss: 0.5498 - val_accuracy: 0.7333\n", + "Epoch 6/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3239 - accuracy: 0.9012 - val_loss: 0.5666 - val_accuracy: 0.6667\n", + "Epoch 7/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.4043 - accuracy: 0.8568 - val_loss: 0.5883 - val_accuracy: 0.6667\n", + "Epoch 8/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3351 - accuracy: 0.8903 - val_loss: 0.5906 - val_accuracy: 0.6667\n", + "Epoch 9/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3624 - accuracy: 0.8761 - val_loss: 0.6198 - val_accuracy: 0.6667\n", + "Epoch 10/15\n", + "9/9 [==============================] - 0s 4ms/step - loss: 0.4015 - accuracy: 0.8622 - val_loss: 0.6087 - val_accuracy: 0.6667\n", + "Epoch 11/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3387 - accuracy: 0.8846 - val_loss: 0.6604 - val_accuracy: 0.6667\n", + "Epoch 12/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3579 - accuracy: 0.8694 - val_loss: 0.6177 - val_accuracy: 0.7333\n", + "Epoch 13/15\n", + "9/9 [==============================] - 0s 5ms/step - loss: 0.3304 - accuracy: 0.8830 - val_loss: 0.6274 - val_accuracy: 0.6667\n", + "Epoch 14/15\n", + "9/9 [==============================] - 0s 4ms/step - loss: 0.3193 - accuracy: 0.8807 - val_loss: 0.6472 - val_accuracy: 0.6667\n", + "Epoch 15/15\n", + "9/9 [==============================] - 0s 4ms/step - loss: 0.3552 - accuracy: 0.8546 - val_loss: 0.6596 - val_accuracy: 0.6667\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 45 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BmHNhUU8bqEX" + }, + "source": [ + "## Функции ошибки при классификации\r\n", + "\r\n", + "При использовании Keras для классификации важно правильно указать функцию ошибки и передаточную функцию на последнем слое. Основные правила:\r\n", + "* Если у сети один выход, то передаточная функция - сигмоида, если несколько - softmax\r\n", + "* Если на ожидаемый выход подается в виде one-hot-encoding, то функция ошибки - cross entropy loss (categorical cross-entropy), если номер класса - sparse categorical cross-entropy, для бинарной классификации с одним выходом - binary cross-entropy (она же log loss)\r\n", + "\r\n", + "В целом бинарную классификацию можно рассматривать как частный случай мультиклассовой, подавая на выход one hot encoded вектор.\r\n", + "\r\n", + "| Классификация | Формат входных данных | Передат.функция | Функция ошибки |\r\n", + "|---------------|-----------------------|-----------------|----------|\r\n", + "| Бинарная | Вероятность 1-го класса | sigmoid | binary crossentropy |\r\n", + "| Бинарная | One-hot encoding (2 выхода) | softmax | categorical crossentropy |\r\n", + "| Мультикласс | One-hot encoding | softmax | categorical crossentropy |\r\n", + "| Мультикласс | Номер класса | softmax | sparse categorical crossentropy" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yX6hqiafwHl9" + }, + "source": [ + "## Выводы\n", + "\n", + "* Tensorflow позволяет более гибко определять структуру графа вычислений, описывать свои функции и конфигурации.\n", + "* Есть более удобные средства для работы с данными (`td.Data`), со слоями (`tf.layers`)\n", + "* Для массового использования нейросетей Google рекомендует **Keras**, который позволяет собирать нейросети как конструктор\n", + "* При этом возможно реализовать свой слой для Keras, и потом использовать его в своих моделях.\n", + "* Для типовых задач имеет смысл использовать Keras\n", + "* Также стоит посмотреть на PyTorch, это \"восходящая звезда\"\n", + "\n", + "Хороший Notebook про Keras и Tensorflow 2.0 от создателя Keras - [тут](https://t.co/k694J95PI8)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gZ-kWx84bMDH" + }, + "source": [ + "**Задание 3**: \r\n", + "Используйте Keras для обучения классификатора на сети MNIST. При этом:\r\n", + "* Обратите внимание, что в keras заложены типовые датасеты, включая MNIST. Для обращения к нему достаточно пары строчек кода (см, например, [тут](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/mnist))\r\n", + "* Попробуйте несколько конфигураций сети с несколькими полносвязными слоями, передаточными функциями, и разным количеством нейронов\r\n", + "\r\n", + "Какой точности вам удалось достичь?\r\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "MG64NKzawHl-" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/3-NeuralNetworks/05-Frameworks/IntroPyTorch.ipynb b/3-NeuralNetworks/05-Frameworks/IntroPyTorch.ipynb new file mode 100644 index 00000000..b37041ac --- /dev/null +++ b/3-NeuralNetworks/05-Frameworks/IntroPyTorch.ipynb @@ -0,0 +1,1726 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "celltoolbar": "Slideshow", + "kernelspec": { + "name": "python3", + "display_name": "Python 3", + "language": "python" + }, + "language_info": { + "mimetype": "text/x-python", + "nbconvert_exporter": "python", + "name": "python", + "file_extension": ".py", + "version": "3.7.4-final", + "pygments_lexer": "ipython3", + "codemirror_mode": { + "version": 3, + "name": "ipython" + } + }, + "livereveal": { + "start_slideshow_at": "selected" + }, + "colab": { + "name": "IntroPyTorch.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "En2vX4FuwHlu" + }, + "source": [ + "# Введение в нейронные сети\n", + "\n", + "## Эпизод 2b: Многослойный персептрон на PyTorch\n", + "\n", + "Дмитрий Сошников | dmitri@soshnikov.com\n", + "\n", + "http://github.com/shwars/NeuroWorkshop\n", + "-> Notebooks -> IntroPyTorch.ipynb" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fVjlmWBwwHl2" + }, + "source": [ + "## Нейросетевые фреймворки\n", + "\n", + "Мы видели, что для обучения нейросетей нужно:\n", + "* Быстро умножать матрицы (тензоры)\n", + "* Считать производные для вычисления градиента для метода обратного распространения ошибки\n", + "\n", + "Что позволяют делать нейросетевые фреймворки:\n", + "* Оперировать с тензорами, как на CPU, так и на GPU\n", + "* Автоматически вычислять производные (они вручную прописаны для всех элементарных функций)\n", + "\n", + "Опционально:\n", + "* Конструктор для нейросетей (описание сети как набора слоёв)\n", + "* Простые функции для обучения (`fit`, как в Scikit Learn)\n", + "* Набор алгоритмов оптимизации\n", + "* Набор абстракций для работы с данными" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8cACQoFMwHl3" + }, + "source": [ + "## Основные фреймворки\n", + "\n", + "* Tensorflow 1.0 - первый, получивший широкое распространение (Google). Позволял определять статический computation graph, и затем в явном виде выполнять вычисления\n", + "* PyTorch - Facebook\n", + "* Keras - надстройка над Tensorflow/PyTorch для унификации (Francois Chollet)\n", + "* Tensorflow 2.0 + Keras - динамический вычислительный граф, код получается похожим на обычные вычисления в numpy\n", + "\n", + "Мы рассмотрим PyTorch. Для начала рекомендуется установить PyTorch [по инструкции на сайте](https://pytorch.org/get-started/locally/). Либо можно выполнять код в [Google Colab](https://colab.research.google.com/), в котором PyTorch уже установлен." + ] + }, + { + "cell_type": "code", + "metadata": { + "tags": [], + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "id": "xwqVx9-bwHl3", + "outputId": "38564a63-0567-4406-ee1a-1d3618f27351" + }, + "source": [ + "import torch\n", + "torch.__version__" + ], + "execution_count": 168, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'1.8.0+cu101'" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 168 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6tp2xGV7wHl4" + }, + "source": [ + "## Основные понятия в PyTorch\n", + "\n", + "**Тензор** - это многомерный массив произвольной размерности. Удобно использовать при обучении нейросетей, например:\n", + "* 400x400 - чёрно-белая картинка\n", + "* 400x400x3 - цветная картинка\n", + "* 16x400x400x3 - minibatch из 16 картинок, используемый для одного шага обучения\n", + "* 25x400x400x3 - секунда видео\n", + "* 8x25x400x400x3 - minibatch из 8 1-секундных видео" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qG2bsaR7wHl4" + }, + "source": [ + "### Простые тензоры" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ybpnk08HwHl4", + "outputId": "54e2c89b-b373-4389-b285-49b0510be931" + }, + "source": [ + "a = torch.tensor([[1,2],[3,4]])\n", + "print(a)\n", + "a = torch.randn(size=(10,3))\n", + "print(a)" + ], + "execution_count": 169, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[1, 2],\n", + " [3, 4]])\n", + "tensor([[ 0.1288, -0.8088, -0.2092],\n", + " [ 0.0789, -1.4706, -0.8880],\n", + " [-0.0601, 0.6659, -3.2120],\n", + " [ 0.8023, -0.4045, -1.0164],\n", + " [-1.5578, -1.3014, -0.8372],\n", + " [ 1.3626, -1.0825, -2.7018],\n", + " [ 0.2258, -0.7675, -0.4102],\n", + " [-1.9663, 0.0386, -2.6183],\n", + " [ 0.7666, -1.3057, -0.4486],\n", + " [-0.9006, 0.7092, 0.8018]])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AXFMsV3r09Ux" + }, + "source": [ + "С тензорами можно производить обычные вычисления, которые производятся поэлементно (как в numpy). При этом тензоры автоматически дополняются до нужной размерности. Можно извлечь numpy-массив из тензора при помощи `.numpy()`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "e5Nu5Xgj1DnQ", + "outputId": "c1fbcd86-dde6-40b6-8edf-7a37f9d60901" + }, + "source": [ + "print(a-a[0])\n", + "print(torch.exp(a)[0].numpy())" + ], + "execution_count": 170, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[ 0.0000, 0.0000, 0.0000],\n", + " [-0.0499, -0.6618, -0.6788],\n", + " [-0.1889, 1.4747, -3.0028],\n", + " [ 0.6734, 0.4044, -0.8072],\n", + " [-1.6867, -0.4926, -0.6280],\n", + " [ 1.2338, -0.2737, -2.4927],\n", + " [ 0.0970, 0.0413, -0.2010],\n", + " [-2.0951, 0.8475, -2.4091],\n", + " [ 0.6377, -0.4969, -0.2395],\n", + " [-1.0295, 1.5180, 1.0110]])\n", + "[1.1375165 0.44538715 0.81125516]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uQ5zN6cVyrG7" + }, + "source": [ + "## In-place и out-of-place операции\n", + "\n", + "Обычно операции с тензорами возвращают новые тензоры. Однако для большинства операций сущетвуют аналогичные варианты, которые модифицируют исходный тензор:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Mjkbcw3-ACKS", + "outputId": "ca021008-9ab6-4b09-c5a5-bbe854cd1493" + }, + "source": [ + "u = torch.tensor(5)\n", + "print(\"Result when adding out-of-place:\",u.add(torch.tensor(3)))\n", + "u.add_(torch.tensor(3))\n", + "print(\"Result after adding in-place:\", u)" + ], + "execution_count": 171, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Result when adding out-of-place: tensor(8)\n", + "Result after adding in-place: tensor(8)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DLPUcVsXACKT" + }, + "source": [ + "Например, вот там можно \"наивно\" посчитать сумму строк тензора `a`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7pu0UZ-_yqfB", + "outputId": "bd2e8c6a-39e1-4f29-990b-9591e866936c" + }, + "source": [ + "s = torch.zeros_like(a[0])\n", + "for i in a:\n", + " s.add_(i)\n", + "\n", + "print(s)" + ], + "execution_count": 172, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([ -1.1197, -5.7273, -11.5398])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rIh1EHcezlNo" + }, + "source": [ + "Умный способ:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aQIdWZ1kzn6P", + "outputId": "89000bb4-f45e-493b-a7b0-39fa4e7d92c1" + }, + "source": [ + "torch.sum(a,axis=0)" + ], + "execution_count": 176, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "tensor([ -1.1197, -5.7273, -11.5398])" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 176 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5UzUmEZhACKT" + }, + "source": [ + "Подробнее про тензоры в PyTorch смотрите [в официальном руководстве](https://pytorch.org/tutorials/beginner/basics/tensorqs_tutorial.html)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U-auwezDwHl6" + }, + "source": [ + "## Вычисляем производные\n", + "\n", + "Для обратного распространения ошибки, нам нужно уметь вычислять градиенты. Мы можем пометить любой тензор в PyTorch атрибутом `requires_grad`, и впоследствии автоматически будут вычисляться все градиенты при операциях с этим тензором. Для вычисления производной необходимо вызвать метод `backward()`:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "m8vFOXr7wHl6", + "outputId": "7054c2b1-0b61-4938-937d-813f75f0b195" + }, + "source": [ + "a = torch.randn(size=(2, 2), requires_grad=True)\n", + "b = torch.randn(size=(2, 2))\n", + "\n", + "c = torch.mean(torch.sqrt(torch.square(a) + torch.square(b))) # Do some math using `a`\n", + "c.backward() # call backward() to compute all gradients\n", + "# What's the gradient of `c` with respect to `a`?\n", + "print(a.grad)" + ], + "execution_count": 179, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[0.0227, 0.1661],\n", + " [0.1980, 0.2253]])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nPj3rtrtACKU" + }, + "source": [ + "На самом деле, PyTorch может таким образом вычислять \"накапливаемые\" градиенты. Если при вызове `backward` указать `retain_graph=True`, то граф вычислений будет сохраняться, и градиенты - накапливаться. Чтобы начать их вычислять заново, нужно в явном виде обнулить поле `grad`: " + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "z_VIw8MoACKU", + "outputId": "36a28b11-6919-47ab-c3f9-c7f1d8500423" + }, + "source": [ + "c = torch.mean(torch.sqrt(torch.square(a) + torch.square(b)))\n", + "c.backward(retain_graph=True)\n", + "c.backward(retain_graph=True)\n", + "print(a.grad)\n", + "a.grad.zero_()\n", + "c.backward()\n", + "print(a.grad)" + ], + "execution_count": 180, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[0.0681, 0.4983],\n", + " [0.5939, 0.6758]])\n", + "tensor([[0.0227, 0.1661],\n", + " [0.1980, 0.2253]])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HM9sUkVgCiG9" + }, + "source": [ + "Для вычисления градиентов PyTorch создаёт и поддерживает **граф вычислений**. Для каждого тензора, который вычисляется с использованием тензоров с установленным флагом `requires_grad`, устанавливается специальная функция `grad_fn`, представляющая собой функцию для вычисления производной по правилу дифференциирования сложной функции:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PcxHb-7jC7Vv", + "outputId": "3b3fa138-6d09-4636-8a71-f4a4051c7827" + }, + "source": [ + "print(c)" + ], + "execution_count": 181, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor(1.0358, grad_fn=)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rvLfNiblACKV" + }, + "source": [ + "На самом деле, PyTorch может вычислять градиенты только для скалярных функций. Если речь идет о вычислении производной тензора по тензору, то PyTorch позволяет нам вычислять произведение якобиана на вектор.\n", + "\n", + "Например, пусть есть векторная функция $\\vec{y}=f(\\vec{x})$, где\n", + "$\\vec{x}=\\langle x_1,\\dots,x_n\\rangle$ и\n", + "$\\vec{y}=\\langle y_1,\\dots,y_m\\rangle$, тогда градиент $\\vec{y}$ по $\\vec{x}$ задаётся **якобианом**:\n", + "\n", + "$$\n", + "\\begin{align}J=\\left(\\begin{array}{ccc}\n", + " \\frac{\\partial y_{1}}{\\partial x_{1}} & \\cdots & \\frac{\\partial y_{1}}{\\partial x_{n}}\\\\\n", + " \\vdots & \\ddots & \\vdots\\\\\n", + " \\frac{\\partial y_{m}}{\\partial x_{1}} & \\cdots & \\frac{\\partial y_{m}}{\\partial x_{n}}\n", + "\\end{array}\\right)\\end{align}\n", + "$$\n", + "\n", + "Вместо вычисления якобиана, PyTorch вычисляет произведение $v^T\\cdot J$ на некоторый вектор\n", + "$v=(v_1 \\dots v_m)$. Для этого необходимо вызвать ``backward``, передав `v` в качестве аргумента. Размер `v` должен совпадать с размером исходного тензора, по которому мы вычисляем производную.\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "VUNYiQCOACKV", + "outputId": "e3127c21-fce6-420d-f347-ec40cc827e7e" + }, + "source": [ + "c = torch.sqrt(torch.square(a) + torch.square(b))\n", + "c.backward(torch.eye(2))\n", + "print(a.grad)" + ], + "execution_count": 182, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[0.1135, 0.1661],\n", + " [0.1980, 1.1263]])\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dGHlkVlvACKV" + }, + "source": [ + "Подробнее про вычисление градиентов в PyTorch читайте [в официальном руководстве](https://pytorch.org/tutorials/beginner/basics/autogradqs_tutorial.html)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FnVvj4LkD15r" + }, + "source": [ + "# Пример 0: Оптимизация функции методом градиентного спуска\r\n", + "\r\n", + "Попробуем использовать автоматическое дифференциирования для оптимизации простой функции двух переменных $f(x_1,x_2)=(x_1-3)^2+(x_2+2)^2$. Пусть тензор `x` представляет собой текущие координато точки. Мы начнем с некоторой начальной точки $x^{(0)}=(0,0)$, и будет вычислять следующую точку по формуле:\r\n", + "$$\r\n", + "x^{(n+1)} = x^{(n)} - \\eta\\nabla f\r\n", + "$$\r\n", + "Здесь $\\eta$ - т.н. **learning rage** (назоём его `lr` в коде), а $\\nabla f = (\\frac{\\partial f}{\\partial x_1},\\frac{\\partial f}{\\partial x_2})$ - градиент функции $f$.\r\n", + "\r\n", + "Для начала определим стартовое значение `x` и функцию `f`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nDw5mV9KEeOa" + }, + "source": [ + "x = torch.zeros(2,requires_grad=True)\r\n", + "f = lambda x : (x-torch.tensor([3,-2])).pow(2).sum()\r\n", + "lr = 0.1" + ], + "execution_count": 184, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Wt815LWdEj77" + }, + "source": [ + "Теперь проделаем 15 итераций градиентного спуска. На каждой итерации, мы будем обновлять координаты `x` и печатать их, чтобы убедиться, что мы достигаем минимума (3,-2):" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "KfwMf555EyWJ", + "outputId": "67e2199c-61ff-4ad1-9c48-b4a646bf8bbd" + }, + "source": [ + "for i in range(15):\r\n", + " y = f(x)\r\n", + " y.backward()\r\n", + " gr = x.grad\r\n", + " x.data.add_(-lr*gr)\r\n", + " x.grad.zero_()\r\n", + " print(\"Step {}: x[0]={}, x[1]={}\".format(i,x[0],x[1]))" + ], + "execution_count": 186, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Step 0: x[0]=1.2000000476837158, x[1]=-0.800000011920929\n", + "Step 1: x[0]=1.5600000619888306, x[1]=-1.0399999618530273\n", + "Step 2: x[0]=1.8480000495910645, x[1]=-1.2319999933242798\n", + "Step 3: x[0]=2.078400135040283, x[1]=-1.385599970817566\n", + "Step 4: x[0]=2.2627201080322266, x[1]=-1.5084799528121948\n", + "Step 5: x[0]=2.4101760387420654, x[1]=-1.6067839860916138\n", + "Step 6: x[0]=2.5281407833099365, x[1]=-1.685427188873291\n", + "Step 7: x[0]=2.6225125789642334, x[1]=-1.7483417987823486\n", + "Step 8: x[0]=2.698009967803955, x[1]=-1.798673391342163\n", + "Step 9: x[0]=2.7584080696105957, x[1]=-1.8389387130737305\n", + "Step 10: x[0]=2.8067264556884766, x[1]=-1.8711509704589844\n", + "Step 11: x[0]=2.845381259918213, x[1]=-1.8969208002090454\n", + "Step 12: x[0]=2.876305103302002, x[1]=-1.9175366163253784\n", + "Step 13: x[0]=2.9010441303253174, x[1]=-1.9340293407440186\n", + "Step 14: x[0]=2.920835256576538, x[1]=-1.947223424911499\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8sfjBMBu59B5" + }, + "source": [ + "## Пример 1: Линейная регрессия\r\n", + "\r\n", + "Попробуем с помощью полученных знаний решить классическую задачу линейной регрессии. Для этого сгенерируем небольшой синтетический датасет:" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "j723455WwHl7" + }, + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.datasets import make_classification, make_regression\n", + "from sklearn.model_selection import train_test_split\n", + "import random" + ], + "execution_count": 187, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "WJNK_J6v6I-Z", + "outputId": "09e6386e-a6d4-4b81-c8d2-153f0acf9696" + }, + "source": [ + "np.random.seed(13) # pick the seed for reproducability - change it to explore the effects of random variations\n", + "\n", + "train_x = np.linspace(0, 3, 120)\n", + "train_labels = 2 * train_x + 0.9 + np.random.randn(*train_x.shape) * 0.5\n", + "\n", + "plt.scatter(train_x,train_labels)" + ], + "execution_count": 188, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 188 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAb2ElEQVR4nO3df4xlZXkH8O+zwwUuP8pVmVoYXZemzZIKZZedEJo1RtZYbFHcgFEb0xbbZpM2bYVQ2tE/ivYfJjFWbW1qJpQWW6pLXdhQUJFkN6FuFDvDLgKyNChuZbBllB0QZsrODE//uPcsd86c97zvuec957zvvd9PsnF25+yZ9+yV5773eZ/3eUVVQURE4drU9ACIiCgfAzURUeAYqImIAsdATUQUOAZqIqLAnVLFTc8991zdsmVLFbcmIhpKc3NzP1HV8azvVRKot2zZgtnZ2SpuTUQ0lETkmOl7TH0QEQWOgZqIKHAM1EREgWOgJiIKHAM1EVHgKqn6ICIaJfsPz+NT9z+JZxeXcX6njZuu3Ird2ye83Z+BmoiohP2H5/Gxux7F8soaAGB+cRkfu+tRAPAWrJn6ICIq4VP3P3kySCeWV9bwqfuf9PYzGKiJiEp4dnG50J8PgoGaiKiE8zvtQn8+CAZqIiJ0c807pw/ggqn7sHP6APYfnnf6ezdduRXt1ti6P2u3xnDTlVu9jY2LiUQ08sosCCbfZ9UHEVGF8hYEXQLu7u0TXgNzGlMfRDTy6lgQLMMaqEVkq4gc6fv1oohcX8fgiIjqUMeCYBnWQK2qT6rqNlXdBmAHgCUAd1c+MiKimtSxIFhG0Rz1OwF8X1WNDa6JiGJTZEGwf7v4Oe0WRIDFpZVKFhEToqruF4vcBuBhVf18xvf2ANgDAJs3b95x7BhjORENl3R1SFq7NYZbrrl4oGAtInOqOpn1PefFRBE5FcDVAP4t6/uqOqOqk6o6OT6eeewXEVHw8uqps6pD+vneOp4okvr4DXRn0//rfRRERAGw1VO7VIFUUSlSpDzvtwB8yfsIiIgCYWuw5FIFUkWliFOgFpEzAbwLwF3eR0BEFAhbPXVWdUi/qipFnFIfqvoygDd4/+lERAE5v9PGfEawTmbJ6eqQuqo+uIWciKjnpiu3bqjqSM+Sq94unoWBmoiop44GS4NgoCYi6tPEjNmGTZmIiALHQE1EFDgGaiKiwDFHTUQjob+ZUiiLhK4YqIloKOQF4kGP2kruOb+4jDERrKliooEgz0BNRIWEODPNC8QAcOOdj2At1SnUdtRW+p7J3y9ynqIvhdqcupqcnNTZ2Vnv9yWiZmW1+SzT2tOXndMHMncUdtotvLL6am7Hu4lOO/NNx3TP/r93aGpX+cH3eGlzSkRka1rUFFOPjsXlldwgLejOkBWvzZSTtqa2Lnh1nqfIQE1EzkI9BHaQjnUCIJ1PKNIpr87zFBmoiciZKTid024Zm+3XwXTm4evOaGVePyayIUgnXDrl1X2eIgM1ETnLCl6tTYKXT6waUwhVSJ/CAgC3XHMxJjptCLr541uuuRg3v/etmQH80x+4BBOWk8d3b584eU+gG9zRd29WfRBRkLKaFi2dWMXxpZV119kqKsowVXjccs3FxsU9U5VKiJ3ysrDqg4hKuWDqvsw0ggB4evoq7z/PVI0xSBVGSKWGeVUfnFETUSm2Zvu++VzQDGXGbMMcNRGVYlrIq2qxzfQGUGcVRt0YqImolP5Ft/6FvKpmqnW/MYSAqQ8iKq3OFEKop7BUySlQi0gHwK0ALkK3Rvz3VPVbVQ6MiMgkltyyL64z6s8B+Lqqvl9ETgVwRoVjIiLKFFKVRp2sgVpEzgHwdgDXAYCqngBwotphERGtN2ir0mHgsph4AYAFAP8oIodF5FYROTN9kYjsEZFZEZldWFjwPlAialZ6N2Dd28RDbQhVB5dAfQqASwH8vapuB/AygKn0Rao6o6qTqjo5Pj7ueZhE1KRkNlvnNvH+n53XcrTphlB1cMlRPwPgGVV9qPf7ryAjUBPR8DLNZj9xz+O5p6qUzSdn9b9OG+b66YQ1UKvq/4jIj0Rkq6o+CeCdAL5X/dCIKBR5/Z4Xl7t9PtKnqvjIJ2e9QfQb9vrphGvVx58AuKNX8fEDAB+pbkhEFBrTNvG0/pyxKZ9cJFDnpTWaOLuwKU6BWlWPAMhsFkJEw++mK7daUxCJvOBaNJ9seoPwfQxW6LiFnIissraJm5ryn99pO/XjcKkiGcXt4lm4hZyInKR3A5oOuk2CaN73XGqik8XI5ZU1jIlgTXWk0h39GKiJRliZygyXnhum7+XVRO/ePrEhkK+pngz0oxakAR4cQDSyTDPiOo6ZMh02AHTTKqaFy6zc9LBsK887OIA5aqIR1eROP1MOW4Dc6pL0YmSTG3HqxEBNNKJ8npRSVNYioQDGWXYiHeBHZVs5AzXRkDNVV5hmtQpU3ssjXUXSabesQTqr2qPJN5s6cTGRaIjlVVfk1Ub77kxnyiP3LxzmMVV71H1eY1MYqImGUBIYs4JYkhpIFuVs15UN1LZSvLxt4rbFzaw3m3ZrDFdcOI6d0weiX2BMMPVB5EHTLUDTY0kW2EyS1MDu7RM4NLULYrmuDFseOe9n2CpQsjbiXLtjAvvm5odqgZEzaqKSQmtob2tkBGxMDfhMIaTTHLb2pHnbxF3+/dIbcXZOH/DSZyQknFETldRE5UHeDN42C85alPO1VTurXM40W0/eBHxvE3dZYAzpE5ALzqiJSqq78sA2g8+bxY6JrHsTSWaYvk72znrTUmSX3i2dWMX+w/PeTxW3fToI7ROQCwZqopLqrjywbb/OWmBrbRJAgJW1brjMCk4+TvY2vTkpuiV4Se9qADi+tLJuDL6CpGmBMZmh2/79QsTUB1FJdXd4s83gsxbYzjr9lJNBOmFKz5RJC5jenCY6bZx52sZ5YRUpoqzn71+UjLH2mjNqopJ8f3S3cZnBp2eoF0zdl3kv05bsQdMCebPZG/YecRqDD3kz9BhrrzmjJvIgKXN7evoqHJraVelH6EFm8C79oYHyC6N5s1nXMVQtxh7XnFETRWaQGbwtb5vwkRYwzWZdx1C1uj8B+cBATRShootvrsGpyrRASAHS5+JlHdiPmihQTfRZbrJH9ajL60fNGTVRgJqq9Q1p1kuvcQrUIvJDAD8DsAZg1RT1iciPJmt9Y0sLjIIiM+orVPUnlY2EiE6KsdaXqsPUB1EA0vnozhktHF9a2XBdyLW+VB3XOmoF8A0RmRORPVkXiMgeEZkVkdmFhQV/IyQaclmNjF76v1W0xta3Mwq91peq4zqjfpuqzovIzwN4QESOquqD/Reo6gyAGaBb9eF5nERDIz17XjqxuiEfvfKqotNu4czTTuGiHrkFalWd7/3vcyJyN4DLADyY/7eIKC2rmsPkheUVHLn51+saGgXMmvoQkTNF5OzkawC/DuCxqgdGNIxcmvonmI+mhMuM+o0A7haR5Pp/VdWvVzoqoiHlWrXBfDT1swZqVf0BgEtqGAvR0DNt0WY+mvKwPI+i1MT2ah9MjYk+cfVbjePvf9Zz2i2IAItLK1E9N5XDQE3RqXN7te0NoegbRtEt2uln7T8hJeu5Y30Do3xsykTR2Tl9wHhq9aGpXd5+jq1BUR0NjEzP2i95bjZUilteUyYeHEDRqWt7ta2Jfh2nj7s8U3JNE6ehUz0YqCk6dZ0UYntDqOMNw+WZkmvYH2R4MVBTdOo6Ssn2hlDHG0bWs/brf+5Qjroi/xioKTq2U6Z9sb0h1PGGkX7WTruF153RynzuKsZT5kRy8oeLiUQ5fFd9ND3eovfi4mR98hYTGaiJIlfVm0Vd1TXUxaO4iCJRNOhWWVPOxclwMEdNlNJUXjarL/XH7no09+dXWZLHxclwMFBTVKoOooMES9f72sY9SNCtctZbV3UN2TFQUzSqCqL9qpihuo57kKBb5ay3ruoasmOOmqJRx8ncVcxQXcdt6qyXF3RNTZ58zXp5InkYGKgpGnXtBCwaLG1M45tfXMbO6QMnFw6vuHAc++bmCwXdok2eKE4M1BSNKoJoWhUzVNO4Ba8dxTW/uIx9c/O4dscEDh5dKBR0OesdfgzUFI2qP+YD+TPUQeuVs8YtANI7GJZX1nDw6EItNcqhbdShfNzwQlFpKsBk7dIDgNed0cLN781u+p/X8N/UulQAPD19VRWPsG5c3HEYHm54oaFh+5hfVSA3HUp7fGklc4NJVsP/dmsMn/ngNuzePmHc9VdHjXIdi7LkF8vzyIsQmvdUWb6Xt2CZVb5nK/NrskaZOw7j4xyoRWRMRA6LyL1VDojiU0d9swtTcLzxzkdKj8U2000HOVswbLJGmTsO41Mk9fFRAE8A+LmKxkKRCuWjtCk4rqmW7n+RtSDYrz/I7T88j00iWMtY/+m/rqlqjToWZckvpxm1iLwJwFUAbq12OBSjUD5K580Iy+4uTGbAnXZrw/f6g1zy6SIrSIcSDLnjMD6uM+rPAvhzAGebLhCRPQD2AMDmzZvLj4yiUUd9swvbrLfsG0cyA85bsDQtOo6JBBUMWXsdF2ugFpH3AHhOVedE5B2m61R1BsAM0C3P8zZCCl4oH6WTwHPjnY9Y0w5lf44pyJneDF5VZWCkgbmkPnYCuFpEfgjgywB2ici/VDoqikpIH6V3b5/Apz9wSWMVFVyooyoU2vDSm1H/maq+J+86bnihpoW0MYabScgFN7zQyGkqB8smSVQFbiGnxjTZb4K9Lig0nFFTcHye9efrnMHZY88X7lxHVAfOqKkRpl4XQHcx0jVIFskJJwE9ryFS/38NzC1TnfJm1Oz1QY3Iq2kusgXd5eis/Yfnse2T38D1e48YgzSQ3XbUxyGxRGUxUFMjbOVqrkHStisymXEvLq8UH2TO/YnqxEBNznx2yMvqHpfmEiRtdcumnYJpUvD+RHVioCYnvjvk9W+SMXEJkrZ2oS7BfqLTxocv39zYJhkiG1Z9kJMqOuT1984YdAu6rW457zSV9GLh5Ftez5I9ChIDNTmpskNe2U0ieZtbTI2aso7QYqMiChUD9RCoY/NG1R3yqgqS3ClIw4CBOnI+N47kCaVD3iA4U6bYcTExci51xD6E1CGPaNRwRh25Ok9XqWNm2p/GOafdgkj3pO+x3tFWRXYtEg0LBurIhXK6io1LHj2dxunfpJIcBFBVaocoZEx9RM5WRxwC1xps180p3NpNo4aBOnIx5I5NefTr9x5Zt8OxSLqGW7tplDD1MQRCr2pwacAE5G9OSQsttUNUJQZq8i6dj+6c0cLxJXNTpCSVYTtFPBFaaoeoagzU5FVWXXdrk6A1JlhZM/c+f3ZxecPmFFZ9EHUxUJNXWfnolVcVnXYLZ552ijG1kaQy6k7j8EguigEXE8krUz76heUVHJrahc9+cFswVSq+OwISVcUaqEXkdBH5jog8IiKPi8gn6xgYvcZnH+iq2fpDh1SlUteuTqKyXFIfrwDYpaoviUgLwDdF5Guq+u2Kx0aor5eHLy49QUKpUqlzVydRGdZArd3Tb1/q/bbV++X/RFzK5LMPdB352Ji61cWyq5PI6RRyERkDMAfglwD8nar+RcY1ewDsAYDNmzfvOHbsmOehjqYLpu7LfFcUAE9PX+V8n6zm/Mmp21mVFKOwyFbkBHOiquWdQu5U9aGqawC2iUgHwN0icpGqPpa6ZgbADABMTk5yxu2Jr1lf1sw8eZHS6RRbuiWrcdLi0kp0AT2m2T+NtkLleaq6KCIHAbwbwGO264dVnbNNX32gbXnX/nSKbZHN1Dgp9Px5llDy5UR5XKo+xnszaYhIG8C7ABytemChqruky1eVhMsMPAnmeYtstsZJrJog8s9lRn0egNt7eepNAO5U1XurHVa4qjjk1cbHrM9le3YSzE3plk0iTr04WDVB5JdL1cd3AWyvYSxRiLGkK0nVLK+sndyKnSwkJvrTKaagnvX3siQBfxQWJInqwJ2JBdk2dISmP1UDdINtuzWGD1++2ZhOSdItYyIb7qfoVouYJAGfu/6I/GGvj4JCOuTVZcZqStUcPLqAQ1O7jPfevX0CN+w9kvm9pKQvr+pj5/SB2lNERMOKgbqgUEq6XHcslknVmHLVE512bpAv+3OJaD0G6gGEUNLluqhZpg67zKcH7voj8oc56opV1VDJdcZa5kzFMqWBMZzlSBQLzqgrVGVDJdcZa9lUzaCfHkJJERENA6deH0VNTk7q7Oys9/vGZuf0gcxgOiaCT3/gklJBi30qiIZL6V4fNBhTemJNtfTMmjNWotHBQF2hvFO1fZSq+V7U5AYVojBxMbFCWQtq/UIqVeMGFaJwMVBXKG+HHxBWqRqPpSIKF1MfFUtSB3XvZiyaxuAGFaJwMVDXwGXhz2d+eJCyQG5QIQoXA3VN8hb+fNdbD9KKNaQeJkS0HnPUAfCZH95/eN5YaZKXxvB1QAER+ccZdQB85YeTmbmJLY0RQg8TItqIM+oA+OpxnXdMFtMYRPHijLokH4uApvzwFReOY+f0gdKVGwCYxiCKGAN1Cb4WAbOqQq64cBz75ua9VG5MdNoM0kQRG9pAXXamW+b0FNet4Xk/Y5ATUli5QTScrIFaRN4M4IsA3ojuKUwzqvq5qgdWRtmZbh2np9h+xiD3HqRRE/t7EIXPZUa9CuBGVX1YRM4GMCciD6jq9yoe28DKznTrOD3F9jMGvXeRyo0q+2UTkT/Wqg9V/bGqPtz7+mcAngAQ9H/FZcvd6jg9xfYz6jghhf09iOJQqDxPRLYA2A7goYzv7RGRWRGZXVhY8DO6AZUtd3P9+2U2idh+Rh0bUNjfgygOzouJInIWgH0ArlfVF9PfV9UZADNA94QXbyN0kM6zpismgGKz0SKLcoNuEnH5GVVvQGF/D6I4OM2oRaSFbpC+Q1XvqnZIxWT1Ud43N49rd0wMPBsdZDZb9BDbELZs8wBaojhYz0wUEQFwO4DnVfV6l5vWeWai6VzCiU4bh6Z2lb6/S1VEzOcXsuqDKAx5Zya6BOq3AfgPAI8CeLX3xx9X1a+a/k6VgTodWEwNiATA09NXFb5ff6DKCsCCbo3iRN+1Vb9ZENHwK3W4rap+E9341LiscrIkcKa55Flt5WlZVRHJz+q/lotyRFSlqJoymQJn+l3ENc9qK0+zBdrkWl9NlYiIskQVqE2BM0lFCIBOu4XTW5tww94j1kU920zYJdA+u7jMRTkiqlRUgdoUOJNc8Gc+uA2vrL6K40srTidp22bCtlPEk2tDqOAgouEVVVMmW+1x0a3jtvv1987Iyoenr2VgJqIqRBWobU2Hii7quTQx6g/AIZSyhTAGIqqXtTxvEHXWUfcb9jK5mOu1iShfXnleVDlqm1gW9YruYkywiRLRaIoq9WEzSD/mupVpLcp6baLRFHWgNuVrBw3MvvK/efcp0yubTZSIRlO0gdp303tf9yt7cktekOdRW0SjKdocte98ra/72e6TV7ud1Qmwvw6c9dpEoym4GbVr+sF3vtbX/VxObjHNil3SIqzXJho9QQVqU9pg9tjzOHh0YV3w9p2v9XU/233yFjxv2Hsk855cLCQabUEFatOM8o5v//eGrnXX7pgodYpLmq/8b5mTW7hYSERZgspR5zVd6re8soaDRxe85mt95X/L3CeWOnAiqldQOxNNOwuzuB4MEBtuEScaTaUODqhTVtqgzMEAMeJiIRGlBZX6yEobfPjyzUwHENFIC2pGDWTPKCff8nqmA4hoZAUXqLMwHUBEo8waqEXkNgDvAfCcql5U/ZCqw4U6IoqRy4z6nwB8HsAXqx3KRj4Dq+/eIEREdbEuJqrqgwCer2Es69j6XhTFXs5EFKugqj76+Q6s7OVMRLHyFqhFZI+IzIrI7MLCQun7+Q6sthPHiYhC5S1Qq+qMqk6q6uT4+Hjp+/kOrNyeTUSxCjb14TOwJouSyytrGBMBAHTaLZze2oQb9h4pdG4hEVHdrIFaRL4E4FsAtorIMyLy+9UPy1+TpP5FSQBYU0Vrk+DlE6s4vrTiZaGSiKhKQTVlqkKRRk8TnTYOTe2qeERERBvlNWUKNvXhS5HFR1aAEFGIhj5QF1l8ZAUIEYVo6AN11qJka5OgNSbr/owVIEQUqiiaMpVhOqMw68+4lZyIQjT0i4lERDGI4oQXdrYjIsoWRKBmZzsiIrMgFhPZ2Y6IyCyIQM3OdkREZkEEana2IyIyCyJQs7MdEZFZEIuJplpnLiQSEQUSqAGeNE5EZBJE6oOIiMwYqImIAsdATUQUOAZqIqLAMVATEQWuku55IrIA4NiAf/1cAD/xOJwmDcuzDMtzAHyWEA3LcwDlnuUtqjqe9Y1KAnUZIjJravUXm2F5lmF5DoDPEqJheQ6gumdh6oOIKHAM1EREgQsxUM80PQCPhuVZhuU5AD5LiIblOYCKniW4HDUREa0X4oyaiIj6MFATEQWusUAtIu8WkSdF5CkRmcr4/mkisrf3/YdEZEv9o7RzeI7rRGRBRI70fv1BE+O0EZHbROQ5EXnM8H0Rkb/pPed3ReTSusfoyuFZ3iEiL/S9Jn9Z9xhdicibReSgiHxPRB4XkY9mXBP8a+P4HFG8LiJyuoh8R0Qe6T3LJzOu8Ru/VLX2XwDGAHwfwC8COBXAIwB+JXXNHwH4Qu/rDwHY28RYPTzHdQA+3/RYHZ7l7QAuBfCY4fu/CeBrAATA5QAeanrMJZ7lHQDubXqcjs9yHoBLe1+fDeC/Mv4/Fvxr4/gcUbwuvX/ns3pftwA8BODy1DVe41dTM+rLADylqj9Q1RMAvgzgfalr3gfg9t7XXwHwThGRGsfowuU5oqCqDwJ4PueS9wH4onZ9G0BHRM6rZ3TFODxLNFT1x6r6cO/rnwF4AkC6cXvwr43jc0Sh9+/8Uu+3rd6vdFWG1/jVVKCeAPCjvt8/g40v2slrVHUVwAsA3lDL6Ny5PAcAXNv7SPoVEXlzPUPzzvVZY/FrvY+uXxORtzY9GBe9j8/b0Z3B9Yvqtcl5DiCS10VExkTkCIDnADygqsbXxEf84mJi9f4dwBZV/VUAD+C1d1lqzsPo9lW4BMDfAtjf8HisROQsAPsAXK+qLzY9nkFZniOa10VV11R1G4A3AbhMRC6q8uc1FajnAfTPLN/U+7PMa0TkFADnAPhpLaNzZ30OVf2pqr7S++2tAHbUNDbfXF6zKKjqi8lHV1X9KoCWiJzb8LCMRKSFbnC7Q1XvyrgkitfG9hyxvS4AoKqLAA4CeHfqW17jV1OB+j8B/LKIXCAip6KbbL8ndc09AH639/X7ARzQXmY+INbnSOUKr0Y3NxejewD8Tq/C4HIAL6jqj5se1CBE5BeSfKGIXIbufwehTQIAdCs6APwDgCdU9a8NlwX/2rg8Ryyvi4iMi0in93UbwLsAHE1d5jV+NXK4raquisgfA7gf3cqJ21T1cRH5KwCzqnoPui/qP4vIU+guDH2oibHmcXyOPxWRqwGsovsc1zU24Bwi8iV0V93PFZFnANyM7iIJVPULAL6KbnXBUwCWAHykmZHaOTzL+wH8oYisAlgG8KEAJwGJnQB+G8CjvZwoAHwcwGYgqtfG5TlieV3OA3C7iIyh+2Zyp6reW2X84hZyIqLAcTGRiChwDNRERIFjoCYiChwDNRFR4BioiYgCx0BNRBQ4BmoiosD9P3fFq1WlCzEwAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ng4rZmGc6oxk" + }, + "source": [ + "Линейная регрессия вычисляется как $f_{W,b}(x) = Wx+b$, где $W, b$ - параметры модели, которые необходимо найти. Функция ошибки на наборе данных $\\{x_i,y_u\\}_{i=1}^N$ может быть определена как среднеевадратичное отклонение\r\n", + "$$\r\n", + "\\mathcal{L}(W,b) = {1\\over N}\\sum_{i=1}^N (f_{W,b}(x_i)-y_i)^2\r\n", + "$$\r\n", + "\r\n", + "Опишем модель и функцию ошибки:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "QxhI4GlB6aiH" + }, + "source": [ + "input_dim = 1\r\n", + "output_dim = 1\r\n", + "learning_rate = 0.1\r\n", + "\r\n", + "# This is our weight matrix\r\n", + "w = torch.tensor([100.0],requires_grad=True,dtype=torch.float32)\r\n", + "# This is our bias vector\r\n", + "b = torch.zeros(size=(output_dim,),requires_grad=True)\r\n", + "\r\n", + "def f(x):\r\n", + " return torch.matmul(x,w) + b\r\n", + "\r\n", + "def compute_loss(labels, predictions):\r\n", + " return torch.mean(torch.square(labels - predictions))" + ], + "execution_count": 189, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JUxwj3367gD2" + }, + "source": [ + "Обучать модель будем на сериях примеров - minibatches. Для обучения используем градиентный спуск, подстраивая парметры в соответствии с формулой:\r\n", + "$$\r\n", + "\\begin{array}{l}\r\n", + "W^{(n+1)}=W^{(n)}-\\eta\\frac{\\partial\\mathcal{L}}{\\partial W} \\\\\r\n", + "b^{(n+1)}=b^{(n)}-\\eta\\frac{\\partial\\mathcal{L}}{\\partial b} \\\\\r\n", + "\\end{array}\r\n", + "$$" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "-991PErM7fJU" + }, + "source": [ + "def train_on_batch(x, y):\r\n", + " predictions = f(x)\r\n", + " loss = compute_loss(y, predictions)\r\n", + " loss.backward()\r\n", + " w.data.sub_(learning_rate * w.grad)\r\n", + " b.data.sub_(learning_rate * b.grad)\r\n", + " w.grad.zero_()\r\n", + " b.grad.zero_()\r\n", + " return loss" + ], + "execution_count": 190, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "idr2VEWb9rr0" + }, + "source": [ + "Теперь приступаем к обучению: делаем несколько проходов по всему датасету (эпох), разбиваем его на minibatches, и вызываем функцию обучения:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nOuu0qpx-wAp" + }, + "source": [ + "# Shuffle the data.\r\n", + "indices = np.random.permutation(len(train_x))\r\n", + "features = torch.tensor(train_x[indices],dtype=torch.float32)\r\n", + "labels = torch.tensor(train_labels[indices],dtype=torch.float32)" + ], + "execution_count": 191, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3zdIf6c_85Ht", + "outputId": "6520288c-da59-4a9f-c37e-cd99779c3073" + }, + "source": [ + "batch_size = 4\r\n", + "for epoch in range(10):\r\n", + " for i in range(0,len(features),batch_size):\r\n", + " loss = train_on_batch(features[i:i+batch_size].view(-1,1),labels[i:i+batch_size])\r\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 192, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 94.5247\n", + "Epoch 1: last batch loss = 9.3428\n", + "Epoch 2: last batch loss = 1.4166\n", + "Epoch 3: last batch loss = 0.5224\n", + "Epoch 4: last batch loss = 0.3807\n", + "Epoch 5: last batch loss = 0.3495\n", + "Epoch 6: last batch loss = 0.3413\n", + "Epoch 7: last batch loss = 0.3390\n", + "Epoch 8: last batch loss = 0.3384\n", + "Epoch 9: last batch loss = 0.3382\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "US6q0nCBD-LL", + "outputId": "c804b779-3231-4f6f-c854-032d211b2853" + }, + "source": [ + "w,b" + ], + "execution_count": 193, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(tensor([1.8617], requires_grad=True), tensor([1.0711], requires_grad=True))" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 193 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "_e6xRMZFDnyI", + "outputId": "79e6c360-265a-401d-ce39-8f211917a13d" + }, + "source": [ + "plt.scatter(train_x,train_labels)\r\n", + "x = np.array([min(train_x),max(train_x)])\r\n", + "with torch.no_grad():\r\n", + " y = w.numpy()*x+b.numpy()\r\n", + "plt.plot(x,y)" + ], + "execution_count": 194, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 194 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhU1Z3/8ffp7uqFrRulQWhoQFZFZOsxGowKiBhN3KNJRo3JTJiJGbcYlGQWZ/L8ZgTcMyYTiZplYhKNInFiZFFQXFFWUeluEGRp9qWbpffq8/uju9qmum7Vra5b1beaz+t5eNJ03br3XMp876lzvud7jLUWERHxr4zOboCIiESnQC0i4nMK1CIiPqdALSLicwrUIiI+l5WMk/bp08cOGTIkGacWEemSVq9efcBaWxjptaQE6iFDhrBq1apknFpEpEsyxmxzek1DHyIiPqdALSLicwrUIiI+p0AtIuJzCtQiIj6XlKwPEZGTycK1FTywuIxdlTUMKMhj1oxRXDWhyLPzK1CLiCRg4doKfrRgAzUNQQAqKmv40YINAJ4Faw19iIgk4IHFZa1BOqSmIcgDi8s8u4YCtYhIAnZV1sT1+45QoBYRScCAgry4ft8RCtQiIjSPNU+es4yhs19m8pxlLFxb4ep9s2aMIi+QecLv8gKZzJoxyrO2aTJRRE56iUwIhl5X1oeISBJFmxB0E3CvmlDkaWAOp6EPETnppWJCMBExA7UxZpQxZl2bP0eMMXemonEiIqmQignBRMQM1NbaMmvteGvteGASUA28mPSWiYikSComBBMR7xj1NOBTa61jgWsRkXQTz4Rg2+Xi+XkBjIHK6oakTCKGGGut+4ONeRpYY619PMJrM4GZAMXFxZO2bVMsF5GuJTw7JFxeIJP7rxnboWBtjFltrS2J9JrryURjTDZwBfCnSK9ba+dba0ustSWFhRG3/RIR8b1o+dSRskPa8nrpeEg8Qx9fprk3vdfzVoiI+ECsfGo3WSDJyBSJJz3vG8AfPG+BiIhPxCqw5CYLJBmZIq4CtTGmOzAdWOB5C0REfCJWPnWk7JC2kpUp4mrow1p7HDjV86uLiPjIgII8KiIE61AvOTw7JFVZH1pCLiLSYtaMUe2yOsJ7ycleLh6JArWISItUFFjqCAVqEZE2OqPHHIuKMomIeOCjiiqeWZmchX7qUYuIJOCzA8d5cEkZf/lwN3175nDNhIHkZTtnhnSEArWISAfsO1rLf7+2mT+8v51AZga3TR3Ody843fMgDQrUInKSaFtMKZFJwqO1DcxfsYWn3tpKfWMTXz9nELdPG0HfnrlJaHUzBWoR6RKiBeKObrUVOmdFZQ0ZQBOQYaDJwlfO7s8PLxnFkD7dk31rCtQiEh+veqZet8kpEAPc/dx6gmGVQmNttRV+zqaW3zdZyMnK4OIz+qUkSEOcZU7dKikpsatWrfL8vCLSuSKV+UyktKdXJs9ZFnFFYUFegLrGpqgV74oK8iI+dL54/2vsqqqN+r63Z09NvPEtopU5VY9aRFxLdBPYZHGq0VFZ0xD1fQZaA3zbXnjxqd2iBulo10wGBWoRcc2vm8A61eiIxgDh4wk1DUFmL/iQ2oam1rHoaNdMFS14ERHXnIJTfl7Asdh+Kjjtedi7WyDi8ZnGtAvSIbUNTdw9fST3Xz3WsVJeqvdTVKAWEdciBcRAhuF4fSMVlTVYPh9CSGawDt+FBeD+a8ZSVJCHoXn8+P5rxnLfV8dEDOAPXT+OIoeHzmm9crlt2ghuOKe49ZzQHNxpc+5UDvVo6ENEXItUtKi6vpHD1SeOBSdz3Nopw+P+a8Y6Tu6FZ6nMGHMar27c2264JC+Qyewvj279u1/qfijrQ0QSMnT2yxGHEQywdc7lnl/PKcPDTRZGY7CJP63eyaOvlrP3SB1jBvRi/9E69h+t6/RUQ2V9iEjSxCq277WOTGhaa1n88R7mLS5jy/7jTCwu4L+/MZFzhp6SlDZ6TYFaRBLipti+l+J9MLz76UHmLipl3Y5KRvTtwfybJjH9zH6YljHndKBALSIJSXWxfbcPhk92HWHuolLeKN9P//xc5l13NtdOHEhmRvoE6BAFahFJWCon3WI9GHYcquahJWX8ef0ueuUG+PFlo7n5vCHkRtmU1u9cBWpjTAHwJHAWzTni37HWvpvMhomIOIn0YDhwrI7Hl23mmZXbyMww/OOFw/jHC4eRnxc5lzqduO1RPwYsstZeZ4zJBrolsU0iIhFFKgh18Zn9ePLNLfxyxRZqG5u4vmQQd148gn69kld2NNViBmpjTD5wAXALgLW2HqhPbrNERE4UKX961p/WkxPI5FhdI18+6zR+OGMUwwp7dHJLveemRz0U2A/8yhgzDlgN3GGtPd72IGPMTGAmQHFxsdftFJFO1tnlTSMVhGposphgEwu/P5nxgwpS1pZUc7OEPAuYCPyPtXYCcByYHX6QtXa+tbbEWltSWFjocTNFpDOFerOpXCbe9tpOi1wA6hubunSQBnc96p3ATmvtypa/P0+EQC0iXZdTedN/f+njqLuqJNoDj1T/OpxTzY6uJGagttbuMcbsMMaMstaWAdOAT5LfNBHxi2j1nkM1n8N3VenI1lfh/uuvG6MG6VRXsessbrM+bgOeacn42AJ8O3lNEhG/cVvvOVSMKfRzpNfcBOq9R2p59NVN7Dta53hMkU+2AUsFV4HaWrsOiFgsRES6vkirAZ1Eq7kRa4OBqpoGnnjjU55+eyvBJkv37EyO17e/ptfbYPmdViaKSExuy5vC5zU3YtXjaDuG3T8/l5Ihp/BG+X6O1DZw5bgB/GD6KNZsP5zSOiJ+pUAtIq6ErwZ02ug2FESjvRb+3l1Vtby0fhejT+vJ77/7BcYMyG8N5DUNQTKNIWjtSTXc0ZYCtchJLJHMDDfFmJxem7eoNOIwytHaxtYg3TaQB61tDfQnW5AGbRwgctJy6hEne5up97ce4vonnEsFFUWZuIw0Nt3ZC3G8oo0DRKQdp9zoZG2hVbrnCA8sKuO10n2OO3wbIo9th4RPRjptywXxpQH6nQK1yEmqIzuldMTOw9U8vLScF9dW0CMni3svHc2p3bO576WPT3hQGHDcGTwkfHOAVD9sOosCtUgX5zQ04JQbbWnelzDRIYRDx+v52fLN/O+728DAzC+dzvcuGkZBt2wAsrMyWtuVnxdoXTjjJFK2R6oeNp1NgVqkC4s2NBAtNzqRIYTq+kaeenMr81ds4Xh9I9dNGsgZ/XvxZMvv2j4srppQ1NrGaJyyPVK9X2NnUaAW6YJCvehIQSw0NBCalIt1nNtA3RBs4o8f7OCnr21i/9E6LjmzH7NmjOLjXUeijiNHGr4IiTW56bQt15TRhUyesyztJxhDFKhFPOCnzAM3hYxCQwOhXu3Q2S9HHB92M4TQ1GR5ecNuHlpSxmcHqzlnyCn84sZJTBrcG4BbfvVB1HHkaNeIlYESKUVwyuhCXlhd0aUmGBWoRRLkt8yDaD3UkPChgY4OIby16QBzF5WyoaKK0af15OlbSqiqbuD2P6xtDZxOWRyhAO10TFFBnqt/v/CFOJPnLOtyE4xu6lGLSBTRMg+SJVSjeejsl5k8Z9kJdaFj9YIjTcrNmjGKvLDNX6Mt1d6ws4obn1zJjU+t5NDxeh6+fhwv3/4ljtQ08uMXPzqhbrXTnt+hh0C8147FzQRjtH8/P1KPWiRBqc48iNWDj9aLzTTmhIdIqIfpZpUhwGcHjvPgkjL+8uFuencL8K9fOZMbzy0mJyuz9f3hDy1L5NS76vpGFq6tcH1tt2J9O/DbNyA3FKhFEpTqzINYucORJtgCGQYMNASbw2Wk4BRpZ++QfUdr+elrm/jj+zsIZGZw29ThfPeC0+mVe+IO304PJwsUhKXgHa5uOKENXgVJpwnGUA89HXOvFahFEhQrMHgtVg/ebaU7p+DUdmL0tPxcxhbl8+amAzQEm/jGOcXcNm04fXtG3uE72ngz0C5XOhkBMlYPPR1zrxWoRRLk9Vf3WNz04MN7qENnvxzxXLGWZO+uqmV3VS0TBhXwyA3jGdKne9S2RXto3fXsOldt8EK0Hno65l5rMlHEA1dNKOLt2VPZOudy3p49NalfoTsy+eYUhMJ/71TVbt/RuphBGpr/He6/ZixFBXkYmnvSoRQ7t21INq8nL1NBPWqRNNORHnys4RlrLcvL9rGrqjbi++Pp9Tr1ZlM9ROQk1d+AvKBALZKG4p18ixacVm87zNxXSnn/s0NkZhiCEcraedHr9VOA9HLyMhVUj1rEp5K92nHT3qPMW1zG0k/2UtgzhzumjSAvkMm/LPwo5TWqRfWoRdJOMnN9d1XW8Oir5Ty/eifds7P44SUj+c75Q+mW3RwOMjOML3q98jlXgdoY8xlwFAgCjU5RX0S8kYxc38rqen7++qf8+p3PwMJ3Jg/l1inDOaV79gnHpduwwMkgnh71FGvtgaS1RERaeZnrW1Mf5FfvbOV/Xv+UY3WNXDNhIHdNH8HA3t0SbaakiIY+RHwgfDy6oFug3QIViG9SrzHYxHOrdvLYa+XsPVLHtNF9mXXpKEaf1svLpksKuA3UFlhijLHAE9ba+eEHGGNmAjMBiouLvWuhSBcXaTw6kGEIZJrWJd/gPpXNWsuij/bwwJIytuw/zqTBvXn8mxP5myGnJO0eJLncBurzrbUVxpi+wFJjTKm1dkXbA1qC93xozvrwuJ0iXUZ477m6vrHdeHRDk6UgL0D3nKy4JvXe+fQAcxeVsX5HJSP69uCXN5dw8Rl9Mcaphp2kA1eB2lpb0fK/+4wxLwLnACuiv0tEwkXqPTupqmlg3X2XuDrvx7uqmLeojDfK9zMgP5d5153NtRMHkpmhAN0VxAzUxpjuQIa19mjLz5cAP0l6y0S6IDdF/UPcjEdvP1jNQ0vL+PO6XeTnBfjny87gpvMGkxu2RFrSm5sedT/gxZavTlnA7621i5LaKpEuym3WRqzx6APH6nh82WaeWbmNzAzDrRcN4x8uHEZ+XsDxPZK+YgZqa+0WYFwK2iLS5TlVbnM7Hn2srpFfrtjCk29uobaxietLBnHnxSPo1yty2VHpGpSeJ2nJT5vJxsOpMNG/XzHGsf0L11Ywb1Epu6pqT9gppW/PHL4w9BQF6ZOAArWknVRupRTrgRDvAyPewkQL1uzk3hc+bE3Ta5tOte9oXbv7TtcHmESnokySdibPWea4i8jbs6d6dp3wBwKcWKAo1uuJsNbyRvl+vvvbVSfkUkcSuu9ktkeSL1pRJm0cIGknVVspxdpdPFm7j6/bUck3fvket/zqg5hBGj6/787YDV1SQ4Fa0k6qdgqJ9UDw+oHx6f5jfO93q7nqZ2+zed8x/uOKMQzIjz3+HLrvdNwLUNxRoJa0k6qtlGI9ELx6YOypquVHCz7kkkdWsKJ8P3ddPJLXZ03hW18cwj2Xjm53r221vW+/bHUl3lOglrQTbV8+L8V6ICT6wKiqaWDuolIuenA5z6/eyU3nDuaNe6Zwx8Uj6JHTPM8ffq8FeQF6dwtEvO9kPMAWrq1g8pxlDJ39MpPnLGPh2ooOn0s6TpOJIlF4nfUBUNsQ5DfvfMbPX/+UI7UNXDW+iB9MH8mgUxIvO+pl1ocmJ1Mr2mSiArVIijQGm1iwpoJHXi1nd1UtF40q5J4ZozlzQGJlR5OVkpeq7Bpppq24RDqRtZYln+zlgcVlbN53jHGDCnj4+vGcN+zUdsfGG3STvWVXPL+X5FGgFgnjZQ/1/a2HmPPKRtZsr+T0wu784saJzBhzWsSyox0JusnYsivEabm7JidTT5OJklaSPbkVCpYVlTVYPg+W8V6ndM8RvvPrD7j+iXepqKzhhpJB1NYH+d7v1nD+3OURz9eRPOhk9npTlV0jsalHLWkjFUvHE+2h7jxczcNLy3lxbQU9c7K499LRnNo9m/te+jhmuzsSdJPZ6413ubskjwK1pI1kfs0P6WgP9dDxeh5ftpnfvbcNY2DmBadz64XDye8WYPKcZa7a3ZGg61Tkyater3Yk9wcFakkbqZjcijdYVtc38tSbW3lixRaq6xv52qRB3Dl9BP3zPz/eqX0VlTVMnrOstbc6ZXQhL6yuiCvoqtd7clCglrSRiskttz3UhmATf3x/O4+9tpkDx+q45Mx+3HPpKIb37em63YbPt+KqqKzhhdUVXDupiOWl++MKuur1dn0K1JI2kv01H6L3UNvWhc7MMASbLOcMPYUnbprEpMG942p327rSITUNQZaX7k9JjrLKoaYXBWpJG6n6mh+ph7pwbQX3PP8h9cEmAIJNzWG2fM8Rdhyqjhio2wbD/LwAuYEMKqsbHHvYkJoc5VTW8xZvKFBLWon1NT8ZPcUPd1Zy7wufB+m2KmsaIwa58GBYWdNAXiCTR24Yz1UTihxX/aUiRzkVk7LiLeVRiyf8ULzHqxzokK0HjvP936/hisffpq6xfZAOiZTrHCsnujNzlLXiMP247lEbYzKBVUCFtfYryWuSpBu/fJV2Co53P7c+rrbsO1rLT1/bxB/f30F2Vga3Tx3On1bvZHdVreN7woNcrGDYmdkaWnGYfuIZ+rgD2AgkVkFGuhy/fJV2Co5Ba109OI7UNjD/jS089dZWGoJNfPMLxdw2dQSFPXM4vbBHuwnBttoGuYVrK8gwhmCEgmdtj+usbI1UTMqKt1wFamPMQOBy4D+BHyS1RZJ2/PJVOtokXbQHR21DkN+9t42fLd/M4eoGvjpuAHdPH8mQPt1bjwm9799f+pjKmoYT3t82yIW+XUQK0n4Jhsq9Tj9ue9SPAvcA7ZNEWxhjZgIzAYqLixNvmaQNv3yVjtRTbCv8wRFssry4toJHlpZTUVnDl0b04d5LR3NWUX7E94d6wNEmLCN9uwDINMZXdZyVe51eYgZqY8xXgH3W2tXGmIucjrPWzgfmQ3M9as9aKL7nl6/SocBz93Prow47WGtZVrqPeYvKKNt7lLMH5jPvurOZPLyP6+s4BTmnbxFN1iowSoe56VFPBq4wxlwG5AK9jDG/s9bemNymSbrw01fp0DWdHhyrtx1iziulfPDZYYac2o2ffXMil42NXHa0I/zy7UK6lrh2eGnpUf8wVtaHdniRzhY+PHHjucWs3lbJqxv3UtgzhzumjeCGvxlEINPbDFVtXyUdpR1e5KQTGp7YVVnDI0vLeWBxGd2zs5g1YxTfnjyEbtnJ+U/fT98upOvQnonSaZJZb6Kyup6fv/4pv37nM7Bw83mD+f6U4fTunp30a4t0hHrU4jteLpJpG3RPy89lYnFvVmzaz7G6Rq6dOJC7po+kKCzPOdK1V207FHflOpFUUI9aOoVTrQto3uXabZCMNCYMMGZALx6+fjyjTut5wrEPLC5zvG54RTuNLUsqqUctvhNtMUw8vet5i0oj5i1XVje0BumFaysiLlQJF6nsqAoViR+oKJN0iljparE2dQV459MD7HKovxF6EIR63LGCtBMVKhI/UI9aXPNyAi7WKkJwDpIfVVQxb3EZK8r3kxmjpobTSsFwkQr5tz2PSGdSoBZXvK6Q1zaNzWnMODxIbj9YzYNLynhp/S4KugX458vOoKBbgH/788eOqyLd9IiLOrhfoUiqKFCLK8mokNe2dka0Jej7j9bx+LJN/P797WRmGL4/ZRgzLxhGfl4AgEBmhmNPP1qhpvDJwpLBpyhlT3xJgVpcSWaFPKdFIhef2Y+Hl5bz5JtbqGts4oa/GcQd00bQr1duu/c7BVSnIZbe3QLc99UxJ7xPhYrErxSou4BULN5Idg2LtkGyrjHI71du58J5yzl4vJ7Lx/bn7ktGcnphjw6dF7RSUNKbAnWaS9XuKqmokNfUZHlp/S4eXFLGzsM1fHHYqdx76WjGDSpI6LzqKUu6U6BOc6naXSWZPVNrLa+X72feojI27j7Cmf178dvvjOVLI/p4VtVOJJ0pUKe5VO6ukoye6drth5nzSikrtx6i+JRu3HTuYF7buJdvPf0++XkBjIHD1Q2taXjxrFoU6SoUqNNcutQ/Dh9Hv/m8wazdXsmij/fQp0c2P7lyDN0Cmfxrm1S7totUQrnSnbVxrkhn0srENDdrxijyApkn/M5v+b+hcfSKyhoszcH2/ldKWV62j7suHskbs6Zw83lDeOTVTa4Wp7hZtSjSlShQp7mrJhRx/zVjKSrIw9C8eMNvhYScVgfWNTbx3KodLP1kLxDfcI2WdsvJREMfXYCfsxpqG4KOC07gxKGMaItTwvltaEckmRSoxXML11Ywb1Epu6pqyXCRtBEaynBT/wP8N7QjkmwK1OKpF9fs5N4XNlAfbAKgyUKmgYwMQ0PQufb5rsqadimAyvoQaaZALZ5ZueXgCUE6JGihZ3YW3XOyYhZgSvUwjrbkknSgQC0J27j7CPMWlbK8bL/jMVU1Day775KYBZhSKVWrOkUSFTNQG2NygRVATsvxz1tr70t2w+Rzfu317ThUzSNLy3lxXQU9c7KY/eXR/PadzyIW82/bYwZ/1N5I1apOkUS56VHXAVOttceMMQHgLWPMK9ba95LcNsGfvb6Dx+p4fPlmnnlvO8bAzAtO59YLh5PfLcBpvXJj9pj9kqWSylWdIomIGaht8+63x1r+Gmj54/2OuBKRl72+RHvmx+saeeqtrcxfsYXq+kauLxnEHRePoH/+56lyfuoxx5IuqzpFXO1CbozJBFYDw4GfWWvvjXDMTGAmQHFx8aRt27Z53NST09DZL0d8Khpg65zLXZ8n0thwaPupSJkUbYN6//xczht2Km+UH+DAsTpmjOnHrBmjGN63Z/sLpRGn8XK/LRiSk0PCu5Bba4PAeGNMAfCiMeYsa+1HYcfMB+YDlJSUqMftEa96fZF65qEPKXw4JTyA7aqq5YU1FQwr7M78myex/WA133r6gxNS6CqrG3zde44knXr/cnKLK+vDWltpjFkOXAp8FOv4riqVk3te1YGONe7adjjFacl3TX2Q7QerT2hP28JJfhg/j5dfxstFoolZ68MYU9jSk8YYkwdMB0qT3TC/ilRg6EcLNrBwbUVSrudVLQ83PfBdlTV8uLPSMdd5d1VtzF29VTBJxHsxx6iNMWcDvwEyaQ7sz1lrfxLtPSUlJXbVqlWeNdJPJs9ZFjGQFRXk8fbsqZ3QIncijceGywtkUtMQJMM0rygMF1odGEu84+cikuAYtbX2Q2CC561KU+mY0hUaqqlpCLYG29BEYltBa7l92ggG5OfyH//3Sbug7vS+cKHeu1/zv0XSjVYmxindUrrCe9JBa8kLZPLVcf3564Y9HKtrBOD84X145IbxFPbMASA3kMndz61v14O2EDVYh8bP/Zj/LZKuVI86Tn4q1L9wbQWT5yxj6OyXmTxnWcRxcqc87D+t3smxukauGDeAN2ZdxO/+/gutQRqag2mTwzBHKKXPAAV5AXp3C7QbP4+W/y0i8VGPOk5+Sely22N1GpKxFv5y2/mcVZTveA2nbw9uxuPTcYhIxK8UqDvADyldblcs9s/PjVh7o6ggL2qQhsRSA9NtiEjEzzT0kWRuhic6wk2PddVnh8gJG6YB98E2kdRAPw0RiaQ79aiTKJkTatF6rOV7jzJvURmvbtxL3545fG3SQN7efIDdVbVxD9V09NuDX4aIRLoCV7U+4tWV86jj4ZRznWkMD10/LqGgFSkvOicrg3EDC1i17RDds7P4x4uG8e3JQ+iWreexiN8lXOtDOsZpeCJobcI967Y91orKGnrkZFHbEGTdzkr+7vyh3HrRcHp3z+5Yw0XEVxSokyjartpeFKi/ZEw/Kipr+MXrn3K8vpFrJg7krukjKerghJ0WqIj4kwJ1EsXaVbujqWoNwSaeW7WDx17dxL6jdVx8Rj/uuXQUI/t1vOyoFqiI+JcCdRKFAlykFX4Qf6qatZa/btjDg0vK2HrgOCWDe/Pzv51IyZBTEm6rtqUS8S8F6iQLBblES5W+s/kAcxeVsn5nFSP79eDJm0uYdkZfjDERj493GEMLVET8S4E6BdykqjkF1o8qqpi7qJQ3Nx1gQH4uD35tHFdPKCIzI3KADp0r3mEMLVAR8S+l5/mAU6rdmAG9WLO9koJuAf5pynBuPHcwuREWsITrSClWbUsl0rmUnudzkcaH6xqbWLu9ku9PGcY/XDiMXrkBV+dauLbCMdMk2jCGFqiI+JcCtQ84Fk4CZs0Y7fo8oV6xk1jDGH6oYSIi7anWRyerawzSKy9ybznefOho22SpzoZI+lKPOkEdXSTS1GT58/oKHlpSTlVNQ7vtr/ICmUwZXcjkOcsSztwANNYsksYUqBPQkewKay2vl+1n7qJSSvccZcyAXvzX1WM5eKyOB5eUtwblKaMLeWF1hSeZG0UFeQrSImmsywbqRJdDu3l/vItE1mw/zNxXSlm59RCDT+3GzecN5tVP9vKtp99vd43Jc5bFvQAlkfrRIuJfMQO1MWYQ8FugH83zW/OttY8lu2GJSHQ5dKK7p4T/fvO+YzywuJTFH++lT49sfnLlGLoFMvnXP3/seI2OLEDpSOaG6nuI+J+bHnUjcLe1do0xpiew2hiz1Fr7SZLb1mGJLod2+/5Yi0T2VNXy6KvlPLdqB92ys/jB9JH83flD6Z6TFbPH3NEFKPFkbqi+h0h6iJn1Ya3dba1d0/LzUWAj4Ov/Fye6HNrt+512Mfn+lGHc/8pGLnxgOS+s2cm3vjiEN2ZdxO3TRtA9J8vVNVKxQ4o2oBVJD3GNURtjhgATgJURXpsJzAQoLi72oGkdl+hyaLfvDx9q6J+fy6TBvZnzSilH6xq5enwRd00fyaBTusV9jVQsQFF9D5H04DpQG2N6AC8Ad1prj4S/bq2dD8yH5iXknrXQhfBx1vCMCYivNxrPpNxVE4r4ytn9eX71Th59dRP/9+Fupowq5J5LR3NG/14JXSPZC1BU30MkPbha8GKMCdAcpJ+x1i5IbpPiExpnraiswdI8zvrC6gqunVTUoU1Zwf2mrtZaFn20hxmPrmD2gg0cPF4HQPneY5TtOerJNZJJG9CKpIeYRZlMcx3N3wCHrLV3ujlpKosydaQAUTycsiLe23KQuYtKWbu9kr49c6isbjgHR4kAAAn7SURBVKA+2NT6vnQpaKSsDxF/iFaUyU2gPh94E9gAhCLRj621f3V6TzIDdXhgcSpAZICtcy6P+3xtA1WkinJt5ecF+PFlo3ns1U3sqqpt97pXDwsR6foSqp5nrX2L5rjX6SKlkxmak7vDuRlnjZWeFq12BkBdQ5CcrEx2RwjSoEk5EfFGWhVlihQ4Le2fIm7HWWOlp8UKtLWNTTywuMzxoaBJORHxQloF6mjlQEOTcgV5AXIDGdz17Domz1nGwrUVcZ9vV2UNx+sa6ZEbOylmV2WNJuVEJKnSKlA79VBDY8GP3DCeusYmDlc3tGaA/GjBBsdg7XS+XnkBLnxgOUdrG4my41XrOfyQwSEiXVdaBepYPdd4V9pFOp8BqmoaGFbYgwW3fpGHrx/fWhc62hDLVROKeHv2VLbOuZy3Z09VkBYRz6RV9bxYq/XiXWl31YQirLX8v5c3cvB4PQD983P5z2vGctHIQowxTCzufUIWSGensvmhDSKSWmkVqCH6ar14V9qt31HJn1bv5ODxegb2zuPuS0Zy5bgiMhzGOzp7qyoVURI5OaVdoI7G7dLvLfuP8dCScl7esJtTumdz31fP5JtfKCYnK/YO317oaK840aqAIpKeulSgjjU0su9ILY++tolnP9hBTlYGt08bwXe/NJSeLnf49kIivWIVURI5OaV1oHbqmYYHvCO1DTzxxqc89dZWgk2WG79QzD9NHUFhzxxX5/OqXZBYr1hFlEROTmkbqN30TGsbgvzvu9v42eubqaxu4IpxA7j7kpEMPrV7h87nRbti9YqjBXlttSVyckrbQB2tZ/rVcQNYsGYnjywtZ1dVLReMLOSeGaM4qyi/Q+eLJ1DHOk+0XnGsIJ+KGtUi4j++C9Ruhx+ceqYVlTV8+bEVlO89xriB+Tz4tXF8cXifmNf1avzXzc4tTr1iNw+Lzs48EZHU81WgdupRrtp2iOWl+08I3tEq5zUGLT//24l8+azTaK7SGptX47+J7Nxy17PrIp5Tk4UiJzdfBWqnHuUz721vrZAXCt7XTipqt4sLwPUlA/nPq8cSyIxv0aVX47+J7NyiyUIRicRXS8ijFV1qq6YhyNJP9jK2zZhzr9ws5l17NvOuGxd3kAbvdlxJ5Dwq7iQikcTcOKAjOrpxgNNuLU6yszK45YtD+N6Fw+jdPTvu6/mRloiLnJwS2jgglSINGzhtDNAtO5OlP7iwtWBSV6HJQhEJ56uhj0jDBt84ZxCBsNobOVkZ/NfVY7tckBYRicRXPWr4vEdpreXlDbt5aEk5DU2W7MwM6oNNFGk4QEROMr4L1ABvbz7A3EWlfLizipH9evDkzSVMO6Ov61Q7EZGuJGagNsY8DXwF2GetPSuZjfmoooq5i0p5c9MBigryePBr47h6QhGZsbZZcUkTdSKSjtz0qH8NPA78NpkNqapp4Gu/eJecQAb/cvkZ3HjuYBZ9tIcL5i33JLCqlrOIpKuYgdpau8IYMyTZDcnPC/DETZMYX1xAr9yA54FVtZxFJF35KuvjgpGF9GqpDR3v/oexqJaziKQrzwK1MWamMWaVMWbV/v37Ez6f14HVaRm2lmeLiN95FqittfOttSXW2pLCwsKEz+d1YNXybBFJV74a+mjLy8AayvaoaQiS2ZLiV5AXIDeQwV3PrmPynGUsXFvhSbtFRLwWM1AbY/4AvAuMMsbsNMb8XfKb5V2RpNCkZKiGSNBaAhmG4/WNHK5uwPL5RKWCtYj4ka+KMiVDPIWeigryeHv21CS3SESkvWhFmXw79OGVeCYflQEiIn7U5QN1PJOPygARET/q8oE60qRkIMMQyDxxWboyQETEr3xZlMlLTnsURvqdViiKiB91+clEEZF0kBY7vKiynYhIZL4I1KpsJyLizBeTiV4XYBIR6Up8EahV2U5ExJkvArUq24mIOPNFoFZlOxERZ76YTHTKddZEooiITwI1NAdrBWYRkfZ8MfQhIiLOFKhFRHxOgVpExOcUqEVEfE6BWkTE55JSPc8Ysx/Y1sG39wEOeNicztRV7qWr3AfoXvyoq9wHJHYvg621hZFeSEqgToQxZpVTqb9001XupavcB+he/Kir3Ack71409CEi4nMK1CIiPufHQD2/sxvgoa5yL13lPkD34kdd5T4gSffiuzFqERE5kR971CIi0oYCtYiIz3VaoDbGXGqMKTPGbDbGzI7weo4x5tmW11caY4akvpWxubiPW4wx+40x61r+/H1ntDMWY8zTxph9xpiPHF43xpifttznh8aYialuo1su7uUiY0xVm8/k31LdRreMMYOMMcuNMZ8YYz42xtwR4RjffzYu7yMtPhdjTK4x5n1jzPqWe/mPCMd4G7+stSn/A2QCnwKnA9nAeuDMsGNuBX7R8vPXgWc7o60e3MctwOOd3VYX93IBMBH4yOH1y4BXAAOcC6zs7DYncC8XAX/p7Ha6vJf+wMSWn3sC5RH+G/P9Z+PyPtLic2n5d+7R8nMAWAmcG3aMp/Grs3rU5wCbrbVbrLX1wB+BK8OOuRL4TcvPzwPTjDEmhW10w819pAVr7QrgUJRDrgR+a5u9BxQYY/qnpnXxcXEvacNau9tau6bl56PARiC8cLvvPxuX95EWWv6dj7X8NdDyJzwrw9P41VmBugjY0ebvO2n/obUeY61tBKqAU1PSOvfc3AfAtS1fSZ83xgxKTdM85/Ze08V5LV9dXzHGjOnsxrjR8vV5As09uLbS6rOJch+QJp+LMSbTGLMO2AcstdY6fiZexC9NJibf/wFDrLVnA0v5/CkrnWcNzXUVxgH/DSzs5PbEZIzpAbwA3GmtPdLZ7emoGPeRNp+LtTZorR0PDATOMcaclczrdVagrgDa9iwHtvwu4jHGmCwgHziYkta5F/M+rLUHrbV1LX99EpiUorZ5zc1nlhastUdCX12ttX8FAsaYPp3cLEfGmADNwe0Za+2CCIekxWcT6z7S7XMBsNZWAsuBS8Ne8jR+dVag/gAYYYwZaozJpnmw/aWwY14CvtXy83XAMtsyMu8jMe8jbKzwCprH5tLRS8DNLRkG5wJV1trdnd2ojjDGnBYaLzTGnEPz/w/81gkAmjM6gKeAjdbahx0O8/1n4+Y+0uVzMcYUGmMKWn7OA6YDpWGHeRq/OmVzW2ttozHmn4DFNGdOPG2t/dgY8xNglbX2JZo/1P81xmymeWLo653R1mhc3sftxpgrgEaa7+OWTmtwFMaYP9A8697HGLMTuI/mSRKstb8A/kpzdsFmoBr4due0NDYX93Id8D1jTCNQA3zdh52AkMnATcCGljFRgB8DxZBWn42b+0iXz6U/8BtjTCbND5PnrLV/SWb80hJyERGf02SiiIjPKVCLiPicArWIiM8pUIuI+JwCtYiIzylQi4j4nAK1iIjP/X8W5zmilIONHQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0giuwC9GHzi8" + }, + "source": [ + "## Вычислиения на GPU\r\n", + "\r\n", + "Для проведения вычислений на GPU PyTorch поддерживает перемещение тензоров на GPU и автоматическое построение вычислетельного графа там. Традиционный способ вычислений состоит в том, что вначале мы определяем доступное вычислительное устройство `device` (CPU или GPU), и затем перемещаем туда все необходимые тензоры по мере необходимости с помощью вызова `.to(device)`. Мы также можем заранее создавать тензоры на нужно устройстве, указывая параметр `device=...`. Такой код работает без изменений как на GPU, так и на CPU: " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "HK7HPLz3Hyrl", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "7e14cccb-d376-4e59-be66-4ab3f5c3f6f4" + }, + "source": [ + "device = 'cuda' if torch.cuda.is_available() else 'cpu'\r\n", + "\r\n", + "print('Doing computations on '+device)\r\n", + "\r\n", + "### Changes here: indicate device\r\n", + "w = torch.tensor([100.0],requires_grad=True,dtype=torch.float32,device=device)\r\n", + "b = torch.zeros(size=(output_dim,),requires_grad=True,device=device)\r\n", + "\r\n", + "def f(x):\r\n", + " return torch.matmul(x,w) + b\r\n", + "\r\n", + "def compute_loss(labels, predictions):\r\n", + " return torch.mean(torch.square(labels - predictions))\r\n", + "\r\n", + "def train_on_batch(x, y):\r\n", + " predictions = f(x)\r\n", + " loss = compute_loss(y, predictions)\r\n", + " loss.backward()\r\n", + " w.data.sub_(learning_rate * w.grad)\r\n", + " b.data.sub_(learning_rate * b.grad)\r\n", + " w.grad.zero_()\r\n", + " b.grad.zero_()\r\n", + " return loss\r\n", + "\r\n", + "batch_size = 4\r\n", + "for epoch in range(10):\r\n", + " for i in range(0,len(features),batch_size):\r\n", + " ### Changes here: move data to required device\r\n", + " loss = train_on_batch(features[i:i+batch_size].view(-1,1).to(device),labels[i:i+batch_size].to(device))\r\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 195, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Doing computations on cuda\n", + "Epoch 0: last batch loss = 94.5247\n", + "Epoch 1: last batch loss = 9.3428\n", + "Epoch 2: last batch loss = 1.4166\n", + "Epoch 3: last batch loss = 0.5224\n", + "Epoch 4: last batch loss = 0.3807\n", + "Epoch 5: last batch loss = 0.3495\n", + "Epoch 6: last batch loss = 0.3413\n", + "Epoch 7: last batch loss = 0.3390\n", + "Epoch 8: last batch loss = 0.3384\n", + "Epoch 9: last batch loss = 0.3382\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A10prCPowHl7" + }, + "source": [ + "## Пример 2: Задача классификации\n", + "\n", + "Рассмотрим пример двухмерной задачи классификации на 2 класса. Примером такой задачи может быть классификация опухоли на 2 типа - доброкачественная и злокачественная, в зависимости от её размера и возраста.\n", + "\n", + "Сгенерируем тестовые данные случайным образом:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "id": "j0OTPkGpwHl7" + }, + "source": [ + "np.random.seed(0) # pick the seed for reproducability - change it to explore the effects of random variations\n", + "\n", + "n = 100\n", + "X, Y = make_classification(n_samples = n, n_features=2,\n", + " n_redundant=0, n_informative=2, flip_y=0.2,class_sep=1)\n", + "X = X.astype(np.float32)\n", + "Y = Y.astype(np.int32)\n", + "\n", + "split = [ 70*n//100, (15+70)*n//100 ]\n", + "train_x, valid_x, test_x = np.split(X, split)\n", + "train_labels, valid_labels, test_labels = np.split(Y, split)" + ], + "execution_count": 196, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "id": "c-_BjSHPwHl8" + }, + "source": [ + "def plot_dataset(features, labels, W=None, b=None):\n", + " # prepare the plot\n", + " fig, ax = plt.subplots(1, 1)\n", + " ax.set_xlabel('$x_i[0]$ -- (feature 1)')\n", + " ax.set_ylabel('$x_i[1]$ -- (feature 2)')\n", + " colors = ['r' if l else 'b' for l in labels]\n", + " ax.scatter(features[:, 0], features[:, 1], marker='o', c=colors, s=100, alpha = 0.5)\n", + " if W is not None:\n", + " min_x = min(features[:,0])\n", + " max_x = max(features[:,1])\n", + " min_y = min(features[:,1])*(1-.1)\n", + " max_y = max(features[:,1])*(1+.1)\n", + " cx = np.array([min_x,max_x],dtype=np.float32)\n", + " cy = (0.5-W[0]*cx-b)/W[1]\n", + " ax.plot(cx,cy,'g')\n", + " ax.set_ylim(min_y,max_y)\n", + " fig.show()" + ], + "execution_count": 197, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "scrolled": false, + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 283 + }, + "id": "tq0vFchQwHl8", + "outputId": "919f1922-f789-4779-cbdc-4f9e742c358b" + }, + "source": [ + "plot_dataset(train_x, train_labels)" + ], + "execution_count": 198, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd5iU5dX/v2dmtrINdpfOsoBKV9AFsQRFNIgx1thiNJZojDH6mkSNKSbRn0l8TaKmGDUxRo2vFbEiYgsIEaWIIFUEgaXsLmzvOzP374/vPM6WqbvT93yua67dnXn2mTPzzJxz36eKMQaKoiiKYou3AIqiKEpioAZBURRFAaAGQVEURfGgBkFRFEUBoAZBURRF8aAGQVEURQEAOOItQF8oKioypaWl8RZDURQlqVizZs1BY0xx9/uT2iCUlpZi9erV8RZDURQlqRCRXb7uV5eRoiiKAkANgqIoqYgxQHs70NERb0mSiqR2GSmK0gecTmDjRuDNN4GdOwG7HZg+HZgzBxgzBhCJt4Th09YGrFoFLFoE7NvH+w47DDjjDOCoo/gaFb+oQVCiijFAYyN/5uQANt2TJgatrcCDDwKffALk5gKFhYDbDXz4IfD++8BZZwHnnptcRqGpCbj/fmDrVqCoCBg9mvdXVgL33QccfzxwzTWAQ9WeP/SdUaJCRwfwwQdcqFVUUK/k5wPz5wNf+QqQlRVvCfs5TzwBbNgAlJZ2VfrDh3PnsHAhlers2XETMWwefxz4/POeu5tBg4CBA4GVK4Fhw4BzzomfjAlOwqzXRGSUiLwnIptEZKOI3BRvmZTe0d4O/PnPwN//zt9LSnhzOIB//xv43e+AhoZ4S9mPqayktR41yvcOwOEAhg4FXnoJcLliL19vqKqiq2jkSN+vSYTGbvFi7o4UnySMQQDgBPAjY8wkALMAfF9EJsVZJqUXLFwIrF8PjB0L5OV5v58DBvC+8nLgscfiK2O/5uOP+TOQ/27AAKCmhrGFZGDDBvolA72mjAyuUD77LHZyJRkJ4zIyxuwHsN/ze4OIbAYwAsCmuAqmhEVTE/D228CIEf7dzyNHUidVVABDhsRWPgVAbS2Qlhb8OJuNFzRW1NRwlb9vH5CZCUydCkyYEFoguKkp9ICx7hD8kjAGoTMiUgpgOoAP4yuJEi7bt9MFHUjfiHAxt3EjMHgwvRJ2e3LFL5OavLzQ0jGNiU2wx+Wie+r11/mcmZn8EC1ezA/ID35A91YgCgpCc29Z2Q2KTxLOIIhIDoAFAP7HGFPv4/FrAVwLACUlJTGWTglGW1toit3lApYtA155hQtWhwM49ljg1FMZE1SiyLRpwLPPUjn6u1gtLcw+Gjs2+vIsWAC8+iqzgrpnAB06xKDTHXcE3k5OncodjbW68IX1mg4/PHKypxiJFEOAiKSBxuApY8yLvo4xxjxijCkzxpQVF/doxaHEmbw86plANDQAa9YAH30EpKcz0WXoUHoLfv1r4N13YyJq/2XYMODooxnM8XWx3G66bc48M/opmlVVwBtv8EPg67kKC7nKeP11733G0O3TeUdQUADMnQvs2uX7NTmdfE3nnqtppwFImHdGRATAowA2G2P+GG95lN5x2GFML21uBrKzez7ucgHLl/P3adNoEAB+R4cPZ8zvX//i7xMmxEzs/sfVVzNnf9s2pmValvzgQRaOfPWrVLDR5oMPuLIP5P8fOhRYsQI47TQGn5YsYcxABJg5k/ePGwdceCFXGx98wA9fQQFf06FDdJGdcw5w8snRf01JjJhgy7kYISInAngfwAYAbs/dPzXGLPL3P2VlZUab2yUe778PPPIIU027xxLKy4H33gPKyoCJE33/f1UVPRU/+lH0Ze3XtLcDq1dzhV5eTgU7dSqNwaRJsQnqPPAAA0+FhYGP27yZsQW7nXGFzEyuLioruYO47DIaMLebRu6tt1iTIAIceSSrr0eP1kCVBxFZY4wp635/wuwQjDHLAejVSgFOPBGorwdeeIGLv4ED+T2srfWmo44f7///CwsZcG5ooMtXiRLp6azePf54KlKR2CtMh4PPHQinE/j0U+4GOscN7Xa6v9rbWZQ2ciQ/WBMm6PaylySMQVBSBxHga1+jm3r5choBY2goLAUfKF3cZuM5rBigEgPi1VNk+nQGjwLFA3ftotHwl2mUns7MoUWLAq80lKCoQVCixrBhwAUX8Gbxl78AmzbRZe0Pt5sGxFcMQkkxpk/nhW5s9J0Oagw/MIcfHthoFRVx5eHvPEpIJFSWkZL6nHQSA86BqKwEjjlGv9f9gqws4PrrGfg9eLBrhlBbG/DFF/Q5jhsX+DzWtrKtLaripjpqEJSYMnEid/779/t+vKWFGYVnnBFbuZQ4MmUK8POf84OxaxewZw+wezeDTmefzQ9De3vgc7hcNAi6rewT6jJSYorDAdx8M/CHP3DxV1DAtjkdHVwg2mzADTdocVrCcegQW0s4HOxLEkrri3AYNw647Tb2M6mu5vOUlLD/0Cef0B0UKM5QWQnMmqVtdPuIGgQl5gwaBPziF8C6dUwpr6hgFuGZZzLwPHhwvCVUvmTXLuDFF6mQbTZvcOf004F58yJvGIYM6VmRPHmyd1s5bFjP/2lq4opi3rzIytIPSZg6hN6gdQiKEkW2bgXuvZdKv7jYG9RtaaFynjaNfYYibRR8UV3NbWV5eddt5aFD3E18//uciKaERMLXISiKkkC0tXGoRV5ez5SwrCz69NatA955h7uFaDNoEPsZffIJi84qK7mtPO881lEMGhT6uZqaaNDcbm5HCwqiJ3eSoQZBiQvxqoNSQmTdOirOoiLfj4uwpcQbb7AjYSz6A2VksDht5sze/X9DA/Dyy8DSpV37IM2cyR5H2otdDYISO+rr2ZLmrbfoAcjMZMxgzhzGKZUE4uOPgwdos7OZCXDgAKuEE5n6enZN3b+fjbIsN5fLxU6Ln34K/PSnfKwfo2mnSkzYu5eB5Oee42Jy9Gi2qFi6lPevXBlvCZUutLeHNnBGJDnGbD7zDLMXRo/uGvOw27kaMQZ46KHgrXpTHDUIStRpbWU80OXi9zE7m3okLY3fxcGDgYcfZhqqkiCUlASfluZyUYEOHBgbmXpLbS1XHIFW/0VFrH9IlpGhUUINghJ11q6li8ifOzozk+1oliyJrVxh4XJxbu8//gHcdx/w5JPsppmqK8rjjuNrC7T6r6hg7n+gPiSJgLXSCLTjsYJZ27ZFXZxERmMIStRZtowzEgIxeDDw4YfAt7/N2GFCUVHBNs3WrN/0dPbXeecd5sh/73up12djyBDOGVi8mNu67sq0poZK9Otfj4984RCqS8tmC220aAqjBkGJOrW1wZW83c4FaUtLghmE+nrgnnvo9yot7fqYMezT/6c/AbfemnqTuC66iEryzTep/LOyqFxbWxkAuv1234ViiUZhobdjYqC0Nrc7OV5PFEmxT7CSiAwcyKByoDYzTqdX5yQUy5ZxNTx6dM/HRFhBu3UrdwxHHhl7+aKJ3Q5cfDHTSleu5EXMyGCH0smTk8cAjh7N61Rb6z/e0drK1zZ1amxlSzCS5IoqycycOcCDD/qvHWpsZNq7zcaF9ogRHNo1ZUqcdY7bzcBGoF4aInQXvfVW6hkEi6Ii9hVJVkSAb30L+O1vmcnQ3b3X1kZj953vJNj2NPZoUFmJOkcdRZ1aWdnzsV276JHYvZuT1LKzmexx333A//4vjUXcaG1lMVOwbUtuLuMLSuIyfjzwwx/SJ7lrF2snKioYcD54ELjySmD27HhLGXd0h6BEnYwMfhfvvZffv8JCKv59+zh/ecAA7iKGDvUeP2gQR+0+9BBnK8elotnanrjdgYezuFwMNCuJzZFHAn/8I9tfbNrE6zp2LAd8p1pSQC9Rg6DEhKFDgTvv5LTEt97i4mzzZg7CmjKFRqEzlnv+009pROLSDjs9ncLt2BG49fKhQ+zb3wusrFVt4REj+tr+IsVRg6DEjAEDgJNP5q2qivGCkhL/ytAqXvvgg14ahMZGYONGZgoNGABMmhR+I7PTT2eW0cCBvgMaLS3cPZxwQsindDpZ0vDGG9wFiXBw0Lx5jNXGa7yxoqhBUOJCYyOTWIKtjDMz6eINC6eTPfyXLOHvFjYbZ3hefHHowcNJk7j6f+kl+rry8ym0MRSssZF1CP6q7rrR1gb87W8s1svP97YA2rmTLrXZs4ErrkieBB4ltdCPnRIXMjO9nQ8CGYX29jALYd1u4LHHGJwoKemqWV0u4L336OK58cbQtK4IWyyXlACvvMLot83G57GMxRFHhCzes88yo2rMmK6vu7iY9mbZMv4899wwXrOiRAg1CEpcGDqUrWXq6wNXMbe3A8ceG8aJt24Fli9nEVl334vdzpz0devYzXPGjNDOKcJjy8qYKtXWxiBkOD34wTT4//yHsRFfRtBm445h8WJg/nwaTUWJJeqtVOKCCHDWWYwldPbqdMbqqjx+fBgnfvttpjD5c8SLMB6waFHYMkOELR1KSsI2BoA3sSVQS530dBrBLVvCF09R+ooaBCVuzJxJb8zu3Vx4u928v6mJmUW5ucBNN4UZZN26NXj3zYICOu1j3La5oSH0bKKWlujKoii+SCiXkYj8E8CZACqNMVPiLY8SXUToK584kcVpn3zC+wsKgG9+kxk3tbVAXR3dSyG1tQhV48YhzzMvL/TmqIHafChKtEgogwDgXwD+AuCJOMuhxAgr5XLiRC7YnU4agZdf5kwTK6HH4QBOOYUdFALWEE2ezBiBVeXmi+pqFkCEMgAmgkyezKd0Ov3Hs9vamAAVlptMUSJEQrmMjDHLAFTHWw4lPtjtTAC66y4WsI0YwQBsSQmzOhcvZklAQ0OAk8ydS3+LP3eQMdxyzJ8fldcQiLw8GrU9e3zvFNxuoLwc+NrXNKCsxIeEMghK/8YY5ugbQ2PQeQGfns7Eob17geefD3CSww5jH/8vvuByuzMdHYwdzJoVt0Z0F17I2MnOnd6AutPJyu1du2jPzjgjLqIpSsK5jIIiItcCuBYASkpK4iyNEkk+/5wr5ECXdfhwYMUK4Bvf8FOfIMIAxMCBwKuvdjUKaWlMbTr77Ji7izqLcN11LEB7803gs88o8qRJtGMTJmgbCyV+JJ1BMMY8AuARACgrK0vR+YX9EyvVMpBCdDi4g/jiiwCLfJuNfpdTT+VJGxsZpR0/PiGitXY72+7389b7SgKSdAZBSV06OkJfuPurXehCRgZ7byuKEhIJFUMQkacBfABgvIiUi8jV8ZZJiR3DhwdX9MYw+FpYGBuZFKU/kVA7BGPMJfGWQYkM9fXeRqO5uUy5DNSiAqALKCPDm3oJAHZXO4bsX4fDPn8TefXlaOlw4OCYGShxnwygNMqvQlH6FwllEJTkp6ODWUDvvMPMT6uOwGYDvlpWjfOG/Rfp2zfz4IkTgeOP/7INRFYW57o/9hjTTXPQiGM/uB+F1Z+hNbMANWmD0druwpyMFZBfvceD58/XKKyiRAg1CErEcLmAv/8d+PBDKvQvi6+MQen2tzD8vmewbSAw6dhctqPYvJltqi+6iEOURTBnDt1Gzz1rcOT6h5HZuBPl2aVwtgnS0oCZJ9pRMGQELc/TT3M2Z1lZPF+2oqQMahCUiLFpE7ByZc/WziW738f0DU+idsgobKtPR2E7MGwYmBra3g78+9/cHsyeDRHahuNH7EbDDzdg/9DRyLYJiotZfPylkUlLY7XaSy8BxxyjuwRFiQAJFVRWkpslSxgv6Kybba4OTN74PBpyh8M40pGZySlhX5Kezmjyc89x1e8hZ/MqDBtpx9HHCKZPZ9fTHu0e8vI4mFkH3CtKRFCDoESMbdt6NhotOrgF6e0NcKaxM11WFlsJdWndkJXFWoHNm733VVcH798gwuBEU1PfhW9sZN+M1ta+nyuWVFez7HnfPm+7WEXpJeoyUiKGzdazR09WSzXS25swuGID0tsb4LKlQWwjANcQwNGt6KC21vt7fn7P1hPdMYaBi74Um23eDLz+Ov1dIrwdfzwHHI8Y0fvzRpvPPwcWLmQql/XGFxUBX/865zvrYGalF6hBUCLGpEnUq0OG8G+HsxVHbH0FQw98jLaMPLhtaXB2uDHJsRfydiZ7CnUeep+W5v195kxOoQ80Y7OhgU/WW8W9eDHwf/9HP5c1xszpZCDkv/8FfvhDvqhEY+1a4M9/piEcNcqr/BsagEceAXbsAC67TI2CEjb6iVHC4uBB4LXX2ITu0Uepm9rb+dhpp9F743YDMAbT1/wduQ170Z6eg/b0XHQ4stAiA5A9vICKfsWKTv8AtqS2KC1lYx9/8QGnk93hzjmndwHlrVuZpTRqFAcaW+dwOBjTyM8H/vQndkZNJOrqgIceoszFxV2Vfm4uI/rvvAOsWRM/GZWkRXcISki4XMALL3BRLcLFqcvFWfb5+cAPfsBWQaecArz7LjA1eweG71+D6sLxyGyrx4D6/ahHPvLzgQEDAEgWq9a2b2fZ8YwZdHlYiADf+x7wxz/SR15YSIXndtMQtLRw3FpYA5c78eabfBGddyUADdWhQ3yOigrm0V53XZAhDDHkww9pDP25yWw274jQsjLNvlLCQg1CClBVxWZvbjfT8ktLI68Hnn+eOmb06J79hurqgP/9X+AXv6CnorAQqP7jMtS1pKMWghr7UTjc2YiitFoMyM6B9bFz2jNg1m1Ey/zzkP3Ny3t+GPPzgdtvB1avpvto924+eVkZ+0QfdljvXmhbG/Dxx9wddH8hq1YxwGyzMevpySfZIO+882JfBFdfT3k++ojbsJISYMMGP21eO1FQwF7aDQ3Bj1WUTqhBSGIOHmQKvzV6UoRGYeRI4PLLgSOOiNzzLF7s2xgA1NstLSwJ+MEPGNfsWF2OA5tysP4LoL4lA58POxGNTTswuPxzHNrTBIcDaHc50Jxegservov2X+Vi/nzq+S6L9sxM4MQTebMi1n1Vyu3t3gwli8ZGYPly3m/FNTo6+JxDhtC95HZzZFssWLeOfrn2dr7BdjvwwQeMbwwezPiLw0GlX1/P/8nJoQGwguMhdQBUFC9qEJKUQ4eA3/yGeqxzXNEYJuv89rfALbdEJia6ciX1S6BOpIMHc9FdU0OPRYdkYOd2F9w2FpSJpKO1aAK2NR2OAztb4GwDikZmYnTaXhSNyYVUb8MX/+9DvP9wDWafPRCOE2b13AFEanWelcX6h/Z2/gS4C3C5uq6oOzqojNPSuDp/8UUaps6B8GiwfTvwwAN0oQ0Y4L0/N5dWeetW73tRVcWfVo+QgQN50dPSeLwSHk4nd2GLF3t3pEcfDcyZE52td4KhQeUk5bnnuDgcPrzrQleEOmHgQODhhyOzSNyzJ3hmp83GW7VnAOp/mmfC1ljfpVDN7QbK99vhzMyBLS8HTZWNaEgvQtlHf8Npq3+Dsub34dz8GfY9swy4+25avGgEdR0ObkUOHODfra0cxdY9TtDeDowdy9/T0qhwP/oo8vJ058UX+YZ3NgYWpaWUf+1aoLKSBquggD/z8xmkf/ttDlvoHh9RAtPcDPzhD8D99zOZobCQC4SVK4Ff/5oDl3zNPk0h1CAkITU1dC0HmiOfl0ddunFj358vPT20mie32+vFePXADKTnZSK9vfHLx5uavAPm7eJGfkcV9h1MQ379btQVlKIxbziksBAba0bAPWo0AyN/+EN0isXmzKE7qqaGiqC7C6m+ngp28GDvfdnZDHBHk4MHWRvROcDemUGDuJOxZm/62kFlZHAUW4orr4jzz39yp1haSmPgcHgr6UeOZCBt5cp4SxlVwjYIIjJAROIzf1ABwMUsEHyYjMPRrU1EL5k+nTGCQLS0cEE7fDiTc9ocA7Dq2BuQ2VKDnMYDEOP+Mlab7azHsNad2JAxA1vaxqAxz1tHkJbGhXlDo7C+YNcuroYjTVERcOut/H3vXhqd9nZardpaWtTjjuv6JlsWL5rU1fFN8ueaaGykIcvP57H19TRojY2UOy2Nk+JqayNz8fsL+/YxVbekxPd773BwcbBwYUpXhAc1CCJiE5FvisjrIlIJYAuA/SKySUTuFZHDoi+m0plAtVqdsYLMfWXqVK/+8SfP/v0s7rW8FCJA1eDJWHrSHagYPAV59eUY2FSOYR274LSl45Vh38VHOBbt6b7TOb+Ue+BA+nOjQWkp8LvfATfc4A3GFhXREMye3bN1RktL9OdepqcHXtnX11PO4mLGCsaO5Wp2+HDKPXeuNw6ya1d0ZU0l1q71BuP9kZvLmM3u3bGTK8aEstx5D8DbAG4H8Kkxxg0AIjIIwBwA94jIQmPMv6MnptKZoUOpMN3uwMWoHR3UeX0lLQ248UbgnnuoEwcP9j5vSwuNwZFHskupJZ+V5FJXMBqrZt6AjNY6lG+qx9YdaXAVDoaBIG/3Cgwb2rUPkaULv9TFVgO7aJGVRffRLbfQR+zvDWts5BYo2iM5hw9nTKCpyXcMwVoNuFwMund2afk6VgmN6mpvgkEgItU7K0EJxWV0qjHmLmPMessYAIAxptoYs8AYcz6AZ6MnotKd4mIq4IoK/8c0N9Pl7XcQfZiMGwfccQcnn5WX87Z7N5/nwgtpMKzvU04OF9id9XhbZj6yx4/CQcdQuIwNHU6Bw+7C+OzyLs/T1ESDkpXlucPp9BkcdbkivHM/4wyutnft6tJ1FcbQr3/oEHD99d5RbtHCbmfe7oEDvl9gTg6tcHa2/ziDRSL3Yko08vO7Xnd/GNPpw5l6BN0hGGM6RGQCgBEAPjTGfBklFJHTjTGLjTEhvJNKJLnoIuCuu7iDLSrqutNtaqKx+MEPIqu/Ro7kOWtquKCyujz4SmY56yxm7+3dy9kHNhsXvOPGMWbqcABnT6jFoOZ9aEwfDoD1Ym43O1Z8SVUVm7V5Hl+1ijVqe/fynFOncmcyaVIfMwKzsoAf/5g+4v/8hxYHoECHHQZcfDF/xoLZs9m8bulSXlzLndXeTpdRXh6LTPxtD+vr6UYaPz428qYCRx/Nax/IH9vUxN3b6NGxlS2GiAmyrRSRGwF8H8BmANMA3GSMednz2FpjzNFRl9IPZWVlZvXq1fF6+rhTXs7OCpar2EpFz89nxXC8B4nV1ABPPMEaKwur84TdDkzK2olzN/waNVkj0OZOQ0YGe9oVFnoO7uig5v/1r9EwaDTuu496ctAg6kRr8d7YyPjFJZdEqJ9bUxO3P04nhRk2LPb55243M1oWLfJaP4eD7q0jjmDRWnZ2z37j9fV843/8Y27nlNAwhm1SNm/myqc7Lhe/aFddBZx8cszFizQissYY00NDhGIQNgA4zhjTKCKlAF4A8KQx5gER+dgYMz0aAodCfzcIAD/HO3eywaXLRd01aVL0k2HCobLSK19hIXvYHTzIwltZ/AYmfPw08scMRNG4AjjSxFtdV10NXHopzLzTcd99TKHt3m0C4Hm/+AK48kr2UkoprPfC5WJQ09ry7djB1cCBA11XtYMGUWmpMQifhgb4XHVUVdE3evrp3CmmQHFaXwzCRmPM5E5/54BGYROAU4wx0yItbKioQUgBjOEW4qWXuCq32bg6Li0Fzj4bmDYN5eXAz3/Onbq/72JzMz0qv/99YhnDqOJ2M7V0927+PmIE/W3B8pEV/7S1eXtn7d3LD9y0aWzlO2FCShgDwL9BCOWrUyEi04wx6wDAs1M4E8A/AUQ5B09JeURY6DBtGrcSVjR88OAvv3zr1gVOzQf4LwcPcqcQK1d/3LHZ6D6KVNMqhTuwE07gze0OnoqaYoRiEC4H0KUBgjHGCeByEXk4KlIp/Q8R72SdbtTVhdaFQSR4AZ2ihEw/HDAUSpZReYDHVkRWHEXpycCBoWUEut2+U/eTjpoaNrBra6Mfe+LE4POlFSUC9Bdvq5LEHH00m/kFyghsbGTAOhKFeHGjuZkjPf/73641CJmZnAw3b16/cl8osUcNgpKQGMPEGrudhWqzZnFYmK9WM04n6y6uuy6Jd/ltbcxw2b6daY+dA8NtbcBTT9HqnX++GgUlaoRsEEREAFwKYKwx5k4RKQEw1BgTsX7AInI6gAcA2AH8wxjzu0idW0kO6uuZjvrGG8y2dDhYmzB7NvXhhg3MvszP5yL64EFmF11wAXD88fGWvg+sWEE30ZgxPRV+RgZTrF57jZbRV568okSAcHYIDwJwAzgFwJ0AGgAsADAjEoJ4Oqj+FcBpAMoBrBKRV4wxmyJxfiXxOXCAozhratieo7SUu4Q1a6gvL7mE2X9vvslsIrud/dxOOSVwSmrC43azAK1TZlUPHA7eli4FLr00tvIp/YZwDMKxxpijReRjADDG1IhICN2gQmYmgO3GmB0AICLPADgbrHdQUpyODhaKtrV17Qxgt7PYrr2d7vWf/MTbtTplaGxkEV5JSeDjCgpYSasoUSIcj2uHZxVvAEBEisEdQ6QYAWBPp7/LPfd1QUSuFZHVIrK6yhofqCQcLhdX+jU13rZAgfj0U5YhFBf7fjw9nQk3r70WWTkVRfESzg7hTwAWAhgsIncD+AaAn0dFqgAYYx4B8AjASuVYP78SmNZW4P336QGx5ifk5wPz5/seMWCxfHnwlNHCQg60qqvjOVOGnBy2Smhs7DnGszO1tZx3kAg0NbGid8MGWvxx4+i/+7IRVQRoaGDDLrebNSrBursqfSYkg+AJKC8DsAbAXAAC4BxjTCT3r3sBdO5UM9Jzn5IkNDXR7fPZZ/z+Wh6Qpibg3/9mltAPf+hb8dfXB+/MahWNtrSkmEGw2YCvfQ147DG+Ob7iCE4nFe9XvhJ7+bqzZg3wyCP07+XkUP5PPgEWLGB67Fln9S2gU1fHc63oVObkdnMWxYUXssWuEhVCchkZNjxaZIzZYoz5qzHmLxE2BgCwCsDhIjLGE5u4GMArEX4OJYo8/TQb7Y0d21XpDxjA+3buZPakLwYNCl5l7HYzHbUvxWetrRxt0NDQ+3NEheOPZ6+c3bt7+tja2thp88wz459htHEj8Oc/0yKXlnLVPmgQrf+IEcALLzBFrLfU1gK/+Q23jMOGsZuhdduyBbjzTmDPnuDnUXpFOC6jtSIywxizKhqCGGOcInIDgDfBtNN/GmMiMAuADNAAACAASURBVCJeiQU1NaynCjSTZcQIppRecEHPrs2zZtHN1NLCZJqiop7upaoqtjzKzQ1fvv37mZ20YgX1rTEcF/C1rwFTpiRAhlJGBnDzzbSqy5d7C9NEOKvhW9/yjqSLF8ZQvvx8No/qjsNBw7BwIXDSSb2z3M8+y1zi7gF2m40FKYcOAQ89xGEgSVt0kriElWUE4FIR2QWgCXQbGWNMhGZyAcaYRQAWRep8SuzYupU6LFCjTbudOmXrVhoAgP+zZAl1yJ49fCw7m3qwpITKOi2NhqKlhQo8XLZtYxdUY+jKcjj4+969wL33starr16OsLDmjhrDKLo1Azkri62rzzuPb0R7O63fxInRn9QWCrt3800LlA2VlsaUsY8/Bk48Mbzz19TQrxhoVVFYyN3S9u3a1C8KhGMQ5kVNiiTDGCqvigqv4go02rY/0N4e/rHG0FX8yiv0hMybxx1EfT31ys6d1BFjx9KYfO97jF2GQ1MT8MADdHVbehfgdSss5GJ3wQKed8qUEE/qcvHEDofvlbI/GhuZJvXee4wJWMyaRd+7lWJVUAAce2zo540VNTXB284CvHgHDoR/fmvSU7D23SK08moQIk7IBsEYsyuagiQLO3dyCtjOnd4gpzGcXfytb/lPm0x1OivbQIh4jy0vB15/na5oSwfMnk1d8vnn3krkuXPZlsJPM9SArFrFFkH+rovDQXkWLQrBINTVsTBsyRKe1BhO+znjDAY8AynKhgZW3e3Zw6CoNYDa5eKqeP164Kc/pd88UbG2VsFwuXrXjM9qNx0Mmy20XGYlbMJpXXGHr/uNMXdGTpzEZvt24He/486+c2Ws2816obvvBn72s/5pFCZO5GK5tdW/Lmht5Xs3cSL/XrqUi8nOC0KHg7sFK3ba0MDFeG/f0xUrgmckFRby+gXM+qys5MWvqfGmQBpD188f/kD//qWX+vdrP/cc3S3du+/Z7XyxlZX0jf/qVwkQ0PDDmDGUt6PDfz9yY/iFsC5yOBQXewM8gd4Dp5Nb8i1b+OHIzuYQjFB6pCsBCcdl1NTp90wAZ4JzlvsFLhfw8MNUGAUFXR+z2bjo27eP1bQ33RQfGeNJRgZd348/TmPZfWqZ00mPwDHHMLvS7WbsNFhqeW4uXdfNzYFT9P3R0hJcT4jwGvp1e7nd9Du1tHQto7b8TgMHctcwejS3ON2pqwsecS8u5htkpWklIgMGMFj89tv+28pWVfF96M1rGDmSRufgQf/1DM3NDCw/8QSzryyys5mF9dWvarC5D4TjMvpD579F5PdgRlC/YOtWftYDtVceOpTTvQ4e7J81NHPncpX98sv8TlqGs7aWC2mnkxXJVvLJxo3UtdOn++7pFgmGDgU2bQqc8NLRwYWv32O2bvW9urew2bhife01BlK7K6SdO7nqDeQbt/yPW7dGxiBYw7bffZd1A04nUzdPP52pWum97Dpz/vlsJPXZZ3RvWdvBjg76+nJyGOzpzcUUod/1N7+hFe/uh2xpoWEdMICPZWV5H2ttZU5zZSVw2WWJu8tKcPrS/jobLB7rF+zYEXzhYcXb9uwJbBA6Orjbra/n9/KII1Kj0EqEsdFjj+Xqf5OnC9XgwXytY8d2/Q6PG8dYwbp11JWdF98WDQ18L8OJ3XZmzhwW1AbyQlRUACefHCCR56OPgitQayuzb1/PWoHqan4oNm3iqjYjg2/GqFFd3xC7PbzovD+MoXFasIByFxfzw3noEPDXvzLu8T//07stV1YW8KMfAW+9xTzeqiqvMTvpJKaB9aVaedw44JZbgAcfpOFJT+e529poEHJygLKynlvQzEyuKt5+m49PmtR7Gfox4cQQNsDTxwisEygGcFc0hEpEgrk1Lawgs79zLF3K2p2mJu85RViAevHFXfVDsjJsGGsNAO4AbruNmVjdX9vo0TS0AwZw59B9DIAx1DdXXtl7L8CECdR/O3bw/N2vYV0dn/O00wKcpLExtBW1zdbVjQFwJf3oo1RuhYV8UYcOccuUnc1WrVaAxOmMTFD5ww8Zs+juuxs4kNu2HTtYaXzzzb1bSWdlMU/39NO5Ine7+doiNa5uwgTGZTZu5I7J5eLu7KOPuJLqbgwsbDYajLfeUoPQS8LZIZzZ6XcngArPbOV+wahRwRMbrKEuQ4f6fvy11/g9HT686w7C5QKWLaOO+NGPEiPlPFJ8/jldaL5W/3l5VNZbt/K9q6z06kOXi1lIEyb0bc6B3Q7ceCPwpz9RN2dlUW91dHDXkpPDBWnADKYhQ5hXHwjr4neumquupmIrLqby//xzCmSz8fjmZuD55+n7HjKEF/7IPpb1uN3ASy9xW+ZLcYrww7x+Pd/gUaN6HhMq6enRq5xOS6Nra9o0731PPBF891FYyP5Koa7glC6Es+663hizy3Pb66ksvidqkiUYkyfzu97U5P+YqioqMF+tVg4c4A7e10rZbuf9W7bQ1ZJKHDrk/3spwoXckUfSU7JjBxfSX3xBl/1XvsJFbF8NZG6ut2321Kk0CMOGsQbsnnuYoBKQY4/l6j1QymV1Na1b54KU5cv5whoaqPxtNiq69HS+KGtF/e67XA1ffHHfZyeXl9MHFqicW4QfutWr+/ZcscbtDr5VFOk6flQJi3B2CKcBuK3bffN93JeSpKVRgdx/Pxd83d2v1dVcIH7zm77/f/lyrz7whQh1yaJF9HunSqJEsNR1ESpky202YQL15YQJPbO5fFFdzZvDwSQef++v3U7j0ytPwsiR9EuvXu17Ek9LC7cb3/9+1/vffZd9fpYu5cXNy+M2sKOj6wVuaOAH6qSTeiFcNyzDE4yMDL5xycS4cTR4gXYJdXXRy1DoBwQ1CCLyPQDXAxgrIus7PZQL4L/REiwRmT6dK9bHHuMq1mbzLkaGD2cnT39V/Z9+2rN/T3dycph5WF8fmjJMBqyEGWs+si+sBJwzzgjdA7F7N9tdfPKJ9zpkZ9Ot/dWv9j6JxiciwNVXU5GvW8cnys31Dn2w2ZhZM2FC1/+rr6el6ujgbiAtjT+t4grAuyNIT+fFHzOmb7JmZ4e2Qm5vD/6BTDTmzWM73UGDfCt8Y3g9Lrss9rKlCKHsEP4PwBsAfgvgJ53ubzDGJNkSo+9Mm0a38KZNTCgRodIbNy4yq/pUW9gMGsTODB995N9dXVFBXRooTb8z27ax6DctjQbEet9bWxmj2bKFcYOIGoWsLBaYbNvGlf+uXVS+c+b4nwOQl0ffV+eLarMxpcxKK+vooGGx2Rhs6atBGDmSW9iGBv9uI2PoAisr69tzxZopU7gq++QTrrw6f+HcbmZyTZ3KqnGlVwQ1CMaYOgB1AC4RkYEADgcL0yAiMMYsi66IiYfDQb93OPG/CROYERfItdvURB3Sm26eicwll1B/7tpF372lqDs66EEpKAC+853QjGFbG7sv5+X1TFO3Mg/Xr+d7fcYZEX4hNhsvZPedgD9OOYVpnoF8Zs3NPF+wOoUQaGgAVq2yocl+DiYu/xuyJmRhyHBHz9jy3r1Urn0JKMcDh4M7saee8gbb7HZvtsesWcDll2vFch8IJ+30OwBuAmsP1gGYBeADAKdER7TUYvZsYPFiLsz8Zc1VVLAup496IeHIywNuv53xkXff9fZ1E2H+/5lncicRCuvXU/H5qxETodF54w2mksZVN5xwAvDMM1y5+sp6aWvjxR4xgtH3QFWPATCGmZbPPUfdmJV5HMpzKzB12UvYkZGJ8ScWY/BQO/3rVrfA7343ObejmZl0351zDt13dXX8gB11VP/sGRNhwgkq3wRgBoCVxpg5IjIBwG+iI1bqMXw48PWvMyNw5MiumTNuN2NlY8cmxkCsaJCbC1x0EdPXKyu9nZ/DTV3/+OPgRWpZWcz4OnAgzovgwkLgl79kj6OKCv7tcNAiNjfTGBx3HEu5Z8wI3Sp24513mJFZUmLtvgQVg89BxxFTMHzzO3D/Zw0ypriQP2UUM5mmT0/+3ObCwsQZJ5pChGMQWo0xrSICEckwxmwRkfFRkywFOfdcKrOXXmJMz/IkiFAfXH554hemNTQwHvDppzRkhx/OOoFQdZnVGLC3dE/Q8YfN1rXDdNwYP54Nnm65hZFwh4OrXCs/uaaGRSn+0tOC0NzMncGoUd1iJiKoLjwc1ScejkOHgKVFBr+4Q5JyU6DEjnAMQrmIFAB4CcBbIlIDQFtih4HNxmHzc+Yw7byujgu18eOTo/fRihXAv/5FRZuTQ0O2YQPw4otscXPGGdH3QowaxdY8gXC5aKx6ueCOPJMnMzVtwQL247YmBdXV0Zd47rm97l3yySc0koEW/IMGATu/EOzdy6detYq7tJwcNhuMVEKEkvyE09zuXM+vvxKR9wDkA1gcFalSnMxMfhGTibVr2e11+PCutVODBlEhPfMMldKpp0ZXjuOOY7ppoDTWigruuBKqP9SQIcD113NHYE1WGj68zxkEe/f6j0lZWLVaDz/McIbNxmvY3s64VmkpcMMNybEoUaJLyOsCId8SkTuMMUvBwPK0YP+nJD9uN0fpFhf7LqS10j+ff56pn5GkqYktJ7Zt44K6uJh1Bl984buVSG0tFeBZZ0VWjogxcCDdRePHRySdLCMjtJYqW7dyNzF6tHfC38iRNAYVFazYrq/vszhKkhOOy+hBAG4wq+hOAA0AFoCBZiWFCdSPyCIjg8bg008jk95eX8/RmkuXUuFZTQNnzfLOP37zTe8MepeLzz9wIJvphVrTkOyMHx+8Dq2ykklMJ5/s26U3ZAgN7LJlzPhS+i/hGIRjjTFHi8jHAGCMqRGRSJb+KAlKTU1ox9ntzO7pK3V1HE5WUcEUUit11OWi/3vDBk6mO/VUYOVKuk0yMlg0OHlylFJNnU5WI65fz3TRUaPol4pzte9hh9H4BZrBsW4ddwOB2iQNGUIDO39+6qU9K6ETjkHoEBE7PC2wRaQY3DEoKY7DEVqw2O2OTHXwM8/QsHRvA2Kl7FdW0h9+xx0xWtHu3An85S/s/ZOeTkGWL6egZ57JnPg4RWWtrhm/+Q0r54cM8Sr09nYW/tlswYt3rVTd5ubUK4xUQiccg/AnAAsBDBaRuwF8A8DPoyKVklCMG0eDEKwfkTF0YfSFmhqu+gP1NCoupo7etavXtVyhs28ftyuZmT19Zk4nI9wA54fGiZEjaRwXLuQOyuqubbez/U9xcXC3knX9dHfQvwmlud2TxpjLABQBuBXAXAAC4BxjTL+Zqdyfyc9nrcGKFf6b9x04wLnqffXd79zJn8GmTQIMNEfdILz0En/6cg05HDQSr71GB30c81yHDuVO4ZJLuNK35nxnZfH28suBB6RZBcwh1cE0NzPt7N13GcXPy+PrnzEjckNylLgQyj73GBEZDuAqABUAngYb3lWISKJkeitR5uKL6TbfubNrJlF7O1fqubnANdf0vQ4hWMaMRUwKz2pr2fI60PQcK+dz5cooCxMaBQUsFhw3zqvcTziB18VfBpjbTYMwf34I12/vXuDnPwf+8Q+vC62ujsV3P/0pPwxK0hKKQXgIwDsAJgBYA2C152b9rvQDBgxg9s5557Faec8ettuoqmIg97jjqDu3bOnbfBIrMBqoHxzA5/A3mS5iWNN9gsUHsrISWhEWFQFXXEHvV01N1/e2qYlGfs4c4Oijg5yosRH4/e+5CigtpfXJzOQWcvRonvjee2lIlaQklG6nfwLwJxH5mzHme9EQQkQuAPArABMBzDTGqKFJQLKzgbPP5kqyqor1AS++yFTTjRt5jNWj6NpruVINl9JSZhbV1fmfCdHSQlmmTOn1SwmNUB3qbnfCd9j8ylf4fr74oneWhzHU5VdcEeJQpo8+orL3l388aBDbc6xYAXztaxF+BUosCCWGIIb4NQbWMX2Q41MA5wF4uA/nUGJEejoXi48/TiXT3Y9fW8s47O23hzCeshsi7PhqzTvo7pJua2PmzDXXRHjegS+ssuzW1sA5my0tXWf/JihTp9KI7t/PXV5GBt2AIQeSlywJHicpLmbr1VAMwr59dLXt2cP3t6yMQkb9wir+CCXL6D0RWQDgZWPMbutOTw3CiQC+DeA9AP/qrRBWcFq081ZS4HYDjz5KY+CrPURBAY/517+Au+4KP64waRIn0z38MPPrMzK8PnCHg6NMZ85kRs3771O5DRzICZSTJwdv5RAy6elM01mwgFbP1wupr2dQNZzhGHHE6pjRK2pqus6M9kVmJgtIAs0/7ugAnnySF89mo9V3Omkc8vM53cgatafElFC+OqeDAeWnRWQMgFpwQI4dwBIA9xtjPo6eiF0RkWsBXAsAJf5SXpSosn07XUaBKpcHDqT3YOfO3n23jzoKuO8+FlVt3kz3xtixXETW1TGuWVXFzJmMDOqgtWuZgnnzzYHH7obFvHksSNu8mZrU6iLndtNatbcDt97aP1a1Awbw9QZKRerooFHwZwyMYa/upUtpZLsfV1vLPhq/+hV9h0pMCSWG0Aq2rXhQRNLA9NMWY0xYkSMReRuArzDgz4wxL4d6HmPMIwAeAYCysrK+uKmUXrJ/f/Cgr7WY3r+/94u9jAzg2GN5s6ivpzupo6Orq8oqptq3j7rkqqt4n3dGQC/JyKCFWbSIrpDWVioxl4v+l298o2/9vJOJk05i/mqghVhlZeA5BXv3cmfgyxgA3F42NzOV95pr+iyyEh5hba6NMR0A9vfmiYwxUe6DqcSKcFxAkfYCrljBHYIvHVxVxdqE8nLuTAYP5mJ2/nwu9Hsd983IYIvqM87gyZ1ObkH6W3vQE04AXn+dASRfRQ3NzVwpnHyy/3OsWEGfXqAI9pAhdB9dckng4gkl4kTK25o0VFdz1QrQAxDnVjRJiTWFzNdUSAur8jWSE8uMYb8dX5MS9+5lEkxmJnV1YyPdS62tHCCzYwdw/QVVcHz0X7b9dLmYBjV7NoUMxXJlZDDBv79SVAT84AfAAw/QKg8eTCvrdNJn53IB3/9+YFfPnj3Bi9esKHdtrRqEGJMQBkFEzgXwZwDFAF4XkXXGmHmRfI6KCrZnXrvW+903hsWVF1zQ/xZ7faG0lCv0Q4f8v29VVdS3gVpQhIvLRR3RPauprY1Dc3JyqJ9cLubXAzQQY0oNml99F1VvPEVdVVDAD8HSpcDbb7NL3iWXaN+GUJg6FbjzTs7tXLaMxsBu5+7h1FODrwBC6dcNMEYTsewAJVR69Y6LyFBjzIFICWGMWQj2SYoKBw4Ad99NxTFypHe36nLRQGzdyu6ZOqM7NEQ45/zuu+kyLi72Gtn2dhantbQAJ57IwHJJSWRcR3a7dyRxZ11RXt61FKB7WcCIfR/h6L3/wua8URg6M90rS24uPwRvvknf0vnn913IRMUYrs63bOG2afBgZkYFG1Dti+HDgcsu49jP1lYq+VCV94wZtN6BVmCNjdzmBctoUiJOb03wIgDB6hoTAmOYIul09ky3s7pn7t/PnPof/zg+MiYjJSXM9HnySfrtAb6PO3dSzx5xBNPWlyxhLcK11/b9+y3CiugPP+x6Lffv7zpCsrmZzw8AYtyYvOl5tBcMQUNL+pdFbV9it3O7s2gRJ++kYqvPgweBRx7hhbIqr10uRtvPO48Blt5YbLs9/N5F06ZxK2el63bHGG7nr7pK53rGgd6+40lTMFBezoraQMpo6FBW2x6I2J6nfzBqFIvP7r6b33Oru+bXv86up5Zrqbwc+O1vGb/pK3PnchfS3u69z+326rOODv5uJcIMrNmB7OZDaE+nL9pndpTDQQX5ySd9FzDRqK3lm797Ny/G6NG8cKWlXKU/9RTw6quxkyczk3UG9fX8wnXuc9LUxGDP8ccztqPEnN4ahL9HVIoosnNn4OAnwMdEvJ02ldAR4Yp7/Xpg+vSe+f8iNLh1dUxQCURzM/XWnj1dFX5nSkuBb3+bRqaykvpk4EC6AxsbqVPKyrwL14zWOhgRuFxccPodRu9wRGa6T6KxeDEt8dChPb8E6ek0EAsXchcRK444AvjlLxmPKC/nBd+9m0b5yiu5ndR4TlzorcvoxYhKEUWs8Yuh0JembP0Zq9FnoLTOYcMYgzz//J5u6+pqpp2//773GmRkAKedxh1H9+PnzKHL6PXXOT0tLY3G4PDDeeucOeaypwPGoLGRriu/rm6Xq3f+9ESmtRV4773AWT9WCugHH3BrFytGjWJGUl0ddzEOB42WGoK4kvIxhECdiztjjMawestnnwV3JaeleYt7O9c1VVZy2ldDA/WBZVRaWzmKYP16xna6n3/8eN4aG7mzWLiQKe7d3dLVgw5Dc3sasgZ0YOxYPxbLypGNere8GHPoEINnwSrzcnJ4EeNBfr7v/idKXEj5GML48VwxNjb6P6aujoajP6eY9wW7PXjlskXn3ZoxwEMPUfmPGtV1h5GZCYwZw67Szz/v/3w5OTTkV18NnH466xF276Z7urwc2L43C+UTT8VXxpQjK9OPkPv2sQlSr5v8JCjhbI21j5iCfhBDsNuByy/3zovtTmMje3ZdfrkmNfSWyZO9ef/+aGujG6jzLmzXLsYQA+3ihg/3NrALhMPBLMh77wUuugiYNYvupp/+FLjwqXOQNesoBok6n6ilhb2gi4oiM90n0SguZjqtv8k4Fo2NSdOcT4kuvXIZGWMejLQg0WT6dCY2PPooXRTWDrq9nSvMm2+mUlN6x4wZwP/9H/Wrv75n+/dzlkLnoO6WLd6Avj8cDu4kduwIPigeYFB7Xo+SxnRW2P73v0wv3e1p2jtgAHsRnXxyalbEpqVx2/T889xu+cJqITtzZmxlUxKSflMKeMwxdBGvX0/lAjAAOWVK/2hUGU0GDAC+8x3gr39lu/zOLmGnkx6Z0aN7Kuq2ttB3ZX0el5mWxuZsX/kKUx6NYc1BqlfDzp3L6sudO1l00/n11tczqHPddalZf6GETYp/G7qSkcHV7IwZ8ZYk9Zg5k37/p5+mK8ha9YuwYvmii3om8QwZEryLgRXvjdj8epvN/yi2VCQri1H5F15gmpeVdmcML8DNN3MLrSgIbWJaKF9Fd7jtsJXQsDoObN7MFXVxMV0niZgheeSRTC3fuZOppA4HW1/7Kki1jk9Pp+vO3y6tvp5Zk937FylhkJ3NINm553J73NHBTIsxYzRwpnQhlB3CPs8tUMTNDkCn1USYgwc5Neyzz7p2HEhLY8eB009PvDioCI1AKDMQsrPpwn/ySaaidq9jaGmhYfnxjxPvdSYlubmhBWJiSGOjN5anrtv4E4pB2GyMCbinFJGYTUzrL1gdB5qa6H/vrBDb2xnEbWsDzjknfjJGgtNO4+t50VPqOGAAd0WNjVQQN9zAXYeSOhjDWN7rr3OxY7MxG/Ckk/h50Hqg+CEmSAK5iGR6pqb16ZhoUFZWZlavXh3rp40JzzzDJpz+hnE5ncy5v/fe1GjdXV3Niuft26kcpkzp2oJCSQ2MofF/+WUmHwwaxMVORwd72jkcwG23+U+KUiKDiKwxxpR1vz/UEZr+TnqlMeaxeBiDVCbUjgMizKQ866zYyRYtBg3iQDIltfn4Y1agl5Z27VKRlsbW9LW1wP33c6xyZmbcxOy39DWi9OuISKF0IdSOA7m58es4oCjhYgx7Vg0c6L9lUUEBOwesWxdb2RQSSpbRen8PAQixU5ASDqEmfhijSSJK8lBbywy0kiDpJ7m53PnOmhUbuZKKxkbOtWhvp8/t8MMjWksTypmGAJgHoKbb/QLgvxGTRPmSoiJm4ASq/AXYhUE7DijJQns7FzDBMsbS0oK3Qul3tLay4nzp0q61JHl5TNU78cSIpOKFYhBeA5BjjOmxiROR//RZAqUHaWms6tWOA0oqYRVDu1yBu1w3NQETJ8ZGpqSgvR144AH2eulebd7cDPz971wdRiAIF9ThYIy52hiz3M9j3+yzBIpPTj2Vu8Fdu3q2baivZzuIK6/UjgNK8pCdTTdQRYX/Y4xhOrUOTOvEihXAxo30tXV3D2Vns1Xw888HfmNDRD3QCUpmJvDDH3IYzIED7Me2ezcNRHo6Ow4cf3y8pVSU8Jg/n922fXWvtaryJ07kYkgB36zXX2dxhj+XUFoafXErVvT56UIJKq81xgQchhPKMUr4WB0HzjsP+Pxz7hQKCrTjgJK8jBzJyvMHHmA2XUEBF71NTfR+TJzIQWr6+fZQX88inWCR+IICjg8877w+PV0oMYSJATKNAAaXdeRRFMnJSbiOA0osaWqiYkhLY3/vJO/jMWECCypXrWI2UWsrW53MmcNxyzpFsxOhTp4CIjIDOBSDMCGEY4L0rFQUJWz27wfeeINa0xh+4YcNA848k874JF5G5+TQAMyZE29JEpy8PAYKm5sDd7Ssq2Npfx8JpVJ5V+e/ReROz/+tA7DOGLOtz1IoitKVHTtYrut2c9i0NSmovh7429+YcXLFFUltFJQQsNvZxfKZZ/ynHLpc9CdHIBIf9qfJGHMHgAcA1AE4V0SSZpymoiQFbW3s35CV1TXNUITFSGPGsLfJBx/EV04lNpx0EmMI5eU93UIdHRwDO39+RGaCh2wQROQBETovjTEVxpg3jTH3GGOu6asQInKviGwRkfUislBE+tEEE0XpxiefcCfgb5CPzcbBGK++Gp6PWUlOsrOBW29lFeru3TQA1s+qKuDCC4ELLohZYZpFA4BXRORiY0yTiMwDcIcx5oQ+SwG8BeB2Y4xTRO4BcDuA2yJwXkVJPj78MPiM59xcKoWqKu0X3R/IzeVg+IoKTstqbWVHyClTIjotK2SDYIz5uYh8E8B/RKQdQCOAn0RCCGPMkk5/rgTwjUicV1GSEqsMPRDWxKT29tjIpCQGQ4bwFiXCcRnNBXANgCYARQBuNMa8HwWZrgLwRgA5rhWR1SKyuqqqKgpPryhxZsQINjELhDWMOl8zvpXIEU5Q+WcAfmGMORlcwT8rIqeE+s8i8raIfOrjdnanY34GWx6JlAAADjtJREFUwAngKX/nMcY8YowpM8aUFRcXhyG+oiQJJ5zArJFA8YGKCuDYY7V3iRJRwnEZndLp9w0iMh/AAgAhNVAwxpwa6HERuQLAmQDmmmBj3BQllSkpYdfCDz/kJJnuwcL6ehoLnSikRJheN9I2xuz3uJH6jIicDuBWACcZY5ojcU5FSVpEgKuvptJftYoVyjk53DU0NHCu6C23sA+EokSQPk1WMMa0REiOvwDIAPCWJ7N1pTHmugidW1GSj4wM4Prr2c3w/fc5QDsjgzuHo48OPChDUXpJ5Ebt9AFjzGHxlkFREg4RuoxKS+MtidJP0Lp3RVEUBYAaBEVRFMVDQriMFCVROXCALvyNGxnjnTCBPcSGD0/6LtSK0gM1CIriA2M4qGrBAir+gQN5/zvvAG++yYzPb3xDm40qqYUaBEXxwbJlwLPPsiQgLc17f24ui4RffZWZoFoKoKQSur5RlG50dAAvvEC3UGdjYGG3c675yy8DLZFKvFaUBEANgqJ0Y9s2thIKlOqfns6xBRs3xk4uRYk26jJSlG7U1YV2nAhQUxNdWRKGtjZg3ToGUaqr6S876SRgxozgrbqVpEENghJzqqqArVupYwoKgMmTgczMeEvlJT09tOOM6ScFw5WVwO9/z5/5+bxYdXXA448z6v6jH/kf76gkFWoQlJjR0AA88QSwejUnAYrwlp4OnHsuMG9eYqRyHnEEs4ecTv9jCSz5J0yIrWwxp6WFxqChoWvFdFYWjUNtLXDvvcBddwGFhXETU4kMGkNQYkJzM/XK2rUMyI4ZQ/0yejQHPz31FLBwYbylJHl5rDUoL/ffgXrvXqCsDCgqiq1sMWftWu4M/A1lKSjgQJ9ly2IrlxIV1CAoMWHpUo6AHTWqZ+5+RgYNwyuvsBAsEbjoIq7+d+7sOqumuRnYsYOv49vfjp98MeOdd/zPdrYYMgR46y2d75wCqEFQoo7TCSxeDAwd6v8Yh4PpnO9HYwZfL8jMBG6+GbjiCv69ezdv7e3At74F3HZbP4mlVlUFn9mbkUHXko7zTHo0hqBEnbo6uqCtal9/5OcDW7bERqZQyMgATjkFOPlkzqQBWJhmt8dVrNiSk0NFHyjS7nTyTfFVtKEkFbpDUKJOIgSK+4LNRq9JQUE/MwYAgymHDgU+prISOP547eORAugVVKJOfj5vTU2Bj6utZQqqkkAcdxy3Sg0Nvh9vbWVp96kBJ+QqSYIaBCXq2O3A/PmcC+8v7tjRwcdOPDG2silBKCgAbrqJPrPycl4ogA2d9u1jFsB3vsOmT0rSozEEJSbMns3xwNu3cxRwZ9dLSwuwfz8zewYPjp+Mih8mTgTuvBN4912mizmd9AMeeyxw2mlalJZCiEniVLGysjKzevXqeIuhhEhzM/DMM8CKFSzsssjJYSvp2bOTP96Q8jiddBNlZGgQOYkRkTXGmLLu9+sOQYkZ2dnAVVcB55/PBnLt7YwtjB+vuiVpcDj6Sb5t/0QNghJz8vPZE01RlMRCg8qKoigKADUIiqIoigc1CIqiKAoANQiKoiiKBzUIiqIoCoAEMQgicpeIrBeRdSKyRESGx1smRVGU/kZCGAQA9xpjjjTGTAPwGoA74i2QoihKfyMhDIIxpr7TnwMAJG/5tKIoSpKSMIVpInI3gMsB1AGYE+C4awFcCwAl2lBLURQlYsSsl5GIvA3A18ysnxljXu503O0AMo0xvwx2Tu1lpCiKEj5x72VkjAm1YfpTABYBCGoQFEVRlMiREDEEETm8059nA0igQYqKoij9g0SJIfxORMYDcAPYBeC6OMujKIrS70gIg2CMOT/eMiiKovR3EsJlpCiKosQfNQiKoigKADUIiqIoigc1CIqiKAoANQiKoiiKBzUIiqIoCgA1CIqiKIoHNQiKoigKADUIiqIoioeEqFRWoo/TCezYATQ1AVlZwLhxQFpavKVSFCWRUIOQ4hgDLF0KvPQSUFcHiPC+7GzgzDOBr34VsNvjLaWiKImAGoQUxhjgxRdpDIYNA0aP9j7W2go8/TRw4ADw7W8DNnUeKkq/R9VACrNzJ/DqqzQE2dldH8vMBEpLgffeAzZujIt4iqIkGGoQUpj33gPS0wGHn32gzQbk5gJvvhlbuRRFSUzUIKQw69YBhYWBjykq4g7B7Y6NTIqiJC5qEFIYtzu02IAxvCmK0r9Rg5DCjBsH1NYGPqa+Hhg1SjONFEVRg5DSnHYa0NDgf/VvDHDoEDB/fmzlUhQlMVGDkMJMmgSUlQG7dvWMERgD7NkDjB/PYxRFUbQOIYWx24Hvfhd46ing/fd5n8PBqmVjaAiuvJKZSIqiKGoQUpyMDOCqq4Czz2bWUU0NkJcHHHUUMGRIvKVTFCWRUIPQTygsBObOjbcUiqIkMhpDUBRFUQCoQVAURVE8qEFQFEVRAABikrhEVUSqAOyK4lMUATgYxfNHk2SVXeWOPckqe7LKDcRf9tHGmOLudya1QYg2IrLaGJOUWfrJKrvKHXuSVfZklRtIXNnVZaQoiqIAUIOgKIqieFCDEJhH4i1AH0hW2VXu2JOssier3ECCyq4xBEVRFAWA7hAURVEUD2oQFEVRFABqEIIiIneJyHoRWSciS0RkeLxlCgURuVdEtnhkXygiBfGWKVRE5AIR2SgibhFJuNS87ojI6SKyVUS2i8hP4i1PqIjIP0WkUkQ+jbcs4SAio0TkPRHZ5Pmc3BRvmUJBRDJF5CMR+cQj96/jLVN3NIYQBBHJM8bUe36/EcAkY8x1cRYrKCLyVQDvGmOcInIPABhjbouzWCEhIhMBuAE8DODHxpjVcRbJLyJiB7ANwGkAygGsAnCJMWZTXAULARGZDaARwBPGmCnxlidURGQYgGHGmLUikgtgDYBzEv09FxEBMMAY0ygiaQCWA7jJGLMyzqJ9ie4QgmAZAw8DACSFBTXGLDHGOD1/rgQwMp7yhIMxZrMxZmu85QiRmQC2G2N2GGPaATwD4Ow4yxQSxphlAKrjLUe4GGP2G2PWen5vALAZwIj4ShUcQxo9f6Z5bgmlT9QghICI3C0iewBcCuCOeMvTC64C8Ea8hUhRRgDY0+nvciSBckoVRKQUwHQAH8ZXktAQEbuIrANQCeAtY0xCya0GAYCIvC0in/q4nQ0AxpifGWNGAXgKwA3xldZLMLk9x/wMgBOUPWEIRXZFCYSI5ABYAOB/uu3kExZjjMsYMw3csc8UkYRy1emAHADGmFNDPPQpAIsA/DKK4oRMMLlF5AoAZwKYaxIsWBTGe57o7AUwqtPfIz33KVHE44NfAOApY8yL8ZYnXIwxtSLyHoDTASRMUF93CEEQkcM7/Xk2gC3xkiUcROR0ALcCOMsY0xxveVKYVQAOF5ExIpIO4GIAr8RZppTGE5x9FMBmY8wf4y1PqIhIsZXtJyJZYCJCQukTzTIKgogsADAezHrZBeA6Y0zCrwBFZDuADACHPHetTIbsKAAQkXMB/BlAMYBaAOuMMfPiK5V/ROQMAPcDsAP4pzHm7jiLFBIi8jSAk8FWzBUAfmmMeTSuQoWAiJwI4H0AG8DvJQD81BizKH5SBUdEjgTwOPg5sQF4zhhzZ3yl6ooaBEVRFAWAuowURVEUD2oQFEVRFABqEBRFURQPahAURVEUAGoQFEVRFA9qEBRFURQAahCUFEFESkWkxdMnxrqvR1tqEcnytDJvF5GiPj5nlogs9XQ8hYjcKCKbRSSsNiEiUiAi1/dFlhCeo0eraxFJF5FlIqIdCxQAahCU1OJzT58Yqy31XwHMBzAJwCUiMskY0+I5Zl8Enu8qAC8aY1yev68HcJox5tIwz1Pg+d+wEBLqd/hfYJuEL/F0Z30HwEXhPreSmqhBUJIGz1CU0zy//z8R+XOAw2PRlvpSAC975HkIwFgAb4jIzSLyLc8wlHUi8nCnXcRLIrLGMyDlWs95fgdgnOfYez27nc4r+R+LyK88v5d6dj1PgD1wRvl7rs4EaHX9kud1KIoaBCWp+CWAn4nIpWDL4/8JcGxU21J7+haNNcZ8AQCetiD7AMwBsBhcdZ/g2Y244FW6VxljjgFQBuBGESkE8BN4djfGmFtCePrDATxojJkMIDvAc4XCpwBmhHG8ksKo71BJGowxyzyNzX4I4GTLVSMid4HNzmJJEdhnyRdzARwDYBXFRRbY/x6gETjX8/soULkfCPO5d3WashXouYJijHF54im5nmEzSj9GDYKSNIjIVADDAByylJeIDAUnT3Un7LbUIvJ9ANd4/jwDwLmd/zbGdI47tADI9HcqAI8bY27vdv6TAZwK4DhjTLOI/MfPOZzounvvfkxTsOcKkwwArX34fyVFUJeRkhQI5+g+BcYBGj3tvQFgGoB1Pv4l7LbUxpi/etw204wx+7r/3e3YGgB2EfGl0N8B8A0RGeyRfZCIjAaQD6DGYwwmAJjlOb4BQG6n/68AMFhECkUkA5xp4Q9/zxUSHpfVQWNMR6j/o6QuahCUhEdEsgG8COBHxpjNAO6Cd0iRT4PgmSd9A4A3wZm7zxljNkZYtCUATvTx3JsA/BzAEhFZD+AtcGezGIBDRDaDgeSVnuMPAVjhmRh3r0c53wngI8//+u2ZH+C5uuBpdf0BgPEiUi4iV3semgPg9d68eCX10PbXSlIjIo+Cbp0SAK8ZY0IaSSgiXwAoM8Yc7MNzHw3gZmPMZb09R7wRkRcB/MQYsy3esijxR3cISlJjjLnaGOMGs2vyOxem+cIqTAPjDu5Ax4bw3GsBvOcrzTMZ8LjSXlJjoFjoDkFRFEUBoDsERVEUxYMaBEVRFAWAGgRFURTFgxoERVEUBYAaBEVRFMWDGgRFURQFgBoERVEUxYMaBEVRFAUA8P8BBYGhJSJXix0AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SjPlpf2-wHl8" + }, + "source": [ + "## Обучение простейшего одноуровневого персептрона вручную\r\n", + "\r\n", + "Используем возможности tensorflow по вычислению градиента для обучения одноуровневого персептрона.\r\n", + "\r\n", + "Для начала, задаём архитектуру сети, в которой будет 2 входа и один выход. Соответственно, матрица весов $W$ будет иметь размерность $2\\times1$, а вектор сдвига $b$ -- $1$.\r\n", + "\r\n", + "Для удобства сгруппируем все параметры в отдельный класс:\r\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "J1KaixW-cMWJ" + }, + "source": [ + "class Network():\r\n", + " def __init__(self):\r\n", + " self.W = torch.randn(size=(2,1),requires_grad=True)\r\n", + " self.b = torch.zeros(size=(1,),requires_grad=True)\r\n", + "\r\n", + " def forward(self,x):\r\n", + " return torch.matmul(x,self.W)+self.b\r\n", + "\r\n", + " def zero_grad(self):\r\n", + " self.W.data.zero_()\r\n", + " self.b.data.zero_()\r\n", + "\r\n", + " def update(self,lr=0.1):\r\n", + " self.W.data.sub_(lr*self.W.grad)\r\n", + " self.b.data.sub_(lr*self.b)\r\n", + "\r\n", + "net = Network()" + ], + "execution_count": 199, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rQ7W6TOacIAI" + }, + "source": [ + "> Обратите внимание, что мы используем `W.data.zero_()` вместо `W.zero_()`. Это делается потому, что к тензору, состояние которого отслеживается через механизм *Autograd*, нельзя обращаться напрямую.\r\n", + "\r\n", + "Функция обучение будет такая же, как в прошлом примере, но функция ошибки будет представлять собой логистическую функцию ошибки. Для этого нам нужно получить на выходе сети значение **вероятности** класса 1, т.е. необходимо привести выход сети $z$ к диапазону [0,1] с помощью передаточной функции `sigmoid`: $p=\\sigma(z)$.\r\n", + "Далее, если для примера с номером класса $y_i\\in\\{0,1\\}$ был получен выход сети $p_i$, то ошибка вычисляется как $\\mathcal{L_i}=-(y_i\\log p_i + (1-y_i)log(1-p_i))$. \r\n", + "\r\n", + "В PyTorch оба эти этапа (применение сигмоиды и взятие логистической функции ошибки) делается одним вызовом `binary_cross_entropy_with_logits`. Поскольку мы делаем обучение по минибатчам, то необходимо усреднить ошибку по всем компонентам минибатча - это функция `binary_cross_entropy_with_logits` делает сама, возвращая одно число. \r\n", + "\r\n", + "> Следует отметить, что вызов `binary_crossentropy_with_logits` эквивалентен последовательному применению `sigmoid` и `binary_crossentropy`" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "kdDxWeCqwHl8" + }, + "source": [ + "def train_on_batch(net, x, y):\n", + " z = net.forward(x).flatten()\n", + " loss = torch.nn.functional.binary_cross_entropy_with_logits(input=z,target=y)\n", + " net.zero_grad()\n", + " loss.backward()\n", + " net.update()\n", + " return loss" + ], + "execution_count": 200, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zAAgw0h6KzUd" + }, + "source": [ + "Для чтения данных воспользуемся встроенными функциями PyTorch по организации датасетов. Концепция датасетов основана на двух понятиях:\r\n", + "* **Dataset** - собественно источник данных, может быть **Iterable** и **Map-style**\r\n", + "* **Dataloader** отвечает за загрузку данных и разбиение на батчи.\r\n", + "\r\n", + "В нашем случае мы определяем датасет на основе тензора, и далее разбиваем входные данные на минибатчи по 16 элементов. Каждый минибатч включает в себя два тензора, входные данные (размером 16x2) и выходные (вектор длины 16 целого типа - номер класса)." + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PfyqjVb2wHl8", + "outputId": "b3a685a9-304c-4e7e-adf9-2858cc47c3a5" + }, + "source": [ + "# Create a tf.data.Dataset object for easy batched iteration\n", + "dataset = torch.utils.data.TensorDataset(torch.tensor(train_x),torch.tensor(train_labels,dtype=torch.float32))\n", + "dataloader = torch.utils.data.DataLoader(dataset,batch_size=16)\n", + "\n", + "list(dataloader)[0]" + ], + "execution_count": 201, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[tensor([[ 1.3383, -0.9861],\n", + " [ 0.5128, 0.4330],\n", + " [-0.4474, -0.2681],\n", + " [-0.9866, -0.2869],\n", + " [-1.0694, 0.4172],\n", + " [-0.8694, -0.8607],\n", + " [-0.9894, 0.3734],\n", + " [-0.9951, 1.2320],\n", + " [-1.3667, -1.0529],\n", + " [ 0.5239, -1.9724],\n", + " [-1.2474, 0.7092],\n", + " [ 1.8675, -0.2057],\n", + " [-1.1899, 0.4243],\n", + " [ 0.7644, -0.6796],\n", + " [-2.0532, 0.4223],\n", + " [-0.5973, -1.2011]]),\n", + " tensor([1., 1., 0., 0., 0., 0., 1., 0., 0., 1., 0., 1., 0., 0., 0., 0.])]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 201 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xrwgkbQjhkEp" + }, + "source": [ + "Теперь мы можем пройтись по всему датасету и организовать процесс обучения:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "QGchp9D6gVJa", + "outputId": "b4c4751d-cb56-4104-d5b5-f1ae9d3d858d" + }, + "source": [ + "for epoch in range(15):\r\n", + " for (x, y) in dataloader:\r\n", + " loss = train_on_batch(net,x,y)\r\n", + " print('Epoch %d: last batch loss = %.4f' % (epoch, float(loss)))" + ], + "execution_count": 202, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 0.6447\n", + "Epoch 1: last batch loss = 0.6046\n", + "Epoch 2: last batch loss = 0.5817\n", + "Epoch 3: last batch loss = 0.5680\n", + "Epoch 4: last batch loss = 0.5596\n", + "Epoch 5: last batch loss = 0.5542\n", + "Epoch 6: last batch loss = 0.5507\n", + "Epoch 7: last batch loss = 0.5485\n", + "Epoch 8: last batch loss = 0.5470\n", + "Epoch 9: last batch loss = 0.5459\n", + "Epoch 10: last batch loss = 0.5453\n", + "Epoch 11: last batch loss = 0.5448\n", + "Epoch 12: last batch loss = 0.5445\n", + "Epoch 13: last batch loss = 0.5443\n", + "Epoch 14: last batch loss = 0.5442\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5QaDiCQUkFOT", + "outputId": "45b4a66b-1222-40f4-c758-d58f1c7daf8c" + }, + "source": [ + "print(net.W,net.b)" + ], + "execution_count": 203, + "outputs": [ + { + "output_type": "stream", + "text": [ + "tensor([[1.0563],\n", + " [0.4450]], requires_grad=True) tensor([0.], requires_grad=True)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s4_Atvn5K4K9" + }, + "source": [ + "Для демонстрации того, как сработало обучение, построим граничную прямую $W\\times x + b = 0.5$" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 283 + }, + "id": "PgRTHttLwHl9", + "outputId": "d9abf92f-cb70-4c56-ccd0-5e027239da58" + }, + "source": [ + "plot_dataset(train_x,train_labels,net.W.detach().numpy(),net.b.detach().numpy())" + ], + "execution_count": 204, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd3iUVdrG7zOT3nsIISGh92YA6QgIglhZP+xlLWtvKEp1V0AF17oWLKyu3V0bKCKCUqVIly4ISYBUEtLLZDLn++NmnJSpybQk53ddcxFm3nnfZ9p5ztOFlBIKhUKhUGg8LYBCoVAovAOlEBQKhUIBQCkEhUKhUJxHKQSFQqFQAFAKQaFQKBTnUQpBoVAoFAAAH08L0BxiYmJkSkqKp8VQtCCOFx5Hma4MfeP7Qiu0nhZHofAIu3btOiuljG14f4tWCCkpKdi5c6enxVC0IHZm7cTgdwbj/8b9H2aNmuVpcRQKjyCEyDB3f4tWCAqFo6S1T8OUrlPwwtYX8MDQBxDiF+JpkTzL2bPA1q3AiROAVgv07QukpQHBwZ6WrGlICWRkABs2AEePAhoN0L8/MGoU0K6dp6XzekRLrlROS0uTykJQOMq209swbNkwLJmwBI+PeNzT4ngGgwFYvhxYsQIQAggJ4X3l5YCfH/C3vwGDBnlaSseorQU++ghYtw7w9QUiIqggzp3ja7vmGmDyZL7eNo4QYpeUMq3R/UohKFxFdTWwdy9w+DB/l506ec/mc+KHE7Evdx9OPnQSQb5BnhbH/Xz7LfDf/wIdOwI+DRwFFRVAXh7wxBNAz56eka8pfPkllVxKCi2DutTUAJmZVHQjR3pEPG/CkkJQWUYKl7BvH/DII8CbbwLbtgG//gq89x7w8MO05j29D5k/Zj7yyvPw9q63PSuIJygt5cKZnNxYGQBAUBAQHg58/rnnPyh7KS0Fvv+er6mhMgBoMbRrR6VRW+t++VoIXqMQhBBJQoh1QohDQoiDQoiHPC2TomkcOgS89BItgZQU/g7j4/l3TAywbBnwyy+elXFk8khclHIRFv+yGJU1lZ4Vxt3s2cNF0dfX8jEREUB6OnD6tNvEaha//cbXZE7BGQkOBoqKgOPH3SdXC8NrFAIAPYAZUspeAC4EcJ8QopeHZVI4iJTAxx9zPTHnGvL3BxISgE8+AXQ698tXl/lj5iOnLAfL9izzrCDuJjvbujIA6GfXaOh/bwkUFjIobgshgJIS18vTQvEahSClzJZS7j7/dymAwwASPSuVwlHS04GsLHocLBEYCFRWAgcOuE0ss4zpOAajkkfhuc3PoVpf7Vlh3ElAgP1uE2s7bmeTlQX87380L19/Hdi+nYEoewgKsu81ScldicIsXqMQ6iKESAEwEMB2M4/dJYTYKYTYmZ+f727RFDY4e5abMFuJHEIAOTnukcmyDALzRs/DmdIzeG/ve54Vxp307s3F01p8oKaGysAdhZ81NcC//w3Mng2sXg2cPEm/49KlwGOPAceO2T5Hr/POBFuvydcX6NLFOXK3QrxOIQghQgB8CeBhKWUj205K+baUMk1KmRYb26jQTuFhtFr74pB6PfDHH8AzzwAPPcSElhUrgIIC18tYlwmdJuDCDhfi2c3PQlfrYR+Wu+jcmcHXs2fNPy4lcOYMMG4cd96u5qOPgPXrKVOHDkBUFBAba8qAWrLEdiwjIQEYOJBym0NKnmPCBPe8phaKVykEIYQvqAw+llJ+5Wl5FI5j3FBas94rKpiFtH49rYSgIFNa/JNPArt3u0NSIoTA/NHzkVmciQ/3fei+C3sSIYB776X2PnWK2tlIZSV36N26AVde6XpZsrKAjRvNp4oC9D1qNNwt2OK226gYTp5kPQVARVBSwsK7AQPc85paMF5ThyCEEAD+A6BQSvmwPc9RdQjeyZtvAjt3AklJjR8zGIBVq6gwLrussWupooIb1/nz3eOtAAApJYa8OwQFFQU4ev9R+GptBFxbC2fPMlVz40YunFIyE2DSJO6kAwJcL8MXX/ALYe7LYqS2ljv/l16iwvr1V+4kgoNZPNe9u0mZVFQAmzfznMXFfE0JCcCUKcDQobaD6W0Ery9ME0KMBLAJwH4AhvN3z5ZSfm/pOUoheCfFxcCzz7K2KSHB9BusrWWR2sGD/H2Ghpp/fk4Ouw3cc4/7ZF5xdAWu+OwKvH/F+7hlwC3uu7A3UF7OLB2NBoiLc++i+fLL3L1HRVk/LiODRXKHDtGy8fenZVNdTTfTAw/Ub01hMABlZaYqbFWdXA+vVwhNQSkE76WkBPjmG2DTJv42heC/lZX8LVvb/dfW0pPw6qvuq2qWUmLgWwNRUVOBw/cdhlajOqG6haVLWUMQF2f5GCmBH38EEhMZPG7oWsrLY7uNp54CIiNdK28rQVUqK9xKWBhw883cAD75JDBzJvDii9zkRUdbf64xndzoBnYHQgjMHzMfxwqP4fODn7vvwm2dCy6gm8caWVncYfTsaT7OEBfHx3/+2TUytiGUQlC4lOBgoGtXxigjIugmspVaLiWtCXe4sOtyZY8r0SeuDxZuXIhag2pv4Bb69+fuobjY/ONS0seYmmq98Cw+Hli7lqmliiajFILCrYwcSdeuNQoK6BkIC3OPTEY0QoN5o+fh8NnD+PLwl+69eFvFz495x+XltATqpqeVlTG+EBHBHYU1/P2501BVyM1CKQSFW+nXjy4jSynwNTXsU3bppe6Vy8i0ntPQI6YHFmxcAIM02H6Covl07kz/f1oalcKpU+xMCjCV9MILbRe3GLOk3FlZ3QpR757Crfj7A48+CixezN98XJypk0JeHjd5N9xgKjx1N1qNFnNHzcWNX9+I5UeW46qeV3lGkLZGYiJw113AddexAZ2PD78cWi13CZ99Zr0fSkkJs43cbVa2MpSFoHA7iYnA008DV19Nr0BGBtPMBwwA5s4FJk70rHzT+0xH16iueHrj02jJWXgtktBQ1iQkJJhiBkOH8u9KC11ppaTJeemlKr20mSgLQeERIiKAqVP5G66uZuq7Pc0q3YGPxgdzRs3BrctvxXe/f4fLul/maZE8h7F4ZO1atn7w8+MCPWIEe5m7g4gIDrZ57TVaAFFRpoW/ooLdW0eNAoYMcY88rRhVh6BQmKGmtgY9Xu+BqMAo/HrHrxBtcedZXs5F+NAhFneFhFBBGIvY7riD/n13ceQIB9wcP87dg8FABXHppey75C07ihaApToEZSEoFGbw1fpi1shZuPPbO/HD8R8wuetkT4vkXqQE3n6bg+pTUuq7YkJC6L55800WgnXv7h6ZevRgR9S8PKap+vkxbqACyU5DxRAUCgvc3P9mJIcnt81YQkYGOxAmJZn3ywcG0t+/fLl75RKCNQfdulFROaoMpGTzuw0bgHXr6A4zqGwyI0q1KtyOlKZ2++YKT70FP60fZo2chXtW3oOfTv6ECZ0meFok97F1Kz8ga66y6GguqAUFtsvPvYFTpzi/NT2dX0Lja4uJYXpr794eFc8b8OKfo6K1ce4cuxg/8ABjhHfdxbkoGRmelswytw24DYmhifjHhn+0LSshL49WgDWMYzZLS90jU3M4cwZYtAjIz+echdRUWhgpKbQQ/vlPz4/w8wKUQlC4hcxMYN484Ouv6YLu2JHNKbdtY03Spk2eltA8/j7+eGLEE9icuRkbMjZ4Whz3ERpquw2EscdISxhJ+fHHVGCxsY2tHmPm0rvv1p8N0QZRCkHhciorgRde4GayY0dTjyIfH6B9e6acL1vGCWreyB2D7kC7kHZYsHGBp0VxH0OGAFVV1o8pKeGHV7fttDeSnc1MKWsdVUNDWRB3+LD75PJClEJQuJxdu7h2WGp57+9PJbF6tXvlspdA30DMHD4TP5/8GZu3fEbXQkaGfbNCWyo9e1Jb5+WZf7y2lsVg5qYceRvZ2dyN2JJTq/Vu/6UbUApB4XI2brTdUSA2llPWbG1KPYLBgL8VdUacIRALVszg5K6//50pkK21DkarZdO5gAAuksYqYWNVcEYG8//dWYfgalqzgrcTpRAULqekxLab2VhTZKs1ttsxGID330fQJ//DY/oh+DEwC9uSBAfCV1dzis+PP3paStcQH88Az7RprAjOyOCtY0fg8ceB6dO93zoA2CvFGO+whsHAYHMbRqWdKlxOdDSDykFBlo/R62nV20pscTv79gHr1wOpqbhHk4TF8lcswEasFNfT7AkIAD79FOjbl/701kZ4OHuMTJlCK8HHp2UEkesSHw/06QMcO2Y53lFczC9qjx7ulc3LUBaCwuWMHWs7MzE3l+1xvK7odNUqLooaDULgh0cxDN+LY9iFLD7u50dNtqGVZyBpNJx21NKUgZEbbuCXKze3vmtISuZDFxczD7qNt79QCkHhcvr358YsN9f846dPMwnkxx+Bv/4VeOIJbsptTVZ0OdXVwO+/14uG348hiJABWICNpuOio4E9ezwgoMJu2rUD5syhuyszk8VpRhdYSAgwa5b7WnB4Md62H1O0Qvz8gBkzWPuTns7mlcHBXG/37eNvsl8/oEsXbkTLyoD332fW0eOPW85OcjkGA33kdfzkYfDHI7gQT4n12CtzMADtKLQa3ej9JCRwwPeZM1QKUtKd1KlTy4iFuAFlISjcQmwsE3PuvJP90IqLmdGo19NF3acPrXUhmBKeksKOCP/6lwdbzQQEUNjy8np3P4ihCJP+WGi0EoqLKbCiZZCYCAwbBgwfzmltShn8iVIICrcRGMg4wVNPcaFPTTXNWDdH+/bsQ3bsmHvl/BMhgMmTG+XiRyAAD2IIvhSHcUDm0rc1oQ31OVK0WpRCUHiE3Fxa7damIgJ0N23f3owLSQnodE3PMR8xgi2Wz5ypd46HcSFCpB8WVfzAWcAO+p9rahh2+Ppr4JtvgP3723zXBIUXoGIICo9QWWlyEVnD358dBRwmP5+ZP2vXstrN15fpThddRNPDXoKCGMhYupQDWjQawM8P0Tod7g/vjsURB/DUtDHo4UDb1t272dSvrIyJL1Ky8DciArj77jaf+ajwIEohKDxCcDAXwbpdiM1RVdWEzsq//87mSXo9+9f4+3NLvm4db/fdBwwcaP/5IiKY+pSZyW19aSkQHY1Hez+GVz8ejEXbn8eHV31o16n27gVeeYViNZxAWVICLFnChJeuXR14vQqFk1AuI4VHiI1lcse5c5aPMc5NcKg7QlERW0sEB3O4izFv3teXrp+oKOD119nfxhGEYMrilVcCN90ETJmC2I69cE/aPfhk/yc4VmA70KHXM3sqNpbiNSQsjAH1//xHdVFQeAalEBQeQQjgiiuoEHQ688ecOUP3SadODpx4yxbms1qKVBvLpdevd0Rcizw2/DH4af3w7OZnbR575AgTkkJCLB8TEcG6jPR0p4inUDiEUggKj9GvH3Drrdysnz7NdVyvp5I4eZIb+nvvdTArcP162z6m+Hge54RteLuQdvjbBX/DB/s+wMlzJ60em5Nj+5LGmTOWivgUCleiFILCo4wfDyxYwHhvWRljwdHRdPPPnm27S2ojSkuZmmQNX19qn9rapopdj5kjZsJH42PTStBo7NNBUnr3aFFF68WrgspCiH8DmAogT0rZx9PyKNxDhw7AjTfy1hDjAmq3lRAVZcoqskR1NV1HTupb0z60Pe4YdAfe3vU25oyag44RHc0el5LC12EtkC4lb8nJThFNoXAIb9uHvA/gEk8LofAsNTUcM7BgAXD77axufuUV9juyucMePx4oLLR+TF4ecPHFTq1QfWLEEwCAxb8stnhMairj0mfPWj5PTg4bp3r7EDJF68SrFIKUciMAG79mRWumshJ4+WWOGTh7ltZDQgJw9CiweDHw4Yc2WlkMGcJqt4IC848XF9N6GD3aqXInhSfhtgG3YdmeZThTcsbsMUIAd9xB+XNy6r8Og4FB9IAAJjEpFJ7AqxSCQvHhh7QEUlPZRkijoWcnLo6767VrgTVrrJwgJAR47DE+KT2dyf01NQxQZGTQXfTYY00obrDNrFGzYJAGLPllicVjkpKAuXPZyC8z03Q7dYqWwdy51kf/KhSuREgvS3gWQqQA+M5SDEEIcReAuwAgOTn5gow2PgO1NZGfD8ycyUXTUlC1spKtg1580XqYAGVlwI4dwE8/MW0pLIxVykOH2u6X0QxuX347PjnwCU48eAIJodYH5uTm0lIQgv3WXKCjFAqzCCF2SSnTGt7vVUFle5BSvg3gbQBIS0vzLm2maBb79vFfaxk2gYEMARw/zjnwFgkJoQK46CKnymiLWaNm4f197+OfW/6JFya9YPXY+HjeFApvQbmMFF5DSYl9E9OE8ILhORboEtUFN/S9AUt3LUVeeZ7tJygUXoRXKQQhxKcAtgLoLoQ4LYS43dMyKdxHeLj9c2bMtX7wFuaMmoPKmkq8uPVFT4uiUDiEVykEKeV1UsoEKaWvlLKDlHKZp2VSuI/+/bn7t5ZFVFlJZdC5s/vkcpTuMd1xbZ9r8dqvr+FshZUcU4XCy/AqhaBoXZw7x5YUtsoCjMTEcPyAcbqhkaCKs+h29FtcsO11dNvwNm7svgO+hmrXCO0k5oyag4qaCry87WVPi6JQ2E2LCyorvJ8jR4AVK4DDh5n9WVvLds5XXAH0TigEsrJ4YPv2jQYm33gjYwl79wIRobUYnv0luv6xClXVAhUyCKOSDOiyYwtwJAR44AGvHYzeO643pvWahle3v4oZw2YgMjDS0yIpFDbxurRTR0hLS5M7d+70tBiKOmzYACxbxizP6GhTqwbdmXx0/+1/uDhyJxI7nK8QNhiACy4Apk9nT+jz6PXAgQPAiSX/Q8KOFSgMTUFcghadOlF/CAFqjZISYN48Fih4Ifty9mHAWwPw9zF/x1Njn/K0OArFn7SatFOF93LmDPv9JyaaxhAAQEh5LkbtXwSNrhI7czoguKcGERGgQti3jwNt5sz5MwfTxwcYkFyIAQGrgGtTzPccCgtjQOGrr4BHHnHHy3OY/u3648oeV+Ll7S/j4QsfRniA6+ofFApnoGIICqexfj3X7rrKAFJi4N73oK3VoSIiERofDU4au0RrNNQeOh3w3nv1AwfGQcrWGtDFxgK//Wa5TYUXMG/0PBRVFeG1X1/ztCgKhU2UQlA4ja1b63l+AAChpVmIPnsE5cHc/QcHM9Bcj/h4Bh6MsQWAvRwCAqxfUKPhzd6otQcYlDAIU7tNxYvbXkRpdamnxVEorKIUgsJp6HSNC8siitL5x/nOokKYZin/ibHraN0xYf7+9s0rkNK+ajZbVFQwLara+dlL80bPQ2FlId7Y8YbTz43CQr5vWVk2uv4pFLZRMQSF02jXjvNp6rYKEtIAH30Vos7+Dj9dGaqlL/xD20EgBkCD9tN1F7QBA4B166xfsKqKvSwSE5su9NGjwKpVjGUIQRfVqFFsj51gvReRvQxJHIJJnSfhha0v4P4h9yPYzwlVdX/8AXzzDaPvWi3fu9hYYOpU5u6qCTuKJqC+NQqnMWlSfe+NprYGSZmb0T5rJ2LPHkZoWTbCC9PRt/gX4OefqT2MCFHf39S7N1OKzp0zfzEpOXtz0iTbE9IssWYNsGgRg9pJSZxKEx8PbNwIzJ9PN5aTmD9mPvIr8vHWrreaf7K9e4GFC6kUkpLYIzw5melZb78NfPSRU8aDKtoeSiEoHEJKZhPt38821XV7Cl1wAdemrCwe2H/vfxCbfwgVQbHQ+wSgHMGoDQpFUPsI7u43b2amUEkJc1S7dTOdzMeHdQZVVTxhXfdRZSWHLvfpA1zSxHlKx45x4ezQgf2mjTtqHx/eFxbGqTwlJU07fwOGJw3H+NTxWPLLElTWVDb9RMXFwBtvsIqvrtwAEBrKvuFr1gC7djVfaEWbQykEhd0cP84N9dy5wEsvAS+8ADz0EPDxx1QMAQEcNZCYCJQezETs75uRE5iKE2EDYKiohL9Gh+Tk84lDwcEMOhw8yCyhW25p7OZITQWeegoYNIha6PRpBpsrKoD/+z9evKnWwQ8/cIymuedLyfuLixkpdxLzRs9Dbnku3tn9TtNPsn07LQFLzZw0GlpWK1cqK0HhMA4XpgkhggFUSSmdM6G8GajCNPdx4ABnEAQH1ykOA5vRZWVx7X78cbr0DQYg+/kPUfLdRhT4JyIwEEgNyEZE+h5oanSmhb+mhge/9x4nnVmjuJj+KK2Wvn2rwxBsUF0N3H1348ELRvPn+HGgqIgLb0AAMHs2MG4cWDzRPMa8PwbHC4/jjwf/QICPjSwqgJZRfj5liYykJi4osD7TQUr2/3jlFVo6CkUDmlyYJoTQALgWwA0ABgOoBuAvhDgLYCWAt6SUx50sr8JOioqAbdvoVtbrOYlr1Ch2hXDWyOCqKuDNN6kIQkLqP+bry0LhEycYm7366vPlBbp0JA4ORbmWa9OerAQYtHFI8M1FUug5BAVKFBgiUVlSg9+PpqBzFBvWWZQ5PNx5g210Ol6ooTLYu5euqMBAXkuvp9L67jvGFZ58stnDjuePno8JH07Ae3vewz2D77F8oF7Pa65cyTiKUdbcXLq0rL0Xxtem1zdLVkXbw54so3UA1gKYBeCAlNIAAEKIKAAXAVgshPhaSvmR68RUmOOXX7i5rq3l+qDRcMTk6tWcNX/dddbruuxl3z6gvLxxjUFdEhKAH38ELr30fGGary/OZNZi5x98PDAQEFot0qvbY/ex9tDpgKhIibjqU/hulRalP3HDfu+9TkvusUxgIF1COp3JZZSeTmUQGVnf/AkPZ8A2L4+DnhcsaNabOi51HIYnDcezm5/F7YNuh5/WjMtKr6cG/vVXBrmTk033Hz3KYPLEidTO2dm0GKSkxm7fnn/7+jbW3gr7qa3l1D2tlmaxs3ZXXo49CmGClLJRl3opZSGALwF8KYRohv2uaAq//Qa89RZ//3Xrt8LC+F1evZoL8zXXNP9a+/dzDbWGvz/X1+xsICUFOBmdhlN7PkZwQmS9MoGSEv7OhABQVgqZGIf41CD0zdmG2LUbsP/7IoROiUDI1LHsh22rOK0p+PhwktqPP1ILGQzMNAoJqf/D1+mATp34d1wcZzL//ruNUW3WEUJg/uj5uOTjS/Cfvf/BnRfc2fign36iMujUqb48Pj5Mx127ljdjrYbxDc7I4BcjPh647bamx1faMqWlbMi1ejVjVQYDFfKUKUBamnN2WF6MzaCylLJGCNFDCDFeCFFvyyGEuMR4jKsEVDRGSuDzz7khNLdearV046xaRdd7c9Hr7UtrN84ykBL47MQQaAN9ESBNaUh6Pd3hAQFAgL+Eb1E+jkYOw7h1czF411IkGM7AUFOL9C1ngKVL2bguO7v5L8Ac48ZxF11UxEWgqqr+AlpaSu1ad+K9ry/ghJjVxM4TMbj9YDyz+RnU1Db46ej1wPff00wytyuNiuItI4MWTEQEFVlICP+Wkv47FTtwnIICWoBffMGEA2MqcnEx8NprwLvv2lcs2YKx+TMXQjwIYDmABwAcEEJcUefhZ1wlmMIymZkM5FpzI/v4cHHevbv510tNtT2y0lh9HBPDwfFHs8Owf9jfEFyeh8AKujSMGZwBshLtq0/iQNBgFBzJh09NFYojUlAVGAW/iCD8URiF6nYdedHnn6dJ4WxiY4GZMyl4ZiatAZ2O1ywq4gI7bFj9HaGvL31nzUQIgflj5iO9KB0f7/+4/oNZWVRGQUGWTyAlU0zPnaPJVVnJ96ioiKbcRRcB335r//g5Bd/TN97g4p+SYjKJhaCi7dSJPtoff/SomK7GnrTTOwFcIKW8EsBYAPOEEA+df6xtONa8jOJifk9tuTX9/Oj6bi5Dh/Ja1taXnBzgwgu5MS0t5Tqak3gBNo+ajdLQBIQXZyL43Cm0r82Ev6zC2rjr8XPQVJRWalERbApOGC2Rigpw0T53ju4TV9CpE7B4MVNe/f158chIvpAxYxr7yaqq6lsMzeDSrpdiYLuBWLRpEfSGOsFfvd76B3vuHN+cxEQOmejUiTK3b08FNn48XUZlZUzpVdjHyZOMzVhKGhCCVtv337dqRWtPDEEjpSwDAClluhBiLIAvhBAdoRSCR7A341Kvt77RtJeICGYPff45LeiG1z97lsrnivO2o7+/KQW+ILobNo+ajeCyXBzfVYycAl+URSahVvgg9PA2+ISb93P/uSZGR9OfO25c81+IOYKCgMsvZ15tdjavZw4paU1ceKFTLmu0Eq76/Cp8duAz3NjvRj4QEUHTzmAw76czmmp6PZVTjx6WLuCc3UBbYe9e7mKsKePAQPo8MzKYztcKscdCyBVCDDD+57xymAogBkBfVwmmsExqKhdlnc7yMVJyTenTxznXvPRS4PrrmfVo7KV26hT/Dg0FZs36c5wBOnTgulbX01MeEg99p27I0KSiVvhQwJoadA6v37par+fv8s8EmaAg97S3njaNpk2lmSpiKbkIXHghd+JO4vLul6NvXF8s3LgQtYbzvumoKKBfPy485jAuWFLyjbaEMdNIYR9lZfa9X0K4pAGit2CPQrgZQE7dO6SUeinlzQBGu0QqhVUCAtjC58wZy8WoubnM609Jcc41hQAmTwZefhm4/XZu2C+9lIpg0aL6a5NWy013bm79fnXt2jG2odMBZeUC8QElSPar99VCWRnl/jMzSadzTaZRQ7p3B+67j8VvGRn0zZeX02rIyGBfjttuc+olNUKDeaPn4WjBUXxx6AvTA1ddxdddaqZddng47+/Y0XJaqfFL0bmzU+Vt1cTGWt9hAaZdVisO2Nt0GUkpG3avr/vYL84VR2EvU6dyd75nD3fmxrWhupr+/Kgo4J57nJ8+HRLCwjdbjB7NWO3atVzDoqK4AevThy2MwsOBYQOrEJKVh9LwDjAYuAZHRDSwxvPzqf3qUFREN5VWyw17vYE8zWHwYPrlt24FduzgApGWBowdS6FckIs+rdc09IrthQUbF+Ca3tdAIzTU4o8/zrqHggK+6VotFZRez1iBtRedl0d5k5KcLm+rZfBg4L//teyqAxi869DBumXWwlEzlVswNTXApk2McxUWcr3y9WXn5gkTnFfY21SkZFHb99+zlxzAzVXPnlRmVSezcfWeuSjzj4HeN2cw7loAACAASURBVBAdO/KxP7M/Kyu5IC5cCLRrh+xsdnzescO0Nhtf75QptmslvJVP93+K67+6Hl9c8wWm9ZpmeqCykhp/504qp44d2do6MBB49llqxfbtTeZUbS0tmuBgttsw+vAU9vH++2y5npLSWPlXV9MknzGD9TEtHEutK5RCaAUYDEw+MRioBLyxHqmqyhTk1mioLE6fBio2/IrYL95EaJQffBPjuBOureUuV6ejGyctDZmZXANra7nOGbNBq6sZz+jcmY31nBFEdze1hlr0eqMXAnwCsOdve2gl2KK4GFi+nO0tjL9hKWk9XHUV838VjqHTAf/+Ny3EgADuXow/LiGAm29m9lkroNkKQQghwH5GnaSUTwshkgG0k1K6KCfQNkohtBLS05nfvX07FzUhGMCdOBHo2BEGA9sIVVZaTgJKT6dn6brr3Cm48/hg3we45Ztb8M30b3BFjytsP8FIWRmtAimpKT1tFrZ0pGT66fr1TEX18WG33REjWpWSdYZCeBOAAcA4KWVPIUQkgB+llIOdK6r9KIXQyqiu5qofGFjPR374MMsFrAXIa2oYbnj55ZZpJegNenR/rTsiAyKx484dEG2kd47CM1hSCI7MQxgqpbwPQBUASCnPAfBC54SixeLvz6hyg4DpwYO2xyb7+tIllZHhQvlciI/GB3NGzcGu7F1YdXyVp8VRtFEcUQg1QggtAAkAQohY0GJQKFyKTmdfTzEhWnarmZv63YSO4R3x9Ian0ZJje4qWiyMK4VUAXwOIE0IsArAZqpeRwg0kJtqfIm4pxtAS8NX6Yvao2dh+ZjvWnFjjaXEUbRC7FML5gPJGADMBPAsgG8CVUsr/uVA2hQIAY3parfUWMgUFLCFw+SwFF3NLypXo4BeDpz+4HfLRR4DnnjPVRCgULsYuhSBpv34vpTwipXxdSvmalPKwi2VTKACwNcbVV7PQzZxSKC1li5/p090vm1M5dAj+c+bjyezO+EVzGusDcphT+/rrwD/+4Z4WHoo2jSMuo91CCJdmFAkhLhFCHBVCHBdCPOnKaym8G73e1NkZYNuM6dNNnSTy8019laqrWdjbojs1ZGVxXnJwMG6PmoAEGYKn/baxxDslhUVoL73UqjttKjyPPd1OjQwFcIMQIgNAOdjpVEop+zlDkPMB69cBXAzgNIAdQogVUspDzji/omVQUAD8/DOHhul0jA307k2FcOmlwMiR7IZtTBHv25eFo05rX+Ep1qz5c85BAIAnMAIPi9XYKDMwGh3pC0tPZ1fWgQM9La2ileKIQphk+5BmMQTAcSnlCQAQQnwG4AoASiG0ETIzgSVL6P5p144V11Jy8V+8mAW4V17JerVWhU7HHiR1evHfiQvwjNyMBdiINbiJd4aEsLWCUggKF2G3y0hKmWHu5kRZEgGcqvP/0+fvq4cQ4i4hxE4hxM58Sy2CFS2O6moWlWk0nLlgbL8hBBtRJicDX3/N3kitDuPs3jrFFkHwxeMYjrXiBLYafxZBQZbbYisUTsBuhSCEmG/u5krhzCGlfFtKmSalTIuNjbX9BEWLYN8+toyJijL/uI8Pa9a++85yy+8Wi3GikKF+Wc/dSEOMDMICbOQdOp3lltcKhRNwxGVUd5hsADgkx5mZRmcA1O3X2+H8fYoWRmkpMyWPHOH/e/Rgd+HQUMvP2bKFTTqtERnJNjNFRfy71RAYSDfQ4cP1OpSGwA8zMAyzxE/YIc9gcLGeg3y8BZ2OkX0pWQBi6wNUeD12KwQp5Qt1/y+E+CeA1U6UZQeArkKIVFARXAvgeieeX+EGNm4EPviAFcPGnkI7dgCffALcdJPlZpHl5ba7tApBl1KrHFh1ySXArl1cZOu8EfdhMJ6XW7Cg5iesCLmUg3o8TWUlg+CrV/Nv44DvUaM4qMMZ1YGFhdwl/PYbv0zduvH8TpxYp2iMIxZCQ4LAXbxTkFLqhRD3g0pGC+DfUko1JbwFsW0b8M47nB/SMOunuhp4913eb24scVwcR3JaG0ZlbEthzdJosXTrBtxyC7Wpvz/fEK0WodXAw1U9MT9iN/bcMgkDPd25r6ICePFFDrhISGCAB2Ce8MaNwO7dHKNnaVi9Paxfz/cBYPdWIaiAfviBLW3/7/8sD7FRNAtHYgj7hRC/nb8dBHAUwCvOFEZK+b2UspuUsrOUcpEzz61wLXo98OmnXAfMpYD6+/OxTz/lsQ0ZPZozE6zFB/Ly6HpqqmdCSs44OXSIrievS+kfNw6YNw8YMICCZmQARUV4YNhDCPcLw8IT73laQuCrr/jmpabWH23q48MJbTU1wJtvNj3Qs3MnZxLExzOTIDycuwTjpLKVK4EVK5zzWhSNcMRCmFrnbz2AXCmlmZ+2oi1y9CgLyTp2tHxMcDCTZI4c4SjNusTHU2ls3syNZ3x8fUugrIyKZMqUpsl34ADwxRdcY40DeoKDeb6JE+1rnucWOnfm7NM77uDi6u+PCK0WD637A09vfBr7c/ejb3xfz8hWXg5s2GDdbRMbyzf5xAnHKwUNBo6xjI01v6vQaqkkvvuOIwFVgN3pOGJ33Vsn3fTMeRfPYpdJpmhRFBTYvyksLDT9rddzDZgxg4t+cTFdTytXcu3Jz2d9QlkZ8PDDXA8cZcsW1jecO8fnJyXx36AgWixvv+3mLql6PQdfZ2XRLDKHry8FPK+pHrrwIYT6hWLhpoVuFLQBJ0/yjfL1tXyMMdBzqAnlQydO8AO35hM09jlvlfnHnscRC+FiAE80uG+ymfvaBBUVXASFoLvXG8dWuhNb8wrMHSsl8NFHrExOTub9nTuzS8OJE2xTsX8/R2OOGdO0YWCFhfRAtG9f38MBMLknNZUKY+BA87ENp1JVBaxdy2Bs+fmkPV9f4KKLGFSOiLD41KjAKNw/5H48t/k5/H3M39EztqeLhTWDXt941rA5tNqmRf6Li+07v48P/YcKp2PzZyyEuAfAvQA6CSF+q/NQKIAtrhLMWyks5ND4DRtMaeP+/ox1TZzYcge9Nxejd8BgsBzvM75fxmNPnmT8MCXF9ByNhgo2Ls50TEBA0ydDbtnC6zZUBkaEYFLMypXA0KE21qPaWvq7fvqJZoufH580fLgpuGqJykoGY48epU/MmImj01FB7NjBYKyVMY2PDnsUr25/FYs2LcJHV39k/XquIDKSb6ZxzKklamqaFlS2d1dVN4VN4VTscRl9AuAyACvO/2u8XSClvMGFsnkdeXnAggXc0cbF0fWQlMSY11dfmdoutEXi47nLzs62fEx2NvsOGVPt16/nGmAtYSQuDli1qlHNlt3s3m114w2Ayub0adZPWKSyks3lliyhUtBquZgvX86Bz1ts7I2++AI4fpwmSd1dg58fAy/l5fRdWfG7xQTF4N7B9+LTA5/i94LfrV/PFSQnM7BbVGT5mJoaWj1Naa/RubPtPudSmhpcKZyOTYUgpSyWUqZLKa8DUAIgHkBHAH2EEKNdLaC3ICXw1ltcF5KT67tRAwL4O09PB/7XhidE3HwzN5EN21TX1PC+yEjWIhQU0C109KjtnX9wMD0J5eXWj7OEXm87Q9Ho9rYYR5CS+bQHD9KciYvjoh4Swi9DbCywdCkLy8xRVkaTMjHR8s46Ph74/Xfm3lphxrAZ8Nf645lNHphNJQRw7bX8QMztfPR6ftBXXdW0VLCgIGD8eGZYWVKM2dn8sR06RIvqnnuAJ56glVVc7Pg1FfVwJO30DnBIzmoA/zj/799dI5b3kZHBbLs6haSNSExkKrbVnWYrJjISmDMHuPhixgZPneItP58xgGHDgEWL2Kp65kx6Xo4ftz77xbghbOrM+ZQUZj9Zo6qKSt1iLPP0aZoaycnmBTH6tL76yvxCduIEtY21QIuxuMuSUjlPfEg87k67Gx/99hH+KPzD6rEAuHAfOUJllptr+3hb9O4NPPAAF9+MDGr2ggIqgqws4Jpr2Jq2qVx1Fa9x4gQVqZGqKu64NBp+Hp9+yp1GdDTf288+A2bP5vMUTcaRoPJDAAYD2CalvEgI0QNtaITm0aOm36wlfHy4Hpw4QddIWyQ8HLjuOnYlNcb9QkO5gf7pJ1N6OcB15MAB/jtihPlMw5ISKtqm1h6MHWuK91iyFHJzKa/F9XrbNj5o7cOPimKxVn6+KQBipKaGz62sNLlUzAWbtFrLWUd1eHz443hjxxt4dvOzePfyd80fVFFBd9bPP5v8bQYD0LMnC7tSUmxexyIXXED32a5dpkriTp0YS7HUjMpe/P2Bhx5i99dVq6hohKDSnTiRfsagoPq9S0JCeCsqAv75T+46WlVvE/fhiEKoklJWCSEghPCXUh4RQnR3mWReRlWV/cWR5gqv6lJYyN9SXh6/2/370wpu6i7YGwkMNNUk/Oc/XCsbvsYuXbjJLCkB9uxpnOUjJTegf/lL09+b1FSuU7/8Uj94bTx/bi7XsLFjrZwkL89yVNqIEFQapaWNFUJ+PhdOo//KYGDwuHv3+sHo2trGzzVDQmgC7hx0J5buWop5o+ehY0SD4o+KCuD557mjbt/e5N+UkvctXEg3S9euNq9lkaAgtpIYNarp57CEnx9dRxddxFxhKbnTWLuWCtVSHUREBBXIL7+whYbCYRypQzgthIgA8A2ANUKI5QCc2f7aq4mPt52rbmxYaWlzotcDH3/MnPtPPqF76bvvgKef5m/07Fnny+1pSkr4Os25z0ND6R2oraUXoK6HQKfj2pWWxkSepiIE8Ne/sgj41CmuF7m5vF5GBpNhnnzSRiwjLMy+mcbm0pl+/pkfusHAhbm2lgGR48fp9z52jMfpdFQodpqWT4x8AhqhwXObn2v84Lff8s3r2LF+sMuYIx0aCrz2mheWajdAo6FLKCaGr+Onn6xmYQGggl271j3ytUIcaW531fk//y6EWAcgHMAPLpHKC+nXjxuXBr3H6lFSwozC1NTGj0nJnfKGDfyd1q2MNbZUWLyYnQus9fNpaRw71qjVfz26dKGXYPt2uo+Mmz9fX+Dyy3lzpMbBHL6+wK23cuLajh3c8AcHMxGmc2c7rI+0NC5G1igt5WKVkGC6LzMT+PBDZuYUFgJbt/LFGF9QRQX78+j1/FJdd53d6ZQdwjrgrwP+imV7lmH2qNlICj/fKLiykkqorhwNCQ9vmdPXioqsvy6ACjkvr3mBpzaMI0FlIYS4UQgxX0q5AcBeAANcJ5p3ERTEQe+nTpnfWFVW8jd/3XXmv4cZGdwpp6Q0bpMgBHeq+fl0kbYmbG1ChWBMYfBgxiPvuw945BEOy5k2zXpRrKPExrJVxa238lpduti5ZnTvzvxiS0HZ2louQpddVt8ntW4dF//sbH5xkpL44RsMvLCvLxeudevol3cwGPvkyCchIbHklyWmO0+f5ptuK6ff35+B5pZEaKhtS02no7ZXyqBJOOIyegPAMADXnf9/KTgDuc1gbLSYlcXNnzF9MiODrs777qMlYQ57cu7btaMXwVYMoiXhSEFZv35cF/v29bICP40GePBB7goyMqj9AVOQIyODi/nIkfWft20b/YcHDtDsi4ykSRIfz51sQAD/jolpUgZQx4iOuKX/LXhn9zvILj1fAGItet7wNXm7y6ghF11ke2JcXh79g4om4YgxPlRKOUgIsQcApJTnhBBtqmGDENwEjhhB6/+PP/i76tOHXgVrvbZOnrS9OBqt3dLS1pMk0bUrX3d5ueVMIZ2OG9aGDe+sodczFX3tWm6+jUXDI0faFZd1nNhY4KmnmP2yerUphapHDyqDvn0b70qrq7no6vUmN5GPDz9c4wcsJX2N2dlNagg3a+QsvL/3fTy/5Xm8OOlFymkw2FYMVVXWOxF6IyNGsKS8tNR8jnBZGS0wVwS62wiOKIQaIYQWgAQAIUQsgCbWj7ZsoqLoj3YEo6fAGsac+9bU6t3Hh1lCb71Fj0lDT4ZezwX9+uvttwrKyxkTPXSISjg0lF6blSt5++tfuXY4nbAwfvCTJ9NK0GqtZx+1a0chrbkvjGMxhaDP0UGF0DmqM27sdyOW7lyKJ0Y8gfioeAamDx+23D6ipoZfsrQ0h67lcaKj2eHwpZcYT4iL407COLlNo2HKqhqt22QcWXpeBfA1gDghxCIAm9GG6hCay6BB1iv+AVNQujUFlQEuzjfdxAafmZmsaSopoSI4fZo1AJdcYt+5pGSHh6NHGY+JjeWaHBxMhRMbS+Vjo76reWg0vKCtVNRJkxg4ttYGtqLCFMxoYg/u2aNmo7q2Gi9sPT/U8C9/4TXPnWt8sE7HD2HatJb5RevZkyl5U6bwB5WRQUU6aRLvd8TMVDTCnuZ2H0opbwIQA2AmgPEABIArpZSu/Nm1KoYNYyGrsSq2IUZ39N/+1vriYUKwpmjQILb82b+f1tKwYbTuHemDlpEB7N1LZWCtaPjrr7l2eJQhQ0zFFsbJX3UpLeWiHB9PF1SnTk26TLfobpje61q8tv0NBO+dibz0RMRWPYmr019HSkg6QiN9IbQaKgOtluaYvRrYG4mLo0K7+mqahlpt6/vReAh7XEYXCCHaA/grgA8AfGp8QAgRJaUstPhMxZ9ERwO33caWOFFR9deHqiq6kIcM4SLZWomJMaWSNpVt25icY2/RsEe9B4GBLDI5fJiZCKGh9KHp9byFh7MaLz+f/9rqwmeB6mogOWMOKvWfYkXJS7gkYhF04al43XcxYvIP47Kwgxg6sAYiqYPtYFdLwlgMqHAa9rybSwH8BKATgF117hdgPKFp25o2yKhRXBO++IJWu3FyV0AArfzJk71ocpeXkp9vO9ZgbFZXVuYF7uSYGFYhzpzJoLG/PxfkpCR+GXJyWKdw3XW2z2WBjz4Ccvb3Qq+Ua3Ag6F+YgBkIRBQSOmihb9cHb2T2QXUy+0kpFNawqRCklK8CeFUI8aaU8h43yNSqGTCAMb9Tp+hH9/Oj+6OlDNipqWEW5ZEjdPt07szXZMud7izsKRo2Voyb643kERISWFjx7becEQrQLDTOBJ0ypcm79rNnecrkZCAQc3FI/Bfb5Cu4CP8AwA10u3Z0oY0cqTYcCuvYE0MQklhUBsZjnCta68VYjNXSOHCAAduyMpPbZs0aLry33MKeQa5myBDWcVmjtJSLoK2iVrcSHc2KuL/8hWaOsRqxmZp0717+q9EA8eiLnvJqbMcrGIZHEAC6oIKCTFPoUlPZNaO0lJ9bly5q1ozChD0uo3VCiC8BLJdSZhrvPF+DMBLALQDWAXjfJRIqvIKjR4EXXqB/vmH6emUlu5lqNK4fQ2ksGs7JMR+Mrq3lenvvvV4aZzR25nQS587Vd6OPwlwcFl9hu/wXxmDen/dLyUr51183TaqUkor94ouBK65oOVaqwnXYk3Z6CYBaAJ8KIbKEEIeEECcAHAOrll+WUr7vQhkVHkZKtuQJCzNfDxQYyMX5o49cX/xqLBoOCWE7HuOcFoOBiiAjg8WDLp+P7CWEhdVvupiAgegmL8M2vIRqmAZznDrFbti+vlToycn8NyaGDRbfeKN1VcgrmoY9E9OqpJRvSClHgJPSxgMYJKXsKKW8U0q5x+VSKjxKejrrBawlwQQF0ZXkzPY4paXs0/bqq8Arr7DorLDQVDQ8fTozbDIyuOB16sTY7TXXeKl14AL69zeNOTYyGvNQJc5hx/nOMnl5bJ7Yq1fjanFjDGvXLjYYVLRtHMrZklLWALAyNVfRGjG6vG0tshoN02cHOKHl4ZYtwHvvcddqLOTdt48ZWn/5C+OwkyezHqm6msHStujyaNeOmaS7d9OVJgSQiMHoIidjK15AmuF+7N8fguRky+EKIWgpfP8940BtRZkqGqOSeBU2sTczxWBwThbLnj2MSbRvX38Ri4qiS+qzz7j4X3wxlZBbGuHl5XHwyq+/Ms0pKQmYMIHVbx5O3bntNsYFjh411biMkvPwnnY41pxbivYRj6G7jVFWYWFMha6oaPp0OkXLpxV1zVG4CuO0RWu9mIwuiy5dmnctg4HjcmNizO9ofX2Ztv/FF3ZNm3QOmzdzis6339Jk8fUFfv+dU8leesn8wHk3EhzMOdV3380YT0YGIE8NQ1fNBKQnPo9uvSts9scyWoAqV7Bt0ySFIIRwoNmAoqUTHc22E9lWnIVnzzJIaW44kCOcOEEXlbU2O/7+dBPt39+8a9nFwYNsnhQXx0issYdRXBw15cGDwLJlHl9J/fzo7lmwgOIsWwYsu3k+CqvzkJXwjs0+WuXljBGpFNS2TVMthO+dKoXC67nxRroiTp2qn9ViMJgUxV13Nd//XFho3zk0GlMHapchJSu6IiLMmyvGgpJduxh19xK0Wr4/ozqOwtiUsfi5ejEqdFVWLby8PDZytavTrsHAysT33uNQ+3ffZVdXWzNmFV5PU2MIKuzUxoiMBObMYYO+LVtM9xsMtB6uucaxJnWWMA4Rs4VbKpHz8jj0wloVobGfzrZtjCt4GfNHz8e4D8ZBXLAM6TvuQ3Jy/boF4/jWlBQ7CwuLipj2ZWzDERDASrfNm/n6H36YJqWiRdJUhfCOU6VQtAgiI4Hbb+fif+YMF5P4eAYyKyvpSg8MbJ6V0Lkzd6nGJpbmMCoMW4HSZlNaSmFsvaCAANuTvDzE2JSxGJE0AuuLnsOSqXdg7Wr/P99bY7rqoEEMTNsMzldXszoxJ8cUWKpLTg6wZAlzgpXvqUXSVIXwlTOFEEJcA+DvAHoCGCKl3OnM8yucS1iYqafQtm1MV8zJ4bqZkMCU0KFDmzYPOSyM8xOM/XnMkZtLZdChQ/Neh00CAuwzV3Q68xV7XoAQAvPHzMekjyahtPN/8PLLd+G33+iaCwri+ID4eDtPtns3U5EsBYratWPRyvbtHHepaHF4SwzhAICrAWx08nkVLqKyEnjxRbqPdTpuGJOTef9bb7GQrLq6aeeePp3nOnmyfiZRTQ3Xo+Bg4M473ZAv3749051KS60fp9OxyZKXcnGnizE0cSie2fQMAoJqMGIEq7nHj3dAGQAcHWprtmt0NPDDD44JqNfbHieocAtNVQhO/SlKKQ9LKY8685wK1/LJJ8x7T001ZQQJwcBzp05shPff/zbt3MHBrDi+6ip2hM3MZDA7P5+1B/PmcZ024rIEH42Gwxvy8iwHTHNyqL26dnWREM3HaCVkFGfgw98+bPqJsrNtW0IhITThbC3wZWXsjDhjBnDHHbwtXcp4hMJjtLgYghDiLgB3AUByS2wZ2go4d441Wh06mN+lC8HHNmzgeMymeFOCgvjcKVOoCKTk5tPo587P57z7n3/m2hIRwTqxESNsb2IdYsQIaqQffqDmi47mC6yooKKIjQUeeMDrB2FP7jIZFyRcgEWbFuHm/jfDR9OEn76/P3fz1grxamttTzDKz2eswTjBKCWFz9uzB9i6lSbi5MmqZNoDNOlbLKV8w9HnCCHWCiEOmLld4eC135ZSpkkp02I9Pv2kbWKchWBtXfDx4W/8yJHmXcvPD0hMpIIxKoMDB5jxtHIlN6QdO3Kt+uorYO5cupqchhAcXvPYY7QEMjNZ+VVVBfzf/wHz53vBFB7bCCEwb/Q8nDh3Ap/s/6RpJxk61HbwPC+Px1lazGtrORuitJSKwFgWrdUyANWhA0vR9+1rmoyKZuG21hVSygnuupbCtVRW2neclE2PI1giN5fxifDw+l2kAwO5XhcVAc89x/U7Koq3Zm80hQD69eOtupqLWkCA11sFDbm8++XoH98fizYtwg19b4BW42DLjbFjgbVr+R6Yy/mtqeFj48dbPsfhw6Y8V3P4+tLcW7GCnfuUleBWWtY3WuEVWKsirosQzk++Wb+e1om5kQKlpUyPX7eOXpzHH2cG5K+/OjHO4O9Pf1YLUwaAyUr4veB3/PdgEwI8iYmchHTmDEvTjW+qlExbysykNWWtXH3rVtv5rZGRNPMKChyXUdEs7JmYFmXHeQxSShvF8VavcRWAfwGIBbBSCLFXSjmpqedTuJbevenK0eksdxitrubvvkcP511XSi725jJjCgpMBXNRUYwrJCWx6du//gVccglw/XUS4vSp881+zhdRdO3aIhf3pnJVz6vQO7Y3FmxcgOl9pkMjHHztY8eybceKFcwq0GppMXXuzNSvvn2tP7+42HZbWuNQbHtNUYXTsMdllHX+Zs120wJocoRXSvk1gK+b+nx7MFbbr1nD7zHA/vATJjCnXVmm9hMYyLTF//6Xln/DWEJtLTs53HCDc6uJ9XoqmobrSU0N6yH8/Hi92lq6+IWg9yE0FNj2VRbGbFqGDlV/mJ4oBP3/t97KL0MbQCM0mDt6Lq778jp8eehLXNP7GsdP0qsXb4WFbIIUGGgKttsiOho4dsz6Mcah2E6cLKewD3sUwmEp5UBrBwghvHpIjl4P/PvfzIwJDjZloRw8COzYwRqam27yeBfjFsWllzIldM0aun2jovg7PneOC/TUqcDEic69po8PXfcNLZPsbF7TGJ+sra2viMIrsjHx2CJkZUgkTukIoamzcBUXs2vpjBms0moDXNPrGvx9/d+xcNNCTOs1zXErwYgxSOMIw4cz/cwaBQVsK+7UdDGFPdjzTRjmpGM8xldfsfK1Y0dau76+vMXH876ff2bGisJ+NBrg+uvpozf2wDEYmCQSF8f5vY89Bnz+ufUuqY4gBDBuHAPLdTl9ur4CqKhgLYSRvvs/RoBvLbL0caiqbrCLDQ/novbuu617hmRODvC//wEzZkD7wIOYe64vfsv9DSsOfuleObp25c1SM8CqKu40rnAo+VDhJOwaoWnpMSHEbbaO8TRlZcCPPzIDxZyrWKPhIrZypRv767cShGD88LbbmO4ZEcGgblUV11lfX773c+ZQQTiDMWNoydUtHtbrTZ9tZaUpVRUAgstyEZd3AOUh8RDCQn1ZaCgthUOHnCOkt7F5MzB7NmspfHyAiAhcmx+PLjWhePqLByFt9cZ2JhoNcP/9/IDSQR12+gAAGq5JREFU000mZWUlqw/z8tg21+WNqhTmaG407R9OkcKFGLvy+lhxjhkDpIcPu0+u1oSUHNJ+8iRjClFRfE8DAxnYjY83dUhuLnFxwCOPUNFnZnIdCQ6mK7u4mLIMH26yGEJLsyCFBgYpIISVmIZGw2Bza+PoUeCdd/jGJSXxQ/H1hU9UDOb4jMMekYOV/3rAva0jwsO5S7j3XsZwior4A5w0CVi0CBg50n2yKOphT5bRb5YeAuBIJxSPUFZmf8pheblrZWmtnDhBZdqxo/m4or8/14BvvrEeu9XpuDZoNHQfW4rp9OwJPPMMY0Lr1nGNq61lgktSkvnRBcaso6Y03GvRrFjB4KyZN+UG9MU/5AY8XbkKlx47BuHOXbmfH/s/eXEPqLaIPUHleACTAJxrcL8AsKXx4d6FI1143TKbtxWydSt/39aSTKKimFxy9mz9PkQA3T9r1zJAXV1NBR4Rwe4FY8aYz1KMjmabocsv5/Evv8zi1oYWQGlYIvTVBogAA7p0sWIQGwyWi6VaKufO0Syz0OLFF1rMxkjc5f8dfvzpbUzq/oKbBVR4G/a4jL4DECKlzGhwSwew3qXSOYFevbjTtBYvrKmhS6lnT/fJ1ZooKDC/K6+LEPwcysrq319YCCxcCCxfTisiKYnrl1YLfPghRxbbiu0IwXnCAwbQ65OVRfdRQQFwMD8Op6P6YXT3PMtFciUl1ECt7QtQVmZznsMtGICk2hD8o+BLSDVQuc1jT1D5dinlZguPXe98kZxLWBhraU6dMu86kpKPXXyxmunRVMLDbbeokJJunbpWmJR0b587x8153d19cDAD1ocPc4qlLQIDgYceYifU4cPpmk5NpaKY8tH1CI/2YcCy4ZegqIgC3HGH9UBTSyQwkK/XykLvBy1mlQ3EVkMGfj75sxuFU3gjrewXYJ7p07kT3bmTvmljenNhIXeSw4ax1bKiaQwdypYS1igu5s4/Ls503+nTptiDOYRgMsrPP9M1ZKwzsIQQLJjt3LnhI+0YxPz3v01FUUJwoWzXjlrDmSXV3kJ0NLXi2bNWc/r/ei4Fi9odxdMbn8b4Tlb6EClaPfYElXdLKQc19xhP4ucH3Hcf8NtvwKpVppbr3btzJGTfvqoorTl0787FPifH/FxlvZ7um5tuqu+9OHyY/7cWe/D15fP/+IO95ZpMQgJTL0+fpkloMDD9yTizszUiBEvKX3iBgWVzEfXcXPgndMDMUbPw0I+PYEP6BoxJGeN+WRVegT0WQk8rmUYAg8vhTpLHZfj4cHbsoEFcC2wtRAr70WiABx9kwW96Ot01wcF8n/PymBr6l7/wva9LZaV9ilgIxnmajRAMUiQlOeFkLYT+/YFrr2VRWkAAPxytlil1xgj/ww/jzshQPPPLc1iwcYFSCG0YexSCPba0hXFS3klr3RB6kpgY+u+3baMVZkzpT0tjv6hu3Ror4NhY+4qDpWScQtEEhGCfke7dmcplbP0aGcnOpMOHAyEhCAQwc8RMzPhxBn7J/AUjkkd4WnKFBxCOZhYIIZ4GFcleAHullL+7QjB7SEtLkzt37vTU5RUWkNI0WMua8i0rAx5+mJ4bS/UBpaU8z3PPKUXuFAwGU6VmAw1dritH6iupGJQwCD/c6OBcZEWLQgixS0qZ1vB+h39iUsr5AF4BUAzgKiGEx8ZpthXy89naecMGxkGc4j5xIUJwgbe1gIeEMFicmWm+pUR1NV1O//d/Shk4DY3G4ojLYL9gzBg2A6v/WI1fz/zqAeEUnsZuC0EI8QqAh6UXJSu3dguhuBj44ANg9+76mYMhIVwkR41q+XEQgwH44gvg++9pCYSF8bWWlPC13Xwzi9MU7qG0uhQpr6RgeNJwfHvdt54WR+EiLFkIjqSdlgJYIYS4VkpZLoSYBGC+lFI5G11AWRmweDF3yElJ9XfIlZXsDVRV5fwW0+5GozEpt82bmRWq0fB1DRumOiC7m1D/UDx64aOYu24udmfvxqAE1yQPFhWxwn3LFn6P27Xj5M0+fVpfOUhLwqEYghDiegCPANABKAOwUEq5yUWy2aQ1WwhffcU2NJa6Keh0TPN84QW1aCqcS3FVMVJeScHYlLH4errz51b99hvw+uv8DkdFUQGUlzNe1Lkz40r2jmlVNI1mxxCEEOMB3AmgHEAMgAc9qQxaMzod+/okJFg+xs+PrpVt29wnl6JtEB4QjoeHPoxvjnyDfTn7nHrujAz2nQoLY0FiaCgLqmNiWEOXmQm8+qp7m68qTDgSqpsDYJ6UciyAvwD4XAgxziVStXHOnqVSsDV+MjRUtexWuIYHhz6IUL9QLNy00KnnXbWKMW1LVeeJiSwcNY65VbgXuxWClHKcsaeRlHI/gMkAnPttUSgUXkFkYCQeHPogvjz0JQ7mHXTKOcvLWQZRt31JQ4Rg/ZyzBiopHKPJyXxSymwAqvGJC4iJoXVgq2FcaWnra9Cp8B4eufARBPkGYdGmRU45X1mZqeutNYKCmEyhMEN2NvDTT8B33zEi37B9cDNpVjxfSlnpLEEUJvz82H11+XLrQWWNhpk4CoUriA6Kxv1D7seSX5Zg/pj56BHTvAaA/v6MDUhpPV1ap2NqtaIOxcVszrjvfExHqzUVGF56KQt6nNCQTZX7eCkXX0x/6unTjQNsFRW8//rr2cZfoXAVjw57FAE+AXhm0zPNPldEBNC1K7sMW6OsDBihktlNlJezUdihQ4zEp6QwFz0lhWX+X30FfP65/aMhraAUgpcSEgI88QQwcCAzL9LTTbfKSs4hH6dC+goXExcch3vS7sEn+z/B8cLjzT7fZZdxs2uph1VhIVNR+/dv9qVaDxs2sENvYmJj08rXl4rhxx+BM2eafSmHexl5E625DqEuZ88y86KmhjUH3bu3wdnACo+RXZqNTq92wvV9rseyK5Y161xSAitXsvlqcDDjZRoN42W5uYwfPPFE22pIa5XaWuDRRxlptzbj9/Rp4KKLgBtusOu0zqhUVniImJjGc4gVCneREJqAuwbdhTd2voG5o+ciNTK1yecyNl/t3BlYvZoucSEYX5g6ldMNo6KcJ3uLp6SEPjRb1acREcDvze8zqhSCQqGwycwRM7F011I8t/k5vHXZW806lxDMjuvZk5ZBTQ03v2pIlRmMk/1sIaVTOkCqGIJC4c3U1HAbvWED0ww9lI+ZGJaI2wfejvf2vofM4kynndffn/EypQwsEBZG90BpqfXjioo4+rGZKAtBofBGpAQ2bWL2SHl5/VzNQYOAW25x+9SgJ0c+iXd3v4slvyzBa1Nec+u12ywaDTBlClNOQ0LM5+vqdPx+jBzZ/Ms1+wwKhcL5rFkDvPMOF4GUFDb6SUnh8Or9+9kK18lFSbZIDk/GrQNuxTu738GZkuZntCjsZPhw7v4zMhoPQykvZwbS9OnWS8DtRCkEhcLbOHcO+Owzpto0zCzRaJh+mJXFVEM3M2vkLNQaavH8lufdfu02i58fh5ZPmkSXYUaG6abXA3ff7bQ++F7hMhJCPA/gMrCt9h8AbpNSFnlWKoXCQ2zdyn/9/Cwfk5DAGclTp1o/zsmkRqbi5v43461db+HJkU+iXUg7t127TePvzxnYl1/OHHSdjvGFLl2cGoDxFgthDYA+Usp+AH4HMMvD8igUnuPIEbaytYax2dXZs+6RqQ6zR82GrlaHf275p9uv3eYJDmbV3uDBLEhycjTeKxSClPJHKaWxdnEbgA6elEeh8DheXDDaJaoLru97Pd7c+Sbyy/M9LY7CiXiFQmjAXwGssvSgEOIuIcROIcTO/Hz1ZVS0Qnr2tJ1mWF1NK8FDFYtzRs1BZU0lXtz6okeur3ANblMIQoi1QogDZm5X1DlmDgA9gI8tnUdK+baUMk1KmRYbG+sO0RUK9zJsGIPHOp3lY7Ky2AHRjfGDuvSI6YHpfabjtR2voaCiwCMyKJyP2xSClHKClLKPmdtyABBC3ApgKoAbZEtusKRQNJeICLayPXWKrW3rYjDw/qQkp2WWNJU5o+agTFeGl7e97FE5FM7DW7KMLgEwE8AYKWWFreMVilbPuHF0Cf3vf0B+fv3CtMGDgZtusjyH0k30ieuDaT2n4dVfX8WM4TMQEaB6sbd0vKLbqRDiOAB/AEbbc5uU8m5bz2sr3U4VbRi9nllHRUUchtK1KxAd7Wmp/mRvzl4MfGsg/jH2H5g/Zr6nxVHYiaVup16hEJqKUggKhee58rMrsSFjAzIezkCYf5inxVHYgSWF4I1ZRgqFogUxb/Q8FFUV4bVfVX+jlo5SCAqFollc0P4CTOk6BS9ufRFlOvf2V1I4F6UQFApFs5k3eh4KKgvw5o43PS2KohkohaBQ2KCqipmemZmcZ61ozIUdLsTEzhPx/JbnUVGjEgVbKl6RdqpQeCNlZcCqVewhZ+w67OPD0bWTJ7NcQGFi/uj5GPneSLy18y08MuwRT4ujaALKQlAozFBSAjz7LAfCR0VxDEFyMhAby67TixYBhYWeltK7GJE8AuNSx2HJliWorFGmVEtEKQSFwgwffwzk5HAmjb+/6X5fX6BjR6C4GFi2zGPieS3zRs9DTlkO3t39rqdFUTQBpRAUigYUFgI7dnAOjSUSEoBDh9hSSGFiTMcxGJU8Cot/WYxqfbWnxVE4iFIICkUD/viDnSI0Vn4dQvCY48fdJ1dLQAiB+WPm40zpGby39z1Pi6NwEKUQFIoGNBxbaw1rDUlbHTk5wHffAe+/D3z9NUc4mul0MD51PIZ1GIZnNz8LXW1beoNaPirLSOERqqu5mAYGMnPHm4iKsv9YD40jcC9VVVQC27bRbPLzo9ZcvpyzG+6+GwgP//NwIQTmjZ6HKZ9MwQf7PsAdg+7wnOwKh/Cyn6KitXPkCLB6NbB3L90ufn7A+PFM5fSWxbVrVyAykmmnISHmj6msBIKCgF693Cub26mtBd58E9i3j9F0Y8dVgNbBsWPAP/8JzJ5N7X6eS7pcgrT2aXhm0zO4pf8t8NX6ekB4haMol5HCbXz/PVM5jx5lO//kZO7Gf/gBeOopeiC8Aa2W4wjy8rg5bohOx2Dytdd6bD6N+zh8mNq7oTIA+P8OHVixt21bg4cE5o+ej5NFJ/HJ/k/cKLCiOSiFoHALBw8Cn33G9SMuzhSw9fOjcvDxAV56yfwC7AnS0oC//Q0oKADS0znLvqCASis3F7jlFmDUKE9L6QbWrKGZ1FAZ1CU2lhV8DeIJU7tNxYB2A7Bo0yLoDXoLT1Z4E8plpHALK1cCYWHM4zdHZCQX3r17gQsvdKtoFhkxAujXjymohw5xveveHRg6tJ7LvHWTns4PzhohITxOp6tXtGG0Eq7+79X4/MDnuKHfDS4VVdF8lEJQuJzSUnoekpOtHxcWBmza5D0KAQBCQzm8bNw4T0viIXx8OLbTGsZpblpto4eu6HEF+sT1wcJNC3Ftn2uh1TQ+RuE9KJeRwuVUVdFFZM3rANB6KC93j0wKOxk0iL4ya5w9y+i6mXQxjdBg3uh5OHL2CL48/KWLhFQ4C6UQFC7HmKlTW2v9uMpKuqMVXsSYMfzgLBVnGAw0AS+5xOIppvWchp4xPbFg4wIYpA1rQ+FRlEJQuJzAQLqBcnOtH1dRwfVH4UV06ABMn85MopKS+oHj8nLg5ElgwgSgTx+Lp9BqtJg7ei4O5B3AN0e+cYPQiqaiFILCLUyezI1mmYWBWmfOAJ06sc5J4WVMmgQ8+CADxpmZHA6RkUGr4bbbgBtvtOkPnN57OrpGdcWCjQvQkue4t3ZES/5w0tLS5M6dOz0thsJODh4E/vUvVilHRdHlXF5OJZGSAjzySBvK3mmJSEmFUF4OBASwNsFMINkS/9n7H9y6/FasuHYFLut+mQsFVdhCCLFLSpnW6H6lEBTupLgY2L4d2LyZMYN27Vip3Lu35ZRUReugprYGPV7vgajAKPx6x68QtrIMFC7DkkJQaacKtxIeDkycyJuibeGr9cXskbNxx7d34IfjP2By18meFknRABVDUCgUbuOm/v/f3v3HVlXecRx/fwaUFoqiCMqQ4VyMm3OGiTqZZNEozpltzrhfhpnMHyOIRFCZq2MZE9S4kCwmxmUu0UwNbiHC1Dh0oGMgy1DRoKL114hMZCoyyWRqHPDdH+epXmvbeyv2Pue2n1fSpOf23Pt82rTn2+ecc7/POUzYdwJXrr7S1xJKyAXBzOqmaVATbVPaeOjlh7h/0/2541gnLghmVlfnTjyXcSPGsWDNAs8SSsYFwczqaujgobRNaWPtP9eyevPq3HGsgguCmdXdBUdfwNjWsSxYvSB3FKvggmBmddc8uJnLT7icVS+u4sHND+aOY4kLgpllMX3SdMYMH8PCNQtzR7HEBcHMshg2ZBhzJ89l5aaVrNuyrvoTrM+VoiBIWijpCUkbJK2Q9Mncmcys71147IWMahnlWUJJlKIgAIsi4qiImAjcA/w8dyAz63utTa1cNvkylj+/nPVb3YYmt1IUhIj4T8XmcMA3J5sNEBcddxH7Ne/nWUIJlKIgAEi6WtJLwDR6mCFImi5pvaT127Ztq19AM+sT+wzdhznHz+HuZ+9mwysbcscZ0OrW7VTS/cBBXXxpXkTcVbHfFUBzRMyv9prudmrWP+x4ZwcTrpvA1EOncsd378gdp9/rrttp3WYIEXFKRBzZxcddnXZdDJxVr1xmlt/I5pHM/tJslrYvZeNrG3PHGbBKccpI0mEVm2cAz+TKYmZ5zDl+Dq1NrVy15qrcUQasUhQE4FpJGyU9AZwKzM4dyMzqa/+W/Zl17CyWPLWE9m3tueMMSKUoCBFxVjp9dFREfCMiXs6dyczq79LJl9IypIVr1l6TO8qAVIqCYGYGMHr4aGYeM5Pbn7yd57c/nzvOgOOCYGalMvfLc2ka1ORZQgYuCGZWKge2HsiMSTO47fHb2PTGptxxBpS6vQ+hL0jaBmzuwyEOAF7vw9fvS42a3bnrr1GzN2puyJ99QkSM7vxgQxeEviZpfVdv3mgEjZrdueuvUbM3am4ob3afMjIzM8AFwczMEheEnv02d4C90KjZnbv+GjV7o+aGkmb3NQQzMwM8QzAzs8QFwczMABeEqhp1vWdJiyQ9k7L/UdLI3JlqJek7kp6StEdS6W7N60zSaZKelfSCpLbceWol6WZJr0lqqH7TksZLWiXp6fR70hDNMCU1S3pY0uMp95W5M3XmawhVSNqnY4lPSRcDR0TEjMyxqpJ0KvCXiNgl6ZcAEfGTzLFqIulzwB7gRmBuRJR2FSRJg4DngKnAFuAR4OyIeDprsBpI+gqwE7g1Io7MnadWksYCYyPiMUkjgEeBb5X9Zy5JwPCI2ClpCLAWmB0R6zJHe49nCFU06nrPEbEiInalzXXAwTnz9EZEtEfEs7lz1Og44IWI2BQR7wJ/oFjTo/QiYg3w79w5eisi/hURj6XP3wTagXF5U1UXhZ1pc0j6KNXxxAWhBrWu91xi5wH35g7RT40DXqrY3kIDHJz6C0mHAF8EHsqbpDaSBknaALwGrIyIUuV2QaBY7zkt0NP54wyAiJgXEeMplveclTft+6rlTvvMA3ZRZC+NWrKb9URSK7AUmNNpJl9aEbE7IiZSzNiPk1SqU3WDcwcog4g4pcZdFwPLgfl9GKdm1XJL+iHwdeDkKNnFol78zMvuZWB8xfbB6THrQ+kc/FJgcUQsy52ntyJih6RVwGlAaS7qe4ZQRaOu9yzpNOBy4JsR8VbuPP3YI8Bhkj4tqQn4PnB35kz9Wro4exPQHhG/yp2nVpJGd9ztJ6mF4kaEUh1PfJdRFZKWAodT3PWyGZjRCEt8SnoBGApsTw+ta4S7owAknQlcD4wGdgAbIuKreVN1T9LpwHXAIODmiLg6c6SaSPo9cCJFK+ZXgfkRcVPWUDWQNAV4EHiS4u8S4KcRsTxfquokHQXcQvF78glgSUQsyJvqg1wQzMwM8CkjMzNLXBDMzAxwQTAzs8QFwczMABcEMzNLXBDMzAxwQbB+QtIhkt5OfWI6HvtQW2pJLamV+buSDtjLMVskrU4dT5F0saR2Sb1qEyJppKSZe5OlhjE+1OpaUpOkNZLcscAAFwTrX/6R+sR0tKW+AfgacARwtqQjIuLttM/Wj2G884BlEbE7bc8EpkbEtF6+zsj03F5Roda/4d9RtEl4T+rO+gDwvd6Obf2TC4I1jLQoytT0+VWSru9h93q0pZ4G3JXy/AY4FLhX0iWSfpAWQ9kg6caKWcSdkh5NC6RMT69zLfCZtO+iNNup/E9+rqRfpM8PSbOeWyl64IzvbqxKPbS6vjN9H2YuCNZQ5gPzJE2jaHk8p4d9+7QtdepbdGhEvAiQ2oJsBU4C7qP4r/uENBvZzfsH3fMiYhJwDHCxpFFAG2l2ExE/rmH4w4BfR8TngWE9jFWLjcCxvdjf+jGfO7SGERFrUmOzS4ETO07VSFpI0eysng6g6LPUlZOBScAjRVxaKPrfQ1EEzkyfj6c4uL/Sy7E3V6yy1dNYVUXE7nQ9ZURabMYGMBcEaxiSvgCMBbZ3HLwkHUSx8lRnvW5LLeki4Edp83TgzMrtiKi87vA20NzdSwG3RMQVnV7/ROAUYHJEvCXpr928xi4+OHvvvM9/q43VS0OBd/bi+dZP+JSRNQQV6+guprgOsDO19waYCGzo4im9bksdETek0zYTI2Jr5+1O+74BDJLU1QH9AeDbksak7PtLmgDsC7yRisFngePT/m8CIyqe/yowRtIoSUMp1rToTndj1SSdsno9Iv5X63Os/3JBsNKTNAxYBlwWEe3AQt5fpKjLgpDWk54F/Jlizd0lEfHUxxxtBTCli7GfBn4GrJD0BLCSYmZzHzBYUjvFheR1af/twN/SinGL0sF5AfBwem63PfN7GOsDUqvrvwOHS9oi6fz0pZOAP32Ub976H7e/toYm6SaK0zqfAu6JiJqWJJT0InBMRLy+F2MfDVwSEed81NfITdIyoC0insudxfLzDMEaWkScHxF7KO6u2bfyjWld6XhjGsV1hz097VvD2I8Bq7q6zbMRpFNpd7oYWAfPEMzMDPAMwczMEhcEMzMDXBDMzCxxQTAzM8AFwczMEhcEMzMDXBDMzCxxQTAzMwD+D0I+ZtV3h/8cAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1W4TZfXOmIlS" + }, + "source": [ + "Посчитаем точность на тестовом датасете:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HUjdeIefsIsg", + "outputId": "a1a363d4-a307-4769-9ccf-fe8a857b62af" + }, + "source": [ + "pred = torch.sigmoid(net.forward(torch.tensor(valid_x)))\r\n", + "torch.mean(((pred.view(-1)>0.5)==(torch.tensor(valid_labels)>0.5)).type(torch.float32))" + ], + "execution_count": 205, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "tensor(0.8667)" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 205 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_95qF9lY2kHp" + }, + "source": [ + "## Нейросети и оптимизаторы\r\n", + "\r\n", + "В PyTorch реализованы специальные модули `torch.nn.Module` для реализации нейросетей в виде набора слоёв. При этом есть два способа описания нейросетей:\r\n", + "* **Sequential**, в виде перечисления набора слоёв\r\n", + "* В виде **класса**, унаследованного от `Module`\r\n", + "\r\n", + "Первый способ проще позволяет описывать стандартные сети, второй - более гибкий, позволяет описывать произвольные сложные конфигурации вычислений.\r\n", + "\r\n", + "Внутри модулей можно использовать стандартные слои, среди которых:\r\n", + "* `Linear` - линейный слой, аналогичный нашей нейросети, приведенной выше\r\n", + "* `Softmax`, `Sigmoid`, `ReLU` - слои для передаточных функций\r\n", + "\r\n", + "> Обратите внимание, что большинство передаточных функций и функций потерь в PyTorch доступны в двух вариантах: **функциональный** (внутри `torch.nn.functional`) и **в виде класса** (внутри `torch.nn`). Для передаточных функций удобно использовать функции из `torch.nn.functional`, не создавая отдельного слоя.\r\n", + "\r\n", + "Для однослойной сети мы можем просто взять экземпляр слоя `Linear`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "D77pXPR6oFRs", + "outputId": "efa49e5c-72d4-4781-89d4-4ab6597d2b0e" + }, + "source": [ + "net = torch.nn.Linear(2,1) # 2 inputs, 1 output\r\n", + "\r\n", + "print(list(net.parameters()))" + ], + "execution_count": 206, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[Parameter containing:\n", + "tensor([[-0.1892, -0.5979]], requires_grad=True), Parameter containing:\n", + "tensor([0.1538], requires_grad=True)]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0tbe0Et_oiNo" + }, + "source": [ + "Мы видим, что у сети есть метод `parameters()`, возвращающий все параметры, которые необходимо подстраивать.\r\n", + "\r\n", + "Методы обучения, такие, как градиентный спуск, реализованы в виде отдельных объектов, которым передается список параметров:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "B4AxyrFMozh0" + }, + "source": [ + "optim = torch.optim.SGD(net.parameters(),lr=0.05)" + ], + "execution_count": 207, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6eB8v58eo9pp" + }, + "source": [ + "С использованием оптимизатора, наш процесс обучения будет выглядеть так:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ups7nlV22ofp", + "outputId": "503d8ae9-35f3-4ecb-e2ff-4da2ec2914eb" + }, + "source": [ + "val_x = torch.tensor(valid_x)\r\n", + "val_lab = torch.tensor(valid_labels)\r\n", + "\r\n", + "for ep in range(10):\r\n", + " for (x,y) in dataloader:\r\n", + " z = net(x).flatten()\r\n", + " loss = torch.nn.functional.binary_cross_entropy_with_logits(z,y)\r\n", + " optim.zero_grad()\r\n", + " loss.backward()\r\n", + " optim.step()\r\n", + " acc = ((torch.sigmoid(net(val_x).flatten())>0.5).float()==val_lab).float().mean()\r\n", + " print(f\"Epoch {ep}: last batch loss = {loss}, val acc = {acc}\")" + ], + "execution_count": 208, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 1.1270943880081177, val acc = 0.6666666865348816\n", + "Epoch 1: last batch loss = 1.0163036584854126, val acc = 0.7333333492279053\n", + "Epoch 2: last batch loss = 0.9239333271980286, val acc = 0.800000011920929\n", + "Epoch 3: last batch loss = 0.8493780493736267, val acc = 0.9333333373069763\n", + "Epoch 4: last batch loss = 0.7907424569129944, val acc = 0.8666666746139526\n", + "Epoch 5: last batch loss = 0.7453570365905762, val acc = 0.8666666746139526\n", + "Epoch 6: last batch loss = 0.7104158401489258, val acc = 0.8666666746139526\n", + "Epoch 7: last batch loss = 0.6834256649017334, val acc = 0.8666666746139526\n", + "Epoch 8: last batch loss = 0.6623864769935608, val acc = 0.8666666746139526\n", + "Epoch 9: last batch loss = 0.645784318447113, val acc = 0.8666666746139526\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vRLXEQ4Qrcvx" + }, + "source": [ + "> Обратите внимание, что для вызова (применения) нейросети к входному аргументу мы используем синтаксис `net(x)`, вместо `net.forward(x)`, поскольку `nn.Module` реализует метод `__call__()`\r\n", + "\r\n", + "С учётом описанного, можем описать универсальную обучающую функцию:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5c6WsBhlrlIs", + "outputId": "54de8404-4170-4a15-abba-039d06d5e946" + }, + "source": [ + "def train(net, dataloader, val_x, val_lab, epochs=10, lr=0.05):\r\n", + " optim = torch.optim.Adam(net.parameters(),lr=lr)\r\n", + " for ep in range(epochs):\r\n", + " for (x,y) in dataloader:\r\n", + " z = net(x).flatten()\r\n", + " loss = torch.nn.functional.binary_cross_entropy_with_logits(z,y)\r\n", + " optim.zero_grad()\r\n", + " loss.backward()\r\n", + " optim.step()\r\n", + " acc = ((torch.sigmoid(net(val_x).flatten())>0.5).float()==val_lab).float().mean()\r\n", + " print(f\"Epoch {ep}: last batch loss = {loss}, val acc = {acc}\")\r\n", + "\r\n", + "net = torch.nn.Linear(2,1)\r\n", + "\r\n", + "train(net,dataloader,val_x,val_lab,lr=0.03)" + ], + "execution_count": 214, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 0.6797666549682617, val acc = 0.06666667014360428\n", + "Epoch 1: last batch loss = 0.6262387037277222, val acc = 0.06666667014360428\n", + "Epoch 2: last batch loss = 0.5835054516792297, val acc = 0.20000000298023224\n", + "Epoch 3: last batch loss = 0.5517064332962036, val acc = 0.2666666805744171\n", + "Epoch 4: last batch loss = 0.5291333794593811, val acc = 0.2666666805744171\n", + "Epoch 5: last batch loss = 0.5140607357025146, val acc = 0.3333333432674408\n", + "Epoch 6: last batch loss = 0.5050028562545776, val acc = 0.4000000059604645\n", + "Epoch 7: last batch loss = 0.5006228089332581, val acc = 0.6000000238418579\n", + "Epoch 8: last batch loss = 0.4996645152568817, val acc = 0.6000000238418579\n", + "Epoch 9: last batch loss = 0.5009827017784119, val acc = 0.7333333492279053\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KzuIDqJ8sFYm" + }, + "source": [ + "## Описание сети в виде набора слоёв\r\n", + "\r\n", + "Простейшую многослойную нейросеть можно описать в виде набора последовательно-применяемых слоёв. При этом такая сеть будет обладать всеми характеристиками вышеописанных - она автоматически соберёт в методе `parameters` параметры всех промежуточных слоёв" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "tBtytmEAsq-O", + "outputId": "06ad840b-c2b7-409e-e01e-a9170548151d" + }, + "source": [ + "net = torch.nn.Sequential(torch.nn.Linear(2,5),torch.nn.Sigmoid(),torch.nn.Linear(5,1))\r\n", + "print(net)" + ], + "execution_count": 215, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Sequential(\n", + " (0): Linear(in_features=2, out_features=5, bias=True)\n", + " (1): Sigmoid()\n", + " (2): Linear(in_features=5, out_features=1, bias=True)\n", + ")\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5r5RbLB1s6YB" + }, + "source": [ + "Обучать такую сеть мы можем с помощью описанного ранее метода `train`:" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ogXKdcfIs_ND", + "outputId": "957ccd8d-0076-4e9b-89f1-edc1de75f18e" + }, + "source": [ + "train(net,dataloader,val_x,val_lab)" + ], + "execution_count": 216, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 0.6622872352600098, val acc = 0.6666666865348816\n", + "Epoch 1: last batch loss = 0.6727332472801208, val acc = 0.800000011920929\n", + "Epoch 2: last batch loss = 0.6326467990875244, val acc = 0.800000011920929\n", + "Epoch 3: last batch loss = 0.5876082181930542, val acc = 0.7333333492279053\n", + "Epoch 4: last batch loss = 0.5610458254814148, val acc = 0.7333333492279053\n", + "Epoch 5: last batch loss = 0.5546905398368835, val acc = 0.7333333492279053\n", + "Epoch 6: last batch loss = 0.5626044273376465, val acc = 0.7333333492279053\n", + "Epoch 7: last batch loss = 0.5754281878471375, val acc = 0.800000011920929\n", + "Epoch 8: last batch loss = 0.5850474238395691, val acc = 0.800000011920929\n", + "Epoch 9: last batch loss = 0.5874522924423218, val acc = 0.800000011920929\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jY4R1XEGtEzJ" + }, + "source": [ + "## Описание нейросети в виде класса\r\n", + "\r\n", + "Это более гибкий способ описания нейросетей, поскольку он позволяет выполнять произвольные вычисления." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "SlsJmGu0tMsZ", + "outputId": "240d5c89-096c-4392-99cd-1ade5ff3e3e1" + }, + "source": [ + "class MyNet(torch.nn.Module):\r\n", + " def __init__(self,hidden_size=10,func=torch.nn.Sigmoid()):\r\n", + " super().__init__()\r\n", + " self.fc1 = torch.nn.Linear(2,hidden_size)\r\n", + " self.func = func\r\n", + " self.fc2 = torch.nn.Linear(hidden_size,1)\r\n", + "\r\n", + " def forward(self,x):\r\n", + " x = self.fc1(x)\r\n", + " x = self.func(x)\r\n", + " x = self.fc2(x)\r\n", + " return x\r\n", + " \r\n", + "net = MyNet(func=torch.nn.ReLU())\r\n", + "print(net)" + ], + "execution_count": 230, + "outputs": [ + { + "output_type": "stream", + "text": [ + "MyNet(\n", + " (fc1): Linear(in_features=2, out_features=10, bias=True)\n", + " (func): ReLU()\n", + " (fc2): Linear(in_features=10, out_features=1, bias=True)\n", + ")\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HwdapRxft-7M", + "outputId": "6eb900cf-4902-4a04-c62b-497b68455406" + }, + "source": [ + "train(net,dataloader,val_x,val_lab,lr=0.005)" + ], + "execution_count": 231, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 0: last batch loss = 0.6930208206176758, val acc = 0.8666666746139526\n", + "Epoch 1: last batch loss = 0.6779811382293701, val acc = 0.9333333373069763\n", + "Epoch 2: last batch loss = 0.6627538800239563, val acc = 0.9333333373069763\n", + "Epoch 3: last batch loss = 0.6472628712654114, val acc = 0.9333333373069763\n", + "Epoch 4: last batch loss = 0.6321150660514832, val acc = 0.9333333373069763\n", + "Epoch 5: last batch loss = 0.618022620677948, val acc = 0.9333333373069763\n", + "Epoch 6: last batch loss = 0.6052160859107971, val acc = 0.9333333373069763\n", + "Epoch 7: last batch loss = 0.5925578474998474, val acc = 0.9333333373069763\n", + "Epoch 8: last batch loss = 0.5803073048591614, val acc = 0.9333333373069763\n", + "Epoch 9: last batch loss = 0.569009006023407, val acc = 0.9333333373069763\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dvAiaj_JndyP" + }, + "source": [ + "**Задание 1**: Постройте графики ошибок на обучающей и тестовой выборке в процессе обучения\r\n", + "\r\n", + "**Задание 2**: Попробуйте решить задачу классификации на датасете MNIST с помощью этого кода. Подсказка: используйте `crossentropy_with_logits` в качестве функции ошибки. При этом в первом случае на выход сети необходимо подавать целевые значения в формате *one hot encoding*, а во втором - в виде целочисленного номера класса." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gZ-kWx84bMDH" + }, + "source": [ + "**Задание 3**: \r\n", + "Используйте PyTorch для обучения классификатора на сети MNIST. При этом:\r\n", + "* Обратите внимание, что в keras заложены типовые датасеты, включая MNIST. Для обращения к нему достаточно пары строчек кода с использованием библиотеки `torchvision` (см, например, [тут](https://pytorch.org/vision/0.8/datasets.html))\r\n", + "* Попробуйте несколько конфигураций сети с несколькими полносвязными слоями, передаточными функциями, и разным количеством нейронов\r\n", + "\r\n", + "Какой точности вам удалось достичь?\r\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "trusted": true, + "id": "MG64NKzawHl-" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/3-NeuralNetworks/README.md b/3-NeuralNetworks/README.md new file mode 100644 index 00000000..cb710a41 --- /dev/null +++ b/3-NeuralNetworks/README.md @@ -0,0 +1,44 @@ +# Introduction to Neural Networks + +As we have discussed in the introduction, one of the ways to achieve intelligence is to train a **computer model** or an **artificial brain**. Since the middle of 20th century, researchers tried different mathematical models, until in recent years this direction proved to by hugely successful. Such mathematical models of the brain are called **Neural Networks** (sometimes *Artificial Neural Networks*, ANNs, in order to indicate that we are talking about models, not real networks of neurons). + +## Machine Learning + +Neural Networks are a part of larger discipline called **Machine Learning**, whose goal is to use data to train computer models that are able to solve problems. Machine Learning constitutes a large part of Artificial Intelligence, however, we do not cover classical ML in this curricula. + +> Visit our separate **[Machine Learning for Beginners](http://github.com/microsoft/ml-for-beginners)** curriculum to learn more about Machine Learning. + +In Machine Learning, we assume that we have some dataset of examples **X**, and corresponding output values **Y**. Examples are often N-dimensional vectors that consist of **features**, and outputs are called **labels**. + +We will consider two most common machine learning problems: +* **Classification**, where we need to classify an input object into two or more classes. +* **Regression**, where we need to predict a numerical number for each of the input samples. + +> When representing inputs and outputs as tensors, input dataset is a matrix of size M×N, where M is number of samples, N is the number of features. Output labels Y is the vector of size M. + +In this curricula, we will only focus on neural network models. + +## A Model of a Neuron + +From biology we know that our brain consists of neural cells, each of them having multiple "inputs" (axons), and an output (dendrite). Axons and dendrites can conduct electrical signals, and connections between axons and dendrites can exhibit different degrees of conductivity (controlled by neuromediators). + +![Model of a Neuron](images/synapse-wikipedia.JPG) | ![Model of a Neuron](images/artneuron.png) +----|---- +Real Neuron | Artificial Neuron + +Thus, simples mathematical model of a neuron contains several inputs X1, ..., XN and an output Y, and a series of weights W1, ..., WN. An output is calculated as + +Y = f\left(\sum_{i=1}^N X_iW_i\right) + +where f is some non-linear **activation function**. + +> Early models of neuron were described in classical paper [A logical calculus of the ideas immanent in nervous activity](http://www.springerlink.com/content/61446605110620kg/fulltext.pdf) by Warren McCullock and Walter Pitts in 1943. Donald Hebb in his book "[The Organization of Behavior: A Neuropsychological Theory](https://books.google.com/books?id=VNetYrB8EBoC)" proposed the way those networks can be trained. + + +## In this Section + +In this section we will learn about: +* [Perceptron](03-Perceptron/README.md), one of the earliest neural network models for two-class classification +* [Modern multi-layered networks](04-OwnFramework/README.md) and [how to build our own framework](04-OwnFramework/OwnFramework.ipynb) +* [Neural Network Frameworks](05-Frameworks/README.md), such as [PyTorch](05-Frameworks/IntroPyTorch.ipynb) and [Keras/Tensorflow](05-Frameworks/IntroKerasTF.ipynb) + diff --git a/3-NeuralNetworks/images/artneuron.png b/3-NeuralNetworks/images/artneuron.png new file mode 100644 index 00000000..29ce4111 Binary files /dev/null and b/3-NeuralNetworks/images/artneuron.png differ diff --git a/3-NeuralNetworks/images/synapse-wikipedia.jpg b/3-NeuralNetworks/images/synapse-wikipedia.jpg new file mode 100644 index 00000000..0ab40f43 Binary files /dev/null and b/3-NeuralNetworks/images/synapse-wikipedia.jpg differ diff --git a/README.md b/README.md index ac2d3e00..6a5713d7 100644 --- a/README.md +++ b/README.md @@ -29,27 +29,30 @@ For a gentle introduction to *AI in the Cloud* topic you may consider taking [Ge # Content - + - + - - + + + + + + + - + @@ -69,7 +72,7 @@ For a gentle introduction to *AI in the Cloud* topic you may consider taking [Ge - +
NoLessonIntroPyTorchTensorflowLab
NoLessonIntroPyTorchKeras/TensorflowLab
IIntroduction to AIPAT
1Introduction and History of AIText
IISymbolic AIPAT
2 Knowledge Representation and Expert SystemsText
IIIIntroduction to Neural NetworksPAT
IIIIntroduction to Neural NetworksPAT
3Perceptron Text - Perceptron
- Own Framework
4 Intro to Frameworks (PyTorch/Tensorflow)TextPyTorchTensorflow
5 Multi-Layered PerceptronTextPyTorchTensorflow
Notebook
4 Multi-Layered Perceptron and Creating our own FrameworkTextNotebook
5Intro to Frameworks (PyTorch/Tensorflow)TextPyTorchKeras/Tensorflow
IVComputer Vision MS Learn MS Learn PAT
6Intro to Computer Vision. OpenCVTextNotebook
7Convolutional Neural NetworksTextPyTorchTensorflow
8Pre-trained Networks and Transfer LearningTextPyTorchTensorflow
8Pre-trained Networks and Transfer LearningTextPyTorchTensorflow
9Autoencoders and VAEsTextPyTorchTensorflow
10 Generative Adversarial NetworksTextPyTorchTensorflow
11Object DetectionTextPyTorchTensorflow
VIOther AI TechniquesPAT
21Genetic AlgorithmsTextNotebook
22Deep Reinforcement LearningTextPyTorchTensorflow
23Multi-Agent SystemsText
23Multi-Agent SystemsText
VIIAI EthicsPAT
24AI Ethics and Responsible AIText