diff --git a/lessons/3-NeuralNetworks/03-Perceptron/Perceptron.ipynb b/lessons/3-NeuralNetworks/03-Perceptron/Perceptron.ipynb index 3d3f57ac..cbcbc45c 100644 --- a/lessons/3-NeuralNetworks/03-Perceptron/Perceptron.ipynb +++ b/lessons/3-NeuralNetworks/03-Perceptron/Perceptron.ipynb @@ -232,7 +232,7 @@ }, "outputs": [], "source": [ - "def train(positive_examples, negative_examples, num_iterations = 100):\n", + "def train(positive_examples, negative_examples, num_iterations = 100, learning_rate = 0.01):\n", " num_dims = positive_examples.shape[1]\n", " \n", " # Initialize weights. \n", @@ -251,11 +251,11 @@ "\n", " z = np.dot(pos, weights) \n", " if z < 0: # positive example was classified as negative\n", - " weights = weights + pos.reshape(weights.shape)\n", + " weights = weights + learning_rate * pos.reshape(weights.shape)\n", "\n", " z = np.dot(neg, weights)\n", " if z >= 0: # negative example was classified as positive\n", - " weights = weights - neg.reshape(weights.shape)\n", + " weights = weights - learning_rate * neg.reshape(weights.shape)\n", " \n", " # Periodically, print out the current accuracy on all examples \n", " if i % report_frequency == 0: \n", @@ -268,6 +268,19 @@ " return weights" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note on Learning Rate**: The `learning_rate` parameter (default `0.01`) controls how much we adjust the weights during each training step. This implements the gradient descent update formula:\n", + "\n", + "$$\\mathbf{w}^{\\tau + 1}=\\mathbf{w}^{\\tau} + \\eta \\mathbf{x}_{n} t_{n}$$\n", + "\n", + "- A larger learning rate (e.g., `1.0`) makes the perceptron learn faster but may overshoot the optimal solution\n", + "- A smaller learning rate (e.g., `0.001`) learns more slowly but may converge more precisely\n", + "- You can experiment by calling: `train(pos_examples, neg_examples, learning_rate=0.1)`\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -371,6 +384,119 @@ "plot_boundary(pos_examples,neg_examples,wts)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Experimenting with Learning Rates\n", + "\n", + "Now let's explore how different learning rates affect the training process. The learning rate controls the step size in gradient descent - a crucial hyperparameter that affects both convergence speed and stability.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Compare different learning rates\n", + "learning_rates = [0.001, 0.01, 0.1, 1.0]\n", + "fig, axes = pylab.subplots(2, 2, figsize=(12, 10))\n", + "fig.suptitle('Effect of Different Learning Rates', fontsize=16)\n", + "\n", + "for idx, lr in enumerate(learning_rates):\n", + " ax = axes[idx // 2, idx % 2]\n", + " \n", + " # Train with this learning rate\n", + " weights_lr = train(pos_examples, neg_examples, num_iterations=100, learning_rate=lr)\n", + " \n", + " # Plot decision boundary\n", + " if np.isclose(weights_lr[1], 0):\n", + " if np.isclose(weights_lr[0], 0):\n", + " x = y = np.array([-6, 6], dtype='float32')\n", + " else:\n", + " y = np.array([-6, 6], dtype='float32')\n", + " x = -(weights_lr[1] * y + weights_lr[2])/weights_lr[0]\n", + " else:\n", + " x = np.array([-6, 6], dtype='float32')\n", + " y = -(weights_lr[0] * x + weights_lr[2])/weights_lr[1]\n", + " \n", + " ax.set_xlim(-6, 6)\n", + " ax.set_ylim(-6, 6)\n", + " ax.plot(pos_examples[:, 0], pos_examples[:, 1], 'bo', label='Positive', alpha=0.7)\n", + " ax.plot(neg_examples[:, 0], neg_examples[:, 1], 'ro', label='Negative', alpha=0.7)\n", + " ax.plot(x, y, 'g-', linewidth=2)\n", + " ax.set_title(f'Learning Rate = {lr}')\n", + " ax.set_xlabel('Feature 1')\n", + " ax.set_ylabel('Feature 2')\n", + " ax.legend()\n", + " ax.grid(True, alpha=0.3)\n", + "\n", + "pylab.tight_layout()\n", + "pylab.show()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Interactive Learning Rate Experiment\n", + "\n", + "Use the slider below to interactively experiment with different learning rates and see how they affect the decision boundary:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def train_and_plot_with_lr(learning_rate=0.01):\n", + " \"\"\"Train perceptron with specified learning rate and plot results\"\"\"\n", + " weights_lr = train(pos_examples, neg_examples, num_iterations=100, learning_rate=learning_rate)\n", + " \n", + " fig, (ax1, ax2) = pylab.subplots(1, 2, figsize=(14, 5))\n", + " \n", + " # Plot 1: Decision boundary\n", + " if np.isclose(weights_lr[1], 0):\n", + " if np.isclose(weights_lr[0], 0):\n", + " x = y = np.array([-6, 6], dtype='float32')\n", + " else:\n", + " y = np.array([-6, 6], dtype='float32')\n", + " x = -(weights_lr[1] * y + weights_lr[2])/weights_lr[0]\n", + " else:\n", + " x = np.array([-6, 6], dtype='float32')\n", + " y = -(weights_lr[0] * x + weights_lr[2])/weights_lr[1]\n", + " \n", + " ax1.set_xlim(-6, 6)\n", + " ax1.set_ylim(-6, 6)\n", + " ax1.plot(pos_examples[:, 0], pos_examples[:, 1], 'bo', label='Positive', s=100, alpha=0.6)\n", + " ax1.plot(neg_examples[:, 0], neg_examples[:, 1], 'ro', label='Negative', s=100, alpha=0.6)\n", + " ax1.plot(x, y, 'g-', linewidth=3, label='Decision Boundary')\n", + " ax1.set_title(f'Decision Boundary (lr={learning_rate})', fontsize=14)\n", + " ax1.set_xlabel('Feature 1')\n", + " ax1.set_ylabel('Feature 2')\n", + " ax1.legend()\n", + " ax1.grid(True, alpha=0.3)\n", + " \n", + " # Plot 2: Weight values\n", + " ax2.bar(['w0', 'w1', 'bias'], weights_lr.flatten(), color=['blue', 'green', 'red'], alpha=0.7)\n", + " ax2.set_title('Final Weight Values', fontsize=14)\n", + " ax2.set_ylabel('Weight Value')\n", + " ax2.grid(True, alpha=0.3, axis='y')\n", + " ax2.axhline(y=0, color='black', linestyle='-', linewidth=0.5)\n", + " \n", + " pylab.tight_layout()\n", + " pylab.show()\n", + " \n", + " print(f\"Final weights: {weights_lr.flatten()}\")\n", + "\n", + "# Create interactive widget\n", + "interact(train_and_plot_with_lr, \n", + " learning_rate=widgets.FloatSlider(value=0.01, min=0.001, max=1.0, step=0.001, \n", + " description='Learning Rate:', continuous_update=False))\n" + ] + }, { "cell_type": "markdown", "metadata": { @@ -435,7 +561,7 @@ }, "outputs": [], "source": [ - "def train_graph(positive_examples, negative_examples, num_iterations = 100):\n", + "def train_graph(positive_examples, negative_examples, num_iterations = 100, learning_rate = 0.01):\n", " num_dims = positive_examples.shape[1]\n", " weights = np.zeros((num_dims,1)) # initialize weights\n", " \n", @@ -451,11 +577,11 @@ "\n", " z = np.dot(pos, weights) \n", " if z < 0:\n", - " weights = weights + pos.reshape(weights.shape)\n", + " weights = weights + learning_rate * pos.reshape(weights.shape)\n", "\n", " z = np.dot(neg, weights)\n", " if z >= 0:\n", - " weights = weights - neg.reshape(weights.shape)\n", + " weights = weights - learning_rate * neg.reshape(weights.shape)\n", " \n", " if i % report_frequency == 0: \n", " pos_out = np.dot(positive_examples, weights)\n", @@ -1077,4 +1203,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file