From 2ba3a610d5596f8f731cfeb62edc1e6f73e88461 Mon Sep 17 00:00:00 2001 From: franzi Date: Sat, 13 Aug 2022 18:02:20 +0200 Subject: [PATCH] update versions --- notebooks/1_visualize_text.ipynb | 13 ++- notebooks/2_image_quantization.ipynb | 4 +- notebooks/3_supervised_comparison.ipynb | 101 ++++++++++++------ ..._toydata.ipynb => 4_analyze_toydata.ipynb} | 25 +---- ...ures.ipynb => 5_hard_drive_failures.ipynb} | 4 +- ...al.ipynb => 6_information_retrieval.ipynb} | 4 +- ..._mnist_keras.ipynb => 7_mnist_keras.ipynb} | 4 +- ..._mnist_torch.ipynb => 7_mnist_torch.ipynb} | 4 +- notebooks/8_rl_gridmove.ipynb | 4 +- requirements.txt | 20 ++-- test_installation.ipynb | 24 ++--- 11 files changed, 112 insertions(+), 95 deletions(-) rename notebooks/{6_analyze_toydata.ipynb => 4_analyze_toydata.ipynb} (95%) rename notebooks/{7_hard_drive_failures.ipynb => 5_hard_drive_failures.ipynb} (99%) rename notebooks/{4_information_retrieval.ipynb => 6_information_retrieval.ipynb} (99%) rename notebooks/{5_mnist_keras.ipynb => 7_mnist_keras.ipynb} (99%) rename notebooks/{5_mnist_torch.ipynb => 7_mnist_torch.ipynb} (99%) diff --git a/notebooks/1_visualize_text.ipynb b/notebooks/1_visualize_text.ipynb index 4084840..46ee186 100644 --- a/notebooks/1_visualize_text.ipynb +++ b/notebooks/1_visualize_text.ipynb @@ -223,15 +223,14 @@ "metadata": {}, "outputs": [], "source": [ - "# lambdas = eigenvalues\n", - "print(kpca.lambdas_[:10])\n", + "print(kpca.eigenvalues_[:10])\n", "# check how much \"information\" we would keep if we were to reduce the dimensionality to 20\n", "# (this is not 100% accurate, since we only computed the first 100 kPCA components, i.e.,\n", - "# normally lambda_ should contain all eigenvalues - but this should be close enough)\n", - "print(\"Percentage of variance retained with 20 components:\", 100*(sum(kpca.lambdas_[:20])/sum(kpca.lambdas_)))\n", + "# normally kpca.eigenvalues_ should contain all eigenvalues - but this should be close enough)\n", + "print(\"Percentage of variance retained with 20 components:\", 100*(sum(kpca.eigenvalues_[:20])/sum(kpca.eigenvalues_)))\n", "# plot eigenvalue spectrum\n", "plt.figure()\n", - "plt.plot(range(1, len(kpca.lambdas_)+1), kpca.lambdas_)\n", + "plt.plot(range(1, len(kpca.eigenvalues_)+1), kpca.eigenvalues_)\n", "plt.xlabel(\"PCs\")\n", "plt.ylabel(\"Eigenvalue\");\n", "# observe how the first value is extremely large" @@ -359,7 +358,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -373,7 +372,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.10.2" } }, "nbformat": 4, diff --git a/notebooks/2_image_quantization.ipynb b/notebooks/2_image_quantization.ipynb index 43713b6..fb764ba 100644 --- a/notebooks/2_image_quantization.ipynb +++ b/notebooks/2_image_quantization.ipynb @@ -185,7 +185,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -199,7 +199,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.10.2" } }, "nbformat": 4, diff --git a/notebooks/3_supervised_comparison.ipynb b/notebooks/3_supervised_comparison.ipynb index ea8a9e9..6170c25 100644 --- a/notebooks/3_supervised_comparison.ipynb +++ b/notebooks/3_supervised_comparison.ipynb @@ -22,6 +22,8 @@ "import matplotlib.pyplot as plt\n", "from matplotlib.colors import ListedColormap\n", "from sklearn.datasets import make_moons\n", + "from sklearn.preprocessing import StandardScaler\n", + "from sklearn.inspection import DecisionBoundaryDisplay\n", "# don't get unneccessary warnings\n", "import warnings\n", "warnings.simplefilter(action='ignore', category=FutureWarning)" @@ -54,39 +56,22 @@ "def plot_classification(X, Y, model=None):\n", " # plot a classification dataset (and model predictions)\n", " plt.figure()\n", - " x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5\n", - " y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5\n", - " xx, yy = np.meshgrid(np.linspace(x_min, x_max, 250),\n", - " np.linspace(y_min, y_max, 250))\n", - " cm = plt.cm.RdBu\n", - " cm_bright = ListedColormap(['#FF0000', '#0000FF'])\n", " if model is not None:\n", - " try:\n", - " Z = model.decision_function(np.c_[xx.ravel(), yy.ravel()])\n", - " alpha = 0.8\n", - " except:\n", - " # decision tree\n", - " Z = model.predict(np.c_[xx.ravel(), yy.ravel()])\n", - " alpha = 0.4\n", - " # Put the result into a color plot\n", - " Z = Z.reshape(xx.shape)\n", - " plt.contourf(xx, yy, Z, cmap=cm, alpha=alpha)\n", - " # Plot the training points\n", - " plt.scatter(X[:, 0], X[:, 1], s=20, c=Y, cmap=cm_bright, label=\"data samples\")\n", - " plt.xlim(xx.min(), xx.max())\n", - " plt.ylim(yy.min(), yy.max())\n", - " plt.xlabel(\"feature 1\")\n", - " plt.ylabel(\"feature 2\")\n", + " DecisionBoundaryDisplay.from_estimator(\n", + " model, X, cmap=plt.cm.RdBu, alpha=0.8, eps=0.5, xlabel=\"feature 1\", ylabel=\"feature 2\", ax=plt.gca()\n", + " )\n", + " # plot the training points\n", + " plt.scatter(X[:, 0], X[:, 1], s=20, c=Y, cmap=ListedColormap(['#FF0000', '#0000FF']), label=\"data samples\")\n", " plt.title(\"Classification Problem\")\n", " plt.colorbar()\n", - "\n", + " \n", "def get_linear_regression():\n", " # generate noisy linear regression dataset\n", " np.random.seed(15)\n", " X = np.random.rand(n_train_reg, 1)\n", " y = -2.5 + 5*X\n", " y += np.random.randn(n_train_reg, 1) * 0.4\n", - " return X, y.flatten()\n", + " return StandardScaler(with_std=False).fit_transform(X), y.flatten()\n", "\n", "def get_linear_outlier():\n", " # generate linear regression dataset with outliers\n", @@ -95,7 +80,7 @@ " y = -2.5 + 5*X\n", " y += np.random.randn(n_train_reg, 1) * 0.05\n", " y[(X>0.7) & (X<0.73)] = 10\n", - " return X, y.flatten()\n", + " return StandardScaler(with_std=False).fit_transform(X), y.flatten()\n", "\n", "def get_nonlinear_regression():\n", " # generate noisy non-linear regression dataset\n", @@ -103,7 +88,7 @@ " X = np.random.rand(n_train_reg, 1) * np.pi * 2.\n", " y = np.sin(X)\n", " y += np.random.randn(n_train_reg, 1) * 0.2\n", - " return X, y.flatten()\n", + " return StandardScaler().fit_transform(X), y.flatten()\n", "\n", "def get_linear_classification_1f():\n", " # generate classification dataset with 1 informative feature\n", @@ -117,7 +102,7 @@ " y = np.zeros(n_train_clf, dtype=int)\n", " y[n_train_clf//2:] = 1\n", " rndidx = np.random.permutation(len(y))\n", - " return X[rndidx], y[rndidx]\n", + " return StandardScaler().fit_transform(X[rndidx]), y[rndidx]\n", "\n", "def get_linear_classification_2f():\n", " # generate classification dataset with 2 informative features\n", @@ -132,11 +117,12 @@ " y = np.zeros(n_train_clf, dtype=int)\n", " y[n_train_clf//2:] = 1\n", " rndidx = np.random.permutation(len(y))\n", - " return X[rndidx], y[rndidx]\n", + " return StandardScaler().fit_transform(X[rndidx]), y[rndidx]\n", "\n", "def get_nonlinear_classification():\n", " # generate non-linear classification dataset\n", - " return make_moons(n_samples=n_train_clf, noise=0.3, random_state=1)" + " X, y = make_moons(n_samples=n_train_clf, noise=0.3, random_state=1)\n", + " return StandardScaler().fit_transform(X), y" ] }, { @@ -248,6 +234,59 @@ "print(f\"f(x) = sigmoid({model.intercept_[0]:.3f} + {model.coef_[0, 0]:.3f} * x_1 + {model.coef_[0, 1]:.3f} * x_2)\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feed-Forward Neural Network (Multi-Layer Perceptron)\n", + "\n", + "After reading the chapter on neural networks, test them here on different datasets and experiment with their hyperparameter settings.\n", + "\n", + "**Questions:**\n", + "- What do you observe when you change the activation function to `'relu'` for the 3rd regression dataset?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.neural_network import MLPRegressor, MLPClassifier" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Neural network for regression:\n", + "# alpha (> 0): regularization (higher values = more regularization)\n", + "# hidden_layer_sizes (tuple of ints): number of units in the hidden layers\n", + "# activation (one of 'identity', 'logistic', 'tanh', 'relu'): non-linear activation function between layers\n", + "X, y = X_reg_3, y_reg_3\n", + "model = MLPRegressor(alpha=1e-05, hidden_layer_sizes=(15,), activation=\"tanh\", solver=\"lbfgs\", random_state=1)\n", + "model.fit(X, y)\n", + "plot_regression(X, y, model)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Neural network for classification:\n", + "# alpha (> 0): regularization (higher values = more regularization)\n", + "# hidden_layer_sizes (tuple of ints): number of units in the hidden layers\n", + "# activation (one of 'identity', 'logistic', 'tanh', 'relu'): non-linear activation function between layers\n", + "X, y = X_clf_3, y_clf_3\n", + "model = MLPClassifier(alpha=1e-05, hidden_layer_sizes=(25, 5), activation=\"relu\", solver=\"adam\", max_iter=1000, random_state=1)\n", + "model.fit(X, y)\n", + "plot_classification(X, y, model)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -468,7 +507,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -482,7 +521,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.10.2" } }, "nbformat": 4, diff --git a/notebooks/6_analyze_toydata.ipynb b/notebooks/4_analyze_toydata.ipynb similarity index 95% rename from notebooks/6_analyze_toydata.ipynb rename to notebooks/4_analyze_toydata.ipynb index a19c929..5e6d87b 100644 --- a/notebooks/6_analyze_toydata.ipynb +++ b/notebooks/4_analyze_toydata.ipynb @@ -840,27 +840,6 @@ "While it was a bit more work to set up the logistic regression model appropriately, incl. extra data preprocessing steps, we now even got a balanced accuracy on the test set that is slightly higher than that of the decision tree (0.938 instead of 0.935)." ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Advanced Exercise (optional)\n", - "\n", - "Use a neural network (either using the `torch`/`skorch` (recommended) or `tensorflow`/`keras` libraries) to solve this task.\n", - "\n", - "Start with a linear network (i.e., a FFNN without hidden layers, i.e., the same number of trainable parameters as the logistic regression model used above) and try to get approximately the same performance as the LogReg model.\n", - "\n", - "Then use a deeper network (e.g., one additional hidden layer) and see if this improves the performance.\n", - "\n", - "**Tips:**\n", - "- Make sure to use scaled data!\n", - "- Since the faulty products are underrepresented, samples from this class should get a higher weight during training (similar to what we're doing with `class_weight=\"balanced\"` in sklearn models). \n", - "\n", - "**Using `torch` & `skorch`:**\n", - "- Use a skorch [`NeuralNetBinaryClassifier`](https://skorch.readthedocs.io/en/latest/classifier.html). Here the torch network shoud predict the output without any non-linear activation function at the end (i.e., *don't* use a sigmoid function to convert the output into probabilities) as the skorch model takes care of this conversion for you!\n", - "- The sample weights can be set by passing `criterion__pos_weight=torch.Tensor([np.sum(y_train==0)/np.sum(y_train==1)])` as an argument to the `NeuralNetBinaryClassifier` (see also the documentation for the torch [`BCEWithLogitsLoss`](https://pytorch.org/docs/stable/generated/torch.nn.BCEWithLogitsLoss.html) loss function, which is used internally by the skorch model)" - ] - }, { "cell_type": "code", "execution_count": null, @@ -871,7 +850,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -885,7 +864,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.10.2" } }, "nbformat": 4, diff --git a/notebooks/7_hard_drive_failures.ipynb b/notebooks/5_hard_drive_failures.ipynb similarity index 99% rename from notebooks/7_hard_drive_failures.ipynb rename to notebooks/5_hard_drive_failures.ipynb index 78647cc..fd5dd46 100644 --- a/notebooks/7_hard_drive_failures.ipynb +++ b/notebooks/5_hard_drive_failures.ipynb @@ -161,7 +161,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -175,7 +175,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.10.2" } }, "nbformat": 4, diff --git a/notebooks/4_information_retrieval.ipynb b/notebooks/6_information_retrieval.ipynb similarity index 99% rename from notebooks/4_information_retrieval.ipynb rename to notebooks/6_information_retrieval.ipynb index dd0faae..33f8696 100644 --- a/notebooks/4_information_retrieval.ipynb +++ b/notebooks/6_information_retrieval.ipynb @@ -408,7 +408,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -422,7 +422,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.10.2" } }, "nbformat": 4, diff --git a/notebooks/5_mnist_keras.ipynb b/notebooks/7_mnist_keras.ipynb similarity index 99% rename from notebooks/5_mnist_keras.ipynb rename to notebooks/7_mnist_keras.ipynb index 8a7e1db..54d5329 100644 --- a/notebooks/5_mnist_keras.ipynb +++ b/notebooks/7_mnist_keras.ipynb @@ -617,7 +617,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -631,7 +631,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.10.2" } }, "nbformat": 4, diff --git a/notebooks/5_mnist_torch.ipynb b/notebooks/7_mnist_torch.ipynb similarity index 99% rename from notebooks/5_mnist_torch.ipynb rename to notebooks/7_mnist_torch.ipynb index f0b20ef..6df8fe0 100644 --- a/notebooks/5_mnist_torch.ipynb +++ b/notebooks/7_mnist_torch.ipynb @@ -667,7 +667,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -681,7 +681,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.10.2" } }, "nbformat": 4, diff --git a/notebooks/8_rl_gridmove.ipynb b/notebooks/8_rl_gridmove.ipynb index 5219b6f..1949f54 100644 --- a/notebooks/8_rl_gridmove.ipynb +++ b/notebooks/8_rl_gridmove.ipynb @@ -335,7 +335,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -349,7 +349,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.10.2" } }, "nbformat": 4, diff --git a/requirements.txt b/requirements.txt index 218deda..97b86e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -numpy>=1.18.5 -pandas>=1.2.1 -scipy>=1.4.1 -scikit-learn>=0.24.1 -matplotlib>=3.3.1 -pillow>=8.1.0 -plotly>=4.9.0 -torch>=1.6.0 -torchvision>=0.8.1 -skorch>=0.9.0 +numpy>=1.22.3 +pandas>=1.4.2 +scipy>=1.8.0 +scikit-learn>=1.1.2 +matplotlib>=3.5.1 +pillow>=9.1.0 +plotly>=5.7.0 +torch>=1.12.1 +torchvision>=0.13.1 +skorch>=0.11.0 diff --git a/test_installation.ipynb b/test_installation.ipynb index 3360acd..5a7c389 100755 --- a/test_installation.ipynb +++ b/test_installation.ipynb @@ -16,19 +16,19 @@ "# check versions of the libraries\n", "# they should not be too much behind the ones in the comments...\n", "import numpy\n", - "print(\"numpy\", numpy.__version__) # >= 1.18.5\n", + "print(\"numpy\", numpy.__version__) # >= 1.22.3\n", "import pandas\n", - "print(\"pandas\", pandas.__version__) # >= 1.2.1\n", + "print(\"pandas\", pandas.__version__) # >= 1.4.2\n", "import scipy\n", - "print(\"scipy\", scipy.__version__) # >= 1.4.1\n", + "print(\"scipy\", scipy.__version__) # >= 1.8.0\n", "import sklearn\n", - "print(\"sklearn\", sklearn.__version__) # >= 0.24.1\n", + "print(\"sklearn\", sklearn.__version__) # >= 1.1.2\n", "import matplotlib\n", - "print(\"matplotlib\", matplotlib.__version__) # >= 3.3.1\n", + "print(\"matplotlib\", matplotlib.__version__) # >= 3.5.1\n", "import PIL\n", - "print(\"pillow\", PIL.__version__) # >= 8.1.0\n", + "print(\"pillow\", PIL.__version__) # >= 9.1.0\n", "import plotly\n", - "print(\"plotly\", plotly.__version__) # >= 4.9.0\n", + "print(\"plotly\", plotly.__version__) # >= 5.7.0\n", "print(\"Congratulations! Your installation of the basic libraries was successful!\")\n", "# the following libraries are needed for the neural network example \n", "# (if you're working with the recommended pytorch, not keras/tensorflow)\n", @@ -38,11 +38,11 @@ "# (if you're working on a normal laptop and you don't know what GPU means,\n", "# don't worry about it and just execute `$ pip install torch torchvision skorch`)\n", "import torch\n", - "print(\"torch\", torch.__version__) # >= 1.6.0\n", + "print(\"torch\", torch.__version__) # >= 1.12.1\n", "import torchvision\n", - "print(\"torchvision\", torchvision.__version__) # >= 0.8.1\n", + "print(\"torchvision\", torchvision.__version__) # >= 0.13.1\n", "import skorch\n", - "print(\"skorch\", skorch.__version__) # >= 0.9.0\n", + "print(\"skorch\", skorch.__version__) # >= 0.11.0\n", "print(\"Congratulations! Your installation of the neural network libraries was successful!\")" ] }, @@ -56,7 +56,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -70,7 +70,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.10.2" } }, "nbformat": 4,