{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "4e6f6cb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import accuracy_score, f1_score, recall_score, classification_report, confusion_matrix, roc_curve\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "4dd5223b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "c1ab7ec9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset shape: (116, 10)\n",
      "   Age        BMI  Glucose  Insulin      HOMA   Leptin  Adiponectin  Resistin  \\\n",
      "0   48  23.500000       70    2.707  0.467409   8.8071     9.702400   7.99585   \n",
      "1   83  20.690495       92    3.115  0.706897   8.8438     5.429285   4.06405   \n",
      "2   82  23.124670       91    4.498  1.009651  17.9393    22.432040   9.27715   \n",
      "3   68  21.367521       77    3.226  0.612725   9.8827     7.169560  12.76600   \n",
      "4   86  21.111111       92    3.549  0.805386   6.6994     4.819240  10.57635   \n",
      "\n",
      "     MCP.1  Classification  \n",
      "0  417.114               1  \n",
      "1  468.786               1  \n",
      "2  554.697               1  \n",
      "3  928.220               1  \n",
      "4  773.920               1  \n"
     ]
    }
   ],
   "source": [
    "# fetch dataset \n",
    "data = pd.read_csv(\"dataR2.csv\")\n",
    "  \n",
    "# data (as pandas dataframes) \n",
    "X = data.drop(columns='Classification')\n",
    "y = data['Classification'].map({2: 1, 1: 0}) # Map 2 to 1 and 1 to 0, 1 = pacient healthy, 0 = pacient sick\n",
    "  \n",
    "print(\"Dataset shape:\", data.shape)\n",
    "print(data.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1004c28",
   "metadata": {},
   "source": [
    "# K-NN classifier "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "082c143b",
   "metadata": {},
   "source": [
    "### 1) No curse of dimention "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "754dce9b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples: 116\n",
      "Number of features: 9\n",
      "Number of classes: 2\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of samples:\", X.shape[0])\n",
    "print(\"Number of features:\", X.shape[1])\n",
    "print(\"Number of classes:\", len(np.unique(y)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01bb817a",
   "metadata": {},
   "source": [
    "Then d is small enough to insure that we are not in the curse of dimention"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f74eaee",
   "metadata": {},
   "source": [
    "### 2) train-test split and rescaling of the feature "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e67beb4",
   "metadata": {},
   "source": [
    "Feature scaling is important for k-NN, as it is a distance-based algorithm and is sensitive to the scale of the features. This means that if some data are much farther from others due to unscaled features, it can damage the quality of the predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "b06c0b07",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # train/test split with 80% training and 20% testing\n",
    "\n",
    "# Feature scaling\n",
    "scaler = StandardScaler()\n",
    "\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "\n",
    "X_test_scaled = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31d5bc79",
   "metadata": {},
   "source": [
    "### 3) Cross validation to find the best k nearest neighbor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "b2e03ac1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of k values to test: 22\n",
      "The best k for k-NN is k = 21\n"
     ]
    }
   ],
   "source": [
    "k_scores = []\n",
    "score = []\n",
    "K_list = np.arange(1, X_train_scaled.shape[0] // 4 )  # concidering 1/2 of the samples as neaighbors is large enough for k-NN to don't overfit\n",
    "\n",
    "print(\"Number of k values to test:\", len(K_list))\n",
    "\n",
    "for k in K_list:\n",
    "    knn = KNeighborsClassifier(n_neighbors=k)\n",
    "    score_f1 = cross_val_score(knn, X_train_scaled, y_train, cv=5, scoring='f1')\n",
    "    score_acc = cross_val_score(knn, X_train_scaled, y_train, cv=5, scoring='accuracy')\n",
    "    score_recall = cross_val_score(knn, X_train_scaled, y_train, cv=5, scoring='recall')\n",
    "    score.append(score_acc.mean())\n",
    "    score.append(score_recall.mean())\n",
    "    k_scores.append(score_f1.mean())\n",
    "\n",
    "k_scores = np.array(k_scores)\n",
    "\n",
    "k_optimal = K_list[np.argmax(k_scores)]\n",
    "print(\"The best k for k-NN is k =\", k_optimal)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "698d03a8",
   "metadata": {},
   "source": [
    "In k-NN classification, to achieve the best prediction performance, we need to find the optimal number of neighbors that maximizes the evaluation score of our models. Here, we use the $f1\\_score$ from sklearn.metrics, as it provides a good balance between precision (e.g., correctly predicting a sick patient as sick, or a healthy patient as healthy) and recall (e.g., correctly identifying sick patients among all those predicted as sick).\n",
    "\n",
    "To determine this hyperparameter, we use 5-fold cross-validation. We chose 5 folds instead of 10 due to the limited amount of data, as this provides a better balance between the sizes of the training and validation sets. After cross-validation, it turns out that the optimal number of neighbors, $k$, is $k = 21$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "4af329ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 100708 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the scores among the k values\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.plot(K_list, k_scores, marker='o', linestyle='-', label='F1 Score')\n",
    "plt.plot(K_list, score[0::2], marker='x', linestyle='--', label='Accuracy')\n",
    "plt.plot(K_list, score[1::2], marker='s', linestyle=':', label='Recall')\n",
    "plt.title('k-NN F1 Score vs. k')\n",
    "plt.xlabel('Number of Neighbors (k)')\n",
    "plt.ylabel('F1 Score')\n",
    "plt.xticks(K_list)\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f53b6e9",
   "metadata": {},
   "source": [
    "With this graphic we can clearly see that the $ f1\\_score $ is a good deal between the recall and the accuracy and why we use it as our cross-validation score. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b2f376f",
   "metadata": {},
   "source": [
    "### 4) Model performance "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "064a5aa7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.7916666666666666\n",
      "F1 Score: 0.7883597883597883\n",
      "Recall: 0.9166666666666666\n",
      "Classification Report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.67      0.76        12\n",
      "           1       0.73      0.92      0.81        12\n",
      "\n",
      "    accuracy                           0.79        24\n",
      "   macro avg       0.81      0.79      0.79        24\n",
      "weighted avg       0.81      0.79      0.79        24\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 20780 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 49624 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=k_optimal) # using the best k founded earlier\n",
    "knn.fit(X_train_scaled, y_train)\n",
    "\n",
    "y_pred = knn.predict(X_test_scaled)\n",
    "\n",
    "acc = accuracy_score(y_test, y_pred)\n",
    "f1 = f1_score(y_test, y_pred, average='weighted')\n",
    "rec = recall_score(y_test, y_pred)\n",
    "class_report = classification_report(y_test, y_pred)\n",
    "\n",
    "# Display differents performance of the model\n",
    "print(\"Accuracy:\", acc)\n",
    "print(\"F1 Score:\", f1)\n",
    "print(\"Recall:\", rec)\n",
    "print(\"Classification Report:\\n\", class_report)\n",
    "\n",
    "# Plotting the confusion matrix\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "plt.figure(figsize=(8, 6))\n",
    "plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n",
    "plt.title('Confusion Matrix')\n",
    "for i in range(cm.shape[0]):\n",
    "    for j in range(cm.shape[1]):\n",
    "        plt.text(j, i, cm[i, j], horizontalalignment='center', color='white' if cm[i, j] > cm.max() / 2 else 'black')\n",
    "plt.xticks(np.arange(len(np.unique(y))),labels = [\"negative\", \"positive\"])\n",
    "plt.yticks(np.arange(len(np.unique(y))),labels = [\"negative\", \"positive\"])\n",
    "plt.xlabel('Predicted label')\n",
    "plt.ylabel('True label')\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# Plotting ROC curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, knn.predict_proba(X_test_scaled)[:, 1])\n",
    "plt.figure(figsize=(8, 6))\n",
    "plt.plot(fpr, tpr, color='blue', label='ROC curve (area = {:.2f})'.format(f1))\n",
    "plt.plot([0, 1], [0, 1], color='red', linestyle='--')\n",
    "plt.xlim([-0.05, 1.05])\n",
    "plt.ylim([-0.05, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver Operating Characteristic (ROC) Curve')\n",
    "plt.legend(loc='lower right')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bf7ed62",
   "metadata": {},
   "source": [
    "In this optimized k-NN classification, we aim to maximize recall while maintaining good accuracy, in order to minimize the number of misclassifications, particularly cases where a sick patient is incorrectly predicted as healthy. We achieve this goal with a recall of 91% and an accuracy of 79%. \n",
    "\n",
    "By the confusion matrix we can see that we only misclassifies 1 patient. Thus our goal to minimize this misclassification is clearly achive."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
