{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Scaling Example\n",
    "\n",
    "You have now seen how feature scaling might change the clusters we obtain from the kmeans algorithm, but it is time to try it out!\n",
    "\n",
    "First let's get some data to work with.  The first cell here will read in the necessary libraries, generate data, and make a plot of the data you will be working with throughout the rest of the notebook.\n",
    "\n",
    "The dataset you will work with through the notebook is then stored in **data**.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 42052 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff2cc2a6128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.cluster import KMeans\n",
    "from IPython.display import Image\n",
    "from sklearn.datasets.samples_generator import make_blobs\n",
    "import tests2 as t\n",
    "\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "# DSND colors: UBlue, Salmon, Gold, Slate\n",
    "plot_colors = ['#02b3e4', '#ee2e76', '#ffb613', '#2e3d49']\n",
    "\n",
    "# Light colors: Blue light, Salmon light\n",
    "plot_lcolors = ['#88d0f3', '#ed8ca1', '#fdd270']\n",
    "\n",
    "# Gray/bg colors: Slate Dark, Gray, Silver\n",
    "plot_grays = ['#1c262f', '#aebfd1', '#fafbfc']\n",
    "\n",
    "\n",
    "def create_data():\n",
    "    n_points = 120\n",
    "    X = np.random.RandomState(3200000).uniform(-3, 3, [n_points, 2])\n",
    "    X_abs = np.absolute(X)\n",
    "\n",
    "    inner_ring_flag = np.logical_and(X_abs[:,0] < 1.2, X_abs[:,1] < 1.2)\n",
    "    outer_ring_flag = X_abs.sum(axis = 1) > 5.3\n",
    "    keep = np.logical_not(np.logical_or(inner_ring_flag, outer_ring_flag))\n",
    "\n",
    "    X = X[keep]\n",
    "    X = X[:60] # only keep first 100\n",
    "    X1 = np.matmul(X, np.array([[2.5, 0], [0, 100]])) + np.array([22.5, 500])\n",
    "    \n",
    "    \n",
    "    plt.figure(figsize = [15,15])\n",
    "\n",
    "    plt.scatter(X1[:,0], X1[:,1], s = 64, c = plot_colors[-1])\n",
    "\n",
    "    plt.xlabel('5k Completion Time (min)', size = 30)\n",
    "    plt.xticks(np.arange(15, 30+5, 5), fontsize = 30)\n",
    "    plt.ylabel('Test Score (raw)', size = 30)\n",
    "    plt.yticks(np.arange(200, 800+200, 200), fontsize = 30)\n",
    "\n",
    "    ax = plt.gca()\n",
    "    ax.spines['top'].set_visible(False)\n",
    "    ax.spines['right'].set_visible(False)\n",
    "    [side.set_linewidth(2) for side in ax.spines.values()]\n",
    "    ax.tick_params(width = 2)\n",
    "    plt.savefig('C18_FeatScalingEx_01.png', transparent = True)\n",
    "    \n",
    "    \n",
    "    data = pd.DataFrame(X1)\n",
    "    data.columns = ['5k_Time', 'Raw_Test_Score']\n",
    "    \n",
    "    return data\n",
    "\n",
    "data = create_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`1.` Take a look at the dataset.  Are there any missing values?  What is the average completion time?  What is the average raw test score?  Use the cells below to find the answers to these questions, and the dictioonary to match values and check against our solution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 60 entries, 0 to 59\n",
      "Data columns (total 2 columns):\n",
      "5k_Time           60 non-null float64\n",
      "Raw_Test_Score    60 non-null float64\n",
      "dtypes: float64(2)\n",
      "memory usage: 1.0 KB\n"
     ]
    }
   ],
   "source": [
    "# cell for work\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>5k_Time</th>\n",
       "      <th>Raw_Test_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>22.899027</td>\n",
       "      <td>511.699603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>4.536244</td>\n",
       "      <td>183.222427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>15.263902</td>\n",
       "      <td>206.597283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>18.820638</td>\n",
       "      <td>361.798208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>24.033613</td>\n",
       "      <td>545.795365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>26.956643</td>\n",
       "      <td>656.346547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>29.867819</td>\n",
       "      <td>797.599192</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         5k_Time  Raw_Test_Score\n",
       "count  60.000000       60.000000\n",
       "mean   22.899027      511.699603\n",
       "std     4.536244      183.222427\n",
       "min    15.263902      206.597283\n",
       "25%    18.820638      361.798208\n",
       "50%    24.033613      545.795365\n",
       "75%    26.956643      656.346547\n",
       "max    29.867819      797.599192"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# another cell for work\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "That looks right!\n"
     ]
    }
   ],
   "source": [
    "# Use the dictionary to match the values to the corresponding statements\n",
    "a = 0\n",
    "b = 60\n",
    "c = 22.9\n",
    "d = 4.53\n",
    "e = 511.7\n",
    "\n",
    "q1_dict = {\n",
    "'number of missing values': a,\n",
    "'the mean 5k time in minutes': c,    \n",
    "'the mean test score as a raw value': e,\n",
    "'number of individuals in the dataset': b\n",
    "}\n",
    "\n",
    "# check your answer against ours here\n",
    "t.check_q1(q1_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`2.` Now, instantiate a kmeans `model` with 2 cluster centers.  Use your model to `fit` and `predict` the the group of each point in your dataset.  Store the predictions in `preds`.  If you correctly created the model and predictions, you should see a top (blue) cluster and bottom (pink) cluster when running the following cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_clusters = 2\n",
    "model = KMeans(n_clusters = n_clusters)\n",
    "preds = model.fit_predict(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 40788 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff2cc2a49e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Run this to see your results\n",
    "\n",
    "def plot_clusters(data, preds, n_clusters):\n",
    "    plt.figure(figsize = [15,15])\n",
    "\n",
    "    for k, col in zip(range(n_clusters), plot_colors[:n_clusters]):\n",
    "        my_members = (preds == k)\n",
    "        plt.scatter(data['5k_Time'][my_members], data['Raw_Test_Score'][my_members], s = 64, c = col)\n",
    "\n",
    "    plt.xlabel('5k Completion Time (min)', size = 30)\n",
    "    plt.xticks(np.arange(15, 30+5, 5), fontsize = 30)\n",
    "    plt.ylabel('Test Score (raw)', size = 30)\n",
    "    plt.yticks(np.arange(200, 800+200, 200), fontsize = 30)\n",
    "\n",
    "    ax = plt.gca()\n",
    "    ax.spines['top'].set_visible(False)\n",
    "    ax.spines['right'].set_visible(False)\n",
    "    [side.set_linewidth(2) for side in ax.spines.values()]\n",
    "    ax.tick_params(width = 2)\n",
    "    \n",
    "plot_clusters(data, preds, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`3.` Now create two new columns to add to your `data` dataframe.  The first is `test_scaled`, which you should create by subtracting the mean test score and dividing by the standard deviation test score.  \n",
    "\n",
    "The second column to create is `5k_time_sec`, which should have the minutes changed to seconds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# your work here\n",
    "data['test_scaled'] = (data['Raw_Test_Score'] - np.mean(data['Raw_Test_Score']))/np.std(data['Raw_Test_Score'])\n",
    "data['5k_time_sec'] = data['5k_Time']*60"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`4.` Now, similar to what you did in question 2, instantiate a kmeans `model` with 2 cluster centers.  Use your model to `fit` and `predict` the the group of each point in your dataset.  Store the predictions in `preds`.  If you correctly created the model and predictions, you should see a right (blue) cluster and left (pink) cluster when running the following cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_clusters = 2\n",
    "model = KMeans(n_clusters = n_clusters)\n",
    "preds = model.fit_predict(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 38784 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff2cc2a1cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Run this to see your results\n",
    "plot_clusters(data, preds, n_clusters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`5.` Match the variable that best describes the way you should think of feature scaling with algorithms that use distance based metrics or regularization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/gif": "[encoded data, 1803424 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# options\n",
    "a = 'We should always use normalizing'\n",
    "b = 'We should always scale our variables between 0 and 1.'\n",
    "c = 'Variable scale will frequently influence your results, so it is important to standardize for all of these algorithms.'\n",
    "d = 'Scaling will not change the results of your output.'\n",
    "\n",
    "best_option = c# best answer variable here\n",
    "\n",
    "\n",
    "# check your answer against ours here\n",
    "t.check_q5(best_option)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
