726 lines
41 KiB
Plaintext
726 lines
41 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Our Mission\n",
|
|
"\n",
|
|
"In this lesson you gained some insight into a number of techniques used to understand how well our model is performing. This notebook is aimed at giving you some practice with the metrics specifically related to classification problems. With that in mind, we will again be looking at the spam dataset from the earlier lessons.\n",
|
|
"\n",
|
|
"First, run the cell below to prepare the data and instantiate a number of different models."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Import our libraries\n",
|
|
"import pandas as pd\n",
|
|
"import numpy as np\n",
|
|
"from sklearn.model_selection import train_test_split\n",
|
|
"from sklearn.feature_extraction.text import CountVectorizer\n",
|
|
"from sklearn.naive_bayes import MultinomialNB\n",
|
|
"from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
|
|
"from sklearn.ensemble import BaggingClassifier, RandomForestClassifier, AdaBoostClassifier\n",
|
|
"from sklearn.svm import SVC\n",
|
|
"import tests as t\n",
|
|
"\n",
|
|
"# Read in our dataset\n",
|
|
"df = pd.read_table('smsspamcollection/SMSSpamCollection',\n",
|
|
" sep='\\t', \n",
|
|
" header=None, \n",
|
|
" names=['label', 'sms_message'])\n",
|
|
"\n",
|
|
"# Fix our response value\n",
|
|
"df['label'] = df.label.map({'ham':0, 'spam':1})\n",
|
|
"\n",
|
|
"# Split our dataset into training and testing data\n",
|
|
"X_train, X_test, y_train, y_test = train_test_split(df['sms_message'], \n",
|
|
" df['label'], \n",
|
|
" random_state=1)\n",
|
|
"\n",
|
|
"# Instantiate the CountVectorizer method\n",
|
|
"count_vector = CountVectorizer()\n",
|
|
"\n",
|
|
"# Fit the training data and then return the matrix\n",
|
|
"training_data = count_vector.fit_transform(X_train)\n",
|
|
"\n",
|
|
"# Transform testing data and return the matrix. Note we are not fitting the testing data into the CountVectorizer()\n",
|
|
"testing_data = count_vector.transform(X_test)\n",
|
|
"\n",
|
|
"# Instantiate a number of our models\n",
|
|
"naive_bayes = MultinomialNB()\n",
|
|
"bag_mod = BaggingClassifier(n_estimators=200)\n",
|
|
"rf_mod = RandomForestClassifier(n_estimators=200)\n",
|
|
"ada_mod = AdaBoostClassifier(n_estimators=300, learning_rate=0.2)\n",
|
|
"svm_mod = SVC()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"> **Step 1**: Now, fit each of the above models to the appropriate data. Answer the following question to assure that you fit the models correctly."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
|
|
" decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
|
|
" max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
|
|
" tol=0.001, verbose=False)"
|
|
]
|
|
},
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"# Fit each of the 4 models\n",
|
|
"# This might take some time to run\n",
|
|
"naive_bayes.fit(training_data, y_train)\n",
|
|
"bag_mod.fit(training_data, y_train)\n",
|
|
"rf_mod.fit(training_data, y_train)\n",
|
|
"ada_mod.fit(training_data, y_train)\n",
|
|
"svm_mod.fit(training_data, y_train)\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"That's right! You need to fit on both parts of the data pertaining to training data!\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# The models you fit above were fit on which data?\n",
|
|
"\n",
|
|
"a = 'X_train'\n",
|
|
"b = 'X_test'\n",
|
|
"c = 'y_train'\n",
|
|
"d = 'y_test'\n",
|
|
"e = 'training_data'\n",
|
|
"f = 'testing_data'\n",
|
|
"\n",
|
|
"# Change models_fit_on to only contain the correct string names\n",
|
|
"# of values that you oassed to the above models\n",
|
|
"\n",
|
|
"models_fit_on = {e, c} # update this to only contain correct letters\n",
|
|
"\n",
|
|
"# Checks your solution - don't change this\n",
|
|
"t.test_one(models_fit_on)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"> **Step 2**: Now make predictions for each of your models on the data that will allow you to understand how well our model will extend to new data. Then correctly add the strings to the set in the following cell."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Make predictions using each of your models\n",
|
|
"nb = naive_bayes.predict(testing_data)\n",
|
|
"bag_pred = bag_mod.predict(testing_data)\n",
|
|
"rf_pred = rf_mod.predict(testing_data)\n",
|
|
"ada_pred = ada_mod.predict(testing_data)\n",
|
|
"svm_pred = svm_mod.predict(testing_data)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"That's right! To see how well our models perform in a new setting, you will want to predict on the test set of data.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Which data was used in the predict method to see how well your\n",
|
|
"# model would work on new data?\n",
|
|
"\n",
|
|
"a = 'X_train'\n",
|
|
"b = 'X_test'\n",
|
|
"c = 'y_train'\n",
|
|
"d = 'y_test'\n",
|
|
"e = 'training_data'\n",
|
|
"f = 'testing_data'\n",
|
|
"\n",
|
|
"# Change models_predict_on to only contain the correct string names\n",
|
|
"# of values that you oassed to the above models\n",
|
|
"\n",
|
|
"models_predict_on = {f} # update this to only contain correct letters\n",
|
|
"\n",
|
|
"# Checks your solution - don't change this\n",
|
|
"t.test_two(models_predict_on)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Now that you have set up all your predictions, let's get to topics addressed in this lesson - measuring how well each of your models performed. First, we will focus on how each metric was calculated for a single model, and then in the final part of this notebook, you will choose models that are best based on a particular metric.\n",
|
|
"\n",
|
|
"You will be writing functions to calculate a number of metrics and then comparing the values to what you get from sklearn. This will help you build intuition for how each metric is calculated.\n",
|
|
"\n",
|
|
"> **Step 3**: As an example of how this will work for the upcoming questions, run the cell below. Fill in the below function to calculate accuracy, and then compare your answer to the built in to assure you are correct."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"0.988513998564\n",
|
|
"0.988513998564\n",
|
|
"Since these match, we correctly calculated our metric!\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# accuracy is the total correct divided by the total to predict\n",
|
|
"def accuracy(actual, preds):\n",
|
|
" '''\n",
|
|
" INPUT\n",
|
|
" preds - predictions as a numpy array or pandas series\n",
|
|
" actual - actual values as a numpy array or pandas series\n",
|
|
" \n",
|
|
" OUTPUT:\n",
|
|
" returns the accuracy as a float\n",
|
|
" '''\n",
|
|
" return np.sum(preds == actual)/len(actual)\n",
|
|
"\n",
|
|
"\n",
|
|
"print(accuracy(y_test, nb))\n",
|
|
"print(accuracy_score(y_test, nb))\n",
|
|
"print(\"Since these match, we correctly calculated our metric!\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"> **Step 4**: Fill in the below function to calculate precision, and then compare your answer to the built in to assure you are correct."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"0.972067039106\n",
|
|
"0.972067039106\n",
|
|
"If the above match, you got it!\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# precision is the true positives over the predicted positive values\n",
|
|
"def precision(actual, preds):\n",
|
|
" '''\n",
|
|
" INPUT\n",
|
|
" (assumes positive = 1 and negative = 0)\n",
|
|
" preds - predictions as a numpy array or pandas series \n",
|
|
" actual - actual values as a numpy array or pandas series\n",
|
|
" \n",
|
|
" OUTPUT:\n",
|
|
" returns the precision as a float\n",
|
|
" '''\n",
|
|
" TP = np.sum((preds == actual) & (preds > 0))\n",
|
|
" FP = np.sum((preds == 1) & (actual == 0))\n",
|
|
" return TP / (TP + FP)\n",
|
|
"\n",
|
|
"\n",
|
|
"print(precision(y_test, nb))\n",
|
|
"print(precision_score(y_test, nb))\n",
|
|
"print(\"If the above match, you got it!\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"> **Step 5**: Fill in the below function to calculate recall, and then compare your answer to the built in to assure you are correct."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"0.940540540541\n",
|
|
"0.940540540541\n",
|
|
"If the above match, you got it!\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# recall is true positives over all actual positive values\n",
|
|
"def recall(actual, preds):\n",
|
|
" '''\n",
|
|
" INPUT\n",
|
|
" preds - predictions as a numpy array or pandas series\n",
|
|
" actual - actual values as a numpy array or pandas series\n",
|
|
" \n",
|
|
" OUTPUT:\n",
|
|
" returns the recall as a float\n",
|
|
" '''\n",
|
|
" TP = np.sum((preds == actual) & (preds > 0))\n",
|
|
" FN = np.sum((preds == 0) & (actual == 1))\n",
|
|
" return TP / (TP + FN)\n",
|
|
"\n",
|
|
"\n",
|
|
"print(recall(y_test, nb))\n",
|
|
"print(recall_score(y_test, nb))\n",
|
|
"print(\"If the above match, you got it!\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"> **Step 6**: Fill in the below function to calculate f1-score, and then compare your answer to the built in to assure you are correct."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"0.956043956044\n",
|
|
"0.956043956044\n",
|
|
"If the above match, you got it!\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# f1_score is 2*(precision*recall)/(precision+recall))\n",
|
|
"def f1(actual, preds):\n",
|
|
" '''\n",
|
|
" INPUT\n",
|
|
" preds - predictions as a numpy array or pandas series\n",
|
|
" actual - actual values as a numpy array or pandas series\n",
|
|
" \n",
|
|
" OUTPUT:\n",
|
|
" returns the f1score as a float\n",
|
|
" '''\n",
|
|
" prec = precision(actual, preds)\n",
|
|
" rec = recall(actual, preds)\n",
|
|
" return 2 * ((prec * rec) / (prec + rec))\n",
|
|
"\n",
|
|
"\n",
|
|
"print(f1(y_test, nb))\n",
|
|
"print(f1_score(y_test, nb))\n",
|
|
"print(\"If the above match, you got it!\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"> **Step 7:** Now that you have calculated a number of different metrics, let's tie that to when we might use one versus another. Use the dictionary below to match a metric to each statement that identifies when you would want to use that metric."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"That's right! It isn't really necessary to memorize these in practice, but it is important to know they exist and know why might use one metric over another for a particular situation.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# add the letter of the most appropriate metric to each statement\n",
|
|
"# in the dictionary\n",
|
|
"a = \"recall\"\n",
|
|
"b = \"precision\"\n",
|
|
"c = \"accuracy\"\n",
|
|
"d = 'f1-score'\n",
|
|
"\n",
|
|
"\n",
|
|
"seven_sol = {\n",
|
|
"'We have imbalanced classes, which metric do we definitely not want to use?': c,\n",
|
|
"'We really want to make sure the positive cases are all caught even if that means we identify some negatives as positives': a,\n",
|
|
"'When we identify something as positive, we want to be sure it is truly positive': b,\n",
|
|
"'We care equally about identifying positive and negative cases': d \n",
|
|
"}\n",
|
|
"\n",
|
|
"t.sol_seven(seven_sol)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"> **Step 8:** Given what you know about the metrics now, use this information to correctly match the appropriate model to when it would be best to use each in the dictionary below."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 61,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"That's right! Naive Bayes was the best model for all of our metrics except precision!\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# use the answers you found to the previous questiona, then match the model that did best for each metric\n",
|
|
"a = \"naive-bayes\"\n",
|
|
"b = \"bagging\"\n",
|
|
"c = \"random-forest\"\n",
|
|
"d = 'ada-boost'\n",
|
|
"e = \"svm\"\n",
|
|
"\n",
|
|
"\n",
|
|
"eight_sol = {\n",
|
|
"'We have imbalanced classes, which metric do we definitely not want to use?': a,\n",
|
|
"'We really want to make sure the positive cases are all caught even if that means we identify some negatives as positives': a, \n",
|
|
"'When we identify something as positive, we want to be sure it is truly positive': c, \n",
|
|
"'We care equally about identifying positive and negative cases': a \n",
|
|
"}\n",
|
|
"\n",
|
|
"t.sol_eight(eight_sol)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 60,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"accuracy_score for nb 0.9885\n",
|
|
"precision_score for nb 0.9721\n",
|
|
"recall_score for nb 0.9405\n",
|
|
"f1_score for nb 0.9560\n",
|
|
"\n",
|
|
"accuracy_score for bag_pred 0.9749\n",
|
|
"precision_score for bag_pred 0.9167\n",
|
|
"recall_score for bag_pred 0.8919\n",
|
|
"f1_score for bag_pred 0.9041\n",
|
|
"\n",
|
|
"accuracy_score for rf_pred 0.9806\n",
|
|
"precision_score for rf_pred 1.0000\n",
|
|
"recall_score for rf_pred 0.8541\n",
|
|
"f1_score for rf_pred 0.9213\n",
|
|
"\n",
|
|
"accuracy_score for ada_pred 0.9770\n",
|
|
"precision_score for ada_pred 0.9693\n",
|
|
"recall_score for ada_pred 0.8541\n",
|
|
"f1_score for ada_pred 0.9080\n",
|
|
"\n",
|
|
"accuracy_score for svm_pred 0.8672\n",
|
|
"precision_score for svm_pred 0.0000\n",
|
|
"recall_score for svm_pred 0.0000\n",
|
|
"f1_score for svm_pred 0.0000\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
|
|
" 'precision', 'predicted', average, warn_for)\n",
|
|
"/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
|
|
" 'precision', 'predicted', average, warn_for)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"models = {'nb': nb,\n",
|
|
" 'bag_pred': bag_pred,\n",
|
|
" 'rf_pred': rf_pred,\n",
|
|
" 'ada_pred': ada_pred,\n",
|
|
" 'svm_pred': svm_pred}\n",
|
|
"metrics = [accuracy_score, precision_score, recall_score, f1_score]\n",
|
|
"\n",
|
|
"for i in models:\n",
|
|
" for j in range(len(metrics)):\n",
|
|
" print(f'{metrics[j].__name__} for {i} {metrics[j](y_test, models[i]):.4f}')\n",
|
|
" print()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# If you get stuck, also notice there is a solution available by hitting the orange button in the top left"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"As a final step in this workbook, let's take a look at the last three metrics you saw, f-beta scores, ROC curves, and AUC.\n",
|
|
"\n",
|
|
"**For f-beta scores:** If you decide that you care more about precision, you should move beta closer to 0. If you decide you care more about recall, you should move beta towards infinity. \n",
|
|
"\n",
|
|
"> **Step 9:** Using the fbeta_score works similar to most of the other metrics in sklearn, but you also need to set beta as your weighting between precision and recall. Use the space below to show that you can use [fbeta in sklearn](http://scikit-learn.org/stable/modules/generated/sklearn.metrics.fbeta_score.html) to replicate your f1-score from above. If in the future you want to use a different weighting, [this article](http://mlwiki.org/index.php/Precision_and_Recall) does an amazing job of explaining how you might adjust beta for different situations."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 66,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"fbeta_score for nb 0.9560439560439562\n",
|
|
"f1_score for nb 0.9560439560439562\n",
|
|
"\n",
|
|
"fbeta_score for bag_pred 0.9041095890410958\n",
|
|
"f1_score for bag_pred 0.9041095890410958\n",
|
|
"\n",
|
|
"fbeta_score for rf_pred 0.9212827988338192\n",
|
|
"f1_score for rf_pred 0.9212827988338192\n",
|
|
"\n",
|
|
"fbeta_score for ada_pred 0.9080459770114943\n",
|
|
"f1_score for ada_pred 0.9080459770114943\n",
|
|
"\n",
|
|
"fbeta_score for svm_pred 0.0\n",
|
|
"f1_score for svm_pred 0.0\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
|
|
" 'precision', 'predicted', average, warn_for)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# import fbeta_score\n",
|
|
"from sklearn.metrics import fbeta_score\n",
|
|
"\n",
|
|
"\n",
|
|
"# Show that you can produce the same f1_score results using fbeta_score\n",
|
|
"beta = 1\n",
|
|
"\n",
|
|
"for i in models:\n",
|
|
" print(f'fbeta_score for {i} {fbeta_score(y_test, models[i], beta)}')\n",
|
|
" print(f'f1_score for {i} {f1_score(y_test, models[i], beta)}')\n",
|
|
" print()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"> **Step 10:** Building ROC curves in python is a pretty involved process on your own. I wrote the function below to assist with the process and make it easier for you to do so in the future as well. Try it out using one of the other classifiers you created above to see how it compares to the random forest model below.\n",
|
|
"\n",
|
|
"Run the cell below to build a ROC curve, and retrieve the AUC for the random forest model."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 67,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"<matplotlib.figure.Figure at 0x7f6dfeea5b38>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"0.94594594594594594"
|
|
]
|
|
},
|
|
"execution_count": 67,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"# Function for calculating auc and roc\n",
|
|
"\n",
|
|
"def build_roc_auc(model, X_train, X_test, y_train, y_test):\n",
|
|
" '''\n",
|
|
" INPUT:\n",
|
|
" model - an sklearn instantiated model\n",
|
|
" X_train - the training data\n",
|
|
" y_train - the training response values (must be categorical)\n",
|
|
" X_test - the test data\n",
|
|
" y_test - the test response values (must be categorical)\n",
|
|
" OUTPUT:\n",
|
|
" auc - returns auc as a float\n",
|
|
" prints the roc curve\n",
|
|
" '''\n",
|
|
" import numpy as np\n",
|
|
" import matplotlib.pyplot as plt\n",
|
|
" from itertools import cycle\n",
|
|
" from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
|
|
" from scipy import interp\n",
|
|
" \n",
|
|
" y_preds = model.fit(X_train, y_train).predict_proba(X_test)\n",
|
|
" # Compute ROC curve and ROC area for each class\n",
|
|
" fpr = dict()\n",
|
|
" tpr = dict()\n",
|
|
" roc_auc = dict()\n",
|
|
" for i in range(len(y_test)):\n",
|
|
" fpr[i], tpr[i], _ = roc_curve(y_test, y_preds[:, 1])\n",
|
|
" roc_auc[i] = auc(fpr[i], tpr[i])\n",
|
|
"\n",
|
|
" # Compute micro-average ROC curve and ROC area\n",
|
|
" fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_preds[:, 1].ravel())\n",
|
|
" roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n",
|
|
" \n",
|
|
" plt.plot(fpr[2], tpr[2], color='darkorange',\n",
|
|
" lw=2, label='ROC curve (area = %0.2f)' % roc_auc[2])\n",
|
|
" plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
|
|
" plt.xlim([0.0, 1.0])\n",
|
|
" plt.ylim([0.0, 1.05])\n",
|
|
" plt.xlabel('False Positive Rate')\n",
|
|
" plt.ylabel('True Positive Rate')\n",
|
|
" plt.title('Receiver operating characteristic example')\n",
|
|
" plt.show()\n",
|
|
" \n",
|
|
" return roc_auc_score(y_test, np.round(y_preds[:, 1]))\n",
|
|
" \n",
|
|
" \n",
|
|
"# Finding roc and auc for the random forest model \n",
|
|
"build_roc_auc(rf_mod, training_data, testing_data, y_train, y_test) "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 71,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmcjXX7wPHPNfuMnUGyhyxJKqEUSkko7ZG0aZmkkh7JT4tESVJkGVoej6c9T4usSaKUQpYiOzGy79uMWa7fH/c9nMbMmTPMmTPnzPV+vbyce7/u75xzrnN/v/f9/YqqYowxxuQkLNABGGOMKdwsURhjjPHKEoUxxhivLFEYY4zxyhKFMcYYryxRGGOM8coSRQgQka4i8k2g4wg0EakmIodFJLwAj1lDRFREIgrqmP4kIitEpPVpbBey70ERaS0iSYGOI5AsUeQzEdkkIsfcL6ztIjJBRIr785iq+oGqtvXnMQojt6yvzpxW1c2qWlxV0wMZV6C4Cav2mexDVc9T1e9zOc4pybGovgeLCksU/nG9qhYHGgMXAv0CHM9pCeSv5FD5hZ4XVt6msLJE4Uequh2YiZMwABCRaBEZJiKbRWSHiCSKSKzH8k4islREDorIehFp584vJSLvisg2EdkqIoMyq1hE5F4R+dF9nSgiwzzjEJGvRKS3+/psEfmfiOwSkY0i8rjHegNEZJKIvC8iB4F7s56TG8dEd/u/RORZEQnziGO+iLwlIgdEZJWItMmyrbdzmC8ib4jIXmCAiNQSke9EZI+I7BaRD0SktLv+f4FqwNfu1dvTWX/pisj3IvKSu99DIvKNiMR7xHO3ew57ROS5rFcoWc47VkRed9c/ICI/ev7dgK7u33S3iPT32K6piPwsIvvd8x4lIlEey1VEHhWRtcBad94IEdnivgcWi8gVHuuHi8j/ue+NQ+7yqiIyz11lmVsed7jrd3TfT/tF5CcRaeSxr00i0ldElgNHRCTCswzc2Be5cewQkeHuppnH2u8e61LP96C77XkiMktE9rrb/l8O5Zrj58GNbYHH3/MRcarGYtzpz8S5aj8gIvNE5DyP/U4QkTEiMt2Ncb6InCUib4rIPve9eWGWsugnIivd5f/OPE42Mef4GQpZqmr/8vEfsAm42n1dBfgdGOGx/E1gMlAWKAF8DbziLmsKHACuwUnilYF67rIvgXFAMaAC8CvwsLvsXuBH93VLYAsg7nQZ4BhwtrvPxcDzQBRwDrABuNZddwCQCtzorhubzflNBL5yY68BrAG6e8SRBjwJRAJ3uOdT1sdzSAMeAyKAWKC2WxbRQHmcL6g3sytrd7oGoECEO/09sB44193f98AQd1kD4DBwuVsWw9xzvzqHv+tod/vKQDhwmRtX5jHfdo9xAZAC1He3uxho7p5TDeBPoJfHfhWYhfN+iHXn3QWUc7d5CtgOxLjL+uC8p+oC4h6vnMe+anvs+yJgJ9DMjfket8yiPcpvKVDV49gnyhT4Gejmvi4ONM+unLN5D5YAtrmxx7jTzXIoV2+fhzD3bz4AqAPsAy702PZ+d5todz9LPZZNAHa75R8DfAdsBO52y2IQMCfLe+kPtyzKAvOBQe6y1kCSR0w5foZC9V/AAwi1f+4b7jBwyP0wzQZKu8sEOALU8lj/UmCj+3oc8EY2+6yI8+UT6zGvS+YbPcuHVIDNQEt3+kHgO/d1M2Bzln33A/7tvh4AzPNybuFuHA085j0MfO8Rx9+4Scqd9yvQzcdz2JzTsd11bgSWZCnr3BLFsx7LewAz3NfPAx95LIsDjpNNonC/HI4BF2SzLPOYVbKcc+cczqEX8IXHtAJX5XLe+zKPDawGOuWwXtZEMRZ4Kcs6q4FWHuV3fzbv38xEMQ94EYjP4ZxzShRdPP9OXs7L6+fB41h7cRJsPy/7Ku3GVMqdngC87bH8MeBPj+nzgf1ZzjvBY7o9sN593ZqTicLrZyhU/1m9pH/cqKrfikgr4EMgHtiP86s4DlgsIpnrCs4XMDi/ZqZls7/qOL/Qt3lsF4Zz5fAPqqoi8jHOh3UecCfwvsd+zhaR/R6bhAM/eEyfsk8P8Ti/ov7ymPcXzq/sTFvV/fR4LD/bx3P4x7FFpAIwErgC55djGM6XZl5s93h9FOeXMW5MJ46nqkdFZE8O+4jH+VW6Pq/HEZFzgeFAE5y/fQTOL1JPWc/7KeABN0YFSroxgPMe8RaHp+rAPSLymMe8KHe/2R47i+7AQGCViGwEXlTVKT4c19cYc/s8oKqbRGQOzhf36BMrOVWWg4Hb3P1kuIvica5iAXZ4HOtYNtNZbzLxLIvM921WvnyGQo61UfiRqs7F+WWT2WawG+cNep6qlnb/lVKn4RucN2qtbHa1BefXeLzHdiVV9bxs1gX4CLhVRKrj/AL6n8d+Nnrso7SqllDV9p5hezml3TjVM9U95lUDtnpMVxaPT727/G8fzyHrsV9x5zVS1ZI4VTLiZf282IZTNQg4bRA41T3Z2Q0kk/3fJjdjgVVAHfcc/o9/ngN4nIfbHtEXuB0oo6qlcb74MrfJ6T2SnS3A4Cx/7zhV/Si7Y2elqmtVtQtONeGrwCQRKeZtmzzGmNvnARFpj3OVMRt4zWPbO4FOwNVAKZwrDzi1bPOiqsfrzPdtVr58hkKOJQr/exO4RkQaq2oGTl32G+6vZUSksohc6677LnCfiLQRkTB3WT1V3QZ8A7wuIiXdZbXcK5ZTqOoSYBfwDjBTVTN//fwKHHQbCWPdhtGGInKJLyeizm2nnwKDRaSEm4h6c/KKBZwvlcdFJFJEbgPqA9Pyeg6uEjjVePtFpDJO/bynHTh1xKdjEnC9iFwmTuPyi+TwJeP+3d4DhrsNmeFuA260D8cpARwEDotIPeARH9ZPw/n7RYjI8zhXFJneAV4SkTriaCQimQkua3m8DSSISDN33WIi0kFESvgQNyJyl4iUd88/8z2U7saWQc5lPwU4S0R6uY3VJUSkWdaVcvs8iHPjwbs4V1f34Py9Mr+QS+D88NiDc1Xysi/nlItHRaSKiJTFSeifZLPOGX2GgpUlCj9T1V04DcDPubP6AuuABeLcWfQtTsMkqvorcB/wBs6vyLmc/PV+N061wUqc6pdJQCUvh/4I59fWhx6xpAPX49yFtRHnF907OL/IfPUYTr3yBuBHd//veSz/BafhcTdO1cCtqppZpZPXc3gRp0H2ADAV+DzL8leAZ8W5o+dfeTgHVHWFey4f41xdHMJp+E3JYZN/4TQiL8SpM38V3z4//8L59XsI50sxuy8fTzOB6Tg3CfyFcyXjWSUyHCdZf4OTgN7FaUQHp43pP2553K6qi3DaqEbhlPc6srmTzYt2wAoROQyMwGl3SVbVozh/2/nusZp7bqSqh3BuQrgep0puLXBlDsfI8fMAjAe+UtVp7nuoO/COmxgnuuWzFef9tCAP55WTD3HKdYP7b1DWFfLpMxR0Mu+MMeaMici9wAOqenmgY8krcR6K3I9TRbQx0PGYgiUim3Deu98GOpbCyK4oTJElIteLSJxb7z4M54phU2CjMqbwsURhirJOOA2Wf+NUl3VWu8Q25hRW9WSMMcYru6IwxhjjVdA9cBcfH681atQIdBjGGBNUFi9evFtVy5/OtkGXKGrUqMGiRYsCHYYxxgQVEfkr97WyZ1VPxhhjvLJEYYwxxitLFMYYY7yyRGGMMcYrSxTGGGO8skRhjDHGK78lChF5T0R2isgfOSwXERkpIutEZLmIXOSvWIwxxpw+f15RTMDppjgn1+H0r1MHeAhngBdjjDH57Pjx9DPa3m8P3KnqPBGp4WWVTsBEtxO2BSJSWkQquQPcGFO0fN4BNmY3Cq4xZ6bP19ew5G9vw77kLpBPZlfmnwOyJLnzTkkUIvIQzlUH1apVK5DgTBFlX9gmxDQ8aycjfzxlgME8CWSiyG7YyWy7slXV8TijXdGkSRPr7rYoKUpf3DXbw81TAx2FCXIrV+7it9+2cdddjQC4W5VWQw5Qs+YpA/b5LJCJIol/DmZehewHMzdFSWFIDPaFbYLQ0aOpDBo0j9de+4nwcKF58yrUrl0WEaFGjdJntO9AJorJQE8R+RhoBhyw9olcFIYv0UCwL25jvJo+fS2PPjqNjRv3A9C9+8WUKxeby1a+81uiEJGPgNZAvIgkAS8AkQCqmghMA9rjDKx+FLjPX7GEhKKUJCwxGOOTrVsP0qvXTCZNWglAo0YVSUzswKWXVs1ly7zx511PXXJZrsCj/jp+QPnzS92+RI0xrkcfncZXX60mLi6SgQNb88QTzYmIyP+nHoJuPIqgYEnCGOMnaWkZJ5LBq69eTWRkOK+/3pZq1Ur57ZiWKLKTX1cET9kNWsaY/HHgQDLPPvsda9bsZcaMrogIdevG89lnt/n92JYoPOVnlVHN9vmzH2NMkaaqfPbZSnr1msG2bYcJDxeWLt3OhRee2UN0eWGJIqfkYNU8xpgAW79+Lz17TmfGjHUAXHppFRITO9KoUcUCjSP4EsX+tfB6ds/q5RNLEMaYQmDYsJ947rk5JCenUbp0DK++ejUPPHARYWF+/P7LQfAlipSD+b9PSw7GmELm6NFUkpPT6NatEcOGtaVChWIBiyX4EkUmayg2xoSQXbuOsHr1Hi6/3OnPrm/fFrRuXYOWLasHODIbuMgYYwIqI0N5553fqFt3FDff/Al79x4DIDo6olAkCQjmKwpjjAlyf/yxk4SEKcyf73Skfc0153D0aCply+Zf9xv5wRKFMcYUsCNHjjNw4FyGD19AWloGFSsW480323HHHechUvCN1bmxRGGMMQXs1ls/Y8aMdYhAjx5NGDy4DaVLxwQ6rBwFZ6Kwh9mMMUGsb98W7NhxmLFjO9CsWZVAh5MrcfrmCx5Nqoou2hJcMRtjiq60tAzeeusXNm3az4gR152Yn5GhBfpMhIgsVtUmp7NtcF5RGGNMEPj11608/PAUli7dDsBDD13MeedVAAjIg3Ony26PNcaYfLZ/fzI9ekylefN3WLp0O9Wrl+Lrr7ucSBLBxq4ojDEmH3388R/06jWDHTuOEBERxlNPXcpzz7WkWLGoQId22ixRGGNMPvrmm/Xs2HGEFi2qMnZsB84/v2A78PMHSxTGGHMGUlLS2Lr1EOecUwaAoUOv4YorqnHPPY2Dqh3CG2ujMMaY0/Tddxtp1CiRDh0+5PjxdADi4+O4774LQyZJgCUKY4zJsx07DtOt2xe0aTORNWv2AJCU5IeerQsJq3oyxhgfZWQob7+9mGeemc3+/cnExETw7LNX0KdPC6KiwgMdnt9YojDGGB/ddNMnTJ68GoBrr63F6NHtqVWrbICj8j+rejLGGB/dfHM9zjqrOJ98civTp3ctEkkCrAsPY4zJ0eTJq0lKOkiPHpcAoKocPnycEiWiAxxZ3lkXHsYYk482bz7A449P56uvVhMdHU67drU555wyiEhQJokzZYnCGGNcqanpjBz5Cy+88D1HjqRSokQUgwZdRfXqpQIdWkBZojDGGGDBgiQefngKy5fvAOC22xrwxhvXUrlyyQBHFniWKIwxBnjuuTksX76DmjVLM2pUe9q3rxPokAoNSxTGmCJJVTl06DglSzptDqNGXcfEicvo378lcXGRAY6ucLG7nowxRc7q1bvp0WMaIjBrVrdCOU51frO7nowxxgfJyWm88soPDBkyn+PH0ylXLpZNm/ZTs2aZQIdWqFmiMMYUCbNmradHj2msW7cXgPvvb8zQoddQrlxcgCMr/Pz6ZLaItBOR1SKyTkSeyWZ5NRGZIyJLRGS5iLT3ZzzGmKJHVbn//q9o2/Z91q3bS4MG5Zk3717efbeTJQkf+e2KQkTCgdHANUASsFBEJqvqSo/VngU+VdWxItIAmAbU8FdMxpiiR0SoUaM0sbERPP98K3r3vjSkO/DzB39WPTUF1qnqBgAR+RjoBHgmCgUyb1IuBfztx3iMMUXE0qXb2bbtENdd59zi2rdvC7p1a2RtEafJn1VPlYEtHtNJ7jxPA4C7RCQJ52risex2JCIPicgiEVnkj0CNMaHh0KEUeveeycUXj+eee75k795jAERHR1iSOAP+TBTZ3W+W9b7WLsAEVa0CtAf+KyKnxKSq41W1yene2mWMCW2qyhdf/EmDBmN4440FANx55/lERloH2fnBn1VPSUBVj+kqnFq11B1oB6CqP4tIDBAP7PRjXMaYEPLXX/vp2XM6U6asAaBJk7MZN64jF11UKcCRhQ5/ptuFQB0RqSkiUUBnYHKWdTYDbQBEpD4QA+zyY0zGmBCiqtxyy6dMmbKGkiWjGTXqOhYs6G5JIp/57YpCVdNEpCcwEwgH3lPVFSIyEFikqpOBp4C3ReRJnGqpezXYHhU3xhS4jAwlLEwQEYYNa0ti4iLeeONaKlUqEejQQpJ14WGMCRp79hzlmWe+BeDtt28IcDTB5Uy68LCWHmNMoaeq/Oc/S6lXbzTvvLOEiROXk5R0MNBhFRnWhYcxplD7889dPPLIVObO/QuA1q1rMHZsB6pUsXEiCoolCmNMoaSqPP/8HF59dT6pqRnEx8fx+utt6datUZHo7bUwsURhjCmURIStWw+RmprBgw9exJAhV1O2bGygwyqSrDHbGFNo/P33IXbvPkqjRhUB2L37KKtX76ZFi2oBjiz4WWO2MSaopadnMGrUr9SvP5rOnSdx/Hg6APHxcZYkCgGrejLGBNRvv23j4YensGiR03FDy5bVOXgwhfh46wK8sPApUbhPVldT1XV+jscYU0QcPJjCc899x6hRC8nIUKpUKcnIke248cZ61lhdyOSaKESkAzAciAJqikhj4AVVvcnfwRljQpOq0rLlv1m2bAfh4ULv3s0ZMKA1JUpEBzo0kw1f2igGAs2A/QCquhSo7c+gjDGhTUR48snmNG1amUWLHuL116+1JFGI+VL1lKqq+7NcCtptR8YYnx0/ns7w4T8THi706dMCgLvvvoC77mpEeLjdU1PY+ZIo/hSR24EwEakJPAEs8G9YxphQ8cMPf5GQMJWVK3cRHR3O3XdfQMWKxRERwsOtLSIY+JLKewIXAxnA50AyTrIwxpgc7d59lPvv/4qWLSewcuUu6tQpy5Qpd1KxYvFAh2byyJcrimtVtS/QN3OGiNyMkzSMMeYfVJUJE5bSp88s9uw5RlRUOP36Xc4zz1xOTIzdkR+MfLmieDabef3zOxBjTOh4//3f2bPnGFddVZPlyxMYMKC1JYkgluNfTkSuxRmmtLKIDPdYVBKnGsoYYwA4ejSVAweSqVSpBCLCmDHtWbjwb7p2Pd+eiQgB3lL8TuAPnDaJFR7zDwHP+DMoY0zwmD59LY8+Oo1zzinDrFndEBHq1o2nbt34QIdm8kmOiUJVlwBLROQDVU0uwJiMMUFg69aD9Oo1k0mTVgJQokQ0e/Ycs643QpAvlYaVRWQw0ACIyZypquf6LSpjTKGVnp7B6NELefbZ7zh06DjFikUycOCVPP54MyIi7JmIUORLopgADAKGAdcB92FtFMYUSRkZSqtWE5g/fwsAN95YjxEj2lGtWqkAR2b8yZf0H6eqMwFUdb2qPgtc6d+wjDGFUViY0LZtLapWLclXX3Xmiy/usCRRBOQ6cJGI/AS0AL4AZgBbgWGqWtf/4Z3KBi4ypuCoKp9+uoKIiDBuuaUBACkpaaSmZlC8eFSAozN5cSYDF/lS9fQkUBx4HBgMlALuP52DGWOCx/r1e+nRYxrffLOe8uXjuOqqmpQpE0t0dATR1n9fkZJrolDVX9yXh4BuACJSxZ9BGWMCJyUljdde+4nBg38gOTmNMmViGDz4KkqVisl9YxOSvCYKEbkEqAz8qKq7ReQ8nK48rgIsWRgTYr7/fhOPPDKVVat2A9CtWyOGDWtLhQrFAhyZCaQcG7NF5BXgA6ArMENE+gNzgGWA3RprTIhJT8+gRw8nSdStW47vvrubiRNvsiRhvF5RdAIuUNVjIlIW+NudXl0woRlj/C0jQ0lOTiMuLpLw8DDGju3AvHl/8fTTLYiOtr6ZjMPbOyFZVY8BqOpeEVllScKY0PH77ztISJhKvXrlePfdTgC0alWDVq1qBDYwU+h4SxTniEhmV+IC1PCYRlVv9mtkxhi/OHLkOAMHzmX48AWkpWWwceM+9u07RpkysYEOzRRS3hLFLVmmR/kzEGOM/3399Wp69pzO5s0HEIEePZoweHAbSpe2O5pMzrx1Cji7IAMxxvhPWloGd9wxic8//xOAxo3PYty4jjRtWjnAkZlgYK1VxhQBERFhlCoVTfHiUbz00pX07NnUOvAzPsu1C48z2rlIO2AEEA68o6pDslnndmAAoMAyVb3T2z6tCw9jfPPLL0kANGvmPPK0Z89Rjh1Lo0qVkoEMywSIv7vwyDxItKqm5GH9cGA0cA2QBCwUkcmqutJjnTpAP6CFqu4TkQq+h26Myc7+/cn06/ct48Ytpl69eJYuTSAqKpxy5WycCHN6cr32FJGmIvI7sNadvkBE3vJh302Bdaq6QVWPAx/jPJvh6UFgtKruA1DVnXmK3hhzgqry4Ye/U6/eKBITFxMeHsYNN9QlPd1GBTBnxpcripFAR+BLAFVdJiK+dDNeGdjiMZ0ENMuyzrkAIjIfp3pqgKrO8GHfxhgPa9fuoUePaXz77QYAWrSoSmJiRxo2tIt0c+Z8SRRhqvpXlgHS033YLrsR1bM2LkQAdYDWOH1H/SAiDVV1/z92JPIQ8BDAxdbDlDH/kJqazlVXTSQp6SBly8YydOjV3HffhYSFZfcRNCbvfEkUW0SkKaBuu8NjwBoftksCqnpMV8HpBiTrOgtUNRXYKCKrcRLHQs+VVHU8MB6cxmwfjm1MyFNVRITIyHAGD76KOXM2MXTo1ZQvb30zmfzly/1xjwC9gWrADqC5Oy83C4E6IlJTRKKAzsDkLOt8iTtanojE41RFbfAtdGOKph07DtOt2xcMGjTvxLy7776Af/+7kyUJ4xe+XFGkqWrnvO5YVdNEpCcwE6f94T1VXSEiA4FFqjrZXdZWRFbiVGf1UdU9eT2WMUVBRoby9tuLeeaZ2ezfn0zp0jH06tWcEiVsFCHjX74MhboeWA18AnyuqocKIrCc2HMUpihatmw7CQlTWbDAeTaiXbvajB7dnnPOKRPgyEyw8OtzFKpaS0Quw6k6elFElgIfq+rHp3NAY4zvUlPT6ddvNm++uYD0dKVSpeKMGNGOW29tQJYbTIzxG5+e4VfVn1T1ceAi4CDOgEbGGD+LiAhjyZLtZGQojz3WlD//fJTbbjvPkoQpULleUYhIcZwH5ToD9YGvgMv8HJcxRdbmzQdIT8+gZs0yiAiJiR04cCCFJk3ODnRopojypTH7D+BrYKiq/uDneIwpslJT0xkx4hdeeOF7Lr20CrNmdUNEqFOnXKBDM0WcL4niHFW1PgCM8aOff95CQsJUli/fAUDZsrEcPZpKsWJRAY7MGC+JQkReV9WngP+JnPqQm41wZ8yZ27fvGM888y3jx/8GQM2apRk9uj3XXVcnwJEZc5K3K4pP3P9tZDtj/CAlJY3GjcexefMBIiPD6NPnMvr3b0lcXGSgQzPmH7yNcPer+7K+qv4jWbgP0tkIeMacgejoCLp3v5DZszcydmwHGjQoH+iQjMmWLw/c/aaqF2WZt0RVL/RrZDmwB+5MsEpOTuOVV36gbt147rzzfMAZojQ8XOx2V+N3fnngTkTuwLkltqaIfO6xqASwP/utjDHZmTVrPT16TGPdur1UqFCMm26qR2xspA1HaoKCtzaKX4E9OL2+jvaYfwhY4s+gjAkV27cfpnfvmXz00R8AnHdeeRITOxIba+0QJnh4a6PYCGwEvi24cIwJDenpGYwbt5j/+7/ZHDiQQmxsBC+80Ionn7yUqKjwQIdnTJ54q3qaq6qtRGQf/xxwSABV1bJ+j86YIJWerrz11q8cOJBC+/Z1GDXqOmrWtA78THDyVvWUOdxpfEEEYkywO3QohfR0pXTpGKKiwnn77evZseMwN99c3xqrTVDLsSXN42nsqkC4qqYDlwIPAzY6ijEuVeXzz/+kfv3RPPXUzBPzL7+8GrfcYr28muDnyy0XX+IMg1oLmIjTMeCHfo3KmCCxadN+brjhY2655VO2bj3EH3/sIjk5LdBhGZOvfEkUGe6Y1jcDb6rqY0Bl/4ZlTOGWmprOq6/+SIMGo5kyZQ0lS0YzatR1/PTT/cTE+NKFmjHBw6ehUEXkNqAbcKM7z+7tM0XW0aOpNG/+Dr//vhOAzp0bMnx4WypVKhHgyIzxD18Sxf1AD5xuxjeISE3gI/+GZUzhFRcXSZMmZ3P0aCpjxnSgbdtagQ7JGL/KtQsPABGJAGq7k+tUNWCVsNaFhyloqsrEicuoVassl19eDYADB5KJigq3B+dM0PDrmNkicgXwX2ArzjMUZ4lIN1WdfzoHNCaY/PnnLh55ZCpz5/5F/frxLF2aQFRUOKVKxQQ6NGMKjC9VT28A7VV1JYCI1MdJHKeVmYwJBseOpTJ48A8MHTqf1NQMypePo1+/y4mMtL6ZTNHjS6KIykwSAKr6p4jYsFsmZM2YsY5HH53Ghg37AHjwwYsYMuRqypaNDXBkxgSGL4niNxEZh3MVAdAV6xTQhKjDh4/TrdsX7N59lIYNK5CY2IEWLaoFOixjAsqXRJEAPA48jdNGMQ94y59BGVOQ0tMzyMhQIiPDKV48ihEj2pGUdJAnn2xOZKR14GeM17ueROR8oBawQlXXFlhUXthdTyY/LV78Nw8/PIVOnery3HOtAh2OMX5zJnc95dgyJyL/h9N9R1dglojcf5rxGVPoHDyYwhNPTKdp03dYvHgb//3vclJT0wMdljGFkreqp65AI1U9IiLlgWnAewUTljH+oapMmrSSJ56YwbZthwkPF3r3bs6LL15p1UzG5MBbokhR1SMAqrpLROy+QBPUDh1K4Y47JjF9+joAmjWrTGJiRxo3PivAkRlTuHlLFOd4jJUtQC3PsbNV9Wa/RmZMPitePIqUlHRKlYpmyJCreeihiwkLsy7AjcmNt0RxS5bpUf4MxBh/mDfvLypVKk6dOuUQEd577wZiYiKoWLF4oEOydgi6AAAVvklEQVQzJmh4GzN7dkEG4rOKFwc6AhMEdu8+ytNPz+Lf/15KmzY1mTWrGyJC9eqlAx2aMUHHOs43ISUjQ5kwYSl9+sxi795jREWFc8UV1UhPVyIirJrJmNPh1wZqEWknIqtFZJ2IPONlvVtFREXE+o8yp23Fip20bj2B7t0ns3fvMdq0qcnvvz/CCy+0JiLC7sUw5nT5fEUhItGqmpKH9cOB0cA1QBKwUEQme/Yb5a5XAufJ71983bcxWR04kEzz5u9y+PBxKlQoxvDhbbnzzvNtvGpj8kGuP7NEpKmI/A6sdacvEBFfuvBoijN2xQZVPQ58DHTKZr2XgKFAsu9hG+PI7FmgVKkY+vZtQULCxaxa9ShduzayJGFMPvHlenwk0BHYA6Cqy4ArfdiuMrDFYzqJLGNti8iFQFVVneJtRyLykIgsEpFFu3bt8uHQJtRt3XqQW2/9lPffX35iXv/+VzB2bEfKlLFeXo3JT74kijBV/SvLPF/6Osju59yJTprcB/jeAJ7KbUeqOl5Vm6hqk/Lly/twaBOq0tIyGDFiAfXqjeZ///uTF174nvT0DAC7gjDGT3xpo9giIk0BddsdHgPW+LBdElDVY7oK8LfHdAmgIfC9+wE/C5gsIjeo6iJfgjdFy8KFW0lImMpvv20D4MYb6zFyZDvCw62h2hh/8iVRPIJT/VQN2AF8687LzUKgjojUxBlGtTNwZ+ZCVT0AxGdOi8j3wL8sSZisjhw5Tt++3zJmzEJUoVq1Urz11nXccEPdQIdmTJGQa6JQ1Z04X/J5oqppItITmAmEA++p6goRGQgsUtXJeY7WFEkREWF8++0GwsKE3r0v5YUXWlGsmA2yaExB8ToeBYCIvI1H20ImVX3IX0F506RJE120yC46Qt369XspXTqGcuXiAKfaKSYmgvPPrxjgyIwJTn4Zj8LDt8Bs9998oALg8/MUxuRFSkoagwbNo2HDsfTt++2J+ZdcUtmShDEB4kvV0yee0yLyX2CW3yIyRdb332/ikUemsmrVbsC5wyk9PcMaq40JsNPp66kmUD2/AzFF186dR+jTZxYTJy4DoG7dcowd24Err6wZ4MiMMeBDohCRfZxsowgD9gI59ttkTF7s3n2U+vVHs3fvMaKjw+nf/wqefroF0dHWX6UxhYXXT6M4DzhcgHN7K0CG5tb6bUwexMfH0alTXZKSDjJmTAdq1y4b6JCMMVl4TRSqqiLyharaIBAmXxw5cpyBA+fSocO5tGzp1GCOGdOB6Ohwe7LamELKl1bCX0XkIr9HYkLe11+vpkGDMQwd+hM9ekwlI8O5OI2JibAkYUwhluMVhYhEqGoacDnwoIisB47g9OGkqmrJw/hky5YDPPHEDL74YhUAF154FuPGdbTxqo0JEt6qnn4FLgJuLKBYTIhJS8tg5MhfeP75ORw5kkrx4lEMGnQljz7a1AYSMiaIeEsUAqCq6wsoFhNiDh5M4ZVXfuTIkVRuuaU+b77ZjipVSgY6LGNMHnlLFOVFpHdOC1V1uB/iMUFu//5kYmMjiI6OoGzZWMaN60h0dDgdOpwb6NCMMafJ2/V/OFAcpzvw7P4Zc4Kq8uGHv1O37iiGDp1/Yv7NN9e3JGFMkPN2RbFNVQcWWCQmaK1Zs4cePaYye/ZGAObN24yq2p1MxoSIXNsojMlJcnIar776Iy+//CPHj6dTtmwsr712Dffe29iShDEhxFuiaFNgUZigs337YVq2/Ddr1+4F4N57G/Paa9cQHx8X4MiMMfktx0ShqnsLMhATXCpWLEbVqqWIiAhj7NgOtGpVI9AhGWP8xHpeMz7JyFDefnsxV15Zk3PPLYeI8OGHN1OmTCxRUeGBDs8Y40f21JPJ1bJl22nR4j0SEqbSo8dUMvuFrFixuCUJY4oAu6IwOTp8+DgDBnzPm28uID1dOfvsEiQknNZIisaYIGaJwmTryy9X8dhj00lKOkhYmPDYY00ZNOgqSpaMDnRoxpgCZonCnGLr1oN07jyJlJR0Lr64EomJHWnS5OxAh2WMCRBLFAaA1NR0IiLCEBEqVy7J4MFXERUVTo8el9iY1cYUcfYNYPjppy1cfPF43n9/+Yl5Tz11GY891syShDHGEkVRtnfvMR5++GtatHiP33/fyZgxi7CRbo0xWVnVUxGkqrz//nKeeuobdu06SmRkGE8/3YL+/a+wrjeMMaewRFHE7NhxmC5d/secOZsAaNWqOmPHdqB+/fKBDcwYU2hZoihiSpeOYdu2w8THxzFs2DXcffcFdhVhjPHKEkURMGvWei66qBLlysURHR3BZ5/dRqVKxSlXzjrwM8bkzhqzQ9i2bYfo0uV/tG37Pn37fntifsOGFSxJGGN8ZlcUISg9PYNx4xbTr99sDh5MITY2grp1y9lgQsaY02KJIsT89ts2EhKmsHDh3wB06FCHUaPaU6NG6QBHZowJVpYoQsimTftp2vRt0tOVypVLMHLkddx0Uz27ijDGnBG/JgoRaQeMAMKBd1R1SJblvYEHgDRgF3C/qv7lz5hCWY0apbnvvsaUKBHNiy+2pkQJ68DPGHPm/NaYLSLhwGjgOqAB0EVEGmRZbQnQRFUbAZOAof6KJxRt2rSf66//iLlzN52YN3789Qwffq0lCWNMvvHnFUVTYJ2qbgAQkY+BTsDKzBVUdY7H+guAu/wYT8hITU1n+PCfefHFuRw7lsbu3Uf5+efuAFbNZIzJd/5MFJWBLR7TSUAzL+t3B6Znt0BEHgIeAqhWrVp+xReUfvxxMwkJU1ixYhcAnTs3ZPjwtgGOyhgTyvyZKLL7aZttj3MichfQBGiV3XJVHQ+MB2jSpEmR7LVu375j9Okzi3ffXQJArVplGDOmA23b1gpwZMaYUOfPRJEEVPWYrgL8nXUlEbka6A+0UtUUP8YT1DIylK++Wk1kZBjPPHM5/fpdTmxsZKDDMsYUAf5MFAuBOiJSE9gKdAbu9FxBRC4ExgHtVHWnH2MJSqtW7aZmzdJER0dQrlwcH3xwM9WqlaJevfhAh2aMKUL8dteTqqYBPYGZwJ/Ap6q6QkQGisgN7mqvAcWBz0RkqYhM9lc8weTo0VT6959No0ZjGTp0/on5bdvWsiRhjClwfn2OQlWnAdOyzHve4/XV/jx+MJoxYx09ekxl48b9AOzefTTAERljijp7MruQ+PvvQ/TqNYPPPnPuHj7//AokJnbkssuq5rKlMcb4lyWKQmDNmj00aTKeQ4eOExcXyYABrejVqzmRkeGBDs0YYyxRFAZ16pTlkksqU6xYJG+9dR3Vq1sHfsaYwsMSRQAcPJjC88/PoUePSzj33HKICJMnd6ZYsahAh2aMMaewRFGAVJVJk1byxBMz2LbtMKtW7WbGDKfXEksSxpjCyhJFAdmwYR89e05j+vR1ADRvXoVXX7WbvowxhZ8lCj87fjydYcN+4qWX5pGcnEbp0jEMGdKGBx+8mLAw68DPGFP4WaLwsy1bDjBw4FxSUtLp2vV8Xn+9LRUrFg90WMYY4zNLFH6wb98xSpeOQUSoVassI0a0o3btsrRpc06gQzPGmDzzWxceRVFGhvLee0uoXfst3n9/+Yn5Dz/cxJKEMSZoWaLIJytW7KR16wl07z6ZvXuPnWi0NsaYYGdVT2fo6NFUXnppLsOG/UxaWgYVKhTjjTeupUuXhoEOzRhj8oUlijOwZs0err32fTZt2o8IJCRczMsvt6FMmdhAh2aMMfnGEsUZqF69FDExEVxwQUUSEzvSvHmVQIdkjDH5zhJFHqSlZZCYuIguXRpSrlwc0dERzJjRlcqVSxIRYc09xpjQZInCR7/+upWEhCksWbKdpUu38847zthL1oGfMSbUWaLIxYEDyfTv/x1jxixEFapVK0WnTnUDHZYxxhQYSxQ5UFU++WQFTz45k+3bDxMREUbv3s15/vlW1oGfMaZIsUSRg2XLdtCly/8AuOyyqiQmduD88ysGOCpjjCl4lig8pKdnEB7uNEo3bnwWTz7ZnAYNynP//RdaB37GmCLLbtVxzZmzkYYNxzJv3l8n5g0ffi0PPHCRJQljTJFW5BPFzp1HuOeeL7nqqomsWrWb4cN/DnRIxhhTqBTZqqeMDOXdd3+jb99v2bcvmejocJ59tiV9+lwW6NCMMaZQKZKJYuPGfdx11xf89NMWANq2rcXo0e2pXbtsgCMzxpjCp0gmipIlo1mzZg9nnVWcN9+8lttvPw8Ra4cwxpjsFJlEMXPmOlq3rkF0dATlysUxeXJnGjQoT6lSMYEOzRhjCrWQb8zesuUAN930Ce3afcBrr/10Yv6ll1a1JGGMMT4I2SuKtLQMRo78heefn8ORI6kULx5F2bLW/bcxxuRVSCaKBQuSSEiYwrJlOwC45Zb6jBjRjsqVSwY4MmOMCT4hlyh++SWJyy57F1WoUaM0o0ZdR4cO5wY6LGOMCVohlyiaNq3MtdfW5sILz+LZZ1sSFxcZ6JCMMSaoBX1j9tq1e+jY8UPWrNkDgIgwdeqdvPxyG0sSxhiTD4L2iiIlJY0hQ37klVd+JCUlnZiYCCZNuh3A+mYyxph85NcrChFpJyKrRWSdiDyTzfJoEfnEXf6LiNTwZb+zZ2+gUaNEBgyYS0pKOvfd15jExI75Hb4xxhj8eEUhIuHAaOAaIAlYKCKTVXWlx2rdgX2qWltEOgOvAnd42+/Gjfu5+ur/AlC/fjyJiR1p2bK6X87BGGOMf68omgLrVHWDqh4HPgY6ZVmnE/Af9/UkoI3k0pfGvn3HiImJ4OWXr2Lp0gRLEsYY42eiqv7ZscitQDtVfcCd7gY0U9WeHuv84a6T5E6vd9fZnWVfDwEPuZMNgT/8EnTwiQd257pW0WBlcZKVxUlWFifVVdUSp7OhPxuzs7syyJqVfFkHVR0PjAcQkUWq2uTMwwt+VhYnWVmcZGVxkpXFSSKy6HS39WfVUxJQ1WO6CvB3TuuISARQCtjrx5iMMcbkkT8TxUKgjojUFJEooDMwOcs6k4F73Ne3At+pv+rCjDHGnBa/VT2papqI9ARmAuHAe6q6QkQGAotUdTLwLvBfEVmHcyXR2Yddj/dXzEHIyuIkK4uTrCxOsrI46bTLwm+N2cYYY0JD0HfhYYwxxr8sURhjjPGq0CYKf3X/EYx8KIveIrJSRJaLyGwRCdmnEHMrC4/1bhURFZGQvTXSl7IQkdvd98YKEfmwoGMsKD58RqqJyBwRWeJ+TtoHIk5/E5H3RGSn+4xadstFREa65bRcRC7yaceqWuj+4TR+rwfOAaKAZUCDLOv0ABLd152BTwIddwDL4kogzn39SFEuC3e9EsA8YAHQJNBxB/B9UQdYApRxpysEOu4AlsV44BH3dQNgU6Dj9lNZtAQuAv7IYXl7YDrOM2zNgV982W9hvaLwS/cfQSrXslDVOap61J1cgPPMSijy5X0B8BIwFEguyOAKmC9l8SAwWlX3AajqzgKOsaD4UhYKZA5xWYpTn+kKCao6D+/PonUCJqpjAVBaRCrltt/CmigqA1s8ppPcedmuo6ppwAGgXIFEV7B8KQtP3XF+MYSiXMtCRC4EqqrqlIIMLAB8eV+cC5wrIvNFZIGItCuw6AqWL2UxALhLRJKAacBjBRNaoZPX7xOg8I5HkW/df4QAn89TRO4CmgCt/BpR4HgtCxEJA94A7i2ogALIl/dFBE71U2ucq8wfRKShqu73c2wFzZey6AJMUNXXReRSnOe3Gqpqhv/DK1RO63uzsF5RWPcfJ/lSFojI1UB/4AZVTSmg2ApabmVRAqfTyO9FZBNOHezkEG3Q9vUz8pWqpqrqRmA1TuIINb6URXfgUwBV/RmIwekwsKjx6fskq8KaKKz7j5NyLQu3umUcTpII1XpoyKUsVPWAqsarag1VrYHTXnODqp52Z2iFmC+fkS9xbnRAROJxqqI2FGiUBcOXstgMtAEQkfo4iWJXgUZZOEwG7nbvfmoOHFDVbbltVCirntR/3X8EHR/L4jWgOPCZ256/WVVvCFjQfuJjWRQJPpbFTKCtiKwE0oE+qroncFH7h49l8RTwtog8iVPVcm8o/rAUkY9wqhrj3faYF4BIAFVNxGmfaQ+sA44C9/m03xAsK2OMMfmosFY9GWOMKSQsURhjjPHKEoUxxhivLFEYY4zxyhKFMcYYryxRmEJHRNJFZKnHvxpe1q2RU0+ZeTzm927vo8vcLi/qnsY+EkTkbvf1vSJytseyd0SkQT7HuVBEGvuwTS8RiTvTY5uiyxKFKYyOqWpjj3+bCui4XVX1ApzOJl/L68aqmqiqE93Je4GzPZY9oKor8yXKk3GOwbc4ewGWKMxps0RhgoJ75fCDiPzm/rssm3XOE5Ff3auQ5SJSx51/l8f8cSISnsvh5gG13W3buGMY/O729R/tzh8iJ8cAGebOGyAi/xKRW3H63PrAPWaseyXQREQeEZGhHjHfKyJvnWacP+PRoZuIjBWRReKMPfGiO+9xnIQ1R0TmuPPaisjPbjl+JiLFczmOKeIsUZjCKNaj2ukLd95O4BpVvQi4AxiZzXYJwAhVbYzzRZ3kdtdwB9DCnZ8OdM3l+NcDv4tIDDABuENVz8fpyeARESkL3AScp6qNgEGeG6vqJGARzi//xqp6zGPxJOBmj+k7gE9OM852ON10ZOqvqk2ARkArEWmkqiNx+vK5UlWvdLvyeBa42i3LRUDvXI5jirhC2YWHKfKOuV+WniKBUW6dfDpOv0VZ/Qz0F5EqwOequlZE2gAXAwvd7k1icZJOdj4QkWPAJpxuqOsCG1V1jbv8P8CjwCicsS7eEZGpgM9dmqvqLhHZ4Pazs9Y9xnx3v3mJsxhOdxWeI5TdLiIP4XyuK+EM0LM8y7bN3fnz3eNE4ZSbMTmyRGGCxZPADuACnCvhUwYlUtUPReQXoAMwU0QewOlW+T+q2s+HY3T17EBQRLId38TtW6gpTidznYGewFV5OJdPgNuBVcAXqqrifGv7HCfOKG5DgNHAzSJSE/gXcImq7hORCTgd32UlwCxV7ZKHeE0RZ1VPJliUAra54wd0w/k1/Q8icg6wwa1umYxTBTMbuFVEKrjrlBXfxxRfBdQQkdrudDdgrlunX0pVp+E0FGd359EhnG7Ps/M5cCPOGAmfuPPyFKeqpuJUITV3q61KAkeAAyJSEbguh1gWAC0yz0lE4kQku6szY06wRGGCxRjgHhFZgFPtdCSbde4A/hCRpUA9nCEfV+J8oX4jIsuBWTjVMrlS1WSc3jU/E5HfgQwgEedLd4q7v7k4VztZTQASMxuzs+x3H7ASqK6qv7rz8hyn2/bxOvAvVV2GMz72CuA9nOqsTOOB6SIyR1V34dyR9ZF7nAU4ZWVMjqz3WGOMMV7ZFYUxxhivLFEYY4zxyhKFMcYYryxRGGOM8coShTHGGK8sURhjjPHKEoUxxhiv/h/kqwKQRP3AygAAAABJRU5ErkJggg==\n",
|
|
"text/plain": [
|
|
"<matplotlib.figure.Figure at 0x7f6dfef289b0>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"0.96820073384642935"
|
|
]
|
|
},
|
|
"execution_count": 71,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"# Your turn here - choose another classifier to see how it compares\n",
|
|
"build_roc_auc(naive_bayes, training_data, testing_data, y_train, y_test) \n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|