completed part 2 implementing gradient descent

python/Deep Learning/Implementing Gradient Descent/Multilayer Perceptron/data_prep.py

@@ -0,0 +1,24 @@
+import numpy as np
+import pandas as pd
+
+admissions = pd.read_csv('binary.csv')
+
+# Make dummy variables for rank
+data = pd.concat([admissions, pd.get_dummies(
+    admissions['rank'], prefix='rank')], axis=1)
+data = data.drop('rank', axis=1)
+
+# Standarize features
+for field in ['gre', 'gpa']:
+    mean, std = data[field].mean(), data[field].std()
+    data.loc[:, field] = (data[field] - mean) / std
+
+# Split off random 10% of the data for testing
+np.random.seed(42)
+sample = np.random.choice(data.index, size=int(len(data) * 0.9), replace=False)
+data, test_data = data.ix[sample], data.drop(sample)
+
+# Split into features and targets
+features, targets = data.drop('admit', axis=1), data['admit']
+features_test, targets_test = test_data.drop(
+    'admit', axis=1), test_data['admit']

python/Deep Learning/Implementing Gradient Descent/Multilayer Perceptron/gradient.py

@@ -0,0 +1,56 @@
+import numpy as np
+
+
+def sigmoid(x):
+    """
+    Calculate sigmoid
+    """
+    return 1 / (1 + np.exp(-x))
+
+
+def sigmoid_prime(x):
+    """
+    # Derivative of the sigmoid function
+    """
+    return sigmoid(x) * (1 - sigmoid(x))
+
+
+learnrate = 0.5
+x = np.array([1, 2, 3, 4])
+y = np.array(0.5)
+
+# Initial weights
+w = np.array([0.5, -0.5, 0.3, 0.1])
+
+# Calculate one gradient descent step for each weight
+# Note: Some steps have been consolidated, so there are
+# fewer variable names than in the above sample code
+
+# TODO: Calculate the node's linear combination of inputs and weights
+h = np.dot(x, w)
+
+# TODO: Calculate output of neural network (y hat)
+nn_output = sigmoid(h)
+
+# TODO: Calculate error of neural network (y - y hat)
+error = y - nn_output
+
+# TODO: Calculate the error term
+#       Remember, this requires the output gradient, which we haven't
+#       specifically added a variable for.
+error_term = error * sigmoid_prime(h)
+# Note: The sigmoid_prime function calculates sigmoid(h) twice,
+#       but you've already calculated it once. You can make this
+#       code more efficient by calculating the derivative directly
+#       rather than calling sigmoid_prime, like this:
+# error_term = error * nn_output * (1 - nn_output)
+
+# TODO: Calculate change in weights
+del_w = learnrate * error_term * x
+
+print('Neural Network output:')
+print(nn_output)
+print('Amount of Error:')
+print(error)
+print('Change in Weights:')
+print(del_w)

python/Deep Learning/Implementing Gradient Descent/Multilayer Perceptron/gradient_2.py

@@ -0,0 +1,71 @@
+import numpy as np
+from data_prep import features, targets, features_test, targets_test
+
+
+def sigmoid(x):
+    """
+    Calculate sigmoid
+    """
+    return 1 / (1 + np.exp(-x))
+
+# TODO: We haven't provided the sigmoid_prime function like we did in
+#       the previous lesson to encourage you to come up with a more
+#       efficient solution. If you need a hint, check out the comments
+#       in solution.py from the previous lecture.
+
+
+# Use to same seed to make debugging easier
+np.random.seed(42)
+
+n_records, n_features = features.shape
+last_loss = None
+
+# Initialize weights
+weights = np.random.normal(scale=1 / n_features**.5, size=n_features)
+
+# Neural Network hyperparameters
+epochs = 1000
+learnrate = 0.5
+
+for e in range(epochs):
+    del_w = np.zeros(weights.shape)
+    for x, y in zip(features.values, targets):
+        # Loop through all records, x is the input, y is the target
+
+        # Note: We haven't included the h variable from the previous
+        #       lesson. You can add it if you want, or you can calculate
+        #       the h together with the output
+
+        # TODO: Calculate the output (y hat)
+        output = sigmoid(np.dot(x, weights))
+
+        # TODO: Calculate the error
+        error = y - output
+
+        # TODO: Calculate the error term
+        error_term = error * output * (1 - output)
+
+        # TODO: Calculate the change in weights for this sample
+        #       and add it to the total weight change
+        del_w += error_term * x
+
+    # TODO: Update weights using the learning rate and the average change in
+    # weights
+    weights += learnrate * del_w / n_records
+
+    # Printing out the mean square error on the training set
+    if e % (epochs / 10) == 0:
+        out = sigmoid(np.dot(features, weights))
+        loss = np.mean((out - targets) ** 2)
+        if last_loss and last_loss < loss:
+            print("Train loss: ", loss, "  WARNING - Loss Increasing")
+        else:
+            print("Train loss: ", loss)
+        last_loss = loss
+
+
+# Calculate accuracy on test data
+tes_out = sigmoid(np.dot(features_test, weights))
+predictions = tes_out > 0.5
+accuracy = np.mean(predictions == targets_test)
+print("Prediction accuracy: {:.3f}".format(accuracy))

python/Deep Learning/Implementing Gradient Descent/Multilayer Perceptron/multilayer.py

@@ -0,0 +1,38 @@
+import numpy as np
+
+
+def sigmoid(x):
+    """
+    Calculate sigmoid
+    """
+    return 1 / (1 + np.exp(-x))
+
+
+# Network size
+N_input = 4
+N_hidden = 3
+N_output = 2
+
+np.random.seed(42)
+# Make some fake data
+X = np.random.randn(4)
+
+weights_input_to_hidden = np.random.normal(
+    0, scale=0.1, size=(N_input, N_hidden))
+weights_hidden_to_output = np.random.normal(
+    0, scale=0.1, size=(N_hidden, N_output))
+
+
+# TODO: Make a forward pass through the network
+
+hidden_layer_in = np.dot(X, weights_input_to_hidden)
+hidden_layer_out = sigmoid(hidden_layer_in)
+
+print('Hidden-layer Output:')
+print(hidden_layer_out)
+
+output_layer_in = np.dot(hidden_layer_out, weights_hidden_to_output)
+output_layer_out = sigmoid(output_layer_in)
+
+print('Output-layer Output:')
+print(output_layer_out)