Neural Network (Week 2) : Logistic Regression

import numpy as np
import matplotlib.pyplot as plt
import h5py
import scipy
from PIL import Image
from scipy import ndimage
from lr_utils import load_dataset


index = 25
plt.imshow(train_set_x_orig[index])


m_train = train_set_x_orig.shape[0]
m_test = test_set_x_orig.shape[0]
num_px = train_set_x_orig.shape[1]


train_set_x_flatten = train_set_x_orig.
    reshape(train_set_x_orig.shape[0], -1).T
test_set_x_flatten = test_set_x_orig.
    reshape(test_set_x_orig.shape[0], -1).T


def initialize_with_zeros(dim):
    w = np.zeros((dim,1))
    b = 0
    return w, b


def propagate(w, b, X, Y):
    m = X.shape[1]
    
    # FORWARD PROPAGATION
    A = sigmoid(np.dot(w.T,X)+b)
    cost = -1/m*(np.dot(Y, np.log(A).T)
        + np.dot((1-Y), np.log(1-A).T))
    
    # BACKWARD PROPAGATION
    dw = 1/m*np.dot(X, (A-Y).T)
    db = 1/m*np.sum(A-Y)

    grads = {"dw": dw,  "db": db}
    
    return grads, cost


def optimize(w, b, X, Y, num_iterations,
    learning_rate, print_cost = False):

    costs = []
    
    for i in range(num_iterations):
        # Cost and gradient calculation
        grads, cost = propagate(w,b,X,Y)
        
        # Retrieve derivatives from grads
        dw = grads["dw"]
        db = grads["db"]
        
        # update rule
        w = w - learning_rate*dw
        b = b - learning_rate*db
        
        # Record the costs
        if i % 100 == 0:
            costs.append(cost)
        
    params = {"w": w, "b": b}
    grads = {"dw": dw, "db": db}
    
    return params, grads, costs


def predict(w, b, X):
    A = sigmoid(np.dot(w.T, X) + b)
    
    for i in range(A.shape[1]):
        
        if (A[0,i] > 0.5):
            Y_prediction[0,i] = 1
        else:
            Y_prediction[0,i] = 0
    
    assert(Y_prediction.shape == (1, m))
    
    return Y_prediction


def model(X_train, Y_train, X_test,
    Y_test, num_iterations = 2000,
    learning_rate = 0.5, print_cost = False):

    # initialize parameters
    w, b = np.zeros((X_train.shape[0],1)), 0

    # Gradient descent (≈ 1 line of code)
    parameters, grads, costs = optimize(
        w,b,X_train, Y_train, num_iterations,
        learning_rate, print_cost)
    
    # Retrieve parameters w and b
    w = parameters["w"]
    b = parameters["b"]
    
    # Predict test/train set examples
    Y_prediction_test = predict(w, b, X_test)
    Y_prediction_train = predict(w,b,X_train)

# Print train/test Errors
    print("train accuracy: {} %".format(
        100 - np.mean(np.abs(
        Y_prediction_train - Y_train)) * 100))
    print("test accuracy: {} %".format(
        100 - np.mean(np.abs(
        Y_prediction_test - Y_test)) * 100))
    
    d = {"costs": costs,
         "Y_prediction_test": Y_prediction_test, 
         "Y_prediction_train" : Y_prediction_train, 
         "w" : w, 
         "b" : b,
         "learning_rate" : learning_rate,
         "num_iterations": num_iterations}
    
    return d


learning_rates = [0.01, 0.001, 0.0001]
models = {}
for i in learning_rates:
    print ("learning rate is: " + str(i))
    models[str(i)] = model(train_set_x,
        train_set_y, test_set_x, test_set_y,
        num_iterations = 1500,
        learning_rate = i, print_cost = False)
    print ('\n' + "----------------------------" + '\n')

for i in learning_rates:
    plt.plot(np.squeeze(models[str(i)]["costs"]),
    label= str(models[str(i)]["learning_rate"]))