Neural Network (Week 4) : Application

# Deep Neural Network for Image Classification: Application

# ## 1 - Packages
import time
import numpy as np
import h5py
import matplotlib.pyplot as plt
import scipy
from PIL import Image
from scipy import ndimage
from dnn_app_utils_v3 import *

# ## 2 - Dataset
train_x_orig, train_y, test_x_orig, test_y, classes = load_data()
index = 10
plt.imshow(train_x_orig[index])

m_train = train_x_orig.shape[0]
num_px = train_x_orig.shape[1]
m_test = test_x_orig.shape[0]


# Reshape the training and test examples
train_x_flatten = train_x_orig.reshape(train_x_orig.shape[0], -1).T
test_x_flatten = test_x_orig.reshape(test_x_orig.shape[0], -1).T


# Standardize data to have feature values between 0 and 1.
train_x = train_x_flatten/255.
test_x = test_x_flatten/255.

### CONSTANTS DEFINING THE MODEL ####
n_x = 12288 # num_px * num_px * 3
n_h = 7
n_y = 1
layers_dims = (n_x, n_h, n_y)


def two_layer_model(X, Y, layers_dims, learning_rate = 0.0075,
    num_iterations = 3000, print_cost=False):
    grads = {}
    costs = []
    m = X.shape[1]
    (n_x, n_h, n_y) = layers_dims
    
    parameters = initialize_parameters(n_x, n_h, n_y)
    
    W1 = parameters["W1"]
    b1 = parameters["b1"]
    W2 = parameters["W2"]
    b2 = parameters["b2"]
    
    # Loop (gradient descent)
    for i in range(0, num_iterations):
        A1, cache1 = linear_activation_forward(X, W1, b1, 'relu')
        A2, cache2 = linear_activation_forward(A1, W2, b2, 'sigmoid')
        
        cost = compute_cost(A2, Y)
        
        dA2 = - (np.divide(Y, A2) - np.divide(1 - Y, 1 - A2))

        dA1, dW2, db2 = linear_activation_backward(dA2, cache2, 'sigmoid')
        dA0, dW1, db1 = linear_activation_backward(dA1, cache1, 'relu')        

        grads['dW1'] = dW1
        grads['db1'] = db1
        grads['dW2'] = dW2
        grads['db2'] = db2
        
        parameters = update_parameters(parameters, grads, learning_rate)

        W1 = parameters["W1"]
        b1 = parameters["b1"]
        W2 = parameters["W2"]
        b2 = parameters["b2"]
        
        # Print the cost every 100 training example
        if print_cost and i % 100 == 0:
            print("Cost after iteration {}: {}".format(i, np.squeeze(cost)))
        if print_cost and i % 100 == 0:
            costs.append(cost)
       
    # plot the cost
    plt.plot(np.squeeze(costs))
    plt.ylabel('cost')
    plt.xlabel('iterations (per tens)')
    plt.title("Learning rate =" + str(learning_rate))
    plt.show()
    
    return parameters


parameters = two_layer_model(
    train_x, train_y, layers_dims = (n_x, n_h, n_y),
    num_iterations = 2500, print_cost=True)

predictions_train = predict(train_x, train_y, parameters)

predictions_test = predict(test_x, test_y, parameters)



layers_dims = [12288, 20, 7, 5, 1]

def L_layer_model(X, Y, layers_dims, learning_rate = 0.0075,
    num_iterations = 3000, print_cost=False):
    costs = []
    parameters = initialize_parameters_deep(layers_dims)
    
    # Loop (gradient descent)
    for i in range(0, num_iterations):
        AL, caches = L_model_forward(X, parameters)
        cost = compute_cost(AL, Y)
        grads = L_model_backward(AL, Y, caches)

        parameters = update_parameters(parameters, grads, learning_rate)
                
        # Print the cost every 100 training example
        if print_cost and i % 100 == 0:
            print ("Cost after iteration %i: %f" %(i, cost))
        if print_cost and i % 100 == 0:
            costs.append(cost)
            
    # plot the cost
    plt.plot(np.squeeze(costs))
    plt.ylabel('cost')
    plt.xlabel('iterations (per tens)')
    plt.title("Learning rate =" + str(learning_rate))
    plt.show()
    
    return parameters

Neural Network (Week 4) : Step By Step

# ## 1 - Packages

import numpy as np

import h5py

import matplotlib.pyplot as plt

from testCases_v4 import *

from dnn_utils_v2 import sigmoid, sigmoid_backward, relu, relu_backward

# ## 2 - Outline of the Assignment

def initialize_parameters_deep(layer_dims):

    parameters = {}

    L = len(layer_dims)

    for l in range(1, L):

        parameters['W' + str(l)] =
            np.random.randn(layer_dims[l], layer_dims[l-1]) * 0.01

        parameters['b' + str(l)] = np.zeros((layer_dims[l], 1))

    return parameters

def linear_forward(A, W, b):

    Z = np.dot(W , A) + b

    cache = (A, W, b)

    return Z, cache

def linear_activation_forward(A_prev, W, b, activation):

    if activation == "sigmoid":

        Z, linear_cache = linear_forward(A_prev, W, b)

        A, activation_cache = sigmoid(Z)

    elif activation == "relu":

        Z, linear_cache = linear_forward(A_prev, W, b)

        A, activation_cache = relu(Z)

    cache = (linear_cache, activation_cache)

    return A, cache

def L_model_forward(X, parameters):

    caches = []

    A = X

    L = len(parameters) // 2

    

    for l in range(1, L):

        A_prev = A 

        A, cache = linear_activation_forward(

            A_prev, parameters['W' + str(l)], parameters['b' + str(l)], 'relu')

        caches.append(cache)

    

    AL, cache = linear_activation_forward(

        A, parameters['W' + str(L)], parameters['b' + str(L)], 'sigmoid')

    caches.append(cache)

            

    return AL, caches

def compute_cost(AL, Y):

    m = Y.shape[1]

    cost = (-1/m) * np.sum(Y * np.log(AL) + (1-Y) * np.log(1-AL), axis=1)

    cost = np.squeeze(cost)

    return cost

def linear_backward(dZ, cache):

    A_prev, W, b = cache

    m = A_prev.shape[1]

    dW = (1/m) * np.dot(dZ , A_prev.T)

    db = (1/m) * np.sum(dZ, axis = 1, keepdims = True)

    dA_prev = np.dot(W.T , dZ)

    

    return dA_prev, dW, db

def linear_activation_backward(dA, cache, activation):

    linear_cache, activation_cache = cache

    

    if activation == "relu":

        dZ = relu_backward(dA, activation_cache)

        dA_prev, dW, db = linear_backward(dZ, linear_cache)

    elif activation == "sigmoid":

        dZ = sigmoid_backward(dA, activation_cache)

        dA_prev, dW, db = linear_backward(dZ, linear_cache)

    return dA_prev, dW, db

def L_model_backward(AL, Y, caches):

    grads = {}

    L = len(caches) # the number of layers

    m = AL.shape[1]

    Y = Y.reshape(AL.shape) # after this line, Y is the same shape as AL

    

    dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))

    

    current_cache = caches[L-1]

    grads["dA" + str(L-1)], grads["dW" + str(L)], grads["db" + str(L)] =

        linear_activation_backward(dAL, current_cache, 'sigmoid')

    

    # Loop from l=L-2 to l=0

    for l in reversed(range(L-1)):

        current_cache = caches[l]

        dA_prev_temp, dW_temp, db_temp =

            linear_activation_backward(grads["dA" + str(l+1)], current_cache, 'relu')

        grads["dA" + str(l)] = dA_prev_temp

        grads["dW" + str(l + 1)] = dW_temp

        grads["db" + str(l + 1)] = db_temp

    return grads

def update_parameters(parameters, grads, learning_rate):

    L = len(parameters) // 2 # number of layers in the neural network

    for l in range(L):

        parameters["W" + str(l+1)] =
            parameters["W" + str(l+1)] - learning_rate * grads["dW" + str(l+1)]

        parameters["b" + str(l+1)] =
            parameters["b" + str(l+1)] - learning_rate * grads["db" + str(l+1)]

    return parameters

Neural Network (Week 3) : One hidden layer

# ## 1 - Packages ##

# Package imports
import numpy as np
import matplotlib.pyplot as plt
from testCases_v2 import *
import sklearn
import sklearn.datasets
import sklearn.linear_model
from planar_utils import plot_decision_boundary, sigmoid, load_planar_dataset, load_extra_datasets

get_ipython().magic('matplotlib inline')
np.random.seed(1)


# ## 2 - Dataset ##
X, Y = load_planar_dataset()
# Visualize the data:
plt.scatter(X[0, :], X[1, :], c=Y, s=40, cmap=plt.cm.Spectral);
shape_X = np.shape(X)
shape_Y = np.shape(Y)
m = np.shape(X)[1]  # training set size


# ## 3 - Simple Logistic Regression
# Train the logistic regression classifier
clf = sklearn.linear_model.LogisticRegressionCV();
clf.fit(X.T, Y.T);
plot_decision_boundary(lambda x: clf.predict(x), X, Y)

# Print accuracy
LR_predictions = clf.predict(X.T)
print ('Accuracy of logistic regression: %d ' % float(
      (np.dot(Y,LR_predictions) +
       np.dot(1-Y,1-LR_predictions))
      /float(Y.size)*100) +
       '% ' + "(percentage of correctly labelled datapoints)")

# ## 4 - Neural Network model
def layer_sizes(X, Y):
    n_x = np.shape(X)[0] # size of input layer
    n_h = 4
    n_y = np.shape(Y)[0] # size of output layer
    return (n_x, n_h, n_y)


# ### 4.2 - Initialize the model's parameters ####
def initialize_parameters(n_x, n_h, n_y):
    np.random.seed(2) 
    
    W1 = np.random.randn(n_h,n_x) * 0.01
    b1 = np.zeros((n_h,1))
    W2 = np.random.randn(n_y,n_h) * 0.01
    b2 = np.zeros((n_y,1))
    
    parameters = {"W1": W1, "b1": b1, "W2": W2,  "b2": b2}
    
    return parameters


# ### 4.3 - The Loop ####
def forward_propagation(X, parameters):
    W1 = parameters["W1"]
    b1 = parameters["b1"]
    W2 = parameters["W2"]
    b2 = parameters["b2"]
    
    Z1 = np.dot(W1, X) + b1
    A1 = np.tanh(Z1)
    Z2 = np.dot(W2, A1) + b2
    A2 = sigmoid(Z2)
    
    cache = {"Z1": Z1, "A1": A1, "Z2": Z2, "A2": A2}
    return A2, cache


def compute_cost(A2, Y, parameters):
    m = Y.shape[1] # number of example

    logprobs = np.multiply(Y, np.log(A2)) + np.multiply(1-Y, np.log(1-A2))
    cost = -(1/m)*np.sum(logprobs)
    
    cost = np.squeeze(cost)
    return cost


def backward_propagation(parameters, cache, X, Y):
    m = X.shape[1]
    
    W1 = parameters["W1"]
    W2 = parameters["W2"]
    A1 = cache["A1"]
    A2 = cache["A2"]
    
    dZ2 = A2 - Y
    dW2 = (1/m) * np.dot(dZ2 , A1.T)
    db2 = (1/m) * np.sum(dZ2, axis = 1, keepdims = True)
    dZ1 = np.dot(W2.T , dZ2) * (1 - np.power(A1, 2))
    dW1 = (1/m) * np.dot(dZ1 , X.T)
    db1 = (1/m) * np.sum(dZ1, axis = 1, keepdims = True)
    
    grads = {"dW1": dW1, "db1": db1, "dW2": dW2, "db2": db2}
    return grads


def update_parameters(parameters, grads, learning_rate = 1.2):
    W1 = parameters["W1"]
    b1 = parameters["b1"]
    W2 = parameters["W2"]
    b2 = parameters["b2"]
    
    dW1 = grads["dW1"]
    db1 = grads["db1"]
    dW2 = grads["dW2"]
    db2 = grads["db2"]
    
    W1 = W1 - learning_rate * dW1
    b1 = b1 - learning_rate * db1
    W2 = W2 - learning_rate * dW2
    b2 = b2 - learning_rate * db2
    
    parameters = {"W1": W1, "b1": b1, "W2": W2, "b2": b2}
    return parameters


def nn_model(X, Y, n_h, num_iterations = 10000, print_cost=False):
    np.random.seed(3)
    n_x = layer_sizes(X, Y)[0]
    n_y = layer_sizes(X, Y)[2]
    
    parameters = initialize_parameters(n_x, n_h, n_y)
    W1 = parameters["W1"]
    b1 = parameters["b1"]
    W2 = parameters["W2"]
    b2 = parameters["b2"]
    
    for i in range(0, num_iterations):
        A2, cache = forward_propagation(X, parameters)
        cost = compute_cost(A2, Y, parameters)
        grads = backward_propagation(parameters, cache, X, Y)

        parameters = update_parameters(parameters, grads)
        return parameters


# ### 4.5 Predictions
def predict(parameters, X):
    A2, cache = forward_propagation(X, parameters)
    predictions = (A2 > 0.5)
    return predictions

# Build a model with a n_h-dimensional hidden layer
    parameters = nn_model(X, Y, n_h = 4, num_iterations = 10000, print_cost=True)

# Plot the decision boundary
    plot_decision_boundary(lambda x: predict(parameters, x.T), X, Y)
    plt.title("Decision Boundary for hidden layer size " + str(4))

# Print accuracy
    predictions = predict(parameters, X)
    print ('Accuracy: %d' % float(
        (np.dot(Y,predictions.T) +
         np.dot(1-Y,1-predictions.T))/
        float(Y.size)*100) + '%')


plt.figure(figsize=(16, 32))
hidden_layer_sizes = [1, 2, 3, 4, 5, 20, 50]
for i, n_h in enumerate(hidden_layer_sizes):
    plt.subplot(5, 2, i+1)
    plt.title('Hidden Layer of size %d' % n_h)
    parameters = nn_model(X, Y, n_h, num_iterations = 5000)
    plot_decision_boundary(lambda x: predict(parameters, x.T), X, Y)
    predictions = predict(parameters, X)
    accuracy = float((np.dot(Y,predictions.T) + np.dot(1-Y,1-predictions.T))/float(Y.size)*100)
    print ("Accuracy for {} hidden units: {} %".format(n_h, accuracy))

Neural Network (Week 2) : Logistic Regression

import numpy as np
import matplotlib.pyplot as plt
import h5py
import scipy
from PIL import Image
from scipy import ndimage
from lr_utils import load_dataset


index = 25
plt.imshow(train_set_x_orig[index])


m_train = train_set_x_orig.shape[0]
m_test = test_set_x_orig.shape[0]
num_px = train_set_x_orig.shape[1]


train_set_x_flatten = train_set_x_orig.
    reshape(train_set_x_orig.shape[0], -1).T
test_set_x_flatten = test_set_x_orig.
    reshape(test_set_x_orig.shape[0], -1).T


def initialize_with_zeros(dim):
    w = np.zeros((dim,1))
    b = 0
    return w, b


def propagate(w, b, X, Y):
    m = X.shape[1]
    
    # FORWARD PROPAGATION
    A = sigmoid(np.dot(w.T,X)+b)
    cost = -1/m*(np.dot(Y, np.log(A).T)
        + np.dot((1-Y), np.log(1-A).T))
    
    # BACKWARD PROPAGATION
    dw = 1/m*np.dot(X, (A-Y).T)
    db = 1/m*np.sum(A-Y)

    grads = {"dw": dw,  "db": db}
    
    return grads, cost


def optimize(w, b, X, Y, num_iterations,
    learning_rate, print_cost = False):

    costs = []
    
    for i in range(num_iterations):
        # Cost and gradient calculation
        grads, cost = propagate(w,b,X,Y)
        
        # Retrieve derivatives from grads
        dw = grads["dw"]
        db = grads["db"]
        
        # update rule
        w = w - learning_rate*dw
        b = b - learning_rate*db
        
        # Record the costs
        if i % 100 == 0:
            costs.append(cost)
        
    params = {"w": w, "b": b}
    grads = {"dw": dw, "db": db}
    
    return params, grads, costs


def predict(w, b, X):
    A = sigmoid(np.dot(w.T, X) + b)
    
    for i in range(A.shape[1]):
        
        if (A[0,i] > 0.5):
            Y_prediction[0,i] = 1
        else:
            Y_prediction[0,i] = 0
    
    assert(Y_prediction.shape == (1, m))
    
    return Y_prediction


def model(X_train, Y_train, X_test,
    Y_test, num_iterations = 2000,
    learning_rate = 0.5, print_cost = False):

    # initialize parameters
    w, b = np.zeros((X_train.shape[0],1)), 0

    # Gradient descent (≈ 1 line of code)
    parameters, grads, costs = optimize(
        w,b,X_train, Y_train, num_iterations,
        learning_rate, print_cost)
    
    # Retrieve parameters w and b
    w = parameters["w"]
    b = parameters["b"]
    
    # Predict test/train set examples
    Y_prediction_test = predict(w, b, X_test)
    Y_prediction_train = predict(w,b,X_train)

# Print train/test Errors
    print("train accuracy: {} %".format(
        100 - np.mean(np.abs(
        Y_prediction_train - Y_train)) * 100))
    print("test accuracy: {} %".format(
        100 - np.mean(np.abs(
        Y_prediction_test - Y_test)) * 100))
    
    d = {"costs": costs,
         "Y_prediction_test": Y_prediction_test, 
         "Y_prediction_train" : Y_prediction_train, 
         "w" : w, 
         "b" : b,
         "learning_rate" : learning_rate,
         "num_iterations": num_iterations}
    
    return d


learning_rates = [0.01, 0.001, 0.0001]
models = {}
for i in learning_rates:
    print ("learning rate is: " + str(i))
    models[str(i)] = model(train_set_x,
        train_set_y, test_set_x, test_set_y,
        num_iterations = 1500,
        learning_rate = i, print_cost = False)
    print ('\n' + "----------------------------" + '\n')

for i in learning_rates:
    plt.plot(np.squeeze(models[str(i)]["costs"]),
    label= str(models[str(i)]["learning_rate"]))

Neural Network (Week 2) : Python Basics

import numpy as np
def sigmoid(x):
    s = 1/(1+np.exp(-x))
    return s

def sigmoid_derivative(x):
    s = 1/(1+np.exp(-x))
    ds = s*(1-s)
    return ds

def image2vector(image):
    v = image.reshape(image.shape[0]*
  image.shape[1]*image.shape[2], 1)
    return v

def normalizeRows(x):
    x_norm = np.linalg.norm(
  x, ord=2, axis=1, keepdims=True)
    x = x/x_norm
    return x

def softmax(x):
    x_exp = np.exp(x)
    x_sum = np.sum(x_exp, axis=1,
keepdims=True)
    s = x_exp / x_sum
    return s

def L1(yhat, y):
    loss = np.sum(abs(y-yhat))
    return loss

def L2(yhat, y):
loss = np.dot((y-yhat),(y-yhat))
    return loss

Neural Network (Week 3) : Summary

dA1 = W.t * dZ
    n-1,m = n-1,n * n,m
    Z = W*A1 에서

dW = (1/m) dZ * A1.t

Machine Learning (Week 3) : Logistic Regression

function g = sigmoid(z)
g = 1./(1.+exp(-z))

function [J, grad] = costFunction(theta, X, y)
A = sigmoid(X*theta);
J = (1/m) * sum(-y.*log(A) - (1.-y).*log(1.-A));
grad = (1/m) .* X' * (A - y);


function p = predict(theta, X)
p=round(sigmoid(X*theta));


%% Load Data
data = load('ex2data1.txt');
X = data(:, [1, 2]); y = data(:, 3);
[m, n] = size(X);

% Add intercept term to x and X_test
X = [ones(m, 1) X];

% Initialize fitting parameters
initial_theta = zeros(n + 1, 1);

%  Set options for fminunc
options = optimset('GradObj', 'on', 'MaxIter', 400);


%  Run fminunc to obtain the optimal theta
%  This function will return theta and the cost 

[theta, cost] = ...
 fminunc(@(t)(costFunction(t, X, y)), initial_theta, options);


function [J, grad] = costFunctionReg(theta, X, y, lambda)
A = sigmoid(X*theta);
theta1 = theta;
theta1(1,1) = 0;
reg = lambda/(2*m) * sum(theta1 .* theta1)
J = (1/m)*sum(-y.*log(A)-(1.-y).*log(1.-A)) + reg;


reg2 = lambda / (m) * theta;
reg2(1,1) = 0;
grad = (1/m).*X'*(A - y) + reg2;