#!/usr/bin/env python
# coding: utf-8

# In[2]:


# Import the TensorFlow and output the verion

get_ipython().system('pip install tensorflow==1.14.0')
import tensorflow as tf
print("\n\nTensorFlow version:", tf.__version__)


# # TensorFlow Implementation
# In TensorFlow, each input is typically represented as a 3D tensor of shape [`height`, `width`, `channels`]. A mini-batch is represented as a 4D tensor of shape [`mini-batch size`, `height`, `width`. `channels`]. The weights of a convolutional layer are represented as a tensor of shape [f$_{h}$, f$_{w}$, f$_{n}$, f$_{n'}$] The bias terms of a convolutional layer are simply represented as a 1D tensor of shape [f$_{n}$].<br>
# The following code loads two sample images, using Scikit-Learn's `load_sample_image()`. Then it creates two 7 X 7 filters (one with a verrtical white line and nother with a horizontal white line), and applites them to bothiamges using a convolutional layer built using TensorFlow's `tf.nn.conv2d()` function (with zero padding and a stride of 2). Finally, it plots one of the resulting feature maps:

# In[3]:


import numpy as np
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
from sklearn.datasets import load_sample_image

# Load sample images
china = load_sample_image("china.jpg")
flower = load_sample_image("flower.jpg")
dataset = np.array([china, flower], dtype=np.float32)
batch_size, height, width, channels = dataset.shape

# Create 2 filters
filters = np.zeros(shape=(7, 7, channels, 2), dtype=np.float32)
filters[:, 3, :, 0] = 1 # vertical line
filters[3, :, :, 1] = 1 # horizontal line

# Create a graph with input X plus a convolutional layer applying the 2 filters
X = tf.placeholder(tf.float32, shape=(None, height, width, channels), name="X")
convolution = tf.nn.conv2d(X, filters, strides=[1, 2, 2, 1], padding="SAME")

with tf.Session() as sess:
    output = sess.run(convolution, feed_dict={X: dataset})
    
plt.imshow(output[0, :, :, 1], cmap="gray") # Plot 1st image's 2nd feature map
plt.show()


# TensorFlow have a `tf.layers.conv2d()` function which creates the filters variable for you (called kernel), and initialized it randomly. For example, the following code creates an input placeholder followed by a convolutional layer with two 7 X 7 feature map, using 2 X 2 strides (note that this function only expects the vertical and horizontal strides), and "SAME" padding:

# In[3]:


tf.reset_default_graph()

china = load_sample_image("china.jpg")
flower = load_sample_image("flower.jpg")
dataset = np.array([china, flower], dtype=np.float32)
batch_size, height, width, channels = dataset.shape

X = tf.placeholder(tf.float32, shape=(None, height, width, channels), name="X")
conv = tf.layers.conv2d(X, filters=2, kernel_size=7, strides=[2, 2], padding="SAME")
init = tf.global_variables_initializer()

with tf.Session() as sess:
    init.run()
    output = sess.run(conv, feed_dict={X: dataset})

plt.imshow(output[0, :, :, 1], cmap="gray")
plt.show()


# # Pooling Layer
# Their goal is to subsample (i.e., shrink) the input image in order to reduce the computational load, the memory usage, and the number of parameters (thereby limiting the risk of overfitting). Reducing the input image size also makes the neural network tolerate a little bit of image shift (location invariance).<br>
# The following layer creates a max pooling layer using a 2 X 2 kernel, stride 2, and no padding, then applies it to all the iamges in the dataset:

# In[4]:


tf.reset_default_graph()

china = load_sample_image("china.jpg")
flower = load_sample_image("flower.jpg")
dataset = np.array([china, flower], dtype=np.float32)
batch_size, height, width, channels = dataset.shape

# Create a graph with input X plus a max pooling layer
X = tf.placeholder(tf.float32, shape=(None, height, width, channels), name="X")
max_pool = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")

with tf.Session() as sess:
    output = sess.run(max_pool, feed_dict={X: dataset})
    
plt.imshow(output[0].astype(np.uint8)) # Plot the output for 1st image
plt.show()


# - The `ksize` argument contains the kernel shape along all four dimensions of the input tensor: `[batch-size, height, width, channels]`. 
# - To create an average pooling layer, just use the `avg_pool()` function instead of `max_pool()`.

# # CNN on MNIST

# In[5]:


tf.reset_default_graph()

height = 28
width = 28
channels = 1
n_inputs = height * width

conv1_fmaps = 32
conv1_ksize = 3
conv1_stride = 1
conv1_pad = "SAME"

conv2_fmaps = 64
conv2_ksize = 3
conv2_stride = 1
conv2_pad = "SAME"
conv2_dropout_rate = 0.25

pool3_fmaps = conv2_fmaps

n_fc1 = 128
fc1_dropout_rate = 0.5

n_outputs = 10

with tf.name_scope("inputs"):
    X = tf.placeholder(tf.float32, shape=[None, n_inputs], name="X")
    X_reshaped = tf.reshape(X, shape=[-1, height, width, channels])
    y = tf.placeholder(tf.int32, shape=[None], name="y")
    training = tf.placeholder_with_default(False, shape=[], name="training")
    
conv1 = tf.layers.conv2d(X_reshaped, filters=conv1_fmaps, kernel_size=conv1_ksize, strides=conv1_stride,
                        padding=conv1_pad, activation=tf.nn.relu, name="conv1")
conv2 = tf.layers.conv2d(conv1, filters=conv2_fmaps, kernel_size=conv2_ksize, strides=conv2_stride,
                        padding=conv2_pad, activation=tf.nn.relu, name="conv2")

with tf.name_scope("pool3"):
    pool3 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")
    pool3_flat = tf.reshape(pool3, shape=[-1, pool3_fmaps * 14 * 14])
    pool3_flat_drop = tf.layers.dropout(pool3_flat, conv2_dropout_rate, training=training)
    
with tf.name_scope("fc1"):
    fc1 = tf.layers.dense(pool3_flat_drop, n_fc1, activation=tf.nn.relu, name="fc1")
    fc1_drop = tf.layers.dropout(fc1, fc1_dropout_rate, training=training)
    
with tf.name_scope("output"):
    logits = tf.layers.dense(fc1_drop, n_outputs, name="output")
    Y_proba = tf.nn.softmax(logits, name="Y_proba")
    
with tf.name_scope("train"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
    loss = tf.reduce_mean(xentropy)
    optimizer = tf.train.AdamOptimizer()
    training_op = optimizer.minimize(loss)

with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

with tf.name_scope("init_and_save"):
    init = tf.global_variables_initializer()
    saver = tf.train.Saver()


# The `get_model_params()` function gets the model's state (i.e., the value of all the variables), and the `restore_model_params()` restores a previous state. This is used to speed up early stopping: instead of storing the best model found so far to disk, we just save it to memory. At the end of training, we roll back to the best model found.

# In[ ]:


def get_model_params():
    gvars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
    return {gvar.op.name: value for gvar, value in zip(gvars, tf.get_default_session().run(gvars))}

def restore_model_params(model_params):
    gvar_names = list(model_params.keys())
    assign_ops = {gvar_name: tf.get_default_graph().get_operation_by_name(gvar_name + "/Assign")
                  for gvar_name in gvar_names}
    init_values = {gvar_name: assign_op.inputs[1] for gvar_name, assign_op in assign_ops.items()}
    feed_dict = {init_values[gvar_name]: model_params[gvar_name] for gvar_name in gvar_names}
    tf.get_default_session().run(assign_ops, feed_dict=feed_dict)


# In[7]:


(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()
X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0
X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0
y_train = y_train.astype(np.int32)
y_test = y_test.astype(np.int32)
X_valid, X_train = X_train[:5000], X_train[5000:]
y_valid, y_train = y_train[:5000], y_train[5000:]


# In[ ]:


def shuffle_batch(X, y, batch_size):
    rnd_idx = np.random.permutation(len(X))
    n_batches = len(X) // batch_size
    for batch_idx in np.array_split(rnd_idx, n_batches):
        X_batch, y_batch = X[batch_idx], y[batch_idx]
        yield X_batch, y_batch


# Now let's train the model! This implementation of Early Stopping works like this:
# - every 500 training iterations, it evaluates the model on the validation set,
# - if the model performs better than the best model found so far, then it saves the model to RAM,
# - if there is no progress for 100 evaluations in a row, then training is interrupted,
# - after training, the code restores the best model found.

# In[9]:


n_epochs = 1000
batch_size = 64
iteration = 0

best_loss_val = np.infty
check_interval = 500
checks_since_last_progress = 0
max_checks_without_progress = 20
best_model_params = None

with tf.Session() as sess:
    init.run()
    for epoch in range(n_epochs):
        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):
            iteration += 1
            sess.run(training_op, feed_dict={X: X_batch, y: y_batch, training: True})
            if iteration % check_interval == 0:
                loss_val = loss.eval(feed_dict={X: X_valid, y: y_valid})
                if loss_val < best_loss_val:
                    best_loss_val = loss_val
                    checks_since_last_progress = 0
                    best_model_params = get_model_params()
                else:
                    checks_since_last_progress += 1
            
        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
        acc_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})
        print("Epoch {}, last batch accuracy: {:.4f}%, valid accuracy: {:.4f}%, valid best loss: {:.6f}".format(epoch, acc_batch * 100, acc_val * 100, best_loss_val))
        if checks_since_last_progress > max_checks_without_progress:
            print("Early stopping!")
            break
        
    if best_model_params:
        restore_model_params(best_model_params)
    acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})
    print("Final accuracy on test set:", acc_test)
    save_path = saver.save(sess, "./my_mnist_model")


# # Classifying large images using Inception

# **Exercise:** Download some images of various animals. Load them in Python, for example using the matplotlib.image.mpimg.imread() function or the scipy.misc.imread() function. Resize and/or crop them to 299 × 299 pixels, and ensure that they have just three channels (RGB), with no transparency channel. The images that the Inception model was trained on were preprocessed so that their values range from -1.0 to 1.0, so you must ensure that your images do too.

# In[ ]:


tf.reset_default_graph()

width = 299
height = 299
channels = 3


# In[57]:


import matplotlib.image as mpimg
test_image = mpimg.imread(os.path.join("rsz_dog.jpg"))[:, :, :channels]
plt.imshow(test_image)
plt.axis("off")
plt.show()


# In[ ]:


test_image = 2 * test_image - 1


# **Exercise:** Download the latest pretrained Inception v4 model at 
# http://download.tensorflow.org/models/inception_v4_2016_09_09.tar.gz.<br>
# The list of class names is available at https://goo.gl/brXRtZ, but you must insert a "background" class at the beginning.
# 

# In[ ]:


import os
import sys
import tarfile
from six.moves import urllib

INCEPTION_V3_URL = "http://download.tensorflow.org/models/inception_v3_2016_08_28.tar.gz"
INCEPTION_PATH = os.path.join("datasets", "inception")
INCEPTION_V3_CHECKPOINT_PATH = os.path.join(INCEPTION_PATH, "inception_v3.ckpt")

def download_progress(count, block_size, total_size):
  percent = count * block_size * 100 // total_size
  sys.stdout.write("\rDownloading: {}%".format(percent))
  sys.stdout.flush()

def fetch_pretrained_model(url=INCEPTION_V4_URL, path=INCEPTION_PATH):
  if os.path.exists(INCEPTION_V4_CHECKPOINT_PATH):
    return
  os.makedirs(path, exist_ok=True)
  tgz_path = os.path.join(path, "inception_v3.tgz")
  urllib.request.urlretrieve(url, tgz_path, reporthook=download_progress)
  inception_tgz = tarfile.open(tgz_path)
  inception_tgz.extractall(path=path)
  inception_tgz.close()
  os.remove(tgz_path)


# In[22]:


fetch_pretrained_model()


# In[23]:


import re

CLASS_NAME_REGEX = re.compile(r"^n\d+\s+(.*)\s*$", re.M | re.U)

def load_class_names():
  path = os.path.join("datasets", "inception", "imagenet_class_names.txt")
  with open(path, encoding="utf=8") as f:
    content = f.read()
    return CLASS_NAME_REGEX.findall(content)

class_names = ["background"] + load_class_names() 
class_names[:5]


# **Exercise:** Create the Inception v4 model by calling the inception_v4() function, as shown below. This must be done within an argument scope created by the inception_v4_arg_scope() function. Also, you must set is_training=False and num_classes=1001 [...]

# In[36]:


from tensorflow.contrib.slim.nets import inception
import tensorflow.contrib.slim as slim

tf.reset_default_graph()

X = tf.placeholder(tf.float32, shape=[None, height, width, channels], name="X")
with slim.arg_scope(inception.inception_v3_arg_scope()):
  logits, end_points = inception.inception_v3(X, num_classes=1001, is_training=False)

predictions = end_points["Predictions"]
saver = tf.train.Saver()


# **Exercise:** Open a session and use the Saver to restore the pretrained model checkpoint you downloaded earlier.
# 

# In[38]:


with tf.Session() as sess:
  saver.restore(sess, INCEPTION_V3_CHECKPOINT_PATH)


# **Exercise:**Run the model to classify the images you prepared.

# In[59]:


X_test = test_image.reshape(-1, height, width, channels)

with tf.Session() as sess:
  saver.restore(sess, INCEPTION_V4_CHECKPOINT_PATH)
  predictions_val = predictions.eval(feed_dict={X: X_test})


# In[60]:


most_likely_class_index = np.argmax(predictions_val[0])
most_likely_class_index


# In[61]:


class_names[most_likely_class_index]


# In[62]:


top_5 = np.argpartition(predictions_val[0], -5)[-5:]
top_5 = reversed(top_5[np.argsort(predictions_val[0][top_5])])
for i in top_5:
    print("{0}: {1:.2f}%".format(class_names[i], 100 * predictions_val[0][i]))


# # Transfer Learning for Large Image Classification

# **Exercise:** Create a training set containing at least 100 images per class. For example, you could classify your own pictures based on the location (beach, mountain, city, etc.), or alternatively you can just use an existing dataset, such as the flowers dataset or MIT's places dataset (requires registration, and it is huge).

# In[ ]:


import sys
import tarfile
from six.moves import urllib

FLOWERS_URL = "http://download.tensorflow.org/example_images/flower_photos.tgz"
FLOWERS_PATH = os.path.join("datasets", "flowers")

def fetch_flowers(url=FLOWERS_URL, path=FLOWERS_PATH):
    if os.path.exists(FLOWERS_PATH):
        return
    os.makedirs(path, exist_ok=True)
    tgz_path = os.path.join(path, "flower_photos.tgz")
    urllib.request.urlretrieve(url, tgz_path, reporthook=download_progress)
    flowers_tgz = tarfile.open(tgz_path)
    flowers_tgz.extractall(path=path)
    flowers_tgz.close()


# In[64]:


fetch_flowers()


# In[65]:


flowers_root_path = os.path.join(FLOWERS_PATH, "flower_photos")
flower_classes = sorted([dirname for dirname in os.listdir(flowers_root_path)
                  if os.path.isdir(os.path.join(flowers_root_path, dirname))])
flower_classes


# In[ ]:


# Let's get the list of all image paths for each class:
from collections import defaultdict

image_paths = defaultdict(list)

for flower_class in flower_classes:
    image_dir = os.path.join(flowers_root_path, flower_class)
    for filepath in os.listdir(image_dir):
        if filepath.endswith(".jpg"):
            image_paths[flower_class].append(os.path.join(image_dir, filepath))


# In[ ]:


# Sort the image paths
for paths in image_paths.values():
    paths.sort()


# In[69]:


# Let's take a peek at the first few images from each class:
n_examples_per_class = 2

for flower_class in flower_classes:
    print("Class:", flower_class)
    plt.figure(figsize=(10,5))
    for index, example_image_path in enumerate(image_paths[flower_class][:n_examples_per_class]):
        example_image = mpimg.imread(example_image_path)[:, :, :channels]
        plt.subplot(100 + n_examples_per_class * 10 + index + 1)
        plt.title("{}x{}".format(example_image.shape[1], example_image.shape[0]))
        plt.imshow(example_image)
        plt.axis("off")
    plt.show()


# **Exercise:** Write a preprocessing step that will resize and crop the image to 299 × 299, with some randomness for data augmentation.

# In[ ]:


from skimage.transform import resize

def prepare_image(image, target_width = 299, target_height = 299, max_zoom = 0.2):
    """Zooms and crops the image randomly for data augmentation."""

    # First, let's find the largest bounding box with the target size ratio that fits within the image
    height = image.shape[0]
    width = image.shape[1]
    image_ratio = width / height
    target_image_ratio = target_width / target_height
    crop_vertically = image_ratio < target_image_ratio
    crop_width = width if crop_vertically else int(height * target_image_ratio)
    crop_height = int(width / target_image_ratio) if crop_vertically else height
        
    # Now let's shrink this bounding box by a random factor (dividing the dimensions by a random number
    # between 1.0 and 1.0 + `max_zoom`.
    resize_factor = np.random.rand() * max_zoom + 1.0
    crop_width = int(crop_width / resize_factor)
    crop_height = int(crop_height / resize_factor)
    
    # Next, we can select a random location on the image for this bounding box.
    x0 = np.random.randint(0, width - crop_width)
    y0 = np.random.randint(0, height - crop_height)
    x1 = x0 + crop_width
    y1 = y0 + crop_height
    
    # Let's crop the image using the random bounding box we built.
    image = image[y0:y1, x0:x1]

    # Let's also flip the image horizontally with 50% probability:
    if np.random.rand() < 0.5:
        image = np.fliplr(image)

    # Now, let's resize the image to the target dimensions.
    # The resize function of scikit-image will automatically transform the image to floats ranging from 0.0 to 1.0
    image = resize(image, (target_width, target_height))
    
    # Finally, let's ensure that the colors are represented as 32-bit floats:
    return image.astype(np.float32)


# In[71]:


# Let's check out the result on each image
plt.figure(figsize=(6, 8))
plt.imshow(example_image)
plt.title("{}x{}".format(example_image.shape[1], example_image.shape[0]))
plt.axis("off")
plt.show()


# In[72]:


prepared_image = prepare_image(example_image)

plt.figure(figsize=(8, 8))
plt.imshow(prepared_image)
plt.title("{}x{}".format(prepared_image.shape[1], prepared_image.shape[0]))
plt.axis("off")
plt.show()


# In[73]:


# Let's look at a few other random images generated from the same original image:
rows, cols = 2, 3

plt.figure(figsize=(14, 8))
for row in range(rows):
    for col in range(cols):
        prepared_image = prepare_image(example_image)
        plt.subplot(rows, cols, row * cols + col + 1)
        plt.title("{}x{}".format(prepared_image.shape[1], prepared_image.shape[0]))
        plt.imshow(prepared_image)
        plt.axis("off")
plt.show()


# Alternatively it is also possible to implement this step directly with TensorFlow, using the functions in tf.image module. <br>
# It does not actually perform the image transformation, but rather creates a set of TensorFlow operations that will perform the transformation when you run the graph.

# In[ ]:


def prepare_image_with_tensorflow(image, target_width = 299, target_height = 299, max_zoom = 0.2):
    """Zooms and crops the image randomly for data augmentation."""

    # First, let's find the largest bounding box with the target size ratio that fits within the image
    image_shape = tf.cast(tf.shape(image), tf.float32)
    height = image_shape[0]
    width = image_shape[1]
    image_ratio = width / height
    target_image_ratio = target_width / target_height
    crop_vertically = image_ratio < target_image_ratio
    crop_width = tf.cond(crop_vertically,
                         lambda: width,
                         lambda: height * target_image_ratio)
    crop_height = tf.cond(crop_vertically,
                          lambda: width / target_image_ratio,
                          lambda: height)

    # Now let's shrink this bounding box by a random factor (dividing the dimensions by a random number
    # between 1.0 and 1.0 + `max_zoom`.
    resize_factor = tf.random_uniform(shape=[], minval=1.0, maxval=1.0 + max_zoom)
    crop_width = tf.cast(crop_width / resize_factor, tf.int32)
    crop_height = tf.cast(crop_height / resize_factor, tf.int32)
    box_size = tf.stack([crop_height, crop_width, 3])   # 3 = number of channels

    # Let's crop the image using a random bounding box of the size we computed
    image = tf.random_crop(image, box_size)

    # Let's also flip the image horizontally with 50% probability:
    image = tf.image.random_flip_left_right(image)

    # The resize_bilinear function requires a 4D tensor (a batch of images)
    # so we need to expand the number of dimensions first:
    image_batch = tf.expand_dims(image, 0)

    # Finally, let's resize the image to the target dimensions. Note that this function
    # returns a float32 tensor.
    image_batch = tf.image.resize_bilinear(image_batch, [target_height, target_width])
    image = image_batch[0] / 255  # back to a single image, and scale the colors from 0.0 to 1.0
    return image


# In[76]:


# Let's test this function
tf.reset_default_graph()

input_image = tf.placeholder(tf.uint8, shape=[None, None, 3])
prepared_image_op = prepare_image_with_tensorflow(input_image)

with tf.Session():
    prepared_image = prepared_image_op.eval(feed_dict={input_image: example_image})
    
plt.figure(figsize=(6, 6))
plt.imshow(prepared_image)
plt.title("{}x{}".format(prepared_image.shape[1], prepared_image.shape[0]))
plt.axis("off")
plt.show()


# **Exercise:** Using the pretrained Inception v3 model from the previous exercise, freeze all layers up to the bottleneck layer (i.e., the last layer before the output layer), and replace the output layer with the appropriate number of outputs for your new classification task (e.g., the flowers dataset has five mutually exclusive classes so the output layer must have five neurons and use the softmax activation function).<br>
# Let's start by fetching the inception v3 graph again. This time, let's use a training placeholder that we will use to tell TensorFlow whether we are training the network or not (this is needed by operations such as dropout and batch normalization).

# In[78]:


from tensorflow.contrib.slim.nets import inception
import tensorflow.contrib.slim as slim

tf.reset_default_graph()

X = tf.placeholder(tf.float32, shape=[None, height, width, channels], name="X")
training = tf.placeholder_with_default(False, shape=[])
with slim.arg_scope(inception.inception_v3_arg_scope()):
    logits, end_points = inception.inception_v3(X, num_classes=1001, is_training=training)

inception_saver = tf.train.Saver()


# Now we need to find the point in the graph where we should attach the new output layer. It should be the layer right before the current output layer. One way to do this is to explore the output layer's inputs:
# 

# In[79]:


logits.op.inputs[0]


# Nope, that's part of the output layer (adding the biases). Let's continue walking backwards in the graph:

# In[80]:


logits.op.inputs[0].op.inputs[0]


# That's also part of the output layer, it's the final layer in the inception layer (if you are not sure you can visualize the graph using TensorBoard). Once again, let's continue walking backwards in the graph:

# In[81]:


logits.op.inputs[0].op.inputs[0].op.inputs[0]


# There we are, this is the output of the dropout layer. This is the very last layer before the output layer in the Inception v3 network, so that's the layer we need to build upon. Note that there was actually a simpler way to find this layer: the inception_v3() function returns a dict of end points:

# In[82]:


end_points


# In[84]:


end_points["PreLogits"]


# We can drop the 2nd and 3rd dimensions using the tf.squeeze() function:

# In[ ]:


prelogits = tf.squeeze(end_points["PreLogits"], axis=[1, 2])


# Then we can add the final fully connected layer on top of this layer:

# In[86]:


n_outputs = len(flower_classes)

with tf.name_scope("new_output_layer"):
    flower_logits = tf.layers.dense(prelogits, n_outputs, name="flower_logits")
    Y_proba = tf.nn.softmax(flower_logits, name="Y_proba")


# There is one important detail, however: since we want to train only the output layer (all other layers must be frozen), we must pass the list of variables to train to the optimizer's minimize() method:

# In[ ]:


y = tf.placeholder(tf.int32, shape=[None])

with tf.name_scope("train"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=flower_logits, labels=y)
    loss = tf.reduce_mean(xentropy)
    optimizer = tf.train.AdamOptimizer()
    flower_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope="flower_logits")
    training_op = optimizer.minimize(loss, var_list=flower_vars)

with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(flower_logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

with tf.name_scope("init_and_save"):
    init = tf.global_variables_initializer()
    saver = tf.train.Saver()


# In[88]:


[v.name for v in flower_vars]


# Notice that we created the inception_saver before adding the new output layer: we will use this saver to restore the pretrained model state, so we don't want it to try to restore new variables (it would just fail saying it does not know the new variables). The second saver will be used to save the final flower model, including both the pretrained variables and the new ones.

# **Exercise:** Split your dataset into a training set and a test set. Train the model on the training set and evaluate it on the test set.

# In[89]:


flower_class_ids = {flower_class: index for index, flower_class in enumerate(flower_classes)}
flower_class_ids


# In[90]:


flower_paths_and_classes = []
for flower_class, paths in image_paths.items():
    for path in paths:
        flower_paths_and_classes.append((path, flower_class_ids[flower_class]))

# Next, lets shuffle the dataset and split it into the training set and the test set:
test_ratio = 0.2
train_size = int(len(flower_paths_and_classes) * (1 - test_ratio))

np.random.shuffle(flower_paths_and_classes)

flower_paths_and_classes_train = flower_paths_and_classes[:train_size]
flower_paths_and_classes_test = flower_paths_and_classes[train_size:]

# Let's look at the first 3 instances in the training set:
flower_paths_and_classes_train[:3]


# Next, we will also need a function to preprocess a set of images. This function will be useful to preprocess the test set, and also to create batches during training. For simplicity, we will use the NumPy/SciPy implementation:

# In[ ]:


from random import sample

def prepare_batch(flower_paths_and_classes, batch_size):
    batch_paths_and_classes = sample(flower_paths_and_classes, batch_size)
    images = [mpimg.imread(path)[:, :, :channels] for path, labels in batch_paths_and_classes]
    prepared_images = [prepare_image(image) for image in images]
    X_batch = 2 * np.stack(prepared_images) - 1 # Inception expects colors ranging from -1 to 1
    y_batch = np.array([labels for path, labels in batch_paths_and_classes], dtype=np.int32)
    return X_batch, y_batch

X_batch, y_batch = prepare_batch(flower_paths_and_classes_train, batch_size=4)


# In[92]:


X_batch.shape


# In[93]:


X_batch.dtype


# In[94]:


y_batch.shape


# In[95]:


y_batch.dtype


# In[ ]:


# Prepare the test set
X_test, y_test = prepare_batch(flower_paths_and_classes_test, batch_size=len(flower_paths_and_classes_test))


# In[97]:


X_test.shape


# And now, we are ready to train the network (or more precisely, the output layer we just added, since all the other layers are frozen).

# In[99]:


n_epochs = 10
batch_size = 64
n_iterations_per_epoch = len(flower_paths_and_classes_train) // batch_size

with tf.Session() as sess:
    init.run()
    inception_saver.restore(sess, INCEPTION_V4_CHECKPOINT_PATH)

    for epoch in range(n_epochs):
        print("Epoch", epoch, end="")
        for iteration in range(n_iterations_per_epoch):
            print(".", end="")
            X_batch, y_batch = prepare_batch(flower_paths_and_classes_train, batch_size)
            sess.run(training_op, feed_dict={X: X_batch, y: y_batch, training: True})

        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
        print("  Last batch accuracy:", acc_batch)

        save_path = saver.save(sess, "./my_flowers_model")


# In[100]:


n_test_batches = 10
X_test_batches = np.array_split(X_test, n_test_batches)
y_test_batches = np.array_split(y_test, n_test_batches)

with tf.Session() as sess:
    saver.restore(sess, "./my_flowers_model")

    print("Computing final accuracy on the test set (this will take a while)...")
    acc_test = np.mean([
        accuracy.eval(feed_dict={X: X_test_batch, y: y_test_batch})
        for X_test_batch, y_test_batch in zip(X_test_batches, y_test_batches)])
    print("Test accuracy:", acc_test)


#!/usr/bin/env python
# coding: utf-8

# In[2]:


# Import the TensorFlow and output the verion

get_ipython().system('pip install tensorflow==1.14.0')
import tensorflow as tf
print("\n\nTensorFlow version:", tf.__version__)


# # TensorFlow Implementation
# In TensorFlow, each input is typically represented as a 3D tensor of shape [`height`, `width`, `channels`]. A mini-batch is represented as a 4D tensor of shape [`mini-batch size`, `height`, `width`. `channels`]. The weights of a convolutional layer are represented as a tensor of shape [f$_{h}$, f$_{w}$, f$_{n}$, f$_{n'}$] The bias terms of a convolutional layer are simply represented as a 1D tensor of shape [f$_{n}$].<br>
# The following code loads two sample images, using Scikit-Learn's `load_sample_image()`. Then it creates two 7 X 7 filters (one with a verrtical white line and nother with a horizontal white line), and applites them to bothiamges using a convolutional layer built using TensorFlow's `tf.nn.conv2d()` function (with zero padding and a stride of 2). Finally, it plots one of the resulting feature maps:

# In[3]:


import numpy as np
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
from sklearn.datasets import load_sample_image

# Load sample images
china = load_sample_image("china.jpg")
flower = load_sample_image("flower.jpg")
dataset = np.array([china, flower], dtype=np.float32)
batch_size, height, width, channels = dataset.shape

# Create 2 filters
filters = np.zeros(shape=(7, 7, channels, 2), dtype=np.float32)
filters[:, 3, :, 0] = 1 # vertical line
filters[3, :, :, 1] = 1 # horizontal line

# Create a graph with input X plus a convolutional layer applying the 2 filters
X = tf.placeholder(tf.float32, shape=(None, height, width, channels), name="X")
convolution = tf.nn.conv2d(X, filters, strides=[1, 2, 2, 1], padding="SAME")

with tf.Session() as sess:
    output = sess.run(convolution, feed_dict={X: dataset})
    
plt.imshow(output[0, :, :, 1], cmap="gray") # Plot 1st image's 2nd feature map
plt.show()


# TensorFlow have a `tf.layers.conv2d()` function which creates the filters variable for you (called kernel), and initialized it randomly. For example, the following code creates an input placeholder followed by a convolutional layer with two 7 X 7 feature map, using 2 X 2 strides (note that this function only expects the vertical and horizontal strides), and "SAME" padding:

# In[3]:


tf.reset_default_graph()

china = load_sample_image("china.jpg")
flower = load_sample_image("flower.jpg")
dataset = np.array([china, flower], dtype=np.float32)
batch_size, height, width, channels = dataset.shape

X = tf.placeholder(tf.float32, shape=(None, height, width, channels), name="X")
conv = tf.layers.conv2d(X, filters=2, kernel_size=7, strides=[2, 2], padding="SAME")
init = tf.global_variables_initializer()

with tf.Session() as sess:
    init.run()
    output = sess.run(conv, feed_dict={X: dataset})

plt.imshow(output[0, :, :, 1], cmap="gray")
plt.show()


# # Pooling Layer
# Their goal is to subsample (i.e., shrink) the input image in order to reduce the computational load, the memory usage, and the number of parameters (thereby limiting the risk of overfitting). Reducing the input image size also makes the neural network tolerate a little bit of image shift (location invariance).<br>
# The following layer creates a max pooling layer using a 2 X 2 kernel, stride 2, and no padding, then applies it to all the iamges in the dataset:

# In[4]:


tf.reset_default_graph()

china = load_sample_image("china.jpg")
flower = load_sample_image("flower.jpg")
dataset = np.array([china, flower], dtype=np.float32)
batch_size, height, width, channels = dataset.shape

# Create a graph with input X plus a max pooling layer
X = tf.placeholder(tf.float32, shape=(None, height, width, channels), name="X")
max_pool = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")

with tf.Session() as sess:
    output = sess.run(max_pool, feed_dict={X: dataset})
    
plt.imshow(output[0].astype(np.uint8)) # Plot the output for 1st image
plt.show()


# - The `ksize` argument contains the kernel shape along all four dimensions of the input tensor: `[batch-size, height, width, channels]`. 
# - To create an average pooling layer, just use the `avg_pool()` function instead of `max_pool()`.

# # CNN on MNIST

# In[5]:


tf.reset_default_graph()

height = 28
width = 28
channels = 1
n_inputs = height * width

conv1_fmaps = 32
conv1_ksize = 3
conv1_stride = 1
conv1_pad = "SAME"

conv2_fmaps = 64
conv2_ksize = 3
conv2_stride = 1
conv2_pad = "SAME"
conv2_dropout_rate = 0.25

pool3_fmaps = conv2_fmaps

n_fc1 = 128
fc1_dropout_rate = 0.5

n_outputs = 10

with tf.name_scope("inputs"):
    X = tf.placeholder(tf.float32, shape=[None, n_inputs], name="X")
    X_reshaped = tf.reshape(X, shape=[-1, height, width, channels])
    y = tf.placeholder(tf.int32, shape=[None], name="y")
    training = tf.placeholder_with_default(False, shape=[], name="training")
    
conv1 = tf.layers.conv2d(X_reshaped, filters=conv1_fmaps, kernel_size=conv1_ksize, strides=conv1_stride,
                        padding=conv1_pad, activation=tf.nn.relu, name="conv1")
conv2 = tf.layers.conv2d(conv1, filters=conv2_fmaps, kernel_size=conv2_ksize, strides=conv2_stride,
                        padding=conv2_pad, activation=tf.nn.relu, name="conv2")

with tf.name_scope("pool3"):
    pool3 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")
    pool3_flat = tf.reshape(pool3, shape=[-1, pool3_fmaps * 14 * 14])
    pool3_flat_drop = tf.layers.dropout(pool3_flat, conv2_dropout_rate, training=training)
    
with tf.name_scope("fc1"):
    fc1 = tf.layers.dense(pool3_flat_drop, n_fc1, activation=tf.nn.relu, name="fc1")
    fc1_drop = tf.layers.dropout(fc1, fc1_dropout_rate, training=training)
    
with tf.name_scope("output"):
    logits = tf.layers.dense(fc1_drop, n_outputs, name="output")
    Y_proba = tf.nn.softmax(logits, name="Y_proba")
    
with tf.name_scope("train"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
    loss = tf.reduce_mean(xentropy)
    optimizer = tf.train.AdamOptimizer()
    training_op = optimizer.minimize(loss)

with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

with tf.name_scope("init_and_save"):
    init = tf.global_variables_initializer()
    saver = tf.train.Saver()


# The `get_model_params()` function gets the model's state (i.e., the value of all the variables), and the `restore_model_params()` restores a previous state. This is used to speed up early stopping: instead of storing the best model found so far to disk, we just save it to memory. At the end of training, we roll back to the best model found.

# In[ ]:


def get_model_params():
    gvars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
    return {gvar.op.name: value for gvar, value in zip(gvars, tf.get_default_session().run(gvars))}

def restore_model_params(model_params):
    gvar_names = list(model_params.keys())
    assign_ops = {gvar_name: tf.get_default_graph().get_operation_by_name(gvar_name + "/Assign")
                  for gvar_name in gvar_names}
    init_values = {gvar_name: assign_op.inputs[1] for gvar_name, assign_op in assign_ops.items()}
    feed_dict = {init_values[gvar_name]: model_params[gvar_name] for gvar_name in gvar_names}
    tf.get_default_session().run(assign_ops, feed_dict=feed_dict)


# In[7]:


(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()
X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0
X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0
y_train = y_train.astype(np.int32)
y_test = y_test.astype(np.int32)
X_valid, X_train = X_train[:5000], X_train[5000:]
y_valid, y_train = y_train[:5000], y_train[5000:]


# In[ ]:


def shuffle_batch(X, y, batch_size):
    rnd_idx = np.random.permutation(len(X))
    n_batches = len(X) // batch_size
    for batch_idx in np.array_split(rnd_idx, n_batches):
        X_batch, y_batch = X[batch_idx], y[batch_idx]
        yield X_batch, y_batch


# Now let's train the model! This implementation of Early Stopping works like this:
# - every 500 training iterations, it evaluates the model on the validation set,
# - if the model performs better than the best model found so far, then it saves the model to RAM,
# - if there is no progress for 100 evaluations in a row, then training is interrupted,
# - after training, the code restores the best model found.

# In[9]:


n_epochs = 1000
batch_size = 64
iteration = 0

best_loss_val = np.infty
check_interval = 500
checks_since_last_progress = 0
max_checks_without_progress = 20
best_model_params = None

with tf.Session() as sess:
    init.run()
    for epoch in range(n_epochs):
        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):
            iteration += 1
            sess.run(training_op, feed_dict={X: X_batch, y: y_batch, training: True})
            if iteration % check_interval == 0:
                loss_val = loss.eval(feed_dict={X: X_valid, y: y_valid})
                if loss_val < best_loss_val:
                    best_loss_val = loss_val
                    checks_since_last_progress = 0
                    best_model_params = get_model_params()
                else:
                    checks_since_last_progress += 1
            
        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
        acc_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})
        print("Epoch {}, last batch accuracy: {:.4f}%, valid accuracy: {:.4f}%, valid best loss: {:.6f}".format(epoch, acc_batch * 100, acc_val * 100, best_loss_val))
        if checks_since_last_progress > max_checks_without_progress:
            print("Early stopping!")
            break
        
    if best_model_params:
        restore_model_params(best_model_params)
    acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})
    print("Final accuracy on test set:", acc_test)
    save_path = saver.save(sess, "./my_mnist_model")


# # Classifying large images using Inception

# **Exercise:** Download some images of various animals. Load them in Python, for example using the matplotlib.image.mpimg.imread() function or the scipy.misc.imread() function. Resize and/or crop them to 299 × 299 pixels, and ensure that they have just three channels (RGB), with no transparency channel. The images that the Inception model was trained on were preprocessed so that their values range from -1.0 to 1.0, so you must ensure that your images do too.

# In[ ]:


tf.reset_default_graph()

width = 299
height = 299
channels = 3


# In[57]:


import matplotlib.image as mpimg
test_image = mpimg.imread(os.path.join("rsz_dog.jpg"))[:, :, :channels]
plt.imshow(test_image)
plt.axis("off")
plt.show()


# In[ ]:


test_image = 2 * test_image - 1


# **Exercise:** Download the latest pretrained Inception v4 model at 
# http://download.tensorflow.org/models/inception_v4_2016_09_09.tar.gz.<br>
# The list of class names is available at https://goo.gl/brXRtZ, but you must insert a "background" class at the beginning.
# 

# In[ ]:


import os
import sys
import tarfile
from six.moves import urllib

INCEPTION_V3_URL = "http://download.tensorflow.org/models/inception_v3_2016_08_28.tar.gz"
INCEPTION_PATH = os.path.join("datasets", "inception")
INCEPTION_V3_CHECKPOINT_PATH = os.path.join(INCEPTION_PATH, "inception_v3.ckpt")

def download_progress(count, block_size, total_size):
  percent = count * block_size * 100 // total_size
  sys.stdout.write("\rDownloading: {}%".format(percent))
  sys.stdout.flush()

def fetch_pretrained_model(url=INCEPTION_V4_URL, path=INCEPTION_PATH):
  if os.path.exists(INCEPTION_V4_CHECKPOINT_PATH):
    return
  os.makedirs(path, exist_ok=True)
  tgz_path = os.path.join(path, "inception_v3.tgz")
  urllib.request.urlretrieve(url, tgz_path, reporthook=download_progress)
  inception_tgz = tarfile.open(tgz_path)
  inception_tgz.extractall(path=path)
  inception_tgz.close()
  os.remove(tgz_path)


# In[22]:


fetch_pretrained_model()


# In[23]:


import re

CLASS_NAME_REGEX = re.compile(r"^n\d+\s+(.*)\s*$", re.M | re.U)

def load_class_names():
  path = os.path.join("datasets", "inception", "imagenet_class_names.txt")
  with open(path, encoding="utf=8") as f:
    content = f.read()
    return CLASS_NAME_REGEX.findall(content)

class_names = ["background"] + load_class_names() 
class_names[:5]


# **Exercise:** Create the Inception v4 model by calling the inception_v4() function, as shown below. This must be done within an argument scope created by the inception_v4_arg_scope() function. Also, you must set is_training=False and num_classes=1001 [...]

# In[36]:


from tensorflow.contrib.slim.nets import inception
import tensorflow.contrib.slim as slim

tf.reset_default_graph()

X = tf.placeholder(tf.float32, shape=[None, height, width, channels], name="X")
with slim.arg_scope(inception.inception_v3_arg_scope()):
  logits, end_points = inception.inception_v3(X, num_classes=1001, is_training=False)

predictions = end_points["Predictions"]
saver = tf.train.Saver()


# **Exercise:** Open a session and use the Saver to restore the pretrained model checkpoint you downloaded earlier.
# 

# In[38]:


with tf.Session() as sess:
  saver.restore(sess, INCEPTION_V3_CHECKPOINT_PATH)


# **Exercise:**Run the model to classify the images you prepared.

# In[59]:


X_test = test_image.reshape(-1, height, width, channels)

with tf.Session() as sess:
  saver.restore(sess, INCEPTION_V4_CHECKPOINT_PATH)
  predictions_val = predictions.eval(feed_dict={X: X_test})


# In[60]:


most_likely_class_index = np.argmax(predictions_val[0])
most_likely_class_index


# In[61]:


class_names[most_likely_class_index]


# In[62]:


top_5 = np.argpartition(predictions_val[0], -5)[-5:]
top_5 = reversed(top_5[np.argsort(predictions_val[0][top_5])])
for i in top_5:
    print("{0}: {1:.2f}%".format(class_names[i], 100 * predictions_val[0][i]))


# # Transfer Learning for Large Image Classification

# **Exercise:** Create a training set containing at least 100 images per class. For example, you could classify your own pictures based on the location (beach, mountain, city, etc.), or alternatively you can just use an existing dataset, such as the flowers dataset or MIT's places dataset (requires registration, and it is huge).

# In[ ]:


import sys
import tarfile
from six.moves import urllib

FLOWERS_URL = "http://download.tensorflow.org/example_images/flower_photos.tgz"
FLOWERS_PATH = os.path.join("datasets", "flowers")

def fetch_flowers(url=FLOWERS_URL, path=FLOWERS_PATH):
    if os.path.exists(FLOWERS_PATH):
        return
    os.makedirs(path, exist_ok=True)
    tgz_path = os.path.join(path, "flower_photos.tgz")
    urllib.request.urlretrieve(url, tgz_path, reporthook=download_progress)
    flowers_tgz = tarfile.open(tgz_path)
    flowers_tgz.extractall(path=path)
    flowers_tgz.close()


# In[64]:


fetch_flowers()


# In[65]:


flowers_root_path = os.path.join(FLOWERS_PATH, "flower_photos")
flower_classes = sorted([dirname for dirname in os.listdir(flowers_root_path)
                  if os.path.isdir(os.path.join(flowers_root_path, dirname))])
flower_classes


# In[ ]:


# Let's get the list of all image paths for each class:
from collections import defaultdict

image_paths = defaultdict(list)

for flower_class in flower_classes:
    image_dir = os.path.join(flowers_root_path, flower_class)
    for filepath in os.listdir(image_dir):
        if filepath.endswith(".jpg"):
            image_paths[flower_class].append(os.path.join(image_dir, filepath))


# In[ ]:


# Sort the image paths
for paths in image_paths.values():
    paths.sort()


# In[69]:


# Let's take a peek at the first few images from each class:
n_examples_per_class = 2

for flower_class in flower_classes:
    print("Class:", flower_class)
    plt.figure(figsize=(10,5))
    for index, example_image_path in enumerate(image_paths[flower_class][:n_examples_per_class]):
        example_image = mpimg.imread(example_image_path)[:, :, :channels]
        plt.subplot(100 + n_examples_per_class * 10 + index + 1)
        plt.title("{}x{}".format(example_image.shape[1], example_image.shape[0]))
        plt.imshow(example_image)
        plt.axis("off")
    plt.show()


# **Exercise:** Write a preprocessing step that will resize and crop the image to 299 × 299, with some randomness for data augmentation.

# In[ ]:


from skimage.transform import resize

def prepare_image(image, target_width = 299, target_height = 299, max_zoom = 0.2):
    """Zooms and crops the image randomly for data augmentation."""

    # First, let's find the largest bounding box with the target size ratio that fits within the image
    height = image.shape[0]
    width = image.shape[1]
    image_ratio = width / height
    target_image_ratio = target_width / target_height
    crop_vertically = image_ratio < target_image_ratio
    crop_width = width if crop_vertically else int(height * target_image_ratio)
    crop_height = int(width / target_image_ratio) if crop_vertically else height
        
    # Now let's shrink this bounding box by a random factor (dividing the dimensions by a random number
    # between 1.0 and 1.0 + `max_zoom`.
    resize_factor = np.random.rand() * max_zoom + 1.0
    crop_width = int(crop_width / resize_factor)
    crop_height = int(crop_height / resize_factor)
    
    # Next, we can select a random location on the image for this bounding box.
    x0 = np.random.randint(0, width - crop_width)
    y0 = np.random.randint(0, height - crop_height)
    x1 = x0 + crop_width
    y1 = y0 + crop_height
    
    # Let's crop the image using the random bounding box we built.
    image = image[y0:y1, x0:x1]

    # Let's also flip the image horizontally with 50% probability:
    if np.random.rand() < 0.5:
        image = np.fliplr(image)

    # Now, let's resize the image to the target dimensions.
    # The resize function of scikit-image will automatically transform the image to floats ranging from 0.0 to 1.0
    image = resize(image, (target_width, target_height))
    
    # Finally, let's ensure that the colors are represented as 32-bit floats:
    return image.astype(np.float32)


# In[71]:


# Let's check out the result on each image
plt.figure(figsize=(6, 8))
plt.imshow(example_image)
plt.title("{}x{}".format(example_image.shape[1], example_image.shape[0]))
plt.axis("off")
plt.show()


# In[72]:


prepared_image = prepare_image(example_image)

plt.figure(figsize=(8, 8))
plt.imshow(prepared_image)
plt.title("{}x{}".format(prepared_image.shape[1], prepared_image.shape[0]))
plt.axis("off")
plt.show()


# In[73]:


# Let's look at a few other random images generated from the same original image:
rows, cols = 2, 3

plt.figure(figsize=(14, 8))
for row in range(rows):
    for col in range(cols):
        prepared_image = prepare_image(example_image)
        plt.subplot(rows, cols, row * cols + col + 1)
        plt.title("{}x{}".format(prepared_image.shape[1], prepared_image.shape[0]))
        plt.imshow(prepared_image)
        plt.axis("off")
plt.show()


# Alternatively it is also possible to implement this step directly with TensorFlow, using the functions in tf.image module. <br>
# It does not actually perform the image transformation, but rather creates a set of TensorFlow operations that will perform the transformation when you run the graph.

# In[ ]:


def prepare_image_with_tensorflow(image, target_width = 299, target_height = 299, max_zoom = 0.2):
    """Zooms and crops the image randomly for data augmentation."""

    # First, let's find the largest bounding box with the target size ratio that fits within the image
    image_shape = tf.cast(tf.shape(image), tf.float32)
    height = image_shape[0]
    width = image_shape[1]
    image_ratio = width / height
    target_image_ratio = target_width / target_height
    crop_vertically = image_ratio < target_image_ratio
    crop_width = tf.cond(crop_vertically,
                         lambda: width,
                         lambda: height * target_image_ratio)
    crop_height = tf.cond(crop_vertically,
                          lambda: width / target_image_ratio,
                          lambda: height)

    # Now let's shrink this bounding box by a random factor (dividing the dimensions by a random number
    # between 1.0 and 1.0 + `max_zoom`.
    resize_factor = tf.random_uniform(shape=[], minval=1.0, maxval=1.0 + max_zoom)
    crop_width = tf.cast(crop_width / resize_factor, tf.int32)
    crop_height = tf.cast(crop_height / resize_factor, tf.int32)
    box_size = tf.stack([crop_height, crop_width, 3])   # 3 = number of channels

    # Let's crop the image using a random bounding box of the size we computed
    image = tf.random_crop(image, box_size)

    # Let's also flip the image horizontally with 50% probability:
    image = tf.image.random_flip_left_right(image)

    # The resize_bilinear function requires a 4D tensor (a batch of images)
    # so we need to expand the number of dimensions first:
    image_batch = tf.expand_dims(image, 0)

    # Finally, let's resize the image to the target dimensions. Note that this function
    # returns a float32 tensor.
    image_batch = tf.image.resize_bilinear(image_batch, [target_height, target_width])
    image = image_batch[0] / 255  # back to a single image, and scale the colors from 0.0 to 1.0
    return image


# In[76]:


# Let's test this function
tf.reset_default_graph()

input_image = tf.placeholder(tf.uint8, shape=[None, None, 3])
prepared_image_op = prepare_image_with_tensorflow(input_image)

with tf.Session():
    prepared_image = prepared_image_op.eval(feed_dict={input_image: example_image})
    
plt.figure(figsize=(6, 6))
plt.imshow(prepared_image)
plt.title("{}x{}".format(prepared_image.shape[1], prepared_image.shape[0]))
plt.axis("off")
plt.show()


# **Exercise:** Using the pretrained Inception v3 model from the previous exercise, freeze all layers up to the bottleneck layer (i.e., the last layer before the output layer), and replace the output layer with the appropriate number of outputs for your new classification task (e.g., the flowers dataset has five mutually exclusive classes so the output layer must have five neurons and use the softmax activation function).<br>
# Let's start by fetching the inception v3 graph again. This time, let's use a training placeholder that we will use to tell TensorFlow whether we are training the network or not (this is needed by operations such as dropout and batch normalization).

# In[78]:


from tensorflow.contrib.slim.nets import inception
import tensorflow.contrib.slim as slim

tf.reset_default_graph()

X = tf.placeholder(tf.float32, shape=[None, height, width, channels], name="X")
training = tf.placeholder_with_default(False, shape=[])
with slim.arg_scope(inception.inception_v3_arg_scope()):
    logits, end_points = inception.inception_v3(X, num_classes=1001, is_training=training)

inception_saver = tf.train.Saver()


# Now we need to find the point in the graph where we should attach the new output layer. It should be the layer right before the current output layer. One way to do this is to explore the output layer's inputs:
# 

# In[79]:


logits.op.inputs[0]


# Nope, that's part of the output layer (adding the biases). Let's continue walking backwards in the graph:

# In[80]:


logits.op.inputs[0].op.inputs[0]


# That's also part of the output layer, it's the final layer in the inception layer (if you are not sure you can visualize the graph using TensorBoard). Once again, let's continue walking backwards in the graph:

# In[81]:


logits.op.inputs[0].op.inputs[0].op.inputs[0]


# There we are, this is the output of the dropout layer. This is the very last layer before the output layer in the Inception v3 network, so that's the layer we need to build upon. Note that there was actually a simpler way to find this layer: the inception_v3() function returns a dict of end points:

# In[82]:


end_points


# In[84]:


end_points["PreLogits"]


# We can drop the 2nd and 3rd dimensions using the tf.squeeze() function:

# In[ ]:


prelogits = tf.squeeze(end_points["PreLogits"], axis=[1, 2])


# Then we can add the final fully connected layer on top of this layer:

# In[86]:


n_outputs = len(flower_classes)

with tf.name_scope("new_output_layer"):
    flower_logits = tf.layers.dense(prelogits, n_outputs, name="flower_logits")
    Y_proba = tf.nn.softmax(flower_logits, name="Y_proba")


# There is one important detail, however: since we want to train only the output layer (all other layers must be frozen), we must pass the list of variables to train to the optimizer's minimize() method:

# In[ ]:


y = tf.placeholder(tf.int32, shape=[None])

with tf.name_scope("train"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=flower_logits, labels=y)
    loss = tf.reduce_mean(xentropy)
    optimizer = tf.train.AdamOptimizer()
    flower_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope="flower_logits")
    training_op = optimizer.minimize(loss, var_list=flower_vars)

with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(flower_logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

with tf.name_scope("init_and_save"):
    init = tf.global_variables_initializer()
    saver = tf.train.Saver()


# In[88]:


[v.name for v in flower_vars]


# Notice that we created the inception_saver before adding the new output layer: we will use this saver to restore the pretrained model state, so we don't want it to try to restore new variables (it would just fail saying it does not know the new variables). The second saver will be used to save the final flower model, including both the pretrained variables and the new ones.

# **Exercise:** Split your dataset into a training set and a test set. Train the model on the training set and evaluate it on the test set.

# In[89]:


flower_class_ids = {flower_class: index for index, flower_class in enumerate(flower_classes)}
flower_class_ids


# In[90]:


flower_paths_and_classes = []
for flower_class, paths in image_paths.items():
    for path in paths:
        flower_paths_and_classes.append((path, flower_class_ids[flower_class]))

# Next, lets shuffle the dataset and split it into the training set and the test set:
test_ratio = 0.2
train_size = int(len(flower_paths_and_classes) * (1 - test_ratio))

np.random.shuffle(flower_paths_and_classes)

flower_paths_and_classes_train = flower_paths_and_classes[:train_size]
flower_paths_and_classes_test = flower_paths_and_classes[train_size:]

# Let's look at the first 3 instances in the training set:
flower_paths_and_classes_train[:3]


# Next, we will also need a function to preprocess a set of images. This function will be useful to preprocess the test set, and also to create batches during training. For simplicity, we will use the NumPy/SciPy implementation:

# In[ ]:


from random import sample

def prepare_batch(flower_paths_and_classes, batch_size):
    batch_paths_and_classes = sample(flower_paths_and_classes, batch_size)
    images = [mpimg.imread(path)[:, :, :channels] for path, labels in batch_paths_and_classes]
    prepared_images = [prepare_image(image) for image in images]
    X_batch = 2 * np.stack(prepared_images) - 1 # Inception expects colors ranging from -1 to 1
    y_batch = np.array([labels for path, labels in batch_paths_and_classes], dtype=np.int32)
    return X_batch, y_batch

X_batch, y_batch = prepare_batch(flower_paths_and_classes_train, batch_size=4)


# In[92]:


X_batch.shape


# In[93]:


X_batch.dtype


# In[94]:


y_batch.shape


# In[95]:


y_batch.dtype


# In[ ]:


# Prepare the test set
X_test, y_test = prepare_batch(flower_paths_and_classes_test, batch_size=len(flower_paths_and_classes_test))


# In[97]:


X_test.shape


# And now, we are ready to train the network (or more precisely, the output layer we just added, since all the other layers are frozen).

# In[99]:


n_epochs = 10
batch_size = 64
n_iterations_per_epoch = len(flower_paths_and_classes_train) // batch_size

with tf.Session() as sess:
    init.run()
    inception_saver.restore(sess, INCEPTION_V4_CHECKPOINT_PATH)

    for epoch in range(n_epochs):
        print("Epoch", epoch, end="")
        for iteration in range(n_iterations_per_epoch):
            print(".", end="")
            X_batch, y_batch = prepare_batch(flower_paths_and_classes_train, batch_size)
            sess.run(training_op, feed_dict={X: X_batch, y: y_batch, training: True})

        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
        print("  Last batch accuracy:", acc_batch)

        save_path = saver.save(sess, "./my_flowers_model")


# In[100]:


n_test_batches = 10
X_test_batches = np.array_split(X_test, n_test_batches)
y_test_batches = np.array_split(y_test, n_test_batches)

with tf.Session() as sess:
    saver.restore(sess, "./my_flowers_model")

    print("Computing final accuracy on the test set (this will take a while)...")
    acc_test = np.mean([
        accuracy.eval(feed_dict={X: X_test_batch, y: y_test_batch})
        for X_test_batch, y_test_batch in zip(X_test_batches, y_test_batches)])
    print("Test accuracy:", acc_test)


#!/usr/bin/env python # coding: utf-8 # In[2]: # Import the TensorFlow and output the verion # # TensorFlow Implementation # In TensorFlow, each input is typically represented as a 3D tensor of shape [`height`, `width`, `channels`]. A mini-batch is represented as a 4D tensor of shape [`mini-batch size`, `height`, `width`. `channels`]. The weights of a convolutional layer are represented as a tensor of shape [f$_{h}$, f$_{w}$, f$_{n}$, f$_{n'}$] The bias terms of a convolutional layer are simply represented as a 1D tensor of shape [f$_{n}$].<br> # The following code loads two sample images, using Scikit-Learn's `load_sample_image()`. Then it creates two 7 X 7 filters (one with a verrtical white line and nother with a horizontal white line), and applites them to bothiamges using a convolutional layer built using TensorFlow's `tf.nn.conv2d()` function (with zero padding and a stride of 2). Finally, it plots one of the resulting feature maps: # In[3]: # Load sample images # Create 2 filters # vertical line # horizontal line # Create a graph with input X plus a convolutional layer applying the 2 filters # Plot 1st image's 2nd feature map # TensorFlow have a `tf.layers.conv2d()` function which creates the filters variable for you (called kernel), and initialized it randomly. For example, the following code creates an input placeholder followed by a convolutional layer with two 7 X 7 feature map, using 2 X 2 strides (note that this function only expects the vertical and horizontal strides), and "SAME" padding: # In[3]: # # Pooling Layer # Their goal is to subsample (i.e., shrink) the input image in order to reduce the computational load, the memory usage, and the number of parameters (thereby limiting the risk of overfitting). Reducing the input image size also makes the neural network tolerate a little bit of image shift (location invariance).<br> # The following layer creates a max pooling layer using a 2 X 2 kernel, stride 2, and no padding, then applies it to all the iamges in the dataset: # In[4]: # Create a graph with input X plus a max pooling layer # Plot the output for 1st image # - The `ksize` argument contains the kernel shape along all four dimensions of the input tensor: `[batch-size, height, width, channels]`. # - To create an average pooling layer, just use the `avg_pool()` function instead of `max_pool()`. # # CNN on MNIST # In[5]: # The `get_model_params()` function gets the model's state (i.e., the value of all the variables), and the `restore_model_params()` restores a previous state. This is used to speed up early stopping: instead of storing the best model found so far to disk, we just save it to memory. At the end of training, we roll back to the best model found. # In[ ]: # In[7]: # In[ ]: # Now let's train the model! This implementation of Early Stopping works like this: # - every 500 training iterations, it evaluates the model on the validation set, # - if the model performs better than the best model found so far, then it saves the model to RAM, # - if there is no progress for 100 evaluations in a row, then training is interrupted, # - after training, the code restores the best model found. # In[9]: # # Classifying large images using Inception # **Exercise:** Download some images of various animals. Load them in Python, for example using the matplotlib.image.mpimg.imread() function or the scipy.misc.imread() function. Resize and/or crop them to 299 × 299 pixels, and ensure that they have just three channels (RGB), with no transparency channel. The images that the Inception model was trained on were preprocessed so that their values range from -1.0 to 1.0, so you must ensure that your images do too. # In[ ]: # In[57]: # In[ ]: # **Exercise:** Download the latest pretrained Inception v4 model at # http://download.tensorflow.org/models/inception_v4_2016_09_09.tar.gz.<br> # The list of class names is available at https://goo.gl/brXRtZ, but you must insert a "background" class at the beginning. # # In[ ]: # In[22]: # In[23]: # **Exercise:** Create the Inception v4 model by calling the inception_v4() function, as shown below. This must be done within an argument scope created by the inception_v4_arg_scope() function. Also, you must set is_training=False and num_classes=1001 [...] # In[36]: # **Exercise:** Open a session and use the Saver to restore the pretrained model checkpoint you downloaded earlier. # # In[38]: # **Exercise:**Run the model to classify the images you prepared. # In[59]: # In[60]: # In[61]: # In[62]: # # Transfer Learning for Large Image Classification # **Exercise:** Create a training set containing at least 100 images per class. For example, you could classify your own pictures based on the location (beach, mountain, city, etc.), or alternatively you can just use an existing dataset, such as the flowers dataset or MIT's places dataset (requires registration, and it is huge). # In[ ]: # In[64]: # In[65]: # In[ ]: # Let's get the list of all image paths for each class: # In[ ]: # Sort the image paths # In[69]: # Let's take a peek at the first few images from each class: # **Exercise:** Write a preprocessing step that will resize and crop the image to 299 × 299, with some randomness for data augmentation. # In[ ]: Zooms and crops the image randomly for data augmentation. # First, let's find the largest bounding box with the target size ratio that fits within the image # Now let's shrink this bounding box by a random factor (dividing the dimensions by a random number # between 1.0 and 1.0 + `max_zoom`. # Next, we can select a random location on the image for this bounding box. # Let's crop the image using the random bounding box we built. # Let's also flip the image horizontally with 50% probability: # Now, let's resize the image to the target dimensions. # The resize function of scikit-image will automatically transform the image to floats ranging from 0.0 to 1.0 # Finally, let's ensure that the colors are represented as 32-bit floats: # In[71]: # Let's check out the result on each image # In[72]: # In[73]: # Let's look at a few other random images generated from the same original image: # Alternatively it is also possible to implement this step directly with TensorFlow, using the functions in tf.image module. <br> # It does not actually perform the image transformation, but rather creates a set of TensorFlow operations that will perform the transformation when you run the graph. # In[ ]: Zooms and crops the image randomly for data augmentation. # First, let's find the largest bounding box with the target size ratio that fits within the image # Now let's shrink this bounding box by a random factor (dividing the dimensions by a random number # between 1.0 and 1.0 + `max_zoom`. # 3 = number of channels # Let's crop the image using a random bounding box of the size we computed # Let's also flip the image horizontally with 50% probability: # The resize_bilinear function requires a 4D tensor (a batch of images) # so we need to expand the number of dimensions first: # Finally, let's resize the image to the target dimensions. Note that this function # returns a float32 tensor. # back to a single image, and scale the colors from 0.0 to 1.0 # In[76]: # Let's test this function # **Exercise:** Using the pretrained Inception v3 model from the previous exercise, freeze all layers up to the bottleneck layer (i.e., the last layer before the output layer), and replace the output layer with the appropriate number of outputs for your new classification task (e.g., the flowers dataset has five mutually exclusive classes so the output layer must have five neurons and use the softmax activation function).<br> # Let's start by fetching the inception v3 graph again. This time, let's use a training placeholder that we will use to tell TensorFlow whether we are training the network or not (this is needed by operations such as dropout and batch normalization). # In[78]: # Now we need to find the point in the graph where we should attach the new output layer. It should be the layer right before the current output layer. One way to do this is to explore the output layer's inputs: # # In[79]: # Nope, that's part of the output layer (adding the biases). Let's continue walking backwards in the graph: # In[80]: # That's also part of the output layer, it's the final layer in the inception layer (if you are not sure you can visualize the graph using TensorBoard). Once again, let's continue walking backwards in the graph: # In[81]: # There we are, this is the output of the dropout layer. This is the very last layer before the output layer in the Inception v3 network, so that's the layer we need to build upon. Note that there was actually a simpler way to find this layer: the inception_v3() function returns a dict of end points: # In[82]: # In[84]: # We can drop the 2nd and 3rd dimensions using the tf.squeeze() function: # In[ ]: # Then we can add the final fully connected layer on top of this layer: # In[86]: # There is one important detail, however: since we want to train only the output layer (all other layers must be frozen), we must pass the list of variables to train to the optimizer's minimize() method: # In[ ]: # In[88]: # Notice that we created the inception_saver before adding the new output layer: we will use this saver to restore the pretrained model state, so we don't want it to try to restore new variables (it would just fail saying it does not know the new variables). The second saver will be used to save the final flower model, including both the pretrained variables and the new ones. # **Exercise:** Split your dataset into a training set and a test set. Train the model on the training set and evaluate it on the test set. # In[89]: # In[90]: # Next, lets shuffle the dataset and split it into the training set and the test set: # Let's look at the first 3 instances in the training set: # Next, we will also need a function to preprocess a set of images. This function will be useful to preprocess the test set, and also to create batches during training. For simplicity, we will use the NumPy/SciPy implementation: # In[ ]: # Inception expects colors ranging from -1 to 1 # In[92]: # In[93]: # In[94]: # In[95]: # In[ ]: # Prepare the test set # In[97]: # And now, we are ready to train the network (or more precisely, the output layer we just added, since all the other layers are frozen). # In[99]: # In[100]: