[AAIS] E06. Grad-CAM

Before start Check resources

!nvidia-smi

Mon Feb  1 09:11:32 2021
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 410.104      Driver Version: 410.104      CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla V100-DGXS...  On   | 00000000:07:00.0 Off |                    0 |
| N/A   65C    P0   286W / 300W |  31558MiB / 32478MiB |     97%      Default |
+-------------------------------+----------------------+----------------------+
|   1  Tesla V100-DGXS...  On   | 00000000:08:00.0 Off |                    0 |
| N/A   61C    P0   228W / 300W |   2630MiB / 32478MiB |     94%      Default |
+-------------------------------+----------------------+----------------------+
|   2  Tesla V100-DGXS...  On   | 00000000:0E:00.0 Off |                    0 |
| N/A   63C    P0   238W / 300W |   5264MiB / 32478MiB |     98%      Default |
+-------------------------------+----------------------+----------------------+
|   3  Tesla V100-DGXS...  On   | 00000000:0F:00.0 Off |                    0 |
| N/A   63C    P0   248W / 300W |  14704MiB / 32478MiB |     96%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
+-----------------------------------------------------------------------------+

# using gpu:/2
import tensorflow as tf
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "2"

gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
  try:
    tf.config.experimental.set_memory_growth(gpus[0], True)
  except RuntimeError as e:
    print(e)

Setup

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

from keras.applications import VGG16
from keras import layers
from keras import models
from keras.applications.vgg16 import preprocess_input, decode_predictions

from keras.preprocessing import image

from keras import backend as K
import cv2

Using VGG16 for Grad-CAM

model = tf.keras.applications.vgg16.VGG16()

model.summary()

Model: "vgg16"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
input_1 (InputLayer)         [(None, 224, 224, 3)]     0
_________________________________________________________________
block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0
_________________________________________________________________
block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856
_________________________________________________________________
block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0
_________________________________________________________________
block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168
_________________________________________________________________
block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080
_________________________________________________________________
block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080
_________________________________________________________________
block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0
_________________________________________________________________
block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160
_________________________________________________________________
block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808
_________________________________________________________________
block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808
_________________________________________________________________
block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0
_________________________________________________________________
block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808
_________________________________________________________________
block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808
_________________________________________________________________
block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808
_________________________________________________________________
block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0
_________________________________________________________________
flatten (Flatten)            (None, 25088)             0
_________________________________________________________________
fc1 (Dense)                  (None, 4096)              102764544
_________________________________________________________________
fc2 (Dense)                  (None, 4096)              16781312
_________________________________________________________________
predictions (Dense)          (None, 1000)              4097000
=================================================================
Total params: 138,357,544
Trainable params: 138,357,544
Non-trainable params: 0
_________________________________________________________________

Preprocess the image

img_path = '../sungjin/dog/test/bo/bo_26.jpg'
img = image.load_img(img_path,target_size=(224,224))
x = image.img_to_array(img)
x = np.expand_dims(x,axis=0)
x = preprocess_input(x)

plt.imshow(img)

<matplotlib.image.AxesImage at 0x7f7a885bc520>

preds = model.predict(x)
print(preds.shape)
print('Predicted:',decode_predictions(preds,top=3)[0])
print(np.argmax(preds[0]))

(1, 1000)
Predicted: [('n02113799', 'standard_poodle', 0.21771519), ('n02111277', 'Newfoundland', 0.19664182), ('n02097130', 'giant_schnauzer', 0.16597004)]
267

About equation

Grad-CAM

For each class c in the CNN, Grad−CAM is calculated with a linear combination of the k-th feature maps of the convolutional layer $A_k$ and the importance weight αc
K, followed by ReLU activation:

# layer which calculate `score class`
conv_layer = model.get_layer("block5_conv3")

heatmap_model = models.Model([model.inputs],[conv_layer.output,model.output])

with tf.GradientTape() as gtape:
    conv_output, predictions = heatmap_model(x) # y^c, c
    loss = predictions[:, np.argmax(predictions[0])] # probability
    grads = gtape.gradient(loss, conv_output) # partial-derivative
    pooled_grads = K.mean(grads, axis=(0, 1, 2)) # a_k^c , weight

heatmap = tf.reduce_mean(tf.multiply(pooled_grads, conv_output), axis=-1) # summation weight * filter
heatmap = np.maximum(heatmap, 0) # ReLu
max_heat = np.max(heatmap) # normalization

if max_heat == 0:
    max_heat = 1e-10
heatmap /= max_heat


plt.matshow(heatmap[0])

<matplotlib.image.AxesImage at 0x7f7c992031c0>

Visualize Grad-CAM

# img = mpimg.imread(img_path)
img = cv2.imread(img_path)
hm = heatmap[0]
hm = cv2.resize(hm, (img.shape[1],img.shape[0]))
hm = np.uint8(255*hm)
hm = cv2.applyColorMap(hm,cv2.COLORMAP_JET)
superimposed_img = hm * 0.8 + img

superimposed_img[superimposed_img > 255] = 255
superimposed_img = superimposed_img.astype('uint8')


plt.imshow(superimposed_img)

<matplotlib.image.AxesImage at 0x7f7c991e5f70>

Return the resources

import IPython
app = IPython.Application.instance()
app.kernel.do_shutdown(True)

{'status': 'ok', 'restart': True}