神經網路中間層輸出

測試中間層輸出

import torch
import numpy as np
from PIL import Image
from torchvision import transforms, models
import matplotlib.cm
from torchinfo import summary
import copy
import cv2
import matplotlib.pyplot as plt

device = 'cuda' if torch.cuda.is_available() else 'cpu'

module_name = []
p_in = []
p_out = []

# 定義hook_fn，顧名思義就是把數值從forward計算過程中“勾出”來
def hook_fn(module, inputs, outputs):
    module_name.append(module.__class__)
    print(module)
    p_in.append(inputs)
    p_out.append(outputs)


# 載入模型，註冊鉤子函式
model = models.detection.fasterrcnn_resnet50_fpn(pretrained=True)
model.rpn.head.cls_logits.register_forward_hook(hook_fn) # faster_rcnn

## Retinanet
# model = models.detection.retinanet_resnet50_fpn(pretrained=True)
# model.head.classification_head.cls_logits.register_forward_hook(hook_fn)

## SSD300 VGG16
# model = models.detection.ssd300_vgg16(pretrained=True)
# model.head.classification_head.register_forward_hook(hook_fn)

# print(summary(model, (1,3,300,300), verbose=1))

# 匯入一張影像
img_file = "elephant_1280p.jpg"
img = Image.open(img_file)
ori_img = img.copy()

# 前處理
transform = transforms.Compose([
    # transforms.Resize((416,416)),
    transforms.ToTensor(),
    transforms.Normalize(
        [0.485, 0.456, 0.406],
        [0.229, 0.224, 0.225]
    )
])

img = transform(img).unsqueeze(0)  # 增加batch維度

model.to(device)
img = img.to(device) # torch.Size([1, 3, 720, 1280])

# EVAL模式
model.eval()
with torch.no_grad():
    model(img)

# 特徵視覺化
def show_feature_map(img_src, conv_features):
    '''視覺化卷積層特徵圖輸出
    img_src:源影像檔案路徑
    conv_feature:得到的卷積輸出,[b, c, h, w]
    '''
    img = Image.open(img_file).convert('RGB')
    height, width = img.size
    conv_features = conv_features.cpu()

    heat = conv_features.squeeze(0)#降維操作,尺寸變為(C,H,W)
    heatmap = torch.mean(heat,dim=0)#對各卷積層(C)求平均值,尺寸變為(H,W)
    # heatmap = torch.max(heat,dim=1).values.squeeze()

    heatmap = heatmap.numpy()#轉換為numpy陣列
    heatmap = np.maximum(heatmap, 0)
    heatmap /= np.max(heatmap)#minmax歸一化處理
    heatmap = cv2.resize(heatmap,(img.size[0],img.size[1]))#變換heatmap影像尺寸,使之與原圖匹配,方便後續視覺化
    heatmap = np.uint8(255*heatmap)#畫素值縮放至(0,255)之間,uint8型別,這也是前面需要做歸一化的原因,否則畫素值會溢位255(也就是8位顏色通道)
    heatmap = cv2.applyColorMap(heatmap,cv2.COLORMAP_HSV)#顏色變換
    plt.imshow(heatmap)
    plt.show()
    # heatmap = np.array(Image.fromarray(heatmap).convert('L'))
    superimg = heatmap*0.4+np.array(img)[:,:,::-1] #影像疊加，注意翻轉通道，cv用的是bgr
    cv2.imwrite('./superimg.jpg',superimg)#儲存結果
    # 視覺化疊加至源影像的結果
    img_ = np.array(Image.open('./superimg.jpg').convert('RGB'))
    plt.imshow(img_)
    plt.show()

model_str = model.__str__()

if model_str.startswith('SSD'):
    for k in range(len(module_name)):
        for j in range(len(p_in[0][0])):
            print(p_in[k][0][j].shape)
            print(p_out[k].shape)
            show_feature_map(img_file, p_in[k][0][j])
            # show_feature_map(img_file, torch.sigmoid(p_out[k]))
            print()

if model_str.startswith('RetinaNet'): # retinanet
    for k in range(len(module_name)):# 不同尺寸的特徵圖
        print(p_in[k][0].shape)
        print(p_out[k].shape)
        # show_feature_map(img_file, p_in[k][j])
        show_feature_map(img_file, torch.sigmoid(p_out[k]))
        print()

if model_str.startswith('FasterRCNN'): # FasterRCNN
    for k in range(len(module_name)):
        print(p_in[k][0].shape)
        print(p_out[k].shape)
        # show_feature_map(img_file, p_in[k][0])
        show_feature_map(img_file, torch.sigmoid(p_out[k]))
        print()

print(summary(model, (1,3,300,300), verbose=1))

測試hook

import torch
import torch.nn as nn
import torch.nn.functional as F
 
class LeNet(nn.Module):
    def __init__(self):
        super(LeNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16*5*5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)
 
    def forward(self, x):
        out = F.relu(self.conv1(x))     #1 
        out = F.max_pool2d(out, 2)      #2
        out = F.relu(self.conv2(out))   #3
        out = F.max_pool2d(out, 2)
        out = out.view(out.size(0), -1)
        out = F.relu(self.fc1(out))
        out = F.relu(self.fc2(out))
        out = self.fc3(out)
        return out
 
features = []
def hook(module, input, output): 
    # module: model.conv2 
    # input :in forward function  [#2]
    # output:is  [#3 self.conv2(out)]
    features.append(output.clone().detach())
    # output is saved in a list 
 
 
net = LeNet() ## 模型例項化 
print(net)

x = torch.randn(2, 3, 32, 32) ## input 
handle = net.conv2.register_forward_hook(hook) ## 獲取整個Lenet模型 conv2的中間結果
y = net(x)  ## 獲取的是 關於 input x 的 conv2 結果 
 
print(features[0].size()) # 即 [#3 self.conv2(out)]
handle.remove() ## hook刪除


"""
import torch 
from torch.autograd import Variable 
 
 
def print_grad(grad):
    print('grad is \n',grad)
 
x = Variable(torch.randn(2, 1), requires_grad=True)
## x = torch.rand(2,1,requires_grad=True) #  等效
print('x value is \n',x)
y = x+3
print('y value is \n',y)
z = torch.mean(torch.pow(y, 1/2))
lr = 1e-3
 
y.register_hook(print_grad) 

z.backward() # 梯度求解

print(x.grad.data)

x.data -= lr*x.grad.data
print('new x is\n',x)
"""

測試net

import torch.nn as nn
 
class AlexNet(nn.Module):
    def __init__(self, num_classes=1000, init_weights=False):
        super(AlexNet, self).__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 48, kernel_size=11, stride=4, padding=2),  # input[3, 224, 224]  output[48, 55, 55]
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),                  # output[48, 27, 27]
            nn.Conv2d(48, 128, kernel_size=5, padding=2),           # output[128, 27, 27]
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),                  # output[128, 13, 13]
            nn.Conv2d(128, 192, kernel_size=3, padding=1),          # output[192, 13, 13]
            nn.ReLU(inplace=True),
            nn.Conv2d(192, 192, kernel_size=3, padding=1),          # output[192, 13, 13]
            nn.ReLU(inplace=True),
            nn.Conv2d(192, 128, kernel_size=3, padding=1),          # output[128, 13, 13]
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),                  # output[128, 6, 6]
        )
        self.classifier = nn.Sequential(
            nn.Dropout(p=0.5),
            nn.Linear(128 * 6 * 6, 2048),
            nn.ReLU(inplace=True),
            nn.Dropout(p=0.5),
            nn.Linear(2048, 2048),
            nn.ReLU(inplace=True),
            nn.Linear(2048, num_classes),
        )
        if init_weights:
            self._initialize_weights()
 
    def forward(self, x):
        x = self.features(x)
        x = self.classifier(x)
        return x
 
    def _initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.normal_(m.weight, 0, 0.01)
                nn.init.constant_(m.bias, 0)
 
if __name__ == '__main__':
    model = AlexNet()

    print('model children: ')
    for module in model.children():
        print(module)
    print("\n")

    print('model modules: ')
    for module in model.modules():
        print(module)
    print("\n")
 
    print('model named children: ')
    for name, module in model.named_children():
        print('name: {}, module: {}'.format(name, module))
    print("\n")
    
    print('model named modules: ')
    for name, module in  model.named_modules():
        print('name: {}, module: {}'.format(name, module))
    print("\n")

    print('parameters: ')
    for parameter in model.parameters():
        print('parameter: {}'.format(parameter.shape))
    print("\n")
 
    print('model named parameters: ')
    for name, parameter in model.named_parameters():
        print('name: {}, parameter: {}'.format(name, parameter.shape))



"""
model children: 
Sequential(
  (0): Conv2d(3, 48, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
  (1): ReLU(inplace=True)
  (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  (3): Conv2d(48, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
  (4): ReLU(inplace=True)
  (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  (6): Conv2d(128, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (7): ReLU(inplace=True)
  (8): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (9): ReLU(inplace=True)
  (10): Conv2d(192, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (11): ReLU(inplace=True)
  (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
)
Sequential(
  (0): Dropout(p=0.5, inplace=False)
  (1): Linear(in_features=4608, out_features=2048, bias=True)
  (2): ReLU(inplace=True)
  (3): Dropout(p=0.5, inplace=False)
  (4): Linear(in_features=2048, out_features=2048, bias=True)
  (5): ReLU(inplace=True)
  (6): Linear(in_features=2048, out_features=1000, bias=True)
)

model modules: 
AlexNet(
  (features): Sequential(
    (0): Conv2d(3, 48, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
    (1): ReLU(inplace=True)
    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
    (3): Conv2d(48, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
    (4): ReLU(inplace=True)
    (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
    (6): Conv2d(128, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (7): ReLU(inplace=True)
    (8): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (9): ReLU(inplace=True)
    (10): Conv2d(192, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (11): ReLU(inplace=True)
    (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (classifier): Sequential(
    (0): Dropout(p=0.5, inplace=False)
    (1): Linear(in_features=4608, out_features=2048, bias=True)
    (2): ReLU(inplace=True)
    (3): Dropout(p=0.5, inplace=False)
    (4): Linear(in_features=2048, out_features=2048, bias=True)
    (5): ReLU(inplace=True)
    (6): Linear(in_features=2048, out_features=1000, bias=True)
  )
)
Sequential(
  (0): Conv2d(3, 48, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
  (1): ReLU(inplace=True)
  (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  (3): Conv2d(48, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
  (4): ReLU(inplace=True)
  (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  (6): Conv2d(128, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (7): ReLU(inplace=True)
  (8): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (9): ReLU(inplace=True)
  (10): Conv2d(192, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (11): ReLU(inplace=True)
  (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
)
Conv2d(3, 48, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
ReLU(inplace=True)
MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
Conv2d(48, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
ReLU(inplace=True)
MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
Conv2d(128, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
ReLU(inplace=True)
Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
ReLU(inplace=True)
Conv2d(192, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
ReLU(inplace=True)
MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
Sequential(
  (0): Dropout(p=0.5, inplace=False)
  (1): Linear(in_features=4608, out_features=2048, bias=True)
  (2): ReLU(inplace=True)
  (3): Dropout(p=0.5, inplace=False)
  (4): Linear(in_features=2048, out_features=2048, bias=True)
  (5): ReLU(inplace=True)
  (6): Linear(in_features=2048, out_features=1000, bias=True)
)
Dropout(p=0.5, inplace=False)
Linear(in_features=4608, out_features=2048, bias=True)
ReLU(inplace=True)
Dropout(p=0.5, inplace=False)
Linear(in_features=2048, out_features=2048, bias=True)
ReLU(inplace=True)
Linear(in_features=2048, out_features=1000, bias=True)

model named children: 
name: features, module: Sequential(
  (0): Conv2d(3, 48, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
  (1): ReLU(inplace=True)
  (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  (3): Conv2d(48, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
  (4): ReLU(inplace=True)
  (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  (6): Conv2d(128, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (7): ReLU(inplace=True)
  (8): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (9): ReLU(inplace=True)
  (10): Conv2d(192, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (11): ReLU(inplace=True)
  (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
)
name: classifier, module: Sequential(
  (0): Dropout(p=0.5, inplace=False)
  (1): Linear(in_features=4608, out_features=2048, bias=True)
  (2): ReLU(inplace=True)
  (3): Dropout(p=0.5, inplace=False)
  (4): Linear(in_features=2048, out_features=2048, bias=True)
  (5): ReLU(inplace=True)
  (6): Linear(in_features=2048, out_features=1000, bias=True)
)

model named modules: 
name: , module: AlexNet(
  (features): Sequential(
    (0): Conv2d(3, 48, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
    (1): ReLU(inplace=True)
    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
    (3): Conv2d(48, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
    (4): ReLU(inplace=True)
    (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
    (6): Conv2d(128, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (7): ReLU(inplace=True)
    (8): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (9): ReLU(inplace=True)
    (10): Conv2d(192, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (11): ReLU(inplace=True)
    (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (classifier): Sequential(
    (0): Dropout(p=0.5, inplace=False)
    (1): Linear(in_features=4608, out_features=2048, bias=True)
    (2): ReLU(inplace=True)
    (3): Dropout(p=0.5, inplace=False)
    (4): Linear(in_features=2048, out_features=2048, bias=True)
    (5): ReLU(inplace=True)
    (6): Linear(in_features=2048, out_features=1000, bias=True)
  )
)
name: features, module: Sequential(
  (0): Conv2d(3, 48, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
  (1): ReLU(inplace=True)
  (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  (3): Conv2d(48, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
  (4): ReLU(inplace=True)
  (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  (6): Conv2d(128, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (7): ReLU(inplace=True)
  (8): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (9): ReLU(inplace=True)
  (10): Conv2d(192, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
  (11): ReLU(inplace=True)
  (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
)
name: features.0, module: Conv2d(3, 48, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
name: features.1, module: ReLU(inplace=True)
name: features.2, module: MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
name: features.3, module: Conv2d(48, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
name: features.4, module: ReLU(inplace=True)
name: features.5, module: MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
name: features.6, module: Conv2d(128, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
name: features.7, module: ReLU(inplace=True)
name: features.8, module: Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
name: features.9, module: ReLU(inplace=True)
name: features.10, module: Conv2d(192, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
name: features.11, module: ReLU(inplace=True)
name: features.12, module: MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
name: classifier, module: Sequential(
  (0): Dropout(p=0.5, inplace=False)
  (1): Linear(in_features=4608, out_features=2048, bias=True)
  (2): ReLU(inplace=True)
  (3): Dropout(p=0.5, inplace=False)
  (4): Linear(in_features=2048, out_features=2048, bias=True)
  (5): ReLU(inplace=True)
  (6): Linear(in_features=2048, out_features=1000, bias=True)
)
name: classifier.0, module: Dropout(p=0.5, inplace=False)
name: classifier.1, module: Linear(in_features=4608, out_features=2048, bias=True)
name: classifier.2, module: ReLU(inplace=True)
name: classifier.3, module: Dropout(p=0.5, inplace=False)
name: classifier.4, module: Linear(in_features=2048, out_features=2048, bias=True)
name: classifier.5, module: ReLU(inplace=True)
name: classifier.6, module: Linear(in_features=2048, out_features=1000, bias=True)

parameters: 
parameter: torch.Size([48, 3, 11, 11])
parameter: torch.Size([48])
parameter: torch.Size([128, 48, 5, 5])
parameter: torch.Size([128])
parameter: torch.Size([192, 128, 3, 3])
parameter: torch.Size([192])
parameter: torch.Size([192, 192, 3, 3])
parameter: torch.Size([192])
parameter: torch.Size([128, 192, 3, 3])
parameter: torch.Size([128])
parameter: torch.Size([2048, 4608])
parameter: torch.Size([2048])
parameter: torch.Size([2048, 2048])
parameter: torch.Size([2048])
parameter: torch.Size([1000, 2048])
parameter: torch.Size([1000])

model named parameters: 
name: features.0.weight, parameter: torch.Size([48, 3, 11, 11])
name: features.0.bias, parameter: torch.Size([48])
name: features.3.weight, parameter: torch.Size([128, 48, 5, 5])
name: features.3.bias, parameter: torch.Size([128])
name: features.6.weight, parameter: torch.Size([192, 128, 3, 3])
name: features.6.bias, parameter: torch.Size([192])
name: features.8.weight, parameter: torch.Size([192, 192, 3, 3])
name: features.8.bias, parameter: torch.Size([192])
name: features.10.weight, parameter: torch.Size([128, 192, 3, 3])
name: features.10.bias, parameter: torch.Size([128])
name: classifier.1.weight, parameter: torch.Size([2048, 4608])
name: classifier.1.bias, parameter: torch.Size([2048])
name: classifier.4.weight, parameter: torch.Size([2048, 2048])
name: classifier.4.bias, parameter: torch.Size([2048])
name: classifier.6.weight, parameter: torch.Size([1000, 2048])
name: classifier.6.bias, parameter: torch.Size([1000])
"""