原文連結：https://blog.csdn.net/zxdd2018/article/details/125505352
另附：imagenet圖對應https://www.cnblogs.com/cpxlll/p/13493247.html
1.（多張圖片）
備註：gram_cam_1
import os
import numpy as np
import torch
import cv2
import matplotlib.pyplot as plt
import torchvision.models as models
from torchvision.transforms import Compose, Normalize, ToTensor
from cifar.resnet import ResNet32
class GradCAM():
'''
Grad-cam: Visual explanations from deep networks via gradient-based localization
Selvaraju R R, Cogswell M, Das A, et al.
https://openaccess.thecvf.com/content_iccv_2017/html/Selvaraju_Grad-CAM_Visual_Explanations_ICCV_2017_paper.html
'''
def __init__(self, model, target_layers, input_size, use_cuda=True):
super(GradCAM).__init__()
self.use_cuda = use_cuda
self.model = model
self.target_layers = target_layers
self.target_layers.register_forward_hook(self.forward_hook)
self.target_layers.register_full_backward_hook(self.backward_hook)
self.activations = []
self.grads = []
self.input_size = input_size
def forward_hook(self, module, input, output):
self.activations.append(output[0])
def backward_hook(self, module, grad_input, grad_output):
self.grads.append(grad_output[0].detach())
def calculate_cam(self, model_input):
if self.use_cuda:
device = torch.device('cuda')
self.model.to(device)
model_input = model_input.to(device)
self.model.eval()
# forward
output, _ = self.model(model_input, 0) # 修改這裡以匹配您模型的輸出
y_hat = output
max_class = np.argmax(y_hat.cpu().data.numpy(), axis=1)
# backward
self.model.zero_grad()
y_c = y_hat[0, max_class]
y_c.backward()
# get activations and gradients
activations = self.activations[0].cpu().data.numpy().squeeze()
grads = self.grads[0].cpu().data.numpy().squeeze()
# calculate weights
weights = np.mean(grads.reshape(grads.shape[0], -1), axis=1)
weights = weights.reshape(-1, 1, 1)
cam = (weights * activations).sum(axis=0)
cam = np.maximum(cam, 0)
cam = cam / cam.max()
return cam
@staticmethod
def show_cam_on_image(image, cam, save_path=None):
h, w = image.shape[:2]
cam = cv2.resize(cam, (w, h)) # 調整熱圖的尺寸與影像相同
cam = cam / cam.max()
heatmap = cv2.applyColorMap((255 * cam).astype(np.uint8), cv2.COLORMAP_JET)
heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)
image = image / image.max()
heatmap = heatmap / heatmap.max()
result = 0.4 * heatmap + 0.6 * image
result = result / result.max()
plt.figure()
plt.imshow((result * 255).astype(np.uint8))
plt.colorbar(shrink=0.8)
plt.tight_layout()
if save_path:
plt.savefig(save_path)
# plt.show()
@staticmethod
def preprocess_image(img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]):
preprocessing = Compose([
ToTensor(),
Normalize(mean=mean, std=std)
])
return preprocessing(img.copy()).unsqueeze(0)
if __name__ == '__main__':
# 載入您的模型
# 假設您的模型儲存在名為custom_model.pth.tar的檔案中
checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint1/151_31.21.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
custom_model = ResNet32(num_classes=100) # 假設你使用的是CIFAR-100資料集
checkpoint = torch.load(checkpoint_path)
custom_model.load_state_dict(checkpoint['state_dict'])
folder_path = '/home/zy/pycharm/project/MetaSAug-main/test/fistcam/new_img/'
image_folders = [f for f in os.listdir(folder_path) if os.path.isdir(os.path.join(folder_path, f))]
for folder_name in image_folders:
folder_image_files = [f for f in os.listdir(os.path.join(folder_path, folder_name)) if
f.endswith(('.png', '.jpg', '.JPEG'))]
print(f"資料夾 {folder_name} 中的圖片檔案：")
for image_file in folder_image_files:
print(image_file)
image_dir = '/home/zy/pycharm/project/MetaSAug-main/test/fistcam/new_img/'+folder_name+'/'+image_file
image = cv2.imread(image_dir)
# 將影像調整為相同的大小
resized_image = cv2.resize(image, (375, 500)) # 修改為你希望的尺寸
input_tensor = GradCAM.preprocess_image(resized_image)
grad_cam = GradCAM(custom_model, custom_model.layer4[-1], (256, 256)) # 替換為您的目標層
cam = grad_cam.calculate_cam(input_tensor)
# 將熱圖調整為相同的大小
resized_cam = cv2.resize(cam, (resized_image.shape[1], resized_image.shape[0]))
save_path = '/home/zy/pycharm/project/MetaSAug-main/test/cam/cam_img/'+folder_name+'_'+image_file
GradCAM.show_cam_on_image(image, cam, save_path)
2.（單個圖片）
備註：gram_cam_2
import os
import numpy as np
import torch
import cv2
import matplotlib.pyplot as plt
import torchvision.models as models
from torchvision.transforms import Compose, Normalize, ToTensor
from cifar.resnet import ResNet32
class GradCAM():
'''
Grad-cam: Visual explanations from deep networks via gradient-based localization
Selvaraju R R, Cogswell M, Das A, et al.
https://openaccess.thecvf.com/content_iccv_2017/html/Selvaraju_Grad-CAM_Visual_Explanations_ICCV_2017_paper.html
'''
def __init__(self, model, target_layers, input_size, use_cuda=True):
super(GradCAM).__init__()
self.use_cuda = use_cuda
self.model = model
self.target_layers = target_layers
self.target_layers.register_forward_hook(self.forward_hook)
self.target_layers.register_full_backward_hook(self.backward_hook)
self.activations = []
self.grads = []
self.input_size = input_size
def forward_hook(self, module, input, output):
self.activations.append(output[0])
def backward_hook(self, module, grad_input, grad_output):
self.grads.append(grad_output[0].detach())
def calculate_cam(self, model_input):
if self.use_cuda:
device = torch.device('cuda')
self.model.to(device)
model_input = model_input.to(device)
self.model.eval()
# forward
output, _ = self.model(model_input, 0) # 修改這裡以匹配您模型的輸出
y_hat = output
max_class = np.argmax(y_hat.cpu().data.numpy(), axis=1)
# backward
self.model.zero_grad()
y_c = y_hat[0, max_class]
y_c.backward()
# get activations and gradients
activations = self.activations[0].cpu().data.numpy().squeeze()
grads = self.grads[0].cpu().data.numpy().squeeze()
# calculate weights
weights = np.mean(grads.reshape(grads.shape[0], -1), axis=1)
weights = weights.reshape(-1, 1, 1)
cam = (weights * activations).sum(axis=0)
cam = np.maximum(cam, 0)
cam = cam / cam.max()
# Resize CAM to match the input size
cam = cv2.resize(cam, (model_input.size(3), model_input.size(2)))
return cam
@staticmethod
def show_cam_on_image(image, cam, save_path=None):
h, w = image.shape[:2]
cam = cv2.resize(cam, (w, h)) # 調整熱圖的大小與原影像相同
cam = cam / cam.max()
heatmap = cv2.applyColorMap((255 * cam).astype(np.uint8), cv2.COLORMAP_JET)
heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)
image = cv2.resize(image, (w, h)) # 調整原始影像的大小與熱圖相同
image = image / image.max()
heatmap = heatmap / heatmap.max()
result = 0.4 * heatmap + 0.6 * image
result = result / result.max()
plt.figure()
plt.imshow((result * 255).astype(np.uint8))
plt.colorbar(shrink=0.8)
plt.tight_layout()
if save_path:
plt.savefig(save_path)
plt.show()
@staticmethod
def preprocess_image(img, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]):
preprocessing = Compose([
ToTensor(),
Normalize(mean=mean, std=std)
])
return preprocessing(img.copy()).unsqueeze(0)
if __name__ == '__main__':
# 載入您的模型
# 假設您的模型儲存在名為custom_model.pth.tar的檔案中
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint2/1_1.21.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint2/3_2.46.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint2/40_20.66.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint2/80_26.27.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint2/120_26.86.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint2/160_32.66.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint1/151_31.21.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint3/3_5.27.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint3/40_20.96.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint3/80_25.3.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint3/120_25.62.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
# checkpoint_path = '/home/zy/pycharm/project/MetaSAug-main/cifar/checkpoint3/160_30.43.pth.tar' # 模型的路徑，你需要替換成你儲存的模型的路徑
custom_model = ResNet32(num_classes=100) # 假設你使用的是CIFAR-100資料集
checkpoint = torch.load(checkpoint_path)
custom_model.load_state_dict(checkpoint['state_dict'])
image_dir = '/home/zy/Desktop/img2/n03792782_22692.JPEG'
image = cv2.imread(image_dir)
resized_image = cv2.resize(image, (256, 256)) # 修改為模型的輸入尺寸
input_tensor = GradCAM.preprocess_image(resized_image)
grad_cam = GradCAM(custom_model, custom_model.layer4[-1], (256, 256)) # 替換為您的目標層
cam = grad_cam.calculate_cam(input_tensor)
save_path = '/home/zy/Desktop/img2/n03792782_22692_eda_1.jpg'
GradCAM.show_cam_on_image(image, cam, save_path)
3.附件（ResNet32）
import torch
import torch.nn as nn
import torch.nn.functional as F
import math
from torch.autograd import Variable
import torch.nn.init as init
import warnings
import matplotlib.pyplot as plt
warnings.filterwarnings('ignore')
# 定義了一個基類MetaModule，它是所有其他模組的父類。
# MetaModule提供了一些用於處理引數和更新引數的方法。
class MetaModule(nn.Module):
# adopted from: Adrien Ecoffet https://github.com/AdrienLE
def params(self):
for name, param in self.named_params(self):
yield param
def named_leaves(self):
return []
def named_submodules(self):
return []
def named_params(self, curr_module=None, memo=None, prefix=''):
if memo is None:
memo = set()
if hasattr(curr_module, 'named_leaves'):
for name, p in curr_module.named_leaves():
if p is not None and p not in memo:
memo.add(p)
yield prefix + ('.' if prefix else '') + name, p
else:
for name, p in curr_module._parameters.items():
if p is not None and p not in memo:
memo.add(p)
yield prefix + ('.' if prefix else '') + name, p
for mname, module in curr_module.named_children():
submodule_prefix = prefix + ('.' if prefix else '') + mname
for name, p in self.named_params(module, memo, submodule_prefix):
yield name, p
def update_params(self, lr_inner, first_order=False, source_params=None, detach=False):
if source_params is not None:
for tgt, src in zip(self.named_params(self), source_params):
name_t, param_t = tgt
grad = src
if first_order:
grad = to_var(grad.detach().data)
tmp = param_t - lr_inner * grad
self.set_param(self, name_t, tmp)
else:
for name, param in self.named_params(self):
if not detach:
grad = param.grad
if first_order:
grad = to_var(grad.detach().data)
tmp = param - lr_inner * grad
self.set_param(self, name, tmp)
else:
param = param.detach_()
self.set_param(self, name, param)
def set_param(self, curr_mod, name, param):
if '.' in name:
n = name.split('.')
module_name = n[0]
rest = '.'.join(n[1:])
for name, mod in curr_mod.named_children():
if module_name == name:
self.set_param(mod, rest, param)
break
else:
setattr(curr_mod, name, param)
def detach_params(self):
for name, param in self.named_params(self):
self.set_param(self, name, param.detach())
def copy(self, other, same_var=False):
for name, param in other.named_params():
if not same_var:
param = to_var(param.data.clone(), requires_grad=True)
self.set_param(name, param)
# 線性層:繼承自MetaModule類，並重寫了前向傳播方法。
class MetaLinear(MetaModule):
def __init__(self, *args, **kwargs):
super().__init__()
ignore = nn.Linear(*args, **kwargs)
self.register_buffer('weight', to_var(ignore.weight.data, requires_grad=True))
self.register_buffer('bias', to_var(ignore.bias.data, requires_grad=True))
def forward(self, x):
return F.linear(x, self.weight, self.bias)
def named_leaves(self):
return [('weight', self.weight), ('bias', self.bias)]
# 歸一化線性層:繼承自MetaModule類，並重寫了前向傳播方法。
class MetaLinear_Norm(MetaModule):
def __init__(self, *args, **kwargs):
super().__init__()
temp = nn.Linear(*args, **kwargs)
temp.weight.data.uniform_(-1, 1).renorm_(2, 1, 1e-5).mul_(1e5)
self.register_buffer('weight', to_var(temp.weight.data.t(), requires_grad=True))
self.weight.data.uniform_(-1, 1).renorm_(2, 1, 1e-5).mul_(1e5)
def forward(self, x):
out = F.normalize(x, dim=1).mm(F.normalize(self.weight, dim=0))
return out
def named_leaves(self):
return [('weight', self.weight)]
# 卷積層:繼承自MetaModule類，並重寫了前向傳播方法。
class MetaConv2d(MetaModule):
def __init__(self, *args, **kwargs):
super().__init__()
ignore = nn.Conv2d(*args, **kwargs)
self.in_channels = ignore.in_channels
self.out_channels = ignore.out_channels
self.stride = ignore.stride
self.padding = ignore.padding
self.dilation = ignore.dilation
self.groups = ignore.groups
self.kernel_size = ignore.kernel_size
self.register_buffer('weight', to_var(ignore.weight.data, requires_grad=True))
if ignore.bias is not None:
self.register_buffer('bias', to_var(ignore.bias.data, requires_grad=True))
else:
self.register_buffer('bias', None)
def forward(self, x):
return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
def named_leaves(self):
return [('weight', self.weight), ('bias', self.bias)]
# 轉置卷積層:繼承自MetaModule類，並重寫了前向傳播方法。
class MetaConvTranspose2d(MetaModule):
def __init__(self, *args, **kwargs):
super().__init__()
ignore = nn.ConvTranspose2d(*args, **kwargs)
self.stride = ignore.stride
self.padding = ignore.padding
self.dilation = ignore.dilation
self.groups = ignore.groups
self.register_buffer('weight', to_var(ignore.weight.data, requires_grad=True))
if ignore.bias is not None:
self.register_buffer('bias', to_var(ignore.bias.data, requires_grad=True))
else:
self.register_buffer('bias', None)
def forward(self, x, output_size=None):
output_padding = self._output_padding(x, output_size)
return F.conv_transpose2d(x, self.weight, self.bias, self.stride, self.padding,
output_padding, self.groups, self.dilation)
def named_leaves(self):
return [('weight', self.weight), ('bias', self.bias)]
# 批歸一化層:繼承自MetaModule類，並重寫了前向傳播方法。
class MetaBatchNorm2d(MetaModule):
def __init__(self, *args, **kwargs):
super().__init__()
ignore = nn.BatchNorm2d(*args, **kwargs)
self.num_features = ignore.num_features
self.eps = ignore.eps
self.momentum = ignore.momentum
self.affine = ignore.affine
self.track_running_stats = ignore.track_running_stats
if self.affine:
self.register_buffer('weight', to_var(ignore.weight.data, requires_grad=True))
self.register_buffer('bias', to_var(ignore.bias.data, requires_grad=True))
if self.track_running_stats:
self.register_buffer('running_mean', torch.zeros(self.num_features))
self.register_buffer('running_var', torch.ones(self.num_features))
else:
self.register_parameter('running_mean', None)
self.register_parameter('running_var', None)
def forward(self, x):
return F.batch_norm(x, self.running_mean, self.running_var, self.weight, self.bias,
self.training or not self.track_running_stats, self.momentum, self.eps)
def named_leaves(self):
return [('weight', self.weight), ('bias', self.bias)]
class LambdaLayer(MetaModule):
def __init__(self, lambd):
super(LambdaLayer, self).__init__()
self.lambd = lambd
def forward(self, x):
return self.lambd(x)
# BasicBlock類，它是ResNet中的基本塊。它繼承自MetaModule類，並重寫了前向傳播方法。
class BasicBlock(MetaModule):
expansion = 1
def __init__(self, in_planes, planes, stride=1, option='A'):
super(BasicBlock, self).__init__()
self.conv1 = MetaConv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
self.bn1 = MetaBatchNorm2d(planes)
self.conv2 = MetaConv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
self.bn2 = MetaBatchNorm2d(planes)
self.shortcut = nn.Sequential()
if stride != 1 or in_planes != planes:
if option == 'A':
self.shortcut = LambdaLayer(lambda x:
F.pad(x[:, :, ::2, ::2], (0, 0, 0, 0, planes // 4, planes // 4), "constant",
0))
elif option == 'B':
self.shortcut = nn.Sequential(
MetaConv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False),
MetaBatchNorm2d(self.expansion * planes)
)
def forward(self, x):
out = F.relu(self.bn1(self.conv1(x)))
out = self.bn2(self.conv2(out))
out += self.shortcut(x)
out = F.relu(out)
return out
# for metamodel
# 定義了ResNet32類，它是一個完整的ResNet模型。
# 它繼承自MetaModule類，並定義了ResNet的整體結構和前向傳播方法。
class ResNet32_meta(MetaModule):
# _first_init_done = False
def __init__(self, num_classes, block=BasicBlock, num_blocks=[5, 5, 5]):
super(ResNet32_meta, self).__init__()
self.in_planes = 16
self.conv1 = MetaConv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)
self.bn1 = MetaBatchNorm2d(16)
self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1)
self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2)
self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2)
self.linear = MetaLinear(64, num_classes)
self.apply(_weights_init)
def _make_layer(self, block, planes, num_blocks, stride):
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
layers.append(block(self.in_planes, planes, stride))
self.in_planes = planes * block.expansion
return nn.Sequential(*layers)
def forward(self, x, epoch):
out = F.relu(self.bn1(self.conv1(x)))
out = self.layer1(out)
out = self.layer2(out)
out = self.layer3(out)
out = F.avg_pool2d(out, out.size()[3])
out = out.view(out.size(0), -1)
y = self.linear(out)
return out, y
# for main
class ResNet32(MetaModule):
def __init__(self, num_classes, block=BasicBlock, num_blocks=[5, 5, 5, 5]):
super(ResNet32, self).__init__()
self.in_planes = 16
self.conv1 = MetaConv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)
self.bn1 = MetaBatchNorm2d(16)
self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1)
self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2)
self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2)
self.layer4 = self._make_layer(block, 128, num_blocks[3], stride=2)
# Add
# print("Using self attention")
# self.modulatedatt = ModulatedAttLayer(in_channels=64 * block.expansion)
#
#
# self.cbam = CBAM(64 * block.expansion, 64)
# self.scse1 = SCse(16*block.expansion)
# self.scse2 = SCse(32*block.expansion)
# self.scse3 = SCse(64*block.expansion)
self.linear = MetaLinear(128, num_classes)
self.apply(_weights_init)
def _make_layer(self, block, planes, num_blocks, stride):
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
layers.append(block(self.in_planes, planes, stride))
self.in_planes = planes * block.expansion
return nn.Sequential(*layers)
def forward(self, x, epoch):
out = F.relu(self.bn1(self.conv1(x)))
out = self.layer1(out)
out = self.layer2(out)
out = self.layer3(out)
out = self.layer4(out)
out = F.avg_pool2d(out, out.size()[3])
# out = F.avg_pool2d(out, kernel_size=(13, 18))
out = out.view(out.size(0), -1)
y = self.linear(out)
return out, y
def to_var(x, requires_grad=True):
if torch.cuda.is_available():
x = x.cuda()
return Variable(x, requires_grad=requires_grad)
def _weights_init(m):
classname = m.__class__.__name__
if isinstance(m, MetaLinear) or isinstance(m, MetaConv2d):
init.kaiming_normal(m.weight)
