def Grad_Cam(model, image, layer_name): # 获取模型提取全链接之前的特征图 new_model = nn.Sequential(*list(model.children())[:44]) print(new_model) new_model.eval() feature_maps = new_model(image) # 获取模型最后一层卷积层 target_layer = model._modules.get(layer_name) # 将模型最后一层卷积层的输出结果作为反向传播的梯度 gradient = torch.zeros(feature_maps.size()) # 返回一个形状与feature_maps相同全为标量 0 的张量 gradient[:, :, feature_maps.size()[2]//2, feature_maps.size()[3]//2] = 1 target_layer.zero_grad() # 将模型中参数的梯度置为0 feature_maps.backward(gradient=gradient) # 获取模型最后一层卷积层的输出结果和梯度 _, _, H, W = feature_maps.size() output_activations = feature_maps.detach().numpy()[0] gradients = target_layer.weight.grad.detach().numpy() # 计算特征图中每个像素点的权重 weights = np.mean(gradients, axis=(2, 3))[0] cam = np.zeros((H, W), dtype=np.float32) for i, w in enumerate(weights): cam += w * output_activations[i, :, :] # 对权重进行归一化处理 cam = np.maximum(cam, 0) cam = cv2.resize(cam, (1440, 1440)) cam = cam - np.min(cam) cam = cam / np.max(cam) # 将热力图叠加到原图上 heatmap = cv2.applyColorMap(np.uint8(255 * cam), cv2.COLORMAP_JET) heatmap = np.float32(heatmap) / 255 image = image.detach().numpy() image = np.transpose(image, (0, 2, 3, 1)) img_CCT = cv2.imread("F:/BaiduSyncdisk/python/svm_CCT/picture CCT_CP/2L5830N023_CCT.png") img_CP = cv2.imread("F:/BaiduSyncdisk/python/svm_CCT/picture CCT_CP/2L5830N023_CP.png") img_CCT = cv2.resize(img_CCT, (1440, 1440)) img_CP = cv2.resize(img_CP, (1440, 1440)) cam_img = heatmap + np.float32(img_CCT[0]) cam_img = cam_img / np.max(cam_img) return np.uint8(255 * cam_img) 上述代码不显示热力图，怎么解决

可能是因为在使用 OpenCV 的 `cv2.imshow()` 函数显示图像时，会因为使用了多线程导致程序崩溃。可以尝试使用 `matplotlib` 库来显示热力图，代码如下：

import matplotlib.pyplot as plt

# 将热力图进行归一化处理
heatmap = cv2.resize(cam, (image.shape[2], image.shape[3]))
heatmap = heatmap - np.min(heatmap)
heatmap = heatmap / np.max(heatmap)

# 将热力图叠加到原图上
heatmap = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)
cam_img = np.float32(heatmap) / 255 + np.transpose(image[0], (1, 2, 0))
cam_img = cam_img / np.max(cam_img)

# 使用 matplotlib 显示热力图
plt.imshow(cam_img)
plt.show()

这样就可以在 Jupyter Notebook 或者 Python IDE 中显示热力图了。如果仍然无法显示，可以尝试将 `plt.show()` 替换为 `plt.savefig('heatmap.png')`，将热力图保存为图片进行查看。

相关问题

首先，基于图像和数值数据作为共同输入，自己创建一个CNN回归模型，其中图像和数值数据在全连接层进行拼接；然后，对CNN模型进行训练和测试；最后，实现Grad-Cam可视化，计算最后一个卷积层中所有特征图对图片类别的权重，以热力图的形式把特征图映射到原始图片中，显示图像中用于预测的重要区域。PyTorch完整代码实现

好的，下面是基于图像和数值数据的CNN回归模型及Grad-Cam可视化的PyTorch代码实现：

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.models as models
import torchvision.transforms as transforms
from torch.utils.data import Dataset, DataLoader
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt


# 定义数据集类
class MyDataset(Dataset):
    def __init__(self, image_paths, numerical_data, targets, transform=None):
        self.image_paths = image_paths
        self.numerical_data = numerical_data
        self.targets = targets
        self.transform = transform
    
    def __len__(self):
        return len(self.targets)
    
    def __getitem__(self, idx):
        img_path = self.image_paths[idx]
        numerical = self.numerical_data[idx]
        target = self.targets[idx]
        
        image = Image.open(img_path).convert('RGB')
        
        if self.transform:
            image = self.transform(image)
            
        return image, numerical, target


# 定义CNN回归模型
class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.Conv2d(16, 32, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )
        self.fc = nn.Sequential(
            nn.Linear(128*7*7 + 1, 512),
            nn.ReLU(),
            nn.Linear(512, 1)
        )
        
    def forward(self, x, numerical):
        x = self.conv(x)
        x = x.view(x.size(0), -1)
        x = torch.cat((x, numerical), dim=1)
        x = self.fc(x)
        return x


# 定义Grad-Cam可视化函数
class GradCam:
    def __init__(self, model, target_layer):
        self.model = model
        self.target_layer = target_layer
        self.feature_maps = None
        self.gradient = None
    
    def get_feature_maps(self, x):
        self.model.eval()
        feature_maps = []
        for name, module in self.model.named_children():
            x = module(x)
            if name == self.target_layer:
                feature_maps = x
        return feature_maps
    
    def get_gradient(self):
        gradient = self.feature_maps.grad.cpu().data.numpy()[0]
        return gradient
    
    def hook_feature_maps(self, module, input, output):
        self.feature_maps = output
    
    def hook_gradient(self, module, grad_in, grad_out):
        self.gradient = grad_out[0]
    
    def __call__(self, x, numerical, index=None):
        self.model.eval()
        feature_maps = self.get_feature_maps(x)
        if index == None:
            index = torch.argmax(self.model(x, numerical)).item()
        self.model.zero_grad()
        target = feature_maps[:, index, :, :].sum()
        target.backward(retain_graph=True)
        gradient = self.get_gradient()
        pooled_gradient = np.mean(gradient, axis=(1,2))
        feature_maps = feature_maps.cpu().data.numpy()[0]
        weighted_feature_maps = np.zeros(feature_maps.shape[1:], dtype=np.float32)
        for i, w in enumerate(pooled_gradient):
            weighted_feature_maps += w * feature_maps[i, :, :]
        weighted_feature_maps = np.maximum(weighted_feature_maps, 0)
        weighted_feature_maps /= np.max(weighted_feature_maps)
        return weighted_feature_maps


# 训练模型
def train(model, train_loader, criterion, optimizer, device):
    model.train()
    running_loss = 0.0
    for i, (images, numerical, targets) in enumerate(train_loader):
        images = images.to(device)
        numerical = numerical.to(device)
        targets = targets.to(device)
        
        optimizer.zero_grad()
        outputs = model(images, numerical)
        loss = criterion(outputs, targets)
        loss.backward()
        optimizer.step()
        
        running_loss += loss.item() * images.size(0)
        
        if (i+1) % 10 == 0:
            print('Training - Epoch: {} | Iter: {} | Loss: {:.4f}'.format(epoch+1, i+1, loss.item()))
        
    epoch_loss = running_loss / len(train_loader.dataset)
    return epoch_loss


# 测试模型
def test(model, test_loader, criterion, device):
    model.eval()
    running_loss = 0.0
    with torch.no_grad():
        for i, (images, numerical, targets) in enumerate(test_loader):
            images = images.to(device)
            numerical = numerical.to(device)
            targets = targets.to(device)

            outputs = model(images, numerical)
            loss = criterion(outputs, targets)

            running_loss += loss.item() * images.size(0)

    epoch_loss = running_loss / len(test_loader.dataset)
    return epoch_loss


# 加载数据集
train_image_paths = ['train_images/'+str(i)+'.jpg' for i in range(1, 2001)]
train_numerical_data = np.load('train_numerical_data.npy')
train_targets = np.load('train_targets.npy')

test_image_paths = ['test_images/'+str(i)+'.jpg' for i in range(1, 501)]
test_numerical_data = np.load('test_numerical_data.npy')
test_targets = np.load('test_targets.npy')

transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])

train_dataset = MyDataset(train_image_paths, train_numerical_data, train_targets, transform)
test_dataset = MyDataset(test_image_paths, test_numerical_data, test_targets, transform)

train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)


# 训练模型
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
model = MyModel().to(device)

criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

num_epochs = 10

for epoch in range(num_epochs):
    train_loss = train(model, train_loader, criterion, optimizer, device)
    print('Training - Epoch: {} | Loss: {:.4f}'.format(epoch+1, train_loss))

    test_loss = test(model, test_loader, criterion, device)
    print('Testing - Epoch: {} | Loss: {:.4f}'.format(epoch+1, test_loss))


# 可视化Grad-Cam结果
grad_cam = GradCam(model, 'conv.11')  # 使用最后一个卷积层进行可视化

img_path = 'test_images/1.jpg'
numerical_data = test_numerical_data[0]
img = Image.open(img_path).convert('RGB')

plt.imshow(img)
plt.axis('off')
plt.show()

img_tensor = transform(img).unsqueeze(0).to(device)
numerical_tensor = torch.from_numpy(numerical_data).unsqueeze(0).float().to(device)

output = model(img_tensor, numerical_tensor)
predicted_class = torch.argmax(output).item()

weighted_feature_maps = grad_cam(img_tensor, numerical_tensor, predicted_class)

plt.imshow(img)
plt.imshow(weighted_feature_maps, alpha=0.5, cmap='jet')
plt.axis('off')
plt.show()

需要注意的是，上述代码中使用了一个数据集，该数据集包含了2000张训练图片和500张测试图片，图片和数值数据是通过函数生成的，因此无法直接运行代码。但是代码中的数据处理部分是通用的，可以根据自己的数据集进行修改。同时，Grad-Cam可视化部分的代码需要注意，其中的`'conv.11'`是指VGG16模型中的最后一个卷积层，如果使用其他模型需要根据实际情况进行修改。

对CNN回归模型中指定卷积层实现Grad-CAM。PyTorch 生成代码

好的，以下是PyTorch生成的代码示例：

import torch
import torch.nn as nn
import cv2
import numpy as np

# 定义CNN模型
class CNNModel(nn.Module):
    def __init__(self):
        super(CNNModel, self).__init__()
        self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1)
        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)
        self.conv3 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1)
        self.fc1 = nn.Linear(128 * 8 * 8, 1024)
        self.fc2 = nn.Linear(1024, 10)

    def forward(self, x):
        x = nn.functional.relu(self.conv1(x))
        x = nn.functional.max_pool2d(x, 2)
        x = nn.functional.relu(self.conv2(x))
        x = nn.functional.max_pool2d(x, 2)
        x = nn.functional.relu(self.conv3(x))
        x = nn.functional.max_pool2d(x, 2)
        x = x.view(-1, 128 * 8 * 8)
        x = nn.functional.relu(self.fc1(x))
        x = self.fc2(x)
        return x

# 加载模型和图像
model = CNNModel()
model.load_state_dict(torch.load('model.pth'))
img = cv2.imread('image.jpg')
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (32, 32))
img = np.transpose(img, (2, 0, 1))
img = torch.tensor(img).unsqueeze(0)

# 定义Grad-CAM方法
def grad_cam(model, img, layer_name):
    # 获取指定卷积层
    conv_layer = None
    for name, module in model.named_modules():
        if name == layer_name:
            conv_layer = module
            break
    if conv_layer is None:
        raise ValueError('指定的卷积层不存在')

    # 前向传播
    out = model(img)
    _, pred = torch.max(out, 1)

    # 反向传播
    model.zero_grad()
    one_hot = torch.zeros((1, out.size()[-1]), dtype=torch.float32)
    one_hot[0][pred] = 1
    out.backward(gradient=one_hot, retain_graph=True)

    # 计算权重和特征图
    weights = torch.mean(conv_layer.weight.grad, axis=(2, 3)).squeeze()
    features = conv_layer(img)
    features = nn.functional.relu(features).squeeze()
    cam = torch.zeros(features.size()[-2:], dtype=torch.float32)
    for i in range(weights.size()[0]):
        cam += weights[i] * features[i]

    # 归一化处理
    cam = nn.functional.relu(cam)
    cam = cam - torch.min(cam)
    cam = cam / torch.max(cam)

    # 转换为图像尺寸
    cam = cv2.resize(cam.detach().numpy(), (img.size()[2], img.size()[3]))
    cam = np.uint8(cam * 255)

    return cam

# 调用Grad-CAM方法并可视化结果
cam = grad_cam(model, img, 'conv3')
heatmap = cv2.applyColorMap(cam, cv2.COLORMAP_JET)
result = cv2.addWeighted(cv2.cvtColor(img.squeeze().numpy().transpose(1, 2, 0), cv2.COLOR_RGB2BGR), 0.5, heatmap, 0.5, 0)
cv2.imwrite('result.jpg', result)

在上述代码中，我们定义了一个CNN模型，加载了预训练的权重和一张输入图像。然后，我们定义了一个Grad-CAM方法，该方法将模型、输入图像和指定的卷积层名称作为输入，返回一个热力图。最后，我们将热力图可视化为彩色图像，并将其保存到本地。