多头注意力机制和交叉注意力机制

多头注意力机制（MHSA）是一种注意力机制，它可以在不同的表示子空间中并行地计算多个注意力分数。这种机制可以帮助模型更好地捕捉输入序列中的不同关系。在图像分割中，MHSA通常被用于编码器的最后一层，以便模型可以同时关注整个图像。而交叉注意力机制则是将注意力机制应用于跳跃连接之后的解码器中，以将高层次语义更丰富的特征图与来自跳跃连接的高分辨率图结合起来，从而提高分割的准确性。

下面是一个简单的例子，展示了如何在PyTorch中实现多头注意力机制和交叉注意力机制：

import torch
import torch.nn as nn

# 多头注意力机制
class MultiHeadAttention(nn.Module):
    def __init__(self, d_model, num_heads):
        super(MultiHeadAttention, self).__init__()
        self.num_heads = num_heads
        self.d_model = d_model
        
        assert d_model % num_heads == 0
        
        self.depth = d_model // num_heads
        
        self.query = nn.Linear(d_model, d_model)
        self.key = nn.Linear(d_model, d_model)
        self.value = nn.Linear(d_model, d_model)
        
        self.fc = nn.Linear(d_model, d_model)
        
    def split_heads(self, x, batch_size):
        x = x.view(batch_size, -1, self.num_heads, self.depth)
        return x.permute(0, 2, 1, 3)
    
    def forward(self, query, key, value, mask=None):
        batch_size = query.size(0)
        
        # 线性变换
        query = self.query(query)
        key = self.key(key)
        value = self.value(value)
        
        # 拆分头
        query = self.split_heads(query, batch_size)
        key = self.split_heads(key, batch_size)
        value = self.split_heads(value, batch_size)
        
        # 计算注意力
        scores = torch.matmul(query, key.transpose(-1, -2)) / torch.sqrt(torch.tensor(self.depth).float())
        if mask is not None:
            scores += mask * -1e9
        attention = nn.Softmax(dim=-1)(scores)
        context = torch.matmul(attention, value)
        
        # 合并头
        context = context.permute(0, 2, 1, 3).contiguous()
        context = context.view(batch_size, -1, self.d_model)
        
        # 线性变换
        output = self.fc(context)
        
        return output, attention

# 交叉注意力机制
class CrossAttention(nn.Module):
    def __init__(self, d_model):
        super(CrossAttention, self).__init__()
        self.query = nn.Linear(d_model, d_model)
        self.key = nn.Linear(d_model, d_model)
        self.value = nn.Linear(d_model, d_model)
        
        self.fc = nn.Linear(d_model, d_model)
        
    def forward(self, query, key, value, mask=None):
        # 线性变换
        query = self.query(query)
        key = self.key(key)
        value = self.value(value)
        
        # 计算注意力
        scores = torch.matmul(query, key.transpose(-1, -2)) / torch.sqrt(torch.tensor(query.size(-1)).float())
        if mask is not None:
            scores += mask * -1e9
        attention = nn.Softmax(dim=-1)(scores)
        context = torch.matmul(attention, value)
        
        # 线性变换
        output = self.fc(context)
        
        return output, attention