Transformer cnn代码

以下是一个简单的 Transformer + CNN 的代码示例，用于文本分类任务：

import torch
import torch.nn as nn
import torch.nn.functional as F

class TransformerCNN(nn.Module):
    def __init__(self, vocab_size, embedding_dim, num_classes, max_seq_len, num_filters, filter_sizes, num_heads, hidden_dim, dropout):
        super(TransformerCNN, self).__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.pos_encoding = PositionalEncoding(embedding_dim, max_seq_len)
        self.transformer_encoder = nn.TransformerEncoder(
            nn.TransformerEncoderLayer(embedding_dim, num_heads, hidden_dim, dropout),
            num_layers=2)
        self.convs = nn.ModuleList([
            nn.Conv2d(1, num_filters, (k, embedding_dim)) for k in filter_sizes
        ])
        self.fc = nn.Linear(num_filters * len(filter_sizes), num_classes)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):
        x = self.embedding(x)
        x = self.pos_encoding(x)
        x = x.permute(1, 0, 2)
        x = self.transformer_encoder(x)
        x = x.permute(1, 0, 2)
        x = x.unsqueeze(1)
        x = [F.relu(conv(x)).squeeze(3) for conv in self.convs]
        x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x]
        x = torch.cat(x, 1)
        x = self.dropout(x)
        logits = self.fc(x)
        return logits

class PositionalEncoding(nn.Module):
    def __init__(self, d_model, max_len=5000):
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(p=0.1)
        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0).transpose(0, 1)
        self.register_buffer('pe', pe)

    def forward(self, x):
        x = x + self.pe[:x.size(0), :]
        return self.dropout(x)

其中，`TransformerCNN` 类继承自 `nn.Module`，包含了一个嵌入层、一个位置编码层、两个 Transformer 编码层、多个卷积层、一个全连接层和一个 dropout 层。在 `forward` 方法中，首先将输入的文本序列进行嵌入和位置编码，然后输入到 Transformer 编码层中进行特征提取。接着，将 Transformer 编码层的输出进行卷积和池化操作，最后通过全连接层得到分类结果。