transformer分类代码

下面是一个使用Transformer模型进行分类的示例代码：

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torchtext.datasets import IMDB
from torchtext.data import Field, LabelField, BucketIterator

class TransformerClassifier(nn.Module):
    def __init__(self, input_dim, embedding_dim, n_heads, hidden_dim, n_layers, output_dim, dropout):
        super(TransformerClassifier, self).__init__()
        self.embedding = nn.Embedding(input_dim, embedding_dim)
        self.encoder = nn.TransformerEncoder(nn.TransformerEncoderLayer(embedding_dim, n_heads, hidden_dim, dropout), n_layers)
        self.fc = nn.Linear(embedding_dim, output_dim)
        self.dropout = nn.Dropout(dropout)
    
    def forward(self, x):
        embedded = self.embedding(x) # [seq_len, batch_size, embedding_dim]
        embedded = self.dropout(embedded)
        encoded = self.encoder(embedded) # [seq_len, batch_size, embedding_dim]
        pooled = F.avg_pool1d(encoded.permute(1, 2, 0), encoded.shape[0]).squeeze(2) # [batch_size, embedding_dim]
        output = self.fc(pooled)
        return output

# 数据预处理
TEXT = Field(tokenize='spacy', lower=True)
LABEL = LabelField(dtype=torch.float)

train_data, test_data = IMDB.splits(TEXT, LABEL)
TEXT.build_vocab(train_data, max_size=25000, vectors="glove.6B.100d", unk_init=torch.Tensor.normal_)
LABEL.build_vocab(train_data)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_iterator, test_iterator = BucketIterator.splits((train_data, test_data), batch_size=64, device=device)

# 模型参数
input_dim = len(TEXT.vocab)
embedding_dim = 100
n_heads = 4
hidden_dim = 256
n_layers = 2
output_dim = 1
dropout = 0.5

model = TransformerClassifier(input_dim, embedding_dim, n_heads, hidden_dim, n_layers, output_dim, dropout).to(device)

# 损失函数和优化器
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(model.parameters())

# 训练和评估函数
def train(model, iterator, optimizer, criterion):
    model.train()
    epoch_loss = 0
    for batch in iterator:
        optimizer.zero_grad()
        predictions = model(batch.text).squeeze(1)
        loss = criterion(predictions, batch.label)
        loss.backward()
        optimizer.step()
        epoch_loss += loss.item()
    return epoch_loss / len(iterator)

def evaluate(model, iterator, criterion):
    model.eval()
    epoch_loss = 0
    with torch.no_grad():
        for batch in iterator:
            predictions = model(batch.text).squeeze(1)
            loss = criterion(predictions, batch.label)
            epoch_loss += loss.item()
    return epoch_loss / len(iterator)

# 训练模型
N_EPOCHS = 5

for epoch in range(N_EPOCHS):
    train_loss = train(model, train_iterator, optimizer, criterion)
    test_loss = evaluate(model, test_iterator, criterion)
    print(f'Epoch: {epoch+1:02}, Train Loss: {train_loss:.3f}, Test Loss: {test_loss:.3f}')

注意：这只是一个简单的示例，可能需要根据你的具体需求进行修改和调整。