transformer模型分类器
时间: 2024-04-14 16:24:23 浏览: 12
Transformer模型可以用作分类器,它在NLP任务中表现出色。下面是一个使用Transformer模型进行文本分类的示例[^2]:
```python
import torch
import torch.nn as nn
import torch.optim as optim
from torchtext.data import Field, TabularDataset, BucketIterator
from torch.nn import TransformerEncoder, TransformerEncoderLayer
# 定义模型
class TransformerClassifier(nn.Module):
def __init__(self, input_dim, hidden_dim, output_dim, n_layers, n_heads, dropout):
super().__init__()
self.embedding = nn.Embedding(input_dim, hidden_dim)
self.encoder_layer = TransformerEncoderLayer(hidden_dim, n_heads, hidden_dim, dropout)
self.encoder = TransformerEncoder(self.encoder_layer, n_layers)
self.fc = nn.Linear(hidden_dim, output_dim)
self.dropout = nn.Dropout(dropout)
def forward(self, text):
embedded = self.embedding(text)
embedded = embedded.permute(1, 0, 2)
output = self.encoder(embedded)
output = output.permute(1, 0, 2)
pooled = torch.mean(output, dim=1)
pooled = self.dropout(pooled)
return self.fc(pooled)
# 定义数据集和字段
TEXT = Field(tokenize='spacy', lower=True)
LABEL = Field(sequential=False, is_target=True)
fields = [('text', TEXT), ('label', LABEL)]
# 加载数据集
train_data, test_data = TabularDataset.splits(
path='data',
train='train.csv',
test='test.csv',
format='csv',
fields=fields,
skip_header=True
)
# 构建词汇表
TEXT.build_vocab(train_data, min_freq=2)
LABEL.build_vocab(train_data)
# 创建迭代器
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_iterator, test_iterator = BucketIterator.splits(
(train_data, test_data),
batch_size=64,
sort_key=lambda x: len(x.text),
sort_within_batch=True,
device=device
)
# 初始化模型和优化器
input_dim = len(TEXT.vocab)
output_dim = len(LABEL.vocab)
hidden_dim = 256
n_layers = 2
n_heads = 4
dropout = 0.5
model = TransformerClassifier(input_dim, hidden_dim, output_dim, n_layers, n_heads, dropout).to(device)
optimizer = optim.Adam(model.parameters())
# 训练模型
def train(model, iterator, optimizer, criterion):
model.train()
epoch_loss = 0
for batch in iterator:
optimizer.zero_grad()
text = batch.text
label = batch.label
predictions = model(text).squeeze(1)
loss = criterion(predictions, label)
loss.backward()
optimizer.step()
epoch_loss += loss.item()
return epoch_loss / len(iterator)
# 测试模型
def evaluate(model, iterator, criterion):
model.eval()
epoch_loss = 0
with torch.no_grad():
for batch in iterator:
text = batch.text
label = batch.label
predictions = model(text).squeeze(1)
loss = criterion(predictions, label)
epoch_loss += loss.item()
return epoch_loss / len(iterator)
# 训练和测试模型
N_EPOCHS = 10
criterion = nn.CrossEntropyLoss().to(device)
for epoch in range(N_EPOCHS):
train_loss = train(model, train_iterator, optimizer, criterion)
test_loss = evaluate(model, test_iterator, criterion)
print(f'Epoch: {epoch+1}, Train Loss: {train_loss:.4f}, Test Loss: {test_loss:.4f}')
# 使用模型进行预测
def predict_sentiment(model, sentence):
model.eval()
tokenized = [tok.text for tok in nlp.tokenizer(sentence)]
indexed = [TEXT.vocab.stoi[t] for t in tokenized]
tensor = torch.LongTensor(indexed).unsqueeze(1).to(device)
prediction = torch.argmax(model(tensor))
return LABEL.vocab.itos[prediction]
# 示例预测
example_sentence = "This movie is great!"
predicted_label = predict_sentiment(model, example_sentence)
print(f'Predicted Label: {predicted_label}')
```
这是一个基本的Transformer模型分类器的示例。你可以根据自己的数据集和任务进行相应的修改和调整。