ELMo 词向量+Glove 词向量在情感分类 任务上的表现pytorch简单代码
时间: 2023-12-13 12:02:35 浏览: 165
以下是使用 PyTorch 实现的 ELMo 词向量和 Glove 词向量在情感分类任务上的简单代码:
```python
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torchtext.datasets import IMDB
from torchtext.data import Field, LabelField, BucketIterator
class ELMo(nn.Module):
def __init__(self, embedding_dim, hidden_dim, num_layers):
super(ELMo, self).__init__()
self.embedding_dim = embedding_dim
self.hidden_dim = hidden_dim
self.num_layers = num_layers
self.embedding = nn.Embedding(vocab_size, embedding_dim)
self.lstm = nn.LSTM(embedding_dim, hidden_dim, num_layers, bidirectional=True)
self.linear = nn.Linear(hidden_dim * 2, 1)
def forward(self, x):
# x: (seq_len, batch_size)
embedded = self.embedding(x)
# embedded: (seq_len, batch_size, embedding_dim)
outputs, _ = self.lstm(embedded)
# outputs: (seq_len, batch_size, hidden_dim * 2)
weights = F.softmax(self.linear(outputs), dim=0)
# weights: (seq_len, batch_size, 1)
embeddings = torch.sum(weights * outputs, dim=0)
# embeddings: (batch_size, hidden_dim * 2)
return embeddings
class Glove(nn.Module):
def __init__(self, embedding_dim):
super(Glove, self).__init__()
self.embedding = nn.Embedding(vocab_size, embedding_dim)
def forward(self, x):
# x: (seq_len, batch_size)
embedded = self.embedding(x)
# embedded: (seq_len, batch_size, embedding_dim)
embeddings = torch.mean(embedded, dim=0)
# embeddings: (batch_size, embedding_dim)
return embeddings
class Classifier(nn.Module):
def __init__(self, input_dim, hidden_dim, output_dim):
super(Classifier, self).__init__()
self.fc1 = nn.Linear(input_dim, hidden_dim)
self.fc2 = nn.Linear(hidden_dim, output_dim)
def forward(self, x):
# x: (batch_size, input_dim)
x = F.relu(self.fc1(x))
# x: (batch_size, hidden_dim)
x = self.fc2(x)
# x: (batch_size, output_dim)
return x
# define Fields
TEXT = Field(tokenize='spacy')
LABEL = LabelField(dtype=torch.float)
# load data
train_data, test_data = IMDB.splits(TEXT, LABEL)
# build vocabulary
TEXT.build_vocab(train_data, max_size=10000, vectors=['glove.6B.100d'])
LABEL.build_vocab(train_data)
# define device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# define hyperparameters
batch_size = 64
embedding_dim = 100
hidden_dim = 256
num_layers = 2
input_dim = hidden_dim * 4
output_dim = 1
lr = 1e-3
num_epochs = 10
# define models
elmo = ELMo(embedding_dim, hidden_dim, num_layers).to(device)
glove = Glove(embedding_dim).to(device)
classifier = Classifier(input_dim, hidden_dim, output_dim).to(device)
# define loss function and optimizer
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(classifier.parameters(), lr=lr)
# define iterators
train_iterator, test_iterator = BucketIterator.splits(
(train_data, test_data), batch_size=batch_size, device=device)
# train models
for epoch in range(num_epochs):
for batch in train_iterator:
elmo_embeddings = elmo(batch.text)
glove_embeddings = glove(batch.text)
embeddings = torch.cat((elmo_embeddings, glove_embeddings), dim=1)
labels = batch.label
optimizer.zero_grad()
outputs = classifier(embeddings)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, loss.item()))
# evaluate models
correct = 0
total = 0
with torch.no_grad():
for batch in test_iterator:
elmo_embeddings = elmo(batch.text)
glove_embeddings = glove(batch.text)
embeddings = torch.cat((elmo_embeddings, glove_embeddings), dim=1)
labels = batch.label
outputs = classifier(embeddings)
predicted = torch.round(torch.sigmoid(outputs))
total += labels.size(0)
correct += (predicted == labels).sum().item()
print('Accuracy: {:.2f}%'.format(100 * correct / total))
```
在这个代码中,我们首先定义了三个模型:ELMo、Glove 和分类器。ELMo 和 Glove 模型分别用于提取 ELMo 词向量和 Glove 词向量,并将两者拼接起来作为分类器的输入。分类器是一个简单的全连接神经网络,用于将拼接后的向量映射到一个二元分类输出(正面或负面情感)。我们使用的数据集是 IMDB 电影评论数据集,其中每个样本都是一个电影评论文本和其对应的情感标签。
在训练过程中,我们首先将每个样本的文本输入到 ELMo 和 Glove 模型中,得到两个向量。然后将这两个向量拼接起来,作为分类器的输入。分类器输出的结果与真实标签计算二元交叉熵损失,并进行反向传播更新模型参数。最终,我们使用测试集评估模型的准确率。