ELMo 词向量+Glove 词向量在情感分类 任务上的表现pytorch简单代码

以下是使用 PyTorch 实现的 ELMo 词向量和 Glove 词向量在情感分类任务上的简单代码：

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torchtext.datasets import IMDB
from torchtext.data import Field, LabelField, BucketIterator

class ELMo(nn.Module):
    def __init__(self, embedding_dim, hidden_dim, num_layers):
        super(ELMo, self).__init__()
        self.embedding_dim = embedding_dim
        self.hidden_dim = hidden_dim
        self.num_layers = num_layers
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.lstm = nn.LSTM(embedding_dim, hidden_dim, num_layers, bidirectional=True)
        self.linear = nn.Linear(hidden_dim * 2, 1)
        
    def forward(self, x):
        # x: (seq_len, batch_size)
        embedded = self.embedding(x)
        # embedded: (seq_len, batch_size, embedding_dim)
        outputs, _ = self.lstm(embedded)
        # outputs: (seq_len, batch_size, hidden_dim * 2)
        weights = F.softmax(self.linear(outputs), dim=0)
        # weights: (seq_len, batch_size, 1)
        embeddings = torch.sum(weights * outputs, dim=0)
        # embeddings: (batch_size, hidden_dim * 2)
        return embeddings

class Glove(nn.Module):
    def __init__(self, embedding_dim):
        super(Glove, self).__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        
    def forward(self, x):
        # x: (seq_len, batch_size)
        embedded = self.embedding(x)
        # embedded: (seq_len, batch_size, embedding_dim)
        embeddings = torch.mean(embedded, dim=0)
        # embeddings: (batch_size, embedding_dim)
        return embeddings

class Classifier(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(Classifier, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, output_dim)
        
    def forward(self, x):
        # x: (batch_size, input_dim)
        x = F.relu(self.fc1(x))
        # x: (batch_size, hidden_dim)
        x = self.fc2(x)
        # x: (batch_size, output_dim)
        return x

# define Fields
TEXT = Field(tokenize='spacy')
LABEL = LabelField(dtype=torch.float)

# load data
train_data, test_data = IMDB.splits(TEXT, LABEL)

# build vocabulary
TEXT.build_vocab(train_data, max_size=10000, vectors=['glove.6B.100d'])
LABEL.build_vocab(train_data)

# define device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# define hyperparameters
batch_size = 64
embedding_dim = 100
hidden_dim = 256
num_layers = 2
input_dim = hidden_dim * 4
output_dim = 1
lr = 1e-3
num_epochs = 10

# define models
elmo = ELMo(embedding_dim, hidden_dim, num_layers).to(device)
glove = Glove(embedding_dim).to(device)
classifier = Classifier(input_dim, hidden_dim, output_dim).to(device)

# define loss function and optimizer
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(classifier.parameters(), lr=lr)

# define iterators
train_iterator, test_iterator = BucketIterator.splits(
    (train_data, test_data), batch_size=batch_size, device=device)

# train models
for epoch in range(num_epochs):
    for batch in train_iterator:
        elmo_embeddings = elmo(batch.text)
        glove_embeddings = glove(batch.text)
        embeddings = torch.cat((elmo_embeddings, glove_embeddings), dim=1)
        labels = batch.label
        optimizer.zero_grad()
        outputs = classifier(embeddings)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
    print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, loss.item()))

# evaluate models
correct = 0
total = 0
with torch.no_grad():
    for batch in test_iterator:
        elmo_embeddings = elmo(batch.text)
        glove_embeddings = glove(batch.text)
        embeddings = torch.cat((elmo_embeddings, glove_embeddings), dim=1)
        labels = batch.label
        outputs = classifier(embeddings)
        predicted = torch.round(torch.sigmoid(outputs))
        total += labels.size(0)
        correct += (predicted == labels).sum().item()
print('Accuracy: {:.2f}%'.format(100 * correct / total))

在这个代码中，我们首先定义了三个模型：ELMo、Glove 和分类器。ELMo 和 Glove 模型分别用于提取 ELMo 词向量和 Glove 词向量，并将两者拼接起来作为分类器的输入。分类器是一个简单的全连接神经网络，用于将拼接后的向量映射到一个二元分类输出（正面或负面情感）。我们使用的数据集是 IMDB 电影评论数据集，其中每个样本都是一个电影评论文本和其对应的情感标签。

在训练过程中，我们首先将每个样本的文本输入到 ELMo 和 Glove 模型中，得到两个向量。然后将这两个向量拼接起来，作为分类器的输入。分类器输出的结果与真实标签计算二元交叉熵损失，并进行反向传播更新模型参数。最终，我们使用测试集评估模型的准确率。