transformer的padding token

在Transformer中，padding token是用来填充输入序列中短于最长序列的位置的特殊标记。在输入序列中，如果某个位置没有输入，则可以用padding token来填充该位置的信息。这样做的好处是可以将所有输入序列的长度都设置为相同的长度，方便进行批处理和并行计算。同时，padding token不会对模型的输出产生影响，因为它们的注意力权重为0，模型不会把它们计算在内。在实现中，通常会使用一个特殊的符号来表示padding token，例如在BERT中，使用[PAD]来表示padding token。

相关问题

transformer输入参数解释

Transformer是一种基于自注意力机制的神经网络模型，主要用于序列到序列的任务，如机器翻译、文本摘要等。Transformer的输入参数包括以下几个：

1. input_ids：输入序列的token id，是一个形状为[batch_size, sequence_length]的整数张量。
2. attention_mask：用于指示哪些token是padding token，哪些是真实的token。如果是padding token，则对应位置为0，否则为1。形状为[batch_size, sequence_length]的张量。
3. token_type_ids：用于区分不同句子的token id，如果只有一个句子则全为0。形状为[batch_size, sequence_length]的张量。
4. position_ids：用于表示每个token在序列中的位置信息，形状为[batch_size, sequence_length]的张量。
5. inputs_embeds：输入序列的嵌入表示，可以直接传入嵌入向量而不是token id。形状为[batch_size, sequence_length, embedding_size]的张量。

transformer中英翻译

Transformer是一种用于机器翻译的神经网络模型，它在处理序列到序列的任务中表现出色。下面是一个使用Transformer进行中英翻译的示例：

import torch
from torchtext.data.metrics import bleu_score
from torchtext.datasets import Multi30k
from torchtext.data import Field, BucketIterator

# 定义源语言和目标语言的Field
SRC = Field(tokenize='spacy', tokenizer_language='en', init_token='<sos>', eos_token='<eos>', lower=True)
TRG = Field(tokenize='spacy', tokenizer_language='de', init_token='<sos>', eos_token='<eos>', lower=True)

# 加载数据集
train_data, valid_data, test_data = Multi30k.splits(exts=('.en', '.de'), fields=(SRC, TRG))

# 构建词汇表
SRC.build_vocab(train_data, min_freq=2)
TRG.build_vocab(train_data, min_freq=2)

# 定义模型
class Transformer(nn.Module):
    def __init__(self, input_dim, output_dim, hid_dim, n_layers, n_heads, pf_dim, dropout, max_length=100):
        super().__init__()
        self.encoder = Encoder(input_dim, hid_dim, n_layers, n_heads, pf_dim, dropout, max_length)
        self.decoder = Decoder(output_dim, hid_dim, n_layers, n_heads, pf_dim, dropout, max_length)
        self.fc_out = nn.Linear(hid_dim, output_dim)
        self.dropout = nn.Dropout(dropout)
        
    def forward(self, src, trg, src_mask, trg_mask, src_padding_mask, trg_padding_mask):
        enc_src = self.encoder(src, src_mask, src_padding_mask)
        output, attention = self.decoder(trg, enc_src, trg_mask, src_mask, trg_padding_mask, src_padding_mask)
        output = self.fc_out(output)
        return output, attention

# 定义训练和评估函数
def train(model, iterator, optimizer, criterion, clip):
    model.train()
    epoch_loss = 0
    for i, batch in enumerate(iterator):
        src = batch.src
        trg = batch.trg
        optimizer.zero_grad()
        output, _ = model(src, trg[:,:-1], src_mask, trg_mask, src_padding_mask, trg_padding_mask)
        output_dim = output.shape[-1]
        output = output.contiguous().view(-1, output_dim)
        trg = trg[:,1:].contiguous().view(-1)
        loss = criterion(output, trg)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
        optimizer.step()
        epoch_loss += loss.item()
    return epoch_loss / len(iterator)

def evaluate(model, iterator, criterion):
    model.eval()
    epoch_loss = 0
    with torch.no_grad():
        for i, batch in enumerate(iterator):
            src = batch.src
            trg = batch.trg
            output, _ = model(src, trg[:,:-1], src_mask, trg_mask, src_padding_mask, trg_padding_mask)
            output_dim = output.shape[-1]
            output = output.contiguous().view(-1, output_dim)
            trg = trg[:,1:].contiguous().view(-1)
            loss = criterion(output, trg)
            epoch_loss += loss.item()
    return epoch_loss / len(iterator)

# 定义超参数
INPUT_DIM = len(SRC.vocab)
OUTPUT_DIM = len(TRG.vocab)
HID_DIM = 256
N_LAYERS = 6
N_HEADS = 8
PF_DIM = 512
DROPOUT = 0.1
CLIP = 1

# 初始化模型和优化器
model = Transformer(INPUT_DIM, OUTPUT_DIM, HID_DIM, N_LAYERS, N_HEADS, PF_DIM, DROPOUT)
optimizer = torch.optim.Adam(model.parameters(), lr=0.0005)
criterion = nn.CrossEntropyLoss(ignore_index=TRG.vocab.stoi[TRG.pad_token])

# 将数据加载到迭代器中
train_iterator, valid_iterator, test_iterator = BucketIterator.splits((train_data, valid_data, test_data), batch_size=128, device=device)

# 训练模型
N_EPOCHS = 10
best_valid_loss = float('inf')
for epoch in range(N_EPOCHS):
    train_loss = train(model, train_iterator, optimizer, criterion, CLIP)
    valid_loss = evaluate(model, valid_iterator, criterion)
    if valid_loss < best_valid_loss:
        best_valid_loss = valid_loss
        torch.save(model.state_dict(), 'tut6-model.pt')
    print(f'Epoch: {epoch+1:02} | Train Loss: {train_loss:.3f} | Val. Loss: {valid_loss:.3f}')

# 加载最佳模型并在测试集上进行评估
model.load_state_dict(torch.load('tut6-model.pt'))
test_loss = evaluate(model, test_iterator, criterion)
print(f'Test Loss: {test_loss:.3f}')

# 使用训练好的模型进行翻译
def translate_sentence(sentence, src_field, trg_field, model, device, max_len=50):
    model.eval()
    if isinstance(sentence, str):
        nlp = spacy.load('en')
        tokens = [token.text.lower() for token in nlp(sentence)]
    else:
        tokens = [token.lower() for token in sentence]
    tokens = [src_field.init_token] + tokens + [src_field.eos_token]
    src_indexes = [src_field.vocab.stoi[token] for token in tokens]
    src_tensor = torch.LongTensor(src_indexes).unsqueeze(0).to(device)
    src_mask = model.make_src_mask(src_tensor)
    with torch.no_grad():
        enc_src = model.encoder(src_tensor, src_mask)
    trg_indexes = [trg_field.vocab.stoi[trg_field.init_token]]
    for i in range(max_len):
        trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(0).to(device)
        trg_mask = model.make_trg_mask(trg_tensor)
        with torch.no_grad():
            output, attention = model.decoder(trg_tensor, enc_src, trg_mask, src_mask)
        pred_token = output.argmax(2)[:,-1].item()
        trg_indexes.append(pred_token)
        if pred_token == trg_field.vocab.stoi[trg_field.eos_token]:
            break
    trg_tokens = [trg_field.vocab.itos[i] for i in trg_indexes]
    return trg_tokens[1:], attention

# 示例翻译
example_sentence = "I love transformers"
translated_sentence, attention = translate_sentence(example_sentence, SRC, TRG, model, device)
print(f'原句: {example_sentence}')
print(f'翻译: {" ".join(translated_sentence[:-1])}')

# 计算BLEU得分
def calculate_bleu(data, src_field, trg_field, model, device, max_len=50):
    trgs = []
    pred_trgs = []
    for datum in data:
        src = vars(datum)['src']
        trg = vars(datum)['trg']
        pred_trg, _ = translate_sentence(src, src_field, trg_field, model, device, max_len)
        pred_trg = pred_trg[:-1]
        pred_trgs.append(pred_trg)
        trgs.append([trg])
    return bleu_score(pred_trgs, trgs)

bleu_score = calculate_bleu(test_data, SRC, TRG, model, device)
print(f'BLEU score = {bleu_score*100:.2f}')