Transformer训练例子

以下是一个使用Transformer模型进行文本分类的训练例子：

1. 导入必要的库

import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Input, Dense, Dropout, GlobalMaxPooling1D, Embedding
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from sklearn.model_selection import train_test_split
import numpy as np
import pandas as pd

2. 加载数据

df = pd.read_csv('data.csv')
texts = df['text'].values
labels = df['label'].values

3. 数据预处理

# 构建Tokenizer
tokenizer = Tokenizer(num_words=10000, oov_token='<OOV>')
tokenizer.fit_on_texts(texts)

# 将文本转换为序列
sequences = tokenizer.texts_to_sequences(texts)

# 对序列进行补齐
max_length = max(len(sequence) for sequence in sequences)
padded_sequences = pad_sequences(sequences, maxlen=max_length, padding='post')

# 将标签进行one-hot编码
num_classes = len(np.unique(labels))
one_hot_labels = tf.keras.utils.to_categorical(labels, num_classes=num_classes)

4. 划分训练集和测试集

X_train, X_test, y_train, y_test = train_test_split(padded_sequences, one_hot_labels, test_size=0.2, random_state=42)

5. 构建模型

# 定义模型参数
input_dim = len(tokenizer.word_index) + 1
embedding_dim = 64
num_heads = 8
dff = 512
num_layers = 4
dropout_rate = 0.1

# 构建输入层
inputs = Input(shape=(max_length,))

# 构建嵌入层
embedding_layer = Embedding(input_dim=input_dim, output_dim=embedding_dim)(inputs)

# 构建Transformer层
encoder_outputs = embedding_layer
for i in range(num_layers):
    # Multi-Head Attention
    attention_output = tf.keras.layers.MultiHeadAttention(num_heads=num_heads, key_dim=embedding_dim)(encoder_outputs, encoder_outputs)
    attention_output = tf.keras.layers.Dropout(rate=dropout_rate)(attention_output)
    attention_output = tf.keras.layers.LayerNormalization(epsilon=1e-6)(encoder_outputs + attention_output)

    # Feed Forward
    ffn = tf.keras.Sequential([
        tf.keras.layers.Dense(dff, activation='relu'),
        tf.keras.layers.Dense(embedding_dim)
    ])
    ffn_output = ffn(attention_output)
    ffn_output = tf.keras.layers.Dropout(rate=dropout_rate)(ffn_output)
    ffn_output = tf.keras.layers.LayerNormalization(epsilon=1e-6)(attention_output + ffn_output)

    encoder_outputs = ffn_output

# 构建池化层
pooling_layer = GlobalMaxPooling1D()(encoder_outputs)

# 构建输出层
outputs = Dense(num_classes, activation='softmax')(pooling_layer)

# 构建模型
model = Model(inputs=inputs, outputs=outputs)

6. 编译模型

model.compile(optimizer=Adam(learning_rate=0.0001),
              loss='categorical_crossentropy',
              metrics=['accuracy'])

7. 训练模型

early_stopping = EarlyStopping(monitor='val_loss', patience=3)
history = model.fit(X_train, y_train,
                    validation_data=(X_test, y_test),
                    epochs=10,
                    callbacks=[early_stopping])

8. 评估模型

loss, accuracy = model.evaluate(X_test, y_test)
print('Test loss:', loss)
print('Test accuracy:', accuracy)