import torch from sklearn.metrics.pairwise import cosine_similarity from transformers import BertTokenizer, BertModel # 加载种子词库 seed_words = [] with open("output/base_words.txt", "r", encoding="utf-8") as f: for line in f: seed_words.append(line.strip()) print(seed_words) # 加载微博文本数据 text_data = [] with open("output/weibo1.txt", "r", encoding="utf-8") as f: for line in f: text_data.append(line.strip()) print(text_data) # 加载BERT模型和分词器 tokenizer = BertTokenizer.from_pretrained('bert-base-chinese') model = BertModel.from_pretrained('bert-base-chinese') # 构建隐私词库 privacy_words = set(seed_words) for text in text_data: # 对文本进行分词，并且添加特殊标记 tokens = ["[CLS]"] + tokenizer.tokenize(text) + ["[SEP]"] token_ids = tokenizer.convert_tokens_to_ids(tokens) segment_ids = [0] * len(token_ids) # 转换为张量，调用BERT模型进行编码 token_tensor = torch.tensor([token_ids]) segment_tensor = torch.tensor([segment_ids]) with torch.no_grad(): outputs = model(token_tensor, segment_tensor) encoded_layers = outputs[0] # 对于每个词，计算它与种子词的相似度 for i in range(1, len(tokens)-1): word = tokens[i] if word in seed_words: continue word_tensor = encoded_layers[0][i].reshape(1, -1) sim = cosine_similarity(encoded_layers[0][1:-1], word_tensor, dense_output=False)[0].max() if sim > 0.5: privacy_words.add(word) # 输出隐私词库 with open("output/privacy_words.txt", "w", encoding="utf-8") as f: for word in privacy_words: f.write(word + "\n") 上述代码中的这两行代码： if sim > 0.5: privacy_words.add(word) 中privacy_words集合写入的词汇不是我想要的，运行之后都是写入privacy_words集合的都是单个字，我需要的是大于等于两个字的中文词汇，并且不包含种子词列表中的词汇，只需要将微博文本数据中与种子词相似度高的词汇写入privacy_words集合中，请帮我正确修改上述代码

可以将代码的 if 判断条件修改为：

if sim > 0.5 and len(word) > 1 and word not in seed_words:

这样就只会将长度大于等于两个字且不包含种子词列表中的词汇添加到 privacy_words 集合中了。修改后的完整代码如下：

import torch
from sklearn.metrics.pairwise import cosine_similarity
from transformers import BertTokenizer, BertModel

# 加载种子词库
seed_words = []
with open("output/base_words.txt", "r", encoding="utf-8") as f:
    for line in f:
        seed_words.append(line.strip())
print(seed_words)

# 加载微博文本数据
text_data = []
with open("output/weibo1.txt", "r", encoding="utf-8") as f:
    for line in f:
        text_data.append(line.strip())
print(text_data)

# 加载BERT模型和分词器
tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
model = BertModel.from_pretrained('bert-base-chinese')

# 构建隐私词库
privacy_words = set(seed_words)
for text in text_data:
    # 对文本进行分词，并且添加特殊标记
    tokens = ["[CLS]"] + tokenizer.tokenize(text) + ["[SEP]"]
    token_ids = tokenizer.convert_tokens_to_ids(tokens)
    segment_ids = [0] * len(token_ids)

    # 转换为张量，调用BERT模型进行编码
    token_tensor = torch.tensor([token_ids])
    segment_tensor = torch.tensor([segment_ids])
    with torch.no_grad():
        outputs = model(token_tensor, segment_tensor)
        encoded_layers = outputs[0]

    # 对于每个词，计算它与种子词的相似度
    for i in range(1, len(tokens)-1):
        word = tokens[i]
        if word in seed_words:
            continue
        word_tensor = encoded_layers[0][i].reshape(1, -1)
        sim = cosine_similarity(encoded_layers[0][1:-1], word_tensor, dense_output=False)[0].max()
        if sim > 0.5 and len(word) > 1 and word not in seed_words:
            privacy_words.add(word)

# 输出隐私词库
with open("output/privacy_words.txt", "w", encoding="utf-8") as f:
    for word in privacy_words:
        f.write(word + "\n")