import ast from dataclasses import dataclass from typing import List import pandas as pd import json ["text", "六十一岁还能办什么保险"] @dataclass class FAQ: title: str sim_questions: List[str] answer: str faq_id: int ori_data = pd.read_csv('baoxianzhidao_filter.csv') data = [] exist_titles = set() for index, row in enumerate(ori_data.iterrows()): row_dict = row[1] title = row_dict['title'] if title not in exist_titles: data.append(FAQ(title=title, answer=row_dict['reply'], sim_questions=[title], faq_id=index)) exist_titles.add(title) from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks pipeline_ins = pipeline(Tasks.faq_question_answering, 'damo/nlp_mgimn_faq-question-answering_chinese-base') bsz = 32 all_sentence_vecs = [] batch = [] sentence_list = [faq.title for faq in data] for i,sent in enumerate(sentence_list): batch.append(sent) if len(batch) == bsz or (i == len(sentence_list)-1 and len(batch)>0): # if i == len(sentence_list)-1 and len(batch)>0: sentence_vecs = pipeline_ins.get_sentence_embedding(batch) all_sentence_vecs.extend(sentence_vecs) batch.clear() import faiss import numpy as np hidden_size = pipeline_ins.model.network.bert.config.hidden_size # hidden_size = pipeline_ins.model.bert.config.hidden_size index = faiss.IndexFlatIP(hidden_size) vecs = np.asarray(all_sentence_vecs, dtype='float32') index.add(vecs) from modelscope.outputs import OutputKeys def ask_faq(input, history=[]): # step1: get sentence vector of query query_vec = pipeline_ins.get_sentence_embedding([input])[0] query_vec = np.asarray(query_vec, dtype='float32').reshape([1, -1]) # step2: faq dense retrieval _, indices = index.search(query_vec, k=30) # step3: build support set support_set = [] for i in indices.tolist()[0]: faq = data[i] support_set.append({"text": faq.title, "label": faq.faq_id, "index": i}) # step4: faq ranking rst = pipeline_ins(input={"query_set": input, "support_set": support_set}) rst = rst[OutputKeys.OUTPUT][0][0] pred_label = rst['label'] pred_score = rst['score'] # get answer by faq_id pred_answer = "" pred_title = "" for faq in data: if faq.faq_id == pred_label: pred_answer = faq.answer pred_title = faq.title break history.append((f'{pred_answer}|(pred_title:{pred_title},pred_score:{pred_score:.3f})')) return history优化这段代码

这段代码是一个Python脚本，用于读取CSV文件中的保险相关问题和答案，构建一个FAQ对象（包含问题、答案、相似问题和FAQ ID），并使用modelscope库中的pipeline进行常见问题解答。其中用到了ast、dataclass、List、pandas和json等Python模块。具体实现过程请参考上述代码。