| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111 |
- # This Python file uses the following encoding: utf-8
- # @author runhey
- # github https://github.com/runhey
- import re
- import rich
- import csv
- import pandas as pd
- from utils import remove_symbols
- class Extracter:
- def __init__(self, data_file: str='data.csv'):
- self.data_file = data_file
- self.df = pd.read_csv(self.data_file)
- @staticmethod
- def parse_one(value: str) -> tuple:
- """
- 解析原始的一行数据,变成OAS使用的格式: question,answer
- :param value:
- :return:
- """
- if not value:
- print(f'value is None {value}')
- return None, None
- if not isinstance(value, str):
- print(f'-----------------------------------------------------------value is not str {value}')
- return None, None
- # 标点符号
- value = value.replace(' ', '').replace('?', '?').replace('!', '!').replace(',', ',').replace('—', '-')
- # 正则匹配
- result = re.search(r"(.*)?-(.*)", value)
- if result:
- return result[1], result[2]
- result = re.search(r"(.*)?--(.*)", value)
- if result:
- return result[1], result[2]
- result = re.search(r"(.*)?-------(.*)", value)
- if result:
- return result[1], result[2]
- result = re.search(r"(.*)?------(.*)", value)
- if result:
- return result[1], result[2]
- result = re.search(r"(.*)?-----(.*)", value)
- if result:
- return result[1], result[2]
- result = re.search(r"(.*)?----(.*)", value)
- if result:
- return result[1], result[2]
- result = re.search(r"(.*)------(.*)", value)
- if result:
- return result[1], result[2]
- result = re.search(r"(.*)-----(.*)", value)
- if result:
- return result[1], result[2]
- result = re.search(r"(.*)----(.*)", value)
- if result:
- return result[1], result[2]
- return None, None
- def xlsx2csv(self, input_file: str='data.xlsx'):
- """
- 将xlsx文件的数据添加到csv文件
- :param input_file:
- :return:
- """
- df_source = pd.read_excel(input_file, usecols='A')
- for index, row in df_source.iterrows():
- item_value = row[0]
- question, answer = self.parse_one(item_value)
- if question is None and answer is None:
- continue
- if len(question) < 3:
- continue
- # print(index, question, answer)
- # 非常重要,去除标点符号
- question = remove_symbols(question)
- answer = remove_symbols(answer)
- if self.appear_in_df(question, answer):
- continue
- print('New', index, question, answer)
- self.df.loc[len(self.df)] = [question, answer]
- self.df.to_csv(self.data_file, index=False, encoding='utf-8-sig')
- def appear_in_df(self, question: str, answer: str) -> bool:
- for index, row in self.df.iterrows():
- if row['question'] == question and row['answer'] == answer:
- return True
- return False
- def clear_symbols(self):
- for index, row in self.df.iterrows():
- row['question'] = remove_symbols(row['question'])
- row['answer'] = remove_symbols(row['answer'])
- self.df.to_csv(self.data_file, index=False, encoding='utf-8-sig')
- if __name__ == "__main__":
- Extracter().xlsx2csv()
|