added functino to clean translated content

This commit is contained in:
brian
2022-12-18 13:36:23 +00:00
parent 73e3a42f88
commit 6c312eb3bc
+22
View File
@@ -7,3 +7,25 @@ def clean_content_df(df):
content_clean = df['content'].apply(clean_content).to_list() content_clean = df['content'].apply(clean_content).to_list()
id_list = df['id'].to_list() id_list = df['id'].to_list()
return content_clean, id_list return content_clean, id_list
def clean_content_en(text_list):
text_list_clean = list()
for text in text_list:
text_clean = clean(
text=text,
fix_unicode=True,
to_ascii=True,
lower=True,
no_line_breaks=True,
no_urls=True,
no_emails=True,
no_phone_numbers=True,
no_numbers=True,
no_digits=True,
no_currency_symbols=True,
no_punct=True,
no_emoji=True,
lang='en'
)
text_list_clean.append(text_clean)
return text_list_clean