From 6c312eb3bc444b9778f9c4228cb63f84b5cfd475 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 18 Dec 2022 13:36:23 +0000 Subject: [PATCH] added functino to clean translated content --- code/clean_data.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/code/clean_data.py b/code/clean_data.py index d34ab8e..29504af 100644 --- a/code/clean_data.py +++ b/code/clean_data.py @@ -6,4 +6,26 @@ def clean_content(text): def clean_content_df(df): content_clean = df['content'].apply(clean_content).to_list() id_list = df['id'].to_list() - return content_clean, id_list \ No newline at end of file + return content_clean, id_list + +def clean_content_en(text_list): + text_list_clean = list() + for text in text_list: + text_clean = clean( + text=text, + fix_unicode=True, + to_ascii=True, + lower=True, + no_line_breaks=True, + no_urls=True, + no_emails=True, + no_phone_numbers=True, + no_numbers=True, + no_digits=True, + no_currency_symbols=True, + no_punct=True, + no_emoji=True, + lang='en' + ) + text_list_clean.append(text_clean) + return text_list_clean \ No newline at end of file