diff --git a/code/clean_data.py b/code/clean_data.py index d34ab8e..29504af 100644 --- a/code/clean_data.py +++ b/code/clean_data.py @@ -6,4 +6,26 @@ def clean_content(text): def clean_content_df(df): content_clean = df['content'].apply(clean_content).to_list() id_list = df['id'].to_list() - return content_clean, id_list \ No newline at end of file + return content_clean, id_list + +def clean_content_en(text_list): + text_list_clean = list() + for text in text_list: + text_clean = clean( + text=text, + fix_unicode=True, + to_ascii=True, + lower=True, + no_line_breaks=True, + no_urls=True, + no_emails=True, + no_phone_numbers=True, + no_numbers=True, + no_digits=True, + no_currency_symbols=True, + no_punct=True, + no_emoji=True, + lang='en' + ) + text_list_clean.append(text_clean) + return text_list_clean \ No newline at end of file