32 lines
918 B
Python
32 lines
918 B
Python
# public packages
|
|
from cleantext import clean
|
|
|
|
def clean_content(text):
|
|
return clean(text, to_ascii=False) # avoid ascii, as it does not have æ,ø,å
|
|
|
|
def clean_content_df(df):
|
|
content_clean_list = df['content'].apply(clean_content).to_list()
|
|
id_list = df['id'].to_list()
|
|
return content_clean_list, id_list
|
|
|
|
def clean_content_en(text_list):
|
|
text_list_clean = list()
|
|
for text in text_list:
|
|
text_clean = clean(
|
|
text=text,
|
|
fix_unicode=True,
|
|
to_ascii=True,
|
|
lower=True,
|
|
no_line_breaks=True,
|
|
no_urls=True,
|
|
no_emails=True,
|
|
no_phone_numbers=True,
|
|
no_numbers=True,
|
|
no_digits=True,
|
|
no_currency_symbols=True,
|
|
no_punct=True,
|
|
no_emoji=True,
|
|
lang='en'
|
|
)
|
|
text_list_clean.append(text_clean)
|
|
return text_list_clean |