diff --git a/code/clean_data.py b/code/clean_data.py new file mode 100644 index 0000000..d34ab8e --- /dev/null +++ b/code/clean_data.py @@ -0,0 +1,9 @@ +from cleantext import clean + +def clean_content(text): + return clean(text, to_ascii=False) + +def clean_content_df(df): + content_clean = df['content'].apply(clean_content).to_list() + id_list = df['id'].to_list() + return content_clean, id_list \ No newline at end of file