From f51f81ba07712879edc0953e693b83f0eae88091 Mon Sep 17 00:00:00 2001
From: brian <mail@brianbjarkejensen.dk>
Date: Sun, 18 Dec 2022 12:01:41 +0000
Subject: [PATCH] added functions to clean content

---
 code/clean_data.py | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 code/clean_data.py

diff --git a/code/clean_data.py b/code/clean_data.py
new file mode 100644
index 0000000..d34ab8e
--- /dev/null
+++ b/code/clean_data.py
@@ -0,0 +1,9 @@
+from cleantext import clean
+
+def clean_content(text):
+    return clean(text, to_ascii=False)
+
+def clean_content_df(df):
+    content_clean = df['content'].apply(clean_content).to_list()
+    id_list = df['id'].to_list()
+    return content_clean, id_list
\ No newline at end of file