added functino to clean translated content

2022-12-18 13:36:23 +00:00
parent 73e3a42f88
commit 6c312eb3bc
1 changed files with 23 additions and 1 deletions
@@ -6,4 +6,26 @@ def clean_content(text):
 def clean_content_df(df):
    content_clean = df['content'].apply(clean_content).to_list()
    id_list = df['id'].to_list()
-    return content_clean, id_list
+    return content_clean, id_list
+
+def clean_content_en(text_list):
+    text_list_clean = list()
+    for text in text_list:
+        text_clean = clean(
+            text=text,
+            fix_unicode=True,
+            to_ascii=True,
+            lower=True,
+            no_line_breaks=True,
+            no_urls=True,
+            no_emails=True,
+            no_phone_numbers=True,
+            no_numbers=True,
+            no_digits=True,
+            no_currency_symbols=True,
+            no_punct=True,
+            no_emoji=True,
+            lang='en'
+        )
+        text_list_clean.append(text_clean)
+    return text_list_clean