added and tested function to spawn process that speed up the translation process

code/translate.py

@@ -1,5 +1,10 @@
 from langdetect import detect
 from google_trans_new import google_translator
+import pandas as pd
+import multiprocessing as mp
+
+from database import Data_table
+from database import insert_translated
 
 def translate(text, target_lang='en'):
     try:
@@ -15,3 +20,56 @@ def translate(text, target_lang='en'):
         print(f'error when translating text: {e}')
         return ''
     return text_trans
+
+def get_untranslated():
+    with Data_table() as dt:
+        query = (
+            f'SELECT * FROM {dt._name} '
+            'WHERE content_en IS NULL '
+            'ORDER BY random() '
+            'LIMIT 1 '
+        )
+        df = pd.read_sql_query(query, dt.con)
+    if len(df)==0: return None
+    val_dict = {
+        'id': df['id'][0],
+        'content': df['content'][0]
+    }
+    return val_dict
+
+def process_translate(val_dict):
+    # translate
+    try:
+        content_en = translate(val_dict['content'])
+    except Exception as e:
+        content_en = ''
+    val_dict['content_en'] = content_en
+    # upload to database
+    insert_translated(val_dict)
+
+def yield_untranslated():
+    val_dict = get_untranslated()
+    while val_dict is not None:
+        yield val_dict
+        val_dict = get_untranslated()
+
+# def continued_translation():
+#     while True:
+#         # get untranslated
+#         val_dict = get_untranslated()
+#         if val_dict is None: break
+#         # translate
+#         try:
+#             content_en = translate(val_dict['content'])
+#         except Exception as e:
+#             content_en = ''
+#         val_dict['content_en'] = content_en
+#         # upload to database
+#         insert_translated(val_dict)
+
+def translate_all_fast():
+    with mp.get_context('spawn').Pool(mp.cpu_count()-1) as p:
+        list(p.map(process_translate, yield_untranslated()))
+
+if __name__ == '__main__':
+    translate_all_fast()