updated code to run on multiple hosts at the same time

code/translate.py

@@ -2,9 +2,13 @@ from langdetect import detect
 from google_trans_new import google_translator
 import pandas as pd
 import multiprocessing as mp
+import numpy as np
+from dotenv import load_dotenv
+from tqdm import tqdm
 
 from database import Data_table
 from database import insert_translated
+from clean_data import clean_content
 
 def translate(text, target_lang='en'):
     try:
@@ -32,44 +36,26 @@ def get_untranslated():
         df = pd.read_sql_query(query, dt.con)
     if len(df)==0: return None
     val_dict = {
-        'id': df['id'][0],
-        'content': df['content'][0]
+        'id': int(df['id'][0]),
+        'content': str(df['content'][0])
     }
     return val_dict
 
-def process_translate(val_dict):
-    # translate
-    try:
-        content_en = translate(val_dict['content'])
-    except Exception as e:
-        content_en = ''
-    val_dict['content_en'] = content_en
-    # upload to database
-    insert_translated(val_dict)
-
-def yield_untranslated():
-    val_dict = get_untranslated()
-    while val_dict is not None:
-        yield val_dict
+def continously_translate():
+    while True:
         val_dict = get_untranslated()
-
-# def continued_translation():
-#     while True:
-#         # get untranslated
-#         val_dict = get_untranslated()
-#         if val_dict is None: break
-#         # translate
-#         try:
-#             content_en = translate(val_dict['content'])
-#         except Exception as e:
-#             content_en = ''
-#         val_dict['content_en'] = content_en
-#         # upload to database
-#         insert_translated(val_dict)
-
-def translate_all_fast():
-    with mp.get_context('spawn').Pool(mp.cpu_count()-1) as p:
-        list(p.map(process_translate, yield_untranslated()))
+        if val_dict is None: break
+        # clean
+        content_clean = clean_content(val_dict['content'])
+        # translate
+        try:
+            content_en = translate(content_clean)
+        except Exception as e:
+            content_en = ''
+        val_dict['content_en'] = content_en
+        # upload to database
+        insert_translated(val_dict)
+    print('no more untranslated content located')
 
 if __name__ == '__main__':
-    translate_all_fast()
+    continously_translate()