commented code

code/translate.py

@@ -1,15 +1,11 @@
+# public packages
 from langdetect import detect
 from google_trans_new import google_translator
-import pandas as pd
+# database-related
-import multiprocessing as mp
+from database import Data_table, get_untranslated, insert_translated
-import numpy as np
 from dotenv import load_dotenv
-from tqdm import tqdm
+# own scripts
-import logging
-
-from database import Data_table
 from clean_data import clean_content
-from setup_logging import setup_logging
 
 def translate(text, target_lang='en'):
     try:
@@ -26,65 +22,39 @@ def translate(text, target_lang='en'):
         return ''
     return text_trans
 
-def get_untranslated(dt):
-    query = (
-        f'SELECT * FROM {dt._name} '
-        'WHERE content_en IS NULL '
-        'ORDER BY random() '
-        'LIMIT 1 '
-    )
-    df = pd.read_sql_query(query, dt.con)
-    if len(df)==0: return None
-    val_dict = {
-        'id': int(df['id'][0]),
-        'content': str(df['content'][0])
-    }
-    return val_dict
-
-def insert_translated(dt, val_dict):
-    assert 'id' in val_dict.keys()
-    assert 'content_en' in val_dict.keys()
-    query = (
-        f'UPDATE {dt._name} ' 
-        ' SET content_en = (%(content_en)s) '
-        'WHERE id = (%(id)s) '
-    )
-    dt.execute(query, val_dict)
-
 def continously_translate():
     with Data_table() as dt:
-        logging.info('began translating')
+        print('began translating')
-        while True:
+        while True: # keep going forever
             try:
                 val_dict = get_untranslated(dt)
             except Exception as e:
-                logging.warning(f'error when getting untranslated content from database: {e}')
+                print(f'error when getting untranslated content from database: {e}')
                 continue
+            # if nother left untranslated, stop
             if val_dict is None: break
-            # clean
+            # clean text
             try:
                 content_clean = clean_content(val_dict['content'])
             except Exception as e:
-                logging.warning(f'error when cleaning content: {e}')
+                print(f'error when cleaning content: {e}')
                 continue
-            # translate
+            # translate text
             try:
                 content_en = translate(content_clean)
             except Exception as e:
-                logging.warning(f'error when translating content: {e}')
+                print(f'error when translating content: {e}')
                 continue
             # upload to database
             val_dict['content_en'] = content_en
             try:
                 insert_translated(dt, val_dict)
             except Exception as e:
-                logging.warning(f'error when sending translated content to database: {e}')
+                print(f'error when sending translated content to database: {e}')
                 continue
-            logging.info('translated content')
+            print('translated content')
-        logging.warning('stopped translating')
+        print('stopped translating')
 
 if __name__ == '__main__':
     load_dotenv()
-    setup_logging()
     continously_translate()
-    # later try to set all empty strings back to null and retranslate failed...