diff --git a/code/translate.py b/code/translate.py index 90633c4..8450738 100644 --- a/code/translate.py +++ b/code/translate.py @@ -8,7 +8,6 @@ from tqdm import tqdm import logging from database import Data_table -from database import insert_translated from clean_data import clean_content from setup_logging import setup_logging @@ -27,15 +26,14 @@ def translate(text, target_lang='en'): return '' return text_trans -def get_untranslated(): - with Data_table() as dt: - query = ( - f'SELECT * FROM {dt._name} ' - 'WHERE content_en IS NULL ' - 'ORDER BY random() ' - 'LIMIT 1 ' - ) - df = pd.read_sql_query(query, dt.con) +def get_untranslated(dt): + query = ( + f'SELECT * FROM {dt._name} ' + 'WHERE content_en IS NULL ' + 'ORDER BY random() ' + 'LIMIT 1 ' + ) + df = pd.read_sql_query(query, dt.con) if len(df)==0: return None val_dict = { 'id': int(df['id'][0]), @@ -43,36 +41,47 @@ def get_untranslated(): } return val_dict +def insert_translated(dt, val_dict): + assert 'id' in val_dict.keys() + assert 'content_en' in val_dict.keys() + query = ( + f'UPDATE {dt._name} ' + ' SET content_en = (%(content_en)s) ' + 'WHERE id = (%(id)s) ' + ) + dt.execute(query, val_dict) + def continously_translate(): - logging.info('began translating') - while True: - try: - val_dict = get_untranslated() - except Exception as e: - logging.warning(f'error when getting untranslated content from database: {e}') - continue - if val_dict is None: break - # clean - try: - content_clean = clean_content(val_dict['content']) - except Exception as e: - logging.warning(f'error when cleaning content: {e}') - continue - # translate - try: - content_en = translate(content_clean) - except Exception as e: - logging.warning(f'error when translating content: {e}') - continue - # upload to database - val_dict['content_en'] = content_en - try: - insert_translated(val_dict) - except Exception as e: - logging.warning(f'error when sending translated content to database: {e}') - continue - logging.info('translated content') - logging.info('no more untranslated content located') + with Data_table() as dt: + logging.info('began translating') + while True: + try: + val_dict = get_untranslated(dt) + except Exception as e: + logging.warning(f'error when getting untranslated content from database: {e}') + continue + if val_dict is None: break + # clean + try: + content_clean = clean_content(val_dict['content']) + except Exception as e: + logging.warning(f'error when cleaning content: {e}') + continue + # translate + try: + content_en = translate(content_clean) + except Exception as e: + logging.warning(f'error when translating content: {e}') + continue + # upload to database + val_dict['content_en'] = content_en + try: + insert_translated(dt, val_dict) + except Exception as e: + logging.warning(f'error when sending translated content to database: {e}') + continue + logging.info('translated content') + logging.warning('stopped translating') if __name__ == '__main__': load_dotenv()