brian/twitter_scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added try except statements and logging

Browse Source
This commit is contained in:
simplypower-bbj
2022-12-18 19:36:46 +01:00
parent be529bf189
commit db4c31d75e
1 changed files with 30 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code/translate.py
+30 -19
View File
@@ -5,10 +5,12 @@ import multiprocessing as mp
import numpy as np import numpy as np
from dotenv import load_dotenv from dotenv import load_dotenv
from tqdm import tqdm from tqdm import tqdm
import logging
from database import Data_table from database import Data_table
from database import insert_translated from database import insert_translated
from clean_data import clean_content from clean_data import clean_content
from setup_logging import setup_logging
def translate(text, target_lang='en'): def translate(text, target_lang='en'):
try: try:
@@ -42,29 +44,38 @@ def get_untranslated():
return val_dict return val_dict
def continously_translate(): def continously_translate():
with Data_table() as dt: logging.info('began translating')
query = ( while True:
f'UPDATE {dt._name} ' try:
' SET content_en = (%(content_en)s) '
'WHERE id = (%(id)s) '
)
while True:
val_dict = get_untranslated() val_dict = get_untranslated()
if val_dict is None: break except Exception as e:
# clean logging.warning(f'error when getting untranslated content from database: {e}')
continue
if val_dict is None: break
# clean
try:
content_clean = clean_content(val_dict['content']) content_clean = clean_content(val_dict['content'])
# translate except Exception as e:
try: logging.warning(f'error when cleaning content: {e}')
content_en = translate(content_clean) continue
except Exception as e: # translate
content_en = '' try:
val_dict['content_en'] = content_en content_en = translate(content_clean)
# upload to database except Exception as e:
dt.execute(query, val_dict) logging.warning(f'error when translating content: {e}')
print('translated content') continue
print('no more untranslated content located') # upload to database
val_dict['content_en'] = content_en
try:
insert_translated(val_dict)
except Exception as e:
logging.warning(f'error when sending translated content to database: {e}')
continue
logging.info('translated content')
logging.info('no more untranslated content located')
if __name__ == '__main__': if __name__ == '__main__':
load_dotenv() load_dotenv()
setup_logging()
continously_translate() continously_translate()
# later try to set all empty strings back to null and retranslate failed... # later try to set all empty strings back to null and retranslate failed...