brian/twitter_scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated code to avoid translating already translated content

Browse Source
This commit is contained in:
brian
2022-12-18 14:56:43 +00:00
parent 0c70b39abd
commit f6e8ee0c77
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code/main.py
+4 -4
View File
@@ -14,7 +14,7 @@ import pandas as pd
from database import get_all from database import get_all
from clean_data import clean_content_df from clean_data import clean_content_df
from translate import translate from translate import translate, get_untranslated
from database import Data_table from database import Data_table
# def update(): # def update():
@@ -74,6 +74,8 @@ def translate_content():
# load all content from database # load all content from database
print('loading data from database...') print('loading data from database...')
df = get_all() df = get_all()
# remove already translated content
df = df[df['content_en'].isna()]
# clean content # clean content
print('cleaning content...') print('cleaning content...')
content_clean_list, id_list = clean_content_df(df) content_clean_list, id_list = clean_content_df(df)
@@ -101,9 +103,7 @@ def translate_content():
dt.execute(query, val_dict) dt.execute(query, val_dict)
except Exception as e: except Exception as e:
print(f'error when sending translated content to database: id={id}') print(f'error when sending translated content to database: id={id}')
time.sleep(2)
if __name__ == '__main__': if __name__ == '__main__':