diff --git a/code/main.py b/code/main.py index fe29d0e..d054187 100644 --- a/code/main.py +++ b/code/main.py @@ -7,10 +7,16 @@ import logging from datetime import datetime import time import os +from tqdm import tqdm import snscrape.modules.twitter as sntwitter import pandas as pd +from database import get_all +from clean_data import clean_content_df +from translate import translate +from database import Data_table + # def update(): # topic = 'Odense' # # scrape latest data @@ -64,6 +70,43 @@ def run_snscrape(): if i % 100 == 0: print(f'found {i} tweets') +def translate_content(): + # load all content from database + print('loading data from database...') + df = get_all() + # clean content + print('cleaning content...') + content_clean_list, id_list = clean_content_df(df) + # translate content + print('translating content...') + with Data_table() as dt: + for id, content in tqdm(zip(id_list, content_clean_list), ascii=True, total=len(id_list)): + # translate content + try: + content_trans = translate(content) + except Exception as e: + print(f'error when translating content with id={id}') + continue + # transfer to database + try: + val_dict = { + 'id': id, + 'content_en': content_trans + } + query = ( + f'UPDATE {dt._name} ' + ' SET content_en = (%(content_en)s) ' + 'WHERE id = (%(id)s) ' + ) + dt.execute(query, val_dict) + except Exception as e: + print(f'error when sending translated content to database: id={id}') + + + + + if __name__ == '__main__': load_dotenv() - run_snscrape() \ No newline at end of file + # run_snscrape() + translate_content() \ No newline at end of file