#!/Users/brian/Projects/twitter_scraper/venv/bin/python
from setup_logging import setup_logging
from selenium_scraper import scrape
from database import insert_list
from dotenv import load_dotenv
import logging
from datetime import datetime
import time
import os
from tqdm import tqdm

import snscrape.modules.twitter as sntwitter
import pandas as pd

from database import get_all
from clean_data import clean_content_df
from translate import translate
from database import Data_table

# def update():
#     topic = 'Odense'
#     # scrape latest data
#     username_list, timestamp_list, content_list = scrape(topic, limit=10, headless=False)
#     # send data to database
#     insert_list(username_list, timestamp_list, content_list)

# if __name__ == '__main__':
#     # setup
#     load_dotenv()
#     setup_logging()
#     logging.info('finished setup')
#     # main loop
#     trigger_minute = datetime.now().minute
#     while True:
#         try:
#             now = datetime.now()
#             if now.minute == trigger_minute:
#                 try:
#                     update()
#                 except Exception as e:
#                     logging.warning(e)
#                 else:
#                     logging.info('finished cycle')
#                 time.sleep(60)
#             time.sleep(0.1)
#         except Exception as e:
#             logging.critical(e)
#             break
#         except KeyboardInterrupt:
#             logging.info('KeyboardInterrupt')
#             break

def run_snscrape():
    # prepare variables
    username_list = list()
    timestamp_list = list()
    content_list = list()
    # begin scraping
    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
    for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
        username_list.append(tweet.user.username)
        timestamp_list.append(tweet.date)
        content_list.append(tweet.content)
        if len(content_list) == 100:
            # send data to datbase
            insert_list(username_list, timestamp_list, content_list)
            username_list = list()
            timestamp_list = list()
            content_list = list()
        if i % 100 == 0: 
            print(f'found {i} tweets')

def translate_content():
    # load all content from database
    print('loading data from database...')
    df = get_all()
    # clean content
    print('cleaning content...')
    content_clean_list, id_list = clean_content_df(df)
    # translate content
    print('translating content...')
    with Data_table() as dt:
        for id, content in tqdm(zip(id_list, content_clean_list), ascii=True, total=len(id_list)):
            # translate content
            try:
                content_trans = translate(content)
            except Exception as e:
                print(f'error when translating content with id={id}')
                continue
            # transfer to database
            try:
                val_dict = {
                    'id': id,
                    'content_en': content_trans
                }
                query = (
                    f'UPDATE {dt._name} ' 
                    ' SET content_en = (%(content_en)s) '
                    'WHERE id = (%(id)s) '
                )
                dt.execute(query, val_dict)
            except Exception as e:
                print(f'error when sending translated content to database: id={id}')
            




if __name__ == '__main__':
    load_dotenv()
    # run_snscrape()
    translate_content()