# public packages
import snscrape.modules.twitter as sntwitter
from tqdm import tqdm
import time
# database-related
from database import Data_table, insert_list, get_all
from dotenv import load_dotenv
# own code
from clean_data import clean_content_df
from translate import translate

def run_snscrape():
    # prepare lists to hold information
    username_list = list()
    timestamp_list = list()
    content_list = list()
    # begin scraping
    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
    for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
        # store information in lists
        username_list.append(tweet.user.username)
        timestamp_list.append(tweet.date)
        content_list.append(tweet.content)
        # when 100 tweets have been collected
        if len(content_list) == 100:
            # send data to database
            insert_list(username_list, timestamp_list, content_list)
            # empty lists
            username_list = list()
            timestamp_list = list()
            content_list = list()
        # check in on progress
        if i % 100 == 0:
            print(f'found {i} tweets')

def translate_content():
    # load all content from database
    print('loading data from database...')
    df = get_all()
    # remove already translated content
    df = df[df['content_en'].isna()]
    # clean content
    print('cleaning content...')
    content_clean_list, id_list = clean_content_df(df)
    # translate content
    print('translating content...')
    with Data_table() as dt:
        for id, content in tqdm(zip(id_list, content_clean_list), ascii=True, total=len(id_list)):
            # translate content
            try:
                content_trans = translate(content)
            except Exception:
                print(f'error when translating content with id={id}')
                continue
            # transfer to database
            try:
                val_dict = {
                    'id': id,
                    'content_en': content_trans
                }
                query = (
                    f'UPDATE {dt._name} '
                    ' SET content_en = (%(content_en)s) '
                    'WHERE id = (%(id)s) '
                )
                dt.execute(query, val_dict)
            except Exception:
                print(f'error when sending translated content to database: id={id}')
            # take a break to avoid being banned by Google
            time.sleep(2)


if __name__ == '__main__':
    load_dotenv()
    # run_snscrape()
    translate_content()