#!/Users/brian/Projects/twitter_scraper/venv/bin/python
from setup_logging import setup_logging
from scraper import scrape
from database import insert_list
from dotenv import load_dotenv
import logging
from datetime import datetime
import time
import os

import snscrape.modules.twitter as sntwitter
import pandas as pd

# def update():
#     topic = 'Odense'
#     # scrape latest data
#     username_list, timestamp_list, content_list = scrape(topic, limit=10, headless=False)
#     # send data to database
#     insert_list(username_list, timestamp_list, content_list)

# if __name__ == '__main__':
#     # setup
#     load_dotenv()
#     setup_logging()
#     logging.info('finished setup')
#     # main loop
#     trigger_minute = datetime.now().minute
#     while True:
#         try:
#             now = datetime.now()
#             if now.minute == trigger_minute:
#                 try:
#                     update()
#                 except Exception as e:
#                     logging.warning(e)
#                 else:
#                     logging.info('finished cycle')
#                 time.sleep(60)
#             time.sleep(0.1)
#         except Exception as e:
#             logging.critical(e)
#             break
#         except KeyboardInterrupt:
#             logging.info('KeyboardInterrupt')
#             break

def run_snscrape():
    # prepare variables
    username_list = list()
    timestamp_list = list()
    content_list = list()
    # begin scraping
    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
    for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
        username_list.append(tweet.user.username)
        timestamp_list.append(tweet.date)
        content_list.append(tweet.content)
        if len(content_list) == 100:
            # send data to datbase
            insert_list(username_list, timestamp_list, content_list)
            username_list = list()
            timestamp_list = list()
            content_list = list()
        if i % 100 == 0: 
            print(f'found {i} tweets')

if __name__ == '__main__':
    load_dotenv()
    run_snscrape()