# public packages
import snscrape.modules.twitter as sntwitter
import pandas as pd
# database-related
from database import insert_list
from dotenv import load_dotenv

def scrape(query=''):
    assert len(query) > 0, 'query string must be provided'
    # prepare lists to hold information
    username_list = list()
    timestamp_list = list()
    content_list = list()
    # begin scraping
    # query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
    for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
        # store information in lists
        username_list.append(tweet.user.username)
        timestamp_list.append(tweet.date)
        content_list.append(tweet.content)
        # when 100 tweets have been collected
        if len(content_list) == 100:
            # send data to database
            insert_list(username_list, timestamp_list, content_list)
            # empty lists
            username_list = list()
            timestamp_list = list()
            content_list = list()
        # check in on progress
        if i % 100 == 0:
            print(f'found {i} tweets')

if __name__ == '__main__':
    load_dotenv()
    scrape()