69 lines
2.1 KiB
Python
69 lines
2.1 KiB
Python
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
|
|
from setup_logging import setup_logging
|
|
from scraper import scrape
|
|
from database import insert_list
|
|
from dotenv import load_dotenv
|
|
import logging
|
|
from datetime import datetime
|
|
import time
|
|
import os
|
|
|
|
import snscrape.modules.twitter as sntwitter
|
|
import pandas as pd
|
|
|
|
# def update():
|
|
# topic = 'Odense'
|
|
# # scrape latest data
|
|
# username_list, timestamp_list, content_list = scrape(topic, limit=10, headless=False)
|
|
# # send data to database
|
|
# insert_list(username_list, timestamp_list, content_list)
|
|
|
|
# if __name__ == '__main__':
|
|
# # setup
|
|
# load_dotenv()
|
|
# setup_logging()
|
|
# logging.info('finished setup')
|
|
# # main loop
|
|
# trigger_minute = datetime.now().minute
|
|
# while True:
|
|
# try:
|
|
# now = datetime.now()
|
|
# if now.minute == trigger_minute:
|
|
# try:
|
|
# update()
|
|
# except Exception as e:
|
|
# logging.warning(e)
|
|
# else:
|
|
# logging.info('finished cycle')
|
|
# time.sleep(60)
|
|
# time.sleep(0.1)
|
|
# except Exception as e:
|
|
# logging.critical(e)
|
|
# break
|
|
# except KeyboardInterrupt:
|
|
# logging.info('KeyboardInterrupt')
|
|
# break
|
|
|
|
def run_snscrape():
|
|
# prepare variables
|
|
username_list = list()
|
|
timestamp_list = list()
|
|
content_list = list()
|
|
# begin scraping
|
|
query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
|
|
for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
|
|
username_list.append(tweet.user.username)
|
|
timestamp_list.append(tweet.date)
|
|
content_list.append(tweet.content)
|
|
if len(content_list) == 100:
|
|
# send data to datbase
|
|
insert_list(username_list, timestamp_list, content_list)
|
|
username_list = list()
|
|
timestamp_list = list()
|
|
content_list = list()
|
|
if i % 100 == 0:
|
|
print(f'found {i} tweets')
|
|
|
|
if __name__ == '__main__':
|
|
load_dotenv()
|
|
run_snscrape() |