twitter_scraper/code/scraper.ipynb

In [1]:

from selenium.webdriver.common.action_chains import ActionChains
from dateutil.parser import parse
from selenium import webdriver
import os
import importlib
import scraper

importlib.reload(scraper)
topic = 'Odense'
limit = 10
username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)

print(len(username_list))

12

In [3]:

max([len(c) for c in content_list])

Out [3]:

289

In [ ]:

# headless mode - has proven difficult...
# database access
# logging with warning to discord
# main loop - go once every hour
# containerize