twitter_scraper/test.ipynb

In [12]:

from selenium.webdriver.common.action_chains import ActionChains
from dateutil.parser import parse
from selenium import webdriver
import os
import importlib
import scraper

importlib.reload(scraper)
topic = 'Odense'
limit = 10
username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)

print(len(username_list))

12

In [11]:

driver = scraper.load_topic(topic, headless=True)

---------------------------------------------------------------------------
TimeoutException                          Traceback (most recent call last)
Cell In[11], line 1
----> 1 driver = scraper.load_topic(topic, headless=True)

File ~/Projects/twitter_scraper/scraper.py:26, in load_topic(topic, headless)
     24 timeout = 30 # seconds
     25 xpath_str = "//*[text()='Not now']"
---> 26 element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str)))
     27 element.click()
     28 return driver

File ~/Projects/twitter_scraper/venv/lib/python3.10/site-packages/selenium/webdriver/support/wait.py:95, in WebDriverWait.until(self, method, message)
     93     if time.monotonic() > end_time:
     94         break
---> 95 raise TimeoutException(message, screen, stacktrace)

TimeoutException: Message: 
Stacktrace:
0   chromedriver                        0x000000010546ff38 chromedriver + 4910904
1   chromedriver                        0x00000001053efa03 chromedriver + 4385283
2   chromedriver                        0x0000000105034747 chromedriver + 472903
3   chromedriver                        0x000000010507934c chromedriver + 754508
4   chromedriver                        0x00000001050795a1 chromedriver + 755105
5   chromedriver                        0x00000001050bce94 chromedriver + 1031828
6   chromedriver                        0x000000010509f13d chromedriver + 909629
7   chromedriver                        0x00000001050ba28e chromedriver + 1020558
8   chromedriver                        0x000000010509eee3 chromedriver + 909027
9   chromedriver                        0x000000010506930c chromedriver + 688908
10  chromedriver                        0x000000010506a88e chromedriver + 694414
11  chromedriver                        0x000000010543d1de chromedriver + 4702686
12  chromedriver                        0x0000000105441b19 chromedriver + 4721433
13  chromedriver                        0x000000010544928e chromedriver + 4752014
14  chromedriver                        0x000000010544291a chromedriver + 4725018
15  chromedriver                        0x0000000105416b02 chromedriver + 4545282
16  chromedriver                        0x0000000105461888 chromedriver + 4851848
17  chromedriver                        0x0000000105461a05 chromedriver + 4852229
18  chromedriver                        0x0000000105477e5f chromedriver + 4943455
19  libsystem_pthread.dylib             0x00007ff8032fd4e1 _pthread_start + 125
20  libsystem_pthread.dylib             0x00007ff8032f8f6b thread_start + 15

In [ ]:

# headless mode - has proven difficult...
# database access
# logging with warning to discord
# main loop - go once every hour
# containerize