twitter_scraper/scraper.ipynb at 7eab2f1ef7ab1d1b938b139d9a49baaa2e51571f

brian/twitter_scraper

Fork 0

Files

T

simplypower-bbj 375ca572a1 renamed notebook

2022-12-17 00:08:23 +01:00

7.0 KiB

Raw Blame History

In [12]:

from selenium.webdriver.common.action_chains import ActionChains
from dateutil.parser import parse
from selenium import webdriver
import os
import importlib
import scraper

importlib.reload(scraper)
topic = 'Odense'
limit = 10
username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)

print(len(username_list))

In [11]:

driver = scraper.load_topic(topic, headless=True)

[0;31m---------------------------------------------------------------------------[0m
[0;31mTimeoutException[0m                          Traceback (most recent call last)
Cell [0;32mIn[11], line 1[0m
[0;32m----> 1[0m driver [39m=[39m scraper[39m.[39;49mload_topic(topic, headless[39m=[39;49m[39mTrue[39;49;00m)

File [0;32m~/Projects/twitter_scraper/scraper.py:26[0m, in [0;36mload_topic[0;34m(topic, headless)[0m
[1;32m     24[0m timeout [39m=[39m [39m30[39m [39m# seconds[39;00m
[1;32m     25[0m xpath_str [39m=[39m [39m"[39m[39m//*[text()=[39m[39m'[39m[39mNot now[39m[39m'[39m[39m][39m[39m"[39m
[0;32m---> 26[0m element [39m=[39m WebDriverWait(driver, timeout)[39m.[39;49muntil(EC[39m.[39;49mpresence_of_element_located((By[39m.[39;49mXPATH, xpath_str)))
[1;32m     27[0m element[39m.[39mclick()
[1;32m     28[0m [39mreturn[39;00m driver

File [0;32m~/Projects/twitter_scraper/venv/lib/python3.10/site-packages/selenium/webdriver/support/wait.py:95[0m, in [0;36mWebDriverWait.until[0;34m(self, method, message)[0m
[1;32m     93[0m     [39mif[39;00m time[39m.[39mmonotonic() [39m>[39m end_time:
[1;32m     94[0m         [39mbreak[39;00m
[0;32m---> 95[0m [39mraise[39;00m TimeoutException(message, screen, stacktrace)

[0;31mTimeoutException[0m: Message: 
Stacktrace:
0   chromedriver                        0x000000010546ff38 chromedriver + 4910904
1   chromedriver                        0x00000001053efa03 chromedriver + 4385283
2   chromedriver                        0x0000000105034747 chromedriver + 472903
3   chromedriver                        0x000000010507934c chromedriver + 754508
4   chromedriver                        0x00000001050795a1 chromedriver + 755105
5   chromedriver                        0x00000001050bce94 chromedriver + 1031828
6   chromedriver                        0x000000010509f13d chromedriver + 909629
7   chromedriver                        0x00000001050ba28e chromedriver + 1020558
8   chromedriver                        0x000000010509eee3 chromedriver + 909027
9   chromedriver                        0x000000010506930c chromedriver + 688908
10  chromedriver                        0x000000010506a88e chromedriver + 694414
11  chromedriver                        0x000000010543d1de chromedriver + 4702686
12  chromedriver                        0x0000000105441b19 chromedriver + 4721433
13  chromedriver                        0x000000010544928e chromedriver + 4752014
14  chromedriver                        0x000000010544291a chromedriver + 4725018
15  chromedriver                        0x0000000105416b02 chromedriver + 4545282
16  chromedriver                        0x0000000105461888 chromedriver + 4851848
17  chromedriver                        0x0000000105461a05 chromedriver + 4852229
18  chromedriver                        0x0000000105477e5f chromedriver + 4943455
19  libsystem_pthread.dylib             0x00007ff8032fd4e1 _pthread_start + 125
20  libsystem_pthread.dylib             0x00007ff8032f8f6b thread_start + 15

In [ ]:

# headless mode - has proven difficult...
# database access
# logging with warning to discord
# main loop - go once every hour
# containerize

7.0 KiB Raw Blame History

7.0 KiB

Raw Blame History