brian/twitter_scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ran test

Browse Source
This commit is contained in:
simplypower-bbj
2022-12-17 00:47:24 +01:00
parent 7eab2f1ef7
commit 7a535b3e94
1 changed files with 11 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scraper.ipynb
+11 -14
View File
@@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 12, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@@ -31,25 +31,22 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"ename": "TimeoutException", "data": {
"evalue": "Message: \nStacktrace:\n0 chromedriver 0x000000010546ff38 chromedriver + 4910904\n1 chromedriver 0x00000001053efa03 chromedriver + 4385283\n2 chromedriver 0x0000000105034747 chromedriver + 472903\n3 chromedriver 0x000000010507934c chromedriver + 754508\n4 chromedriver 0x00000001050795a1 chromedriver + 755105\n5 chromedriver 0x00000001050bce94 chromedriver + 1031828\n6 chromedriver 0x000000010509f13d chromedriver + 909629\n7 chromedriver 0x00000001050ba28e chromedriver + 1020558\n8 chromedriver 0x000000010509eee3 chromedriver + 909027\n9 chromedriver 0x000000010506930c chromedriver + 688908\n10 chromedriver 0x000000010506a88e chromedriver + 694414\n11 chromedriver 0x000000010543d1de chromedriver + 4702686\n12 chromedriver 0x0000000105441b19 chromedriver + 4721433\n13 chromedriver 0x000000010544928e chromedriver + 4752014\n14 chromedriver 0x000000010544291a chromedriver + 4725018\n15 chromedriver 0x0000000105416b02 chromedriver + 4545282\n16 chromedriver 0x0000000105461888 chromedriver + 4851848\n17 chromedriver 0x0000000105461a05 chromedriver + 4852229\n18 chromedriver 0x0000000105477e5f chromedriver + 4943455\n19 libsystem_pthread.dylib 0x00007ff8032fd4e1 _pthread_start + 125\n20 libsystem_pthread.dylib 0x00007ff8032f8f6b thread_start + 15\n", "text/plain": [
"output_type": "error", "289"
"traceback": [ ]
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", },
"\u001b[0;31mTimeoutException\u001b[0m Traceback (most recent call last)", "execution_count": 3,
"Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m driver \u001b[39m=\u001b[39m scraper\u001b[39m.\u001b[39;49mload_topic(topic, headless\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n", "metadata": {},
"File \u001b[0;32m~/Projects/twitter_scraper/scraper.py:26\u001b[0m, in \u001b[0;36mload_topic\u001b[0;34m(topic, headless)\u001b[0m\n\u001b[1;32m 24\u001b[0m timeout \u001b[39m=\u001b[39m \u001b[39m30\u001b[39m \u001b[39m# seconds\u001b[39;00m\n\u001b[1;32m 25\u001b[0m xpath_str \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m//*[text()=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mNot now\u001b[39m\u001b[39m'\u001b[39m\u001b[39m]\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m---> 26\u001b[0m element \u001b[39m=\u001b[39m WebDriverWait(driver, timeout)\u001b[39m.\u001b[39;49muntil(EC\u001b[39m.\u001b[39;49mpresence_of_element_located((By\u001b[39m.\u001b[39;49mXPATH, xpath_str)))\n\u001b[1;32m 27\u001b[0m element\u001b[39m.\u001b[39mclick()\n\u001b[1;32m 28\u001b[0m \u001b[39mreturn\u001b[39;00m driver\n", "output_type": "execute_result"
"File \u001b[0;32m~/Projects/twitter_scraper/venv/lib/python3.10/site-packages/selenium/webdriver/support/wait.py:95\u001b[0m, in \u001b[0;36mWebDriverWait.until\u001b[0;34m(self, method, message)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[39mif\u001b[39;00m time\u001b[39m.\u001b[39mmonotonic() \u001b[39m>\u001b[39m end_time:\n\u001b[1;32m 94\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[0;32m---> 95\u001b[0m \u001b[39mraise\u001b[39;00m TimeoutException(message, screen, stacktrace)\n",
"\u001b[0;31mTimeoutException\u001b[0m: Message: \nStacktrace:\n0 chromedriver 0x000000010546ff38 chromedriver + 4910904\n1 chromedriver 0x00000001053efa03 chromedriver + 4385283\n2 chromedriver 0x0000000105034747 chromedriver + 472903\n3 chromedriver 0x000000010507934c chromedriver + 754508\n4 chromedriver 0x00000001050795a1 chromedriver + 755105\n5 chromedriver 0x00000001050bce94 chromedriver + 1031828\n6 chromedriver 0x000000010509f13d chromedriver + 909629\n7 chromedriver 0x00000001050ba28e chromedriver + 1020558\n8 chromedriver 0x000000010509eee3 chromedriver + 909027\n9 chromedriver 0x000000010506930c chromedriver + 688908\n10 chromedriver 0x000000010506a88e chromedriver + 694414\n11 chromedriver 0x000000010543d1de chromedriver + 4702686\n12 chromedriver 0x0000000105441b19 chromedriver + 4721433\n13 chromedriver 0x000000010544928e chromedriver + 4752014\n14 chromedriver 0x000000010544291a chromedriver + 4725018\n15 chromedriver 0x0000000105416b02 chromedriver + 4545282\n16 chromedriver 0x0000000105461888 chromedriver + 4851848\n17 chromedriver 0x0000000105461a05 chromedriver + 4852229\n18 chromedriver 0x0000000105477e5f chromedriver + 4943455\n19 libsystem_pthread.dylib 0x00007ff8032fd4e1 _pthread_start + 125\n20 libsystem_pthread.dylib 0x00007ff8032f8f6b thread_start + 15\n"
]
} }
], ],
"source": [ "source": [
"driver = scraper.load_topic(topic, headless=True)" "max([len(c) for c in content_list])"
] ]
}, },
{ {