From 7a535b3e94689ece609dfae07ae3b3d42a491ced Mon Sep 17 00:00:00 2001 From: simplypower-bbj Date: Sat, 17 Dec 2022 00:47:24 +0100 Subject: [PATCH] ran test --- scraper.ipynb | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/scraper.ipynb b/scraper.ipynb index e47b20c..a0dda97 100644 --- a/scraper.ipynb +++ b/scraper.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 12, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -31,25 +31,22 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 3, "metadata": {}, "outputs": [ { - "ename": "TimeoutException", - "evalue": "Message: \nStacktrace:\n0 chromedriver 0x000000010546ff38 chromedriver + 4910904\n1 chromedriver 0x00000001053efa03 chromedriver + 4385283\n2 chromedriver 0x0000000105034747 chromedriver + 472903\n3 chromedriver 0x000000010507934c chromedriver + 754508\n4 chromedriver 0x00000001050795a1 chromedriver + 755105\n5 chromedriver 0x00000001050bce94 chromedriver + 1031828\n6 chromedriver 0x000000010509f13d chromedriver + 909629\n7 chromedriver 0x00000001050ba28e chromedriver + 1020558\n8 chromedriver 0x000000010509eee3 chromedriver + 909027\n9 chromedriver 0x000000010506930c chromedriver + 688908\n10 chromedriver 0x000000010506a88e chromedriver + 694414\n11 chromedriver 0x000000010543d1de chromedriver + 4702686\n12 chromedriver 0x0000000105441b19 chromedriver + 4721433\n13 chromedriver 0x000000010544928e chromedriver + 4752014\n14 chromedriver 0x000000010544291a chromedriver + 4725018\n15 chromedriver 0x0000000105416b02 chromedriver + 4545282\n16 chromedriver 0x0000000105461888 chromedriver + 4851848\n17 chromedriver 0x0000000105461a05 chromedriver + 4852229\n18 chromedriver 0x0000000105477e5f chromedriver + 4943455\n19 libsystem_pthread.dylib 0x00007ff8032fd4e1 _pthread_start + 125\n20 libsystem_pthread.dylib 0x00007ff8032f8f6b thread_start + 15\n", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTimeoutException\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m driver \u001b[39m=\u001b[39m scraper\u001b[39m.\u001b[39;49mload_topic(topic, headless\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n", - "File \u001b[0;32m~/Projects/twitter_scraper/scraper.py:26\u001b[0m, in \u001b[0;36mload_topic\u001b[0;34m(topic, headless)\u001b[0m\n\u001b[1;32m 24\u001b[0m timeout \u001b[39m=\u001b[39m \u001b[39m30\u001b[39m \u001b[39m# seconds\u001b[39;00m\n\u001b[1;32m 25\u001b[0m xpath_str \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m//*[text()=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mNot now\u001b[39m\u001b[39m'\u001b[39m\u001b[39m]\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m---> 26\u001b[0m element \u001b[39m=\u001b[39m WebDriverWait(driver, timeout)\u001b[39m.\u001b[39;49muntil(EC\u001b[39m.\u001b[39;49mpresence_of_element_located((By\u001b[39m.\u001b[39;49mXPATH, xpath_str)))\n\u001b[1;32m 27\u001b[0m element\u001b[39m.\u001b[39mclick()\n\u001b[1;32m 28\u001b[0m \u001b[39mreturn\u001b[39;00m driver\n", - "File \u001b[0;32m~/Projects/twitter_scraper/venv/lib/python3.10/site-packages/selenium/webdriver/support/wait.py:95\u001b[0m, in \u001b[0;36mWebDriverWait.until\u001b[0;34m(self, method, message)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[39mif\u001b[39;00m time\u001b[39m.\u001b[39mmonotonic() \u001b[39m>\u001b[39m end_time:\n\u001b[1;32m 94\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[0;32m---> 95\u001b[0m \u001b[39mraise\u001b[39;00m TimeoutException(message, screen, stacktrace)\n", - "\u001b[0;31mTimeoutException\u001b[0m: Message: \nStacktrace:\n0 chromedriver 0x000000010546ff38 chromedriver + 4910904\n1 chromedriver 0x00000001053efa03 chromedriver + 4385283\n2 chromedriver 0x0000000105034747 chromedriver + 472903\n3 chromedriver 0x000000010507934c chromedriver + 754508\n4 chromedriver 0x00000001050795a1 chromedriver + 755105\n5 chromedriver 0x00000001050bce94 chromedriver + 1031828\n6 chromedriver 0x000000010509f13d chromedriver + 909629\n7 chromedriver 0x00000001050ba28e chromedriver + 1020558\n8 chromedriver 0x000000010509eee3 chromedriver + 909027\n9 chromedriver 0x000000010506930c chromedriver + 688908\n10 chromedriver 0x000000010506a88e chromedriver + 694414\n11 chromedriver 0x000000010543d1de chromedriver + 4702686\n12 chromedriver 0x0000000105441b19 chromedriver + 4721433\n13 chromedriver 0x000000010544928e chromedriver + 4752014\n14 chromedriver 0x000000010544291a chromedriver + 4725018\n15 chromedriver 0x0000000105416b02 chromedriver + 4545282\n16 chromedriver 0x0000000105461888 chromedriver + 4851848\n17 chromedriver 0x0000000105461a05 chromedriver + 4852229\n18 chromedriver 0x0000000105477e5f chromedriver + 4943455\n19 libsystem_pthread.dylib 0x00007ff8032fd4e1 _pthread_start + 125\n20 libsystem_pthread.dylib 0x00007ff8032f8f6b thread_start + 15\n" - ] + "data": { + "text/plain": [ + "289" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "driver = scraper.load_topic(topic, headless=True)" + "max([len(c) for c in content_list])" ] }, {