brian/twitter_scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

made and tested code to scrape twitter for a specific topic

Browse Source
This commit is contained in:
simplypower-bbj
2022-12-17 00:07:02 +01:00
parent 1b5ed84229
commit 701e55c53d
2 changed files with 183 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test.ipynb
+96
View File
@@ -0,0 +1,96 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"12\n"
]
}
],
"source": [
"from selenium.webdriver.common.action_chains import ActionChains\n",
"from dateutil.parser import parse\n",
"from selenium import webdriver\n",
"import os\n",
"import importlib\n",
"import scraper\n",
"\n",
"importlib.reload(scraper)\n",
"topic = 'Odense'\n",
"limit = 10\n",
"username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)\n",
"\n",
"print(len(username_list))"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"ename": "TimeoutException",
"evalue": "Message: \nStacktrace:\n0 chromedriver 0x000000010546ff38 chromedriver + 4910904\n1 chromedriver 0x00000001053efa03 chromedriver + 4385283\n2 chromedriver 0x0000000105034747 chromedriver + 472903\n3 chromedriver 0x000000010507934c chromedriver + 754508\n4 chromedriver 0x00000001050795a1 chromedriver + 755105\n5 chromedriver 0x00000001050bce94 chromedriver + 1031828\n6 chromedriver 0x000000010509f13d chromedriver + 909629\n7 chromedriver 0x00000001050ba28e chromedriver + 1020558\n8 chromedriver 0x000000010509eee3 chromedriver + 909027\n9 chromedriver 0x000000010506930c chromedriver + 688908\n10 chromedriver 0x000000010506a88e chromedriver + 694414\n11 chromedriver 0x000000010543d1de chromedriver + 4702686\n12 chromedriver 0x0000000105441b19 chromedriver + 4721433\n13 chromedriver 0x000000010544928e chromedriver + 4752014\n14 chromedriver 0x000000010544291a chromedriver + 4725018\n15 chromedriver 0x0000000105416b02 chromedriver + 4545282\n16 chromedriver 0x0000000105461888 chromedriver + 4851848\n17 chromedriver 0x0000000105461a05 chromedriver + 4852229\n18 chromedriver 0x0000000105477e5f chromedriver + 4943455\n19 libsystem_pthread.dylib 0x00007ff8032fd4e1 _pthread_start + 125\n20 libsystem_pthread.dylib 0x00007ff8032f8f6b thread_start + 15\n",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTimeoutException\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m driver \u001b[39m=\u001b[39m scraper\u001b[39m.\u001b[39;49mload_topic(topic, headless\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n",
"File \u001b[0;32m~/Projects/twitter_scraper/scraper.py:26\u001b[0m, in \u001b[0;36mload_topic\u001b[0;34m(topic, headless)\u001b[0m\n\u001b[1;32m 24\u001b[0m timeout \u001b[39m=\u001b[39m \u001b[39m30\u001b[39m \u001b[39m# seconds\u001b[39;00m\n\u001b[1;32m 25\u001b[0m xpath_str \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m//*[text()=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mNot now\u001b[39m\u001b[39m'\u001b[39m\u001b[39m]\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m---> 26\u001b[0m element \u001b[39m=\u001b[39m WebDriverWait(driver, timeout)\u001b[39m.\u001b[39;49muntil(EC\u001b[39m.\u001b[39;49mpresence_of_element_located((By\u001b[39m.\u001b[39;49mXPATH, xpath_str)))\n\u001b[1;32m 27\u001b[0m element\u001b[39m.\u001b[39mclick()\n\u001b[1;32m 28\u001b[0m \u001b[39mreturn\u001b[39;00m driver\n",
"File \u001b[0;32m~/Projects/twitter_scraper/venv/lib/python3.10/site-packages/selenium/webdriver/support/wait.py:95\u001b[0m, in \u001b[0;36mWebDriverWait.until\u001b[0;34m(self, method, message)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[39mif\u001b[39;00m time\u001b[39m.\u001b[39mmonotonic() \u001b[39m>\u001b[39m end_time:\n\u001b[1;32m 94\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[0;32m---> 95\u001b[0m \u001b[39mraise\u001b[39;00m TimeoutException(message, screen, stacktrace)\n",
"\u001b[0;31mTimeoutException\u001b[0m: Message: \nStacktrace:\n0 chromedriver 0x000000010546ff38 chromedriver + 4910904\n1 chromedriver 0x00000001053efa03 chromedriver + 4385283\n2 chromedriver 0x0000000105034747 chromedriver + 472903\n3 chromedriver 0x000000010507934c chromedriver + 754508\n4 chromedriver 0x00000001050795a1 chromedriver + 755105\n5 chromedriver 0x00000001050bce94 chromedriver + 1031828\n6 chromedriver 0x000000010509f13d chromedriver + 909629\n7 chromedriver 0x00000001050ba28e chromedriver + 1020558\n8 chromedriver 0x000000010509eee3 chromedriver + 909027\n9 chromedriver 0x000000010506930c chromedriver + 688908\n10 chromedriver 0x000000010506a88e chromedriver + 694414\n11 chromedriver 0x000000010543d1de chromedriver + 4702686\n12 chromedriver 0x0000000105441b19 chromedriver + 4721433\n13 chromedriver 0x000000010544928e chromedriver + 4752014\n14 chromedriver 0x000000010544291a chromedriver + 4725018\n15 chromedriver 0x0000000105416b02 chromedriver + 4545282\n16 chromedriver 0x0000000105461888 chromedriver + 4851848\n17 chromedriver 0x0000000105461a05 chromedriver + 4852229\n18 chromedriver 0x0000000105477e5f chromedriver + 4943455\n19 libsystem_pthread.dylib 0x00007ff8032fd4e1 _pthread_start + 125\n20 libsystem_pthread.dylib 0x00007ff8032f8f6b thread_start + 15\n"
]
}
],
"source": [
"driver = scraper.load_topic(topic, headless=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# headless mode - has proven difficult...\n",
"# database access\n",
"# logging with warning to discord\n",
"# main loop - go once every hour\n",
"# containerize"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}