brian/twitter_scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ran tests

Browse Source
This commit is contained in:
brian
2023-03-05 15:33:45 +00:00
parent d1d4c8f3c7
commit c1cfb69fa0
1 changed files with 36 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code/sntwitter_scraper.ipynb
+36 -162
View File
@@ -2,24 +2,31 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"found 0 tweets\n", "Error retrieving https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel: non-200 status code\n",
"found 100 tweets\n", "4 requests to https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel failed, giving up.\n"
"found 200 tweets\n", ]
"found 300 tweets\n", },
"found 400 tweets\n", {
"found 500 tweets\n", "ename": "ScraperException",
"found 600 tweets\n", "evalue": "4 requests to https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel failed, giving up.",
"found 700 tweets\n", "output_type": "error",
"found 800 tweets\n", "traceback": [
"found 900 tweets\n", "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"found 1000 tweets\n" "\u001b[0;31mScraperException\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[1], line 10\u001b[0m\n\u001b[1;32m 8\u001b[0m tweet_list \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m()\n\u001b[1;32m 9\u001b[0m \u001b[39m# begin scraping\u001b[39;00m\n\u001b[0;32m---> 10\u001b[0m \u001b[39mfor\u001b[39;00m i, tweet \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(sntwitter\u001b[39m.\u001b[39mTwitterSearchScraper(query)\u001b[39m.\u001b[39mget_items()):\n\u001b[1;32m 11\u001b[0m \u001b[39m# if limit is reached, break out of loop\u001b[39;00m\n\u001b[1;32m 12\u001b[0m \u001b[39mif\u001b[39;00m limit \u001b[39m>\u001b[39m \u001b[39m0\u001b[39m \u001b[39mand\u001b[39;00m i \u001b[39m>\u001b[39m limit: \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m 13\u001b[0m \u001b[39m# if another 100 tweets found, tell it\u001b[39;00m\n",
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/modules/twitter.py:680\u001b[0m, in \u001b[0;36mTwitterSearchScraper.get_items\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 677\u001b[0m \t\u001b[39mdel\u001b[39;00m params[\u001b[39m'\u001b[39m\u001b[39mtweet_search_mode\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[1;32m 678\u001b[0m \t\u001b[39mdel\u001b[39;00m paginationParams[\u001b[39m'\u001b[39m\u001b[39mtweet_search_mode\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[0;32m--> 680\u001b[0m \u001b[39mfor\u001b[39;00m obj \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_iter_api_data(\u001b[39m'\u001b[39m\u001b[39mhttps://api.twitter.com/2/search/adaptive.json\u001b[39m\u001b[39m'\u001b[39m, params, paginationParams, cursor \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_cursor):\n\u001b[1;32m 681\u001b[0m \t\u001b[39myield from\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_instructions_to_tweets(obj)\n",
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/modules/twitter.py:369\u001b[0m, in \u001b[0;36m_TwitterAPIScraper._iter_api_data\u001b[0;34m(self, endpoint, params, paginationParams, cursor, direction)\u001b[0m\n\u001b[1;32m 367\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m 368\u001b[0m \t_logger\u001b[39m.\u001b[39minfo(\u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mRetrieving scroll page \u001b[39m\u001b[39m{\u001b[39;00mcursor\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m)\n\u001b[0;32m--> 369\u001b[0m \tobj \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_get_api_data(endpoint, reqParams)\n\u001b[1;32m 370\u001b[0m \t\u001b[39myield\u001b[39;00m obj\n\u001b[1;32m 372\u001b[0m \t\u001b[39m# No data format test, just a hard and loud crash if anything's wrong :-)\u001b[39;00m\n",
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/modules/twitter.py:339\u001b[0m, in \u001b[0;36m_TwitterAPIScraper._get_api_data\u001b[0;34m(self, endpoint, params)\u001b[0m\n\u001b[1;32m 337\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_get_api_data\u001b[39m(\u001b[39mself\u001b[39m, endpoint, params):\n\u001b[1;32m 338\u001b[0m \t\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ensure_guest_token()\n\u001b[0;32m--> 339\u001b[0m \tr \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_get(endpoint, params \u001b[39m=\u001b[39;49m params, headers \u001b[39m=\u001b[39;49m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_apiHeaders, responseOkCallback \u001b[39m=\u001b[39;49m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_check_api_response)\n\u001b[1;32m 340\u001b[0m \t\u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 341\u001b[0m \t\tobj \u001b[39m=\u001b[39m r\u001b[39m.\u001b[39mjson()\n",
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/base.py:216\u001b[0m, in \u001b[0;36mScraper._get\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_get\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[0;32m--> 216\u001b[0m \t\u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_request(\u001b[39m'\u001b[39;49m\u001b[39mGET\u001b[39;49m\u001b[39m'\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/base.py:212\u001b[0m, in \u001b[0;36mScraper._request\u001b[0;34m(self, method, url, params, data, headers, timeout, responseOkCallback, allowRedirects)\u001b[0m\n\u001b[1;32m 210\u001b[0m \tmsg \u001b[39m=\u001b[39m \u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_retries \u001b[39m+\u001b[39m \u001b[39m1\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m requests to \u001b[39m\u001b[39m{\u001b[39;00mreq\u001b[39m.\u001b[39murl\u001b[39m}\u001b[39;00m\u001b[39m failed, giving up.\u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 211\u001b[0m \tlogger\u001b[39m.\u001b[39mfatal(msg)\n\u001b[0;32m--> 212\u001b[0m \t\u001b[39mraise\u001b[39;00m ScraperException(msg)\n\u001b[1;32m 213\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\u001b[39m'\u001b[39m\u001b[39mReached unreachable code\u001b[39m\u001b[39m'\u001b[39m)\n",
"\u001b[0;31mScraperException\u001b[0m: 4 requests to https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel failed, giving up."
] ]
} }
], ],
@@ -28,16 +35,16 @@
"import pandas as pd\n", "import pandas as pd\n",
"\n", "\n",
"# prepare variables\n", "# prepare variables\n",
"query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n", "# query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
"limit = 1000\n", "query = '(from:Maersk) -filter:replies'\n",
"limit = 10\n",
"tweet_list = list()\n", "tweet_list = list()\n",
"query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
"# begin scraping\n", "# begin scraping\n",
"for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):\n", "for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):\n",
" # if limit is reached, break out of loop\n", " # if limit is reached, break out of loop\n",
" if limit > 0 and i > limit: break\n", " if limit > 0 and i > limit: break\n",
" # if another 100 tweets found, tell it\n", " # if another 100 tweets found, tell it\n",
" if i % 100 == 0: \n", " if i % 10 == 0: \n",
" print(f'found {i} tweets')\n", " print(f'found {i} tweets')\n",
" tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])\n", " tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])\n",
"# create dataframe\n", "# create dataframe\n",
@@ -46,152 +53,19 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "ename": "NameError",
"text/html": [ "evalue": "name 'df' is not defined",
"<div>\n", "output_type": "error",
"<style scoped>\n", "traceback": [
" .dataframe tbody tr th:only-of-type {\n", "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
" vertical-align: middle;\n", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
" }\n", "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df\n",
"\n", "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
" .dataframe tbody tr th {\n", ]
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Datetime</th>\n",
" <th>Id</th>\n",
" <th>Text</th>\n",
" <th>Username</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2022-12-16 22:20:26+00:00</td>\n",
" <td>1603877740972752897</td>\n",
" <td>Danimarka’nın Odense takımı bu yaz müthiş bir ...</td>\n",
" <td>EryilmazFa10</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2022-12-16 21:56:35+00:00</td>\n",
" <td>1603871736579371008</td>\n",
" <td>Highlights: SønderjyskE – Odense https://t.co/...</td>\n",
" <td>trans_rumor</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2022-12-16 21:39:40+00:00</td>\n",
" <td>1603867481101066241</td>\n",
" <td>“Meta Platforms Inc has halted construction of...</td>\n",
" <td>sakak</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>2022-12-16 21:19:09+00:00</td>\n",
" <td>1603862318647070720</td>\n",
" <td>Sponsoreret: Verdensmestre, europamestre og st...</td>\n",
" <td>otwndk</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2022-12-16 21:07:09+00:00</td>\n",
" <td>1603859298249019392</td>\n",
" <td>Meta just halted a major data centre project t...</td>\n",
" <td>techosmo</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>996</th>\n",
" <td>2022-11-23 15:42:37+00:00</td>\n",
" <td>1595442703990493211</td>\n",
" <td>Mohamed Djan Fofana går fra banen, og han erst...</td>\n",
" <td>Odense_Boldklub</td>\n",
" </tr>\n",
" <tr>\n",
" <th>997</th>\n",
" <td>2022-11-23 15:34:36+00:00</td>\n",
" <td>1595440689629859840</td>\n",
" <td>1-1...\\nEfter et hjørnespark passerer bolden t...</td>\n",
" <td>Odense_Boldklub</td>\n",
" </tr>\n",
" <tr>\n",
" <th>998</th>\n",
" <td>2022-11-23 15:31:12+00:00</td>\n",
" <td>1595439830187798529</td>\n",
" <td>Ud er gået Sebastian Wille, Omar Jebali, Law T...</td>\n",
" <td>Odense_Boldklub</td>\n",
" </tr>\n",
" <tr>\n",
" <th>999</th>\n",
" <td>2022-11-23 15:21:18+00:00</td>\n",
" <td>1595437338997542912</td>\n",
" <td>Anden halvleg er netop sparket i gang 🔵⚪\\n#obd...</td>\n",
" <td>Odense_Boldklub</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1000</th>\n",
" <td>2022-11-23 15:05:57+00:00</td>\n",
" <td>1595433476135829506</td>\n",
" <td>Efter en scoring af Franco Tongya går vi nu ti...</td>\n",
" <td>Odense_Boldklub</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>1001 rows × 4 columns</p>\n",
"</div>"
],
"text/plain": [
" Datetime Id \\\n",
"0 2022-12-16 22:20:26+00:00 1603877740972752897 \n",
"1 2022-12-16 21:56:35+00:00 1603871736579371008 \n",
"2 2022-12-16 21:39:40+00:00 1603867481101066241 \n",
"3 2022-12-16 21:19:09+00:00 1603862318647070720 \n",
"4 2022-12-16 21:07:09+00:00 1603859298249019392 \n",
"... ... ... \n",
"996 2022-11-23 15:42:37+00:00 1595442703990493211 \n",
"997 2022-11-23 15:34:36+00:00 1595440689629859840 \n",
"998 2022-11-23 15:31:12+00:00 1595439830187798529 \n",
"999 2022-11-23 15:21:18+00:00 1595437338997542912 \n",
"1000 2022-11-23 15:05:57+00:00 1595433476135829506 \n",
"\n",
" Text Username \n",
"0 Danimarka’nın Odense takımı bu yaz müthiş bir ... EryilmazFa10 \n",
"1 Highlights: SønderjyskE – Odense https://t.co/... trans_rumor \n",
"2 “Meta Platforms Inc has halted construction of... sakak \n",
"3 Sponsoreret: Verdensmestre, europamestre og st... otwndk \n",
"4 Meta just halted a major data centre project t... techosmo \n",
"... ... ... \n",
"996 Mohamed Djan Fofana går fra banen, og han erst... Odense_Boldklub \n",
"997 1-1...\\nEfter et hjørnespark passerer bolden t... Odense_Boldklub \n",
"998 Ud er gået Sebastian Wille, Omar Jebali, Law T... Odense_Boldklub \n",
"999 Anden halvleg er netop sparket i gang 🔵⚪\\n#obd... Odense_Boldklub \n",
"1000 Efter en scoring af Franco Tongya går vi nu ti... Odense_Boldklub \n",
"\n",
"[1001 rows x 4 columns]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
} }
], ],
"source": [ "source": [
@@ -222,12 +96,12 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]" "version": "3.8.10"
}, },
"orig_nbformat": 4, "orig_nbformat": 4,
"vscode": { "vscode": {
"interpreter": { "interpreter": {
"hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe" "hash": "57b41ea9bda188de64800ef31c5cfb4fcfca577257038becb66e4fa0a5ee97a6"
} }
} }
}, },