twitter_scraper/code/sntwitter_scraper.ipynb

In [1]:

import snscrape.modules.twitter as sntwitter
import pandas as pd

# prepare variables
# query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
query = '(from:Maersk) -filter:replies'
limit = 10
tweet_list = list()
# begin scraping
for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
    # if limit is reached, break out of loop
    if limit > 0 and i > limit: break
    # if another 100 tweets found, tell it
    if i % 10 == 0: 
        print(f'found {i} tweets')
    tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])
# create dataframe
df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])

Error retrieving https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel: non-200 status code
4 requests to https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel failed, giving up.

---------------------------------------------------------------------------
ScraperException                          Traceback (most recent call last)
Cell In[1], line 10
      8 tweet_list = list()
      9 # begin scraping
---> 10 for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
     11     # if limit is reached, break out of loop
     12     if limit > 0 and i > limit: break
     13     # if another 100 tweets found, tell it

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/modules/twitter.py:680, in TwitterSearchScraper.get_items(self)
    677 	del params['tweet_search_mode']
    678 	del paginationParams['tweet_search_mode']
--> 680 for obj in self._iter_api_data('https://api.twitter.com/2/search/adaptive.json', params, paginationParams, cursor = self._cursor):
    681 	yield from self._instructions_to_tweets(obj)

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/modules/twitter.py:369, in _TwitterAPIScraper._iter_api_data(self, endpoint, params, paginationParams, cursor, direction)
    367 while True:
    368 	_logger.info(f'Retrieving scroll page {cursor}')
--> 369 	obj = self._get_api_data(endpoint, reqParams)
    370 	yield obj
    372 	# No data format test, just a hard and loud crash if anything's wrong :-)

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/modules/twitter.py:339, in _TwitterAPIScraper._get_api_data(self, endpoint, params)
    337 def _get_api_data(self, endpoint, params):
    338 	self._ensure_guest_token()
--> 339 	r = self._get(endpoint, params = params, headers = self._apiHeaders, responseOkCallback = self._check_api_response)
    340 	try:
    341 		obj = r.json()

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/base.py:216, in Scraper._get(self, *args, **kwargs)
    215 def _get(self, *args, **kwargs):
--> 216 	return self._request('GET', *args, **kwargs)

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/snscrape/base.py:212, in Scraper._request(self, method, url, params, data, headers, timeout, responseOkCallback, allowRedirects)
    210 	msg = f'{self._retries + 1} requests to {req.url} failed, giving up.'
    211 	logger.fatal(msg)
--> 212 	raise ScraperException(msg)
    213 raise RuntimeError('Reached unreachable code')

ScraperException: 4 requests to https://api.twitter.com/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&q=%28from%3AMaersk%29+-filter%3Areplies&tweet_search_mode=live&count=100&query_source=spelling_expansion_revert_click&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel failed, giving up.

In [2]:

df

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[2], line 1
----> 1 df

NameError: name 'df' is not defined

In [ ]: