diff --git a/code/sntwitter_scraper.ipynb b/code/sntwitter_scraper.ipynb new file mode 100644 index 0000000..d026aa5 --- /dev/null +++ b/code/sntwitter_scraper.ipynb @@ -0,0 +1,230 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "found 0 tweets\n", + "found 100 tweets\n", + "found 200 tweets\n", + "found 300 tweets\n", + "found 400 tweets\n", + "found 500 tweets\n", + "found 600 tweets\n", + "found 700 tweets\n", + "found 800 tweets\n", + "found 900 tweets\n", + "found 1000 tweets\n" + ] + } + ], + "source": [ + "import snscrape.modules.twitter as sntwitter\n", + "import pandas as pd\n", + "tweet_list = list()\n", + "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n", + "limit = 1000\n", + "\n", + "for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):\n", + " if i % 100 == 0: print(f'found {i} tweets')\n", + " if i > limit: break\n", + " tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])\n", + "\n", + "df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DatetimeIdTextUsername
02022-12-16 22:20:26+00:001603877740972752897Danimarka’nın Odense takımı bu yaz müthiş bir ...EryilmazFa10
12022-12-16 21:56:35+00:001603871736579371008Highlights: SønderjyskE – Odense https://t.co/...trans_rumor
22022-12-16 21:39:40+00:001603867481101066241“Meta Platforms Inc has halted construction of...sakak
32022-12-16 21:19:09+00:001603862318647070720Sponsoreret: Verdensmestre, europamestre og st...otwndk
42022-12-16 21:07:09+00:001603859298249019392Meta just halted a major data centre project t...techosmo
...............
9962022-11-23 15:42:37+00:001595442703990493211Mohamed Djan Fofana går fra banen, og han erst...Odense_Boldklub
9972022-11-23 15:34:36+00:0015954406896298598401-1...\\nEfter et hjørnespark passerer bolden t...Odense_Boldklub
9982022-11-23 15:31:12+00:001595439830187798529Ud er gået Sebastian Wille, Omar Jebali, Law T...Odense_Boldklub
9992022-11-23 15:21:18+00:001595437338997542912Anden halvleg er netop sparket i gang 🔵⚪\\n#obd...Odense_Boldklub
10002022-11-23 15:05:57+00:001595433476135829506Efter en scoring af Franco Tongya går vi nu ti...Odense_Boldklub
\n", + "

1001 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " Datetime Id \\\n", + "0 2022-12-16 22:20:26+00:00 1603877740972752897 \n", + "1 2022-12-16 21:56:35+00:00 1603871736579371008 \n", + "2 2022-12-16 21:39:40+00:00 1603867481101066241 \n", + "3 2022-12-16 21:19:09+00:00 1603862318647070720 \n", + "4 2022-12-16 21:07:09+00:00 1603859298249019392 \n", + "... ... ... \n", + "996 2022-11-23 15:42:37+00:00 1595442703990493211 \n", + "997 2022-11-23 15:34:36+00:00 1595440689629859840 \n", + "998 2022-11-23 15:31:12+00:00 1595439830187798529 \n", + "999 2022-11-23 15:21:18+00:00 1595437338997542912 \n", + "1000 2022-11-23 15:05:57+00:00 1595433476135829506 \n", + "\n", + " Text Username \n", + "0 Danimarka’nın Odense takımı bu yaz müthiş bir ... EryilmazFa10 \n", + "1 Highlights: SønderjyskE – Odense https://t.co/... trans_rumor \n", + "2 “Meta Platforms Inc has halted construction of... sakak \n", + "3 Sponsoreret: Verdensmestre, europamestre og st... otwndk \n", + "4 Meta just halted a major data centre project t... techosmo \n", + "... ... ... \n", + "996 Mohamed Djan Fofana går fra banen, og han erst... Odense_Boldklub \n", + "997 1-1...\\nEfter et hjørnespark passerer bolden t... Odense_Boldklub \n", + "998 Ud er gået Sebastian Wille, Omar Jebali, Law T... Odense_Boldklub \n", + "999 Anden halvleg er netop sparket i gang 🔵⚪\\n#obd... Odense_Boldklub \n", + "1000 Efter en scoring af Franco Tongya går vi nu ti... Odense_Boldklub \n", + "\n", + "[1001 rows x 4 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/code/sntwitter_scraper.py b/code/sntwitter_scraper.py new file mode 100644 index 0000000..efca04a --- /dev/null +++ b/code/sntwitter_scraper.py @@ -0,0 +1,18 @@ +import snscrape.modules.twitter as sntwitter +import pandas as pd + +def scrape(limit=0): + # prepare variables + tweet_list = list() + query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies' + # begin scraping + for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()): + # if limit is reached, break out of loop + if limit > 0 and i > limit: break + # if another 100 tweets found, tell it + if i % 100 == 0: + print(f'found {i} tweets') + tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username]) + # create dataframe + df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username']) + return df