added code and ran tests using sntwitter
This commit is contained in:
simplypower-bbj
@@ -0,0 +1,230 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"found 0 tweets\n",
|
||||
"found 100 tweets\n",
|
||||
"found 200 tweets\n",
|
||||
"found 300 tweets\n",
|
||||
"found 400 tweets\n",
|
||||
"found 500 tweets\n",
|
||||
"found 600 tweets\n",
|
||||
"found 700 tweets\n",
|
||||
"found 800 tweets\n",
|
||||
"found 900 tweets\n",
|
||||
"found 1000 tweets\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import snscrape.modules.twitter as sntwitter\n",
|
||||
"import pandas as pd\n",
|
||||
"tweet_list = list()\n",
|
||||
"query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
|
||||
"limit = 1000\n",
|
||||
"\n",
|
||||
"for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):\n",
|
||||
" if i % 100 == 0: print(f'found {i} tweets')\n",
|
||||
" if i > limit: break\n",
|
||||
" tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])\n",
|
||||
"\n",
|
||||
"df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>Datetime</th>\n",
|
||||
" <th>Id</th>\n",
|
||||
" <th>Text</th>\n",
|
||||
" <th>Username</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>2022-12-16 22:20:26+00:00</td>\n",
|
||||
" <td>1603877740972752897</td>\n",
|
||||
" <td>Danimarka’nın Odense takımı bu yaz müthiş bir ...</td>\n",
|
||||
" <td>EryilmazFa10</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>2022-12-16 21:56:35+00:00</td>\n",
|
||||
" <td>1603871736579371008</td>\n",
|
||||
" <td>Highlights: SønderjyskE – Odense https://t.co/...</td>\n",
|
||||
" <td>trans_rumor</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>2022-12-16 21:39:40+00:00</td>\n",
|
||||
" <td>1603867481101066241</td>\n",
|
||||
" <td>“Meta Platforms Inc has halted construction of...</td>\n",
|
||||
" <td>sakak</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>2022-12-16 21:19:09+00:00</td>\n",
|
||||
" <td>1603862318647070720</td>\n",
|
||||
" <td>Sponsoreret: Verdensmestre, europamestre og st...</td>\n",
|
||||
" <td>otwndk</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>2022-12-16 21:07:09+00:00</td>\n",
|
||||
" <td>1603859298249019392</td>\n",
|
||||
" <td>Meta just halted a major data centre project t...</td>\n",
|
||||
" <td>techosmo</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>...</th>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>996</th>\n",
|
||||
" <td>2022-11-23 15:42:37+00:00</td>\n",
|
||||
" <td>1595442703990493211</td>\n",
|
||||
" <td>Mohamed Djan Fofana går fra banen, og han erst...</td>\n",
|
||||
" <td>Odense_Boldklub</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>997</th>\n",
|
||||
" <td>2022-11-23 15:34:36+00:00</td>\n",
|
||||
" <td>1595440689629859840</td>\n",
|
||||
" <td>1-1...\\nEfter et hjørnespark passerer bolden t...</td>\n",
|
||||
" <td>Odense_Boldklub</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>998</th>\n",
|
||||
" <td>2022-11-23 15:31:12+00:00</td>\n",
|
||||
" <td>1595439830187798529</td>\n",
|
||||
" <td>Ud er gået Sebastian Wille, Omar Jebali, Law T...</td>\n",
|
||||
" <td>Odense_Boldklub</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>999</th>\n",
|
||||
" <td>2022-11-23 15:21:18+00:00</td>\n",
|
||||
" <td>1595437338997542912</td>\n",
|
||||
" <td>Anden halvleg er netop sparket i gang 🔵⚪\\n#obd...</td>\n",
|
||||
" <td>Odense_Boldklub</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1000</th>\n",
|
||||
" <td>2022-11-23 15:05:57+00:00</td>\n",
|
||||
" <td>1595433476135829506</td>\n",
|
||||
" <td>Efter en scoring af Franco Tongya går vi nu ti...</td>\n",
|
||||
" <td>Odense_Boldklub</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"<p>1001 rows × 4 columns</p>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Datetime Id \\\n",
|
||||
"0 2022-12-16 22:20:26+00:00 1603877740972752897 \n",
|
||||
"1 2022-12-16 21:56:35+00:00 1603871736579371008 \n",
|
||||
"2 2022-12-16 21:39:40+00:00 1603867481101066241 \n",
|
||||
"3 2022-12-16 21:19:09+00:00 1603862318647070720 \n",
|
||||
"4 2022-12-16 21:07:09+00:00 1603859298249019392 \n",
|
||||
"... ... ... \n",
|
||||
"996 2022-11-23 15:42:37+00:00 1595442703990493211 \n",
|
||||
"997 2022-11-23 15:34:36+00:00 1595440689629859840 \n",
|
||||
"998 2022-11-23 15:31:12+00:00 1595439830187798529 \n",
|
||||
"999 2022-11-23 15:21:18+00:00 1595437338997542912 \n",
|
||||
"1000 2022-11-23 15:05:57+00:00 1595433476135829506 \n",
|
||||
"\n",
|
||||
" Text Username \n",
|
||||
"0 Danimarka’nın Odense takımı bu yaz müthiş bir ... EryilmazFa10 \n",
|
||||
"1 Highlights: SønderjyskE – Odense https://t.co/... trans_rumor \n",
|
||||
"2 “Meta Platforms Inc has halted construction of... sakak \n",
|
||||
"3 Sponsoreret: Verdensmestre, europamestre og st... otwndk \n",
|
||||
"4 Meta just halted a major data centre project t... techosmo \n",
|
||||
"... ... ... \n",
|
||||
"996 Mohamed Djan Fofana går fra banen, og han erst... Odense_Boldklub \n",
|
||||
"997 1-1...\\nEfter et hjørnespark passerer bolden t... Odense_Boldklub \n",
|
||||
"998 Ud er gået Sebastian Wille, Omar Jebali, Law T... Odense_Boldklub \n",
|
||||
"999 Anden halvleg er netop sparket i gang 🔵⚪\\n#obd... Odense_Boldklub \n",
|
||||
"1000 Efter en scoring af Franco Tongya går vi nu ti... Odense_Boldklub \n",
|
||||
"\n",
|
||||
"[1001 rows x 4 columns]"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"df"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.6"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import snscrape.modules.twitter as sntwitter
|
||||
import pandas as pd
|
||||
|
||||
def scrape(limit=0):
|
||||
# prepare variables
|
||||
tweet_list = list()
|
||||
query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
|
||||
# begin scraping
|
||||
for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
|
||||
# if limit is reached, break out of loop
|
||||
if limit > 0 and i > limit: break
|
||||
# if another 100 tweets found, tell it
|
||||
if i % 100 == 0:
|
||||
print(f'found {i} tweets')
|
||||
tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])
|
||||
# create dataframe
|
||||
df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])
|
||||
return df
|
||||
Reference in New Issue
Block a user