diff --git a/code/sntwitter_scraper.ipynb b/code/sntwitter_scraper.ipynb index d026aa5..ae449c2 100644 --- a/code/sntwitter_scraper.ipynb +++ b/code/sntwitter_scraper.ipynb @@ -26,15 +26,21 @@ "source": [ "import snscrape.modules.twitter as sntwitter\n", "import pandas as pd\n", - "tweet_list = list()\n", + "\n", + "# prepare variables\n", "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n", "limit = 1000\n", - "\n", + "tweet_list = list()\n", + "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n", + "# begin scraping\n", "for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):\n", - " if i % 100 == 0: print(f'found {i} tweets')\n", - " if i > limit: break\n", + " # if limit is reached, break out of loop\n", + " if limit > 0 and i > limit: break\n", + " # if another 100 tweets found, tell it\n", + " if i % 100 == 0: \n", + " print(f'found {i} tweets')\n", " tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])\n", - "\n", + "# create dataframe\n", "df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])" ] }, @@ -216,7 +222,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.6" + "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]" }, "orig_nbformat": 4, "vscode": { diff --git a/code/sntwitter_scraper.py b/code/sntwitter_scraper.py index efca04a..7269e2e 100644 --- a/code/sntwitter_scraper.py +++ b/code/sntwitter_scraper.py @@ -1,18 +1,34 @@ +# public packages import snscrape.modules.twitter as sntwitter import pandas as pd +# database-related +from database import insert_list +from dotenv import load_dotenv -def scrape(limit=0): - # prepare variables - tweet_list = list() - query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies' +def scrape(): + # prepare lists to hold information + username_list = list() + timestamp_list = list() + content_list = list() # begin scraping + query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies' for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()): - # if limit is reached, break out of loop - if limit > 0 and i > limit: break - # if another 100 tweets found, tell it - if i % 100 == 0: + # store information in lists + username_list.append(tweet.user.username) + timestamp_list.append(tweet.date) + content_list.append(tweet.content) + # when 100 tweets have been collected + if len(content_list) == 100: + # send data to database + insert_list(username_list, timestamp_list, content_list) + # empty lists + username_list = list() + timestamp_list = list() + content_list = list() + # check in on progress + if i % 100 == 0: print(f'found {i} tweets') - tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username]) - # create dataframe - df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username']) - return df + +if __name__ == '__main__': + load_dotenv() + scrape() \ No newline at end of file