updated code and ran tests

code/sntwitter_scraper.ipynb

@@ -26,15 +26,21 @@
    "source": [
     "import snscrape.modules.twitter as sntwitter\n",
     "import pandas as pd\n",
-    "tweet_list = list()\n",
+    "\n",
+    "# prepare variables\n",
     "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
     "limit = 1000\n",
-    "\n",
+    "tweet_list = list()\n",
+    "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
+    "# begin scraping\n",
     "for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):\n",
-    "    if i % 100 == 0: print(f'found {i} tweets')\n",
-    "    if i > limit: break\n",
+    "    # if limit is reached, break out of loop\n",
+    "    if limit > 0 and i > limit: break\n",
+    "    # if another 100 tweets found, tell it\n",
+    "    if i % 100 == 0: \n",
+    "        print(f'found {i} tweets')\n",
     "    tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])\n",
-    "\n",
+    "# create dataframe\n",
     "df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])"
    ]
   },
@@ -216,7 +222,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.6"
+   "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
   },
   "orig_nbformat": 4,
   "vscode": {

code/sntwitter_scraper.py

@@ -1,18 +1,34 @@
+# public packages
 import snscrape.modules.twitter as sntwitter
 import pandas as pd
+# database-related
+from database import insert_list
+from dotenv import load_dotenv
 
-def scrape(limit=0):
-    # prepare variables
-    tweet_list = list()
-    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
+def scrape():
+    # prepare lists to hold information
+    username_list = list()
+    timestamp_list = list()
+    content_list = list()
     # begin scraping
+    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
     for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
-        # if limit is reached, break out of loop
-        if limit > 0 and i > limit: break
-        # if another 100 tweets found, tell it
-        if i % 100 == 0: 
+        # store information in lists
+        username_list.append(tweet.user.username)
+        timestamp_list.append(tweet.date)
+        content_list.append(tweet.content)
+        # when 100 tweets have been collected
+        if len(content_list) == 100:
+            # send data to database
+            insert_list(username_list, timestamp_list, content_list)
+            # empty lists
+            username_list = list()
+            timestamp_list = list()
+            content_list = list()
+        # check in on progress
+        if i % 100 == 0:
             print(f'found {i} tweets')
-        tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])
-    # create dataframe
-    df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])
-    return df
+
+if __name__ == '__main__':
+    load_dotenv()
+    scrape()