updated code and ran tests

code/sntwitter_scraper.py

@@ -1,18 +1,34 @@
+# public packages
 import snscrape.modules.twitter as sntwitter
 import pandas as pd
+# database-related
+from database import insert_list
+from dotenv import load_dotenv
 
-def scrape(limit=0):
-    # prepare variables
-    tweet_list = list()
-    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
+def scrape():
+    # prepare lists to hold information
+    username_list = list()
+    timestamp_list = list()
+    content_list = list()
     # begin scraping
+    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
     for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
-        # if limit is reached, break out of loop
-        if limit > 0 and i > limit: break
-        # if another 100 tweets found, tell it
-        if i % 100 == 0: 
+        # store information in lists
+        username_list.append(tweet.user.username)
+        timestamp_list.append(tweet.date)
+        content_list.append(tweet.content)
+        # when 100 tweets have been collected
+        if len(content_list) == 100:
+            # send data to database
+            insert_list(username_list, timestamp_list, content_list)
+            # empty lists
+            username_list = list()
+            timestamp_list = list()
+            content_list = list()
+        # check in on progress
+        if i % 100 == 0:
             print(f'found {i} tweets')
-        tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])
-    # create dataframe
-    df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])
-    return df
+
+if __name__ == '__main__':
+    load_dotenv()
+    scrape()