updated to run snscraper

code/main.py

@@ -8,35 +8,62 @@ from datetime import datetime
 import time
 import os
 
-def update():
-    topic = 'Odense'
-    # scrape latest data
-    username_list, timestamp_list, content_list = scrape(topic, limit=10, headless=False)
-    # send data to database
-    insert_list(username_list, timestamp_list, content_list)
+import snscrape.modules.twitter as sntwitter
+import pandas as pd
+
+# def update():
+#     topic = 'Odense'
+#     # scrape latest data
+#     username_list, timestamp_list, content_list = scrape(topic, limit=10, headless=False)
+#     # send data to database
+#     insert_list(username_list, timestamp_list, content_list)
+
+# if __name__ == '__main__':
+#     # setup
+#     load_dotenv()
+#     setup_logging()
+#     logging.info('finished setup')
+#     # main loop
+#     trigger_minute = datetime.now().minute
+#     while True:
+#         try:
+#             now = datetime.now()
+#             if now.minute == trigger_minute:
+#                 try:
+#                     update()
+#                 except Exception as e:
+#                     logging.warning(e)
+#                 else:
+#                     logging.info('finished cycle')
+#                 time.sleep(60)
+#             time.sleep(0.1)
+#         except Exception as e:
+#             logging.critical(e)
+#             break
+#         except KeyboardInterrupt:
+#             logging.info('KeyboardInterrupt')
+#             break
+
+def run_snscrape():
+    # prepare variables
+    username_list = list()
+    timestamp_list = list()
+    content_list = list()
+    # begin scraping
+    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
+    for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
+        username_list.append(tweet.user.username)
+        timestamp_list.append(tweet.date)
+        content_list.append(tweet.content)
+        if len(content_list) == 100:
+            # send data to datbase
+            insert_list(username_list, timestamp_list, content_list)
+            username_list = list()
+            timestamp_list = list()
+            content_list = list()
+        if i % 100 == 0: 
+            print(f'found {i} tweets')
 
 if __name__ == '__main__':
-    # setup
     load_dotenv()
-    setup_logging()
-    logging.info('finished setup')
-    # main loop
-    trigger_minute = datetime.now().minute
-    while True:
-        try:
-            now = datetime.now()
-            if now.minute == trigger_minute:
-                try:
-                    update()
-                except Exception as e:
-                    logging.warning(e)
-                else:
-                    logging.info('finished cycle')
-                time.sleep(60)
-            time.sleep(0.1)
-        except Exception as e:
-            logging.critical(e)
-            break
-        except KeyboardInterrupt:
-            logging.info('KeyboardInterrupt')
-            break
+    run_snscrape()