commented code

code/main.py

@@ -1,73 +1,36 @@
-#!/Users/brian/Projects/twitter_scraper/venv/bin/python
+# public packages
-from setup_logging import setup_logging
-from selenium_scraper import scrape
-from database import insert_list
-from dotenv import load_dotenv
-import logging
-from datetime import datetime
-import time
-import os
-from tqdm import tqdm
-
 import snscrape.modules.twitter as sntwitter
-import pandas as pd
+from tqdm import tqdm
-
+import time
-from database import get_all
+# database-related
+from database import Data_table, insert_list, get_all
+from dotenv import load_dotenv
+# own code
 from clean_data import clean_content_df
-from translate import translate, get_untranslated
+from translate import translate
-from database import Data_table
-
-# def update():
-#     topic = 'Odense'
-#     # scrape latest data
-#     username_list, timestamp_list, content_list = scrape(topic, limit=10, headless=False)
-#     # send data to database
-#     insert_list(username_list, timestamp_list, content_list)
-
-# if __name__ == '__main__':
-#     # setup
-#     load_dotenv()
-#     setup_logging()
-#     logging.info('finished setup')
-#     # main loop
-#     trigger_minute = datetime.now().minute
-#     while True:
-#         try:
-#             now = datetime.now()
-#             if now.minute == trigger_minute:
-#                 try:
-#                     update()
-#                 except Exception as e:
-#                     logging.warning(e)
-#                 else:
-#                     logging.info('finished cycle')
-#                 time.sleep(60)
-#             time.sleep(0.1)
-#         except Exception as e:
-#             logging.critical(e)
-#             break
-#         except KeyboardInterrupt:
-#             logging.info('KeyboardInterrupt')
-#             break
 
 def run_snscrape():
-    # prepare variables
+    # prepare lists to hold information
     username_list = list()
     timestamp_list = list()
     content_list = list()
     # begin scraping
     query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
     for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
+        # store information in lists
         username_list.append(tweet.user.username)
         timestamp_list.append(tweet.date)
         content_list.append(tweet.content)
+        # when 100 tweets have been collected
         if len(content_list) == 100:
-            # send data to datbase
+            # send data to database
             insert_list(username_list, timestamp_list, content_list)
+            # empty lists
             username_list = list()
             timestamp_list = list()
             content_list = list()
-        if i % 100 == 0: 
+        # check in on progress
+        if i % 100 == 0:
             print(f'found {i} tweets')
 
 def translate_content():
@@ -86,7 +49,7 @@ def translate_content():
             # translate content
             try:
                 content_trans = translate(content)
-            except Exception as e:
+            except Exception:
                 print(f'error when translating content with id={id}')
                 continue
             # transfer to database
@@ -96,13 +59,14 @@ def translate_content():
                     'content_en': content_trans
                 }
                 query = (
-                    f'UPDATE {dt._name} ' 
+                    f'UPDATE {dt._name} '
                     ' SET content_en = (%(content_en)s) '
                     'WHERE id = (%(id)s) '
                 )
                 dt.execute(query, val_dict)
-            except Exception as e:
+            except Exception:
                 print(f'error when sending translated content to database: id={id}')
+            # take a break to avoid being banned by Google
             time.sleep(2)