From 80410e2b4ebfb9388785dfa21de5e2afbff373a8 Mon Sep 17 00:00:00 2001 From: simplypower-bbj Date: Sat, 17 Dec 2022 01:13:18 +0100 Subject: [PATCH] added entrypoint for service --- code/main.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 code/main.py diff --git a/code/main.py b/code/main.py new file mode 100644 index 0000000..98be484 --- /dev/null +++ b/code/main.py @@ -0,0 +1,41 @@ +#!/Users/brian/Projects/twitter_scraper/venv/bin/python +from setup_logging import setup_logging +from scraper import scrape +from database import insert_list +from dotenv import load_dotenv +import logging +from datetime import datetime +import time + +def update(): + topic = 'Odense' + # scrape latest data + username_list, timestamp_list, content_list = scrape(topic, limit=100, headless=False) + # send data to database + insert_list(username_list, timestamp_list, content_list) + +if __name__ == '__main__': + # setup + load_dotenv() + setup_logging() + logging.info('finished setup') + # main loop + trigger_second = datetime.now().second + while True: + try: + now = datetime.now() + if now.second == trigger_second: + try: + update + except Exception as e: + logging.warning(e) + else: + logging.info('finished cycle') + time.sleep(1) + time.sleep(0.1) + except Exception as e: + logging.critical(e) + break + except KeyboardInterrupt: + logging.info('KeyboardInterrupt') + break \ No newline at end of file