added entrypoint for service
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
|
||||
from setup_logging import setup_logging
|
||||
from scraper import scrape
|
||||
from database import insert_list
|
||||
from dotenv import load_dotenv
|
||||
import logging
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
def update():
|
||||
topic = 'Odense'
|
||||
# scrape latest data
|
||||
username_list, timestamp_list, content_list = scrape(topic, limit=100, headless=False)
|
||||
# send data to database
|
||||
insert_list(username_list, timestamp_list, content_list)
|
||||
|
||||
if __name__ == '__main__':
|
||||
# setup
|
||||
load_dotenv()
|
||||
setup_logging()
|
||||
logging.info('finished setup')
|
||||
# main loop
|
||||
trigger_second = datetime.now().second
|
||||
while True:
|
||||
try:
|
||||
now = datetime.now()
|
||||
if now.second == trigger_second:
|
||||
try:
|
||||
update
|
||||
except Exception as e:
|
||||
logging.warning(e)
|
||||
else:
|
||||
logging.info('finished cycle')
|
||||
time.sleep(1)
|
||||
time.sleep(0.1)
|
||||
except Exception as e:
|
||||
logging.critical(e)
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
logging.info('KeyboardInterrupt')
|
||||
break
|
||||
Reference in New Issue
Block a user