added entrypoint for service
This commit is contained in:
@@ -0,0 +1,41 @@
|
|||||||
|
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
|
||||||
|
from setup_logging import setup_logging
|
||||||
|
from scraper import scrape
|
||||||
|
from database import insert_list
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
|
def update():
|
||||||
|
topic = 'Odense'
|
||||||
|
# scrape latest data
|
||||||
|
username_list, timestamp_list, content_list = scrape(topic, limit=100, headless=False)
|
||||||
|
# send data to database
|
||||||
|
insert_list(username_list, timestamp_list, content_list)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
# setup
|
||||||
|
load_dotenv()
|
||||||
|
setup_logging()
|
||||||
|
logging.info('finished setup')
|
||||||
|
# main loop
|
||||||
|
trigger_second = datetime.now().second
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
now = datetime.now()
|
||||||
|
if now.second == trigger_second:
|
||||||
|
try:
|
||||||
|
update
|
||||||
|
except Exception as e:
|
||||||
|
logging.warning(e)
|
||||||
|
else:
|
||||||
|
logging.info('finished cycle')
|
||||||
|
time.sleep(1)
|
||||||
|
time.sleep(0.1)
|
||||||
|
except Exception as e:
|
||||||
|
logging.critical(e)
|
||||||
|
break
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
logging.info('KeyboardInterrupt')
|
||||||
|
break
|
||||||
Reference in New Issue
Block a user