41 lines
1.2 KiB
Python
41 lines
1.2 KiB
Python
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
|
|
from setup_logging import setup_logging
|
|
from scraper import scrape
|
|
from database import insert_list
|
|
from dotenv import load_dotenv
|
|
import logging
|
|
from datetime import datetime
|
|
import time
|
|
|
|
def update():
|
|
topic = 'Odense'
|
|
# scrape latest data
|
|
username_list, timestamp_list, content_list = scrape(topic, limit=100, headless=False)
|
|
# send data to database
|
|
insert_list(username_list, timestamp_list, content_list)
|
|
|
|
if __name__ == '__main__':
|
|
# setup
|
|
load_dotenv()
|
|
setup_logging()
|
|
logging.info('finished setup')
|
|
# main loop
|
|
trigger_second = datetime.now().second
|
|
while True:
|
|
try:
|
|
now = datetime.now()
|
|
if now.second == trigger_second:
|
|
try:
|
|
update
|
|
except Exception as e:
|
|
logging.warning(e)
|
|
else:
|
|
logging.info('finished cycle')
|
|
time.sleep(1)
|
|
time.sleep(0.1)
|
|
except Exception as e:
|
|
logging.critical(e)
|
|
break
|
|
except KeyboardInterrupt:
|
|
logging.info('KeyboardInterrupt')
|
|
break |