From b3d50ea563e87e5808eab8b5bff68e6c1e517368 Mon Sep 17 00:00:00 2001 From: simplypower-bbj Date: Thu, 16 Mar 2023 19:25:10 +0100 Subject: [PATCH] fixed bug where keyboardinterrupt did not break infinite loop --- code/selenium_scraper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/code/selenium_scraper.py b/code/selenium_scraper.py index 6a187b4..5cac54b 100644 --- a/code/selenium_scraper.py +++ b/code/selenium_scraper.py @@ -250,6 +250,8 @@ class Twitter(object): content['video'] = self._extract_video_url(elem) # get image(s) content['images'] = self._extract_post_images(elem) + except KeyboardInterrupt: + return except Exception: traceback.print_exc() else: @@ -312,6 +314,7 @@ def scrape_account(account: str, until_year: int): num_posts_found = get_period(account, start_date, end_date) except KeyboardInterrupt: logging.warning('KeyboardInterrupt') + return except: date_list.append((start_date, end_date)) logging.warning(f'retrying later due to error getting period {start_date} to {end_date}')