brian/twitter_scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed bug where keyboardinterrupt did not break infinite loop

Browse Source
This commit is contained in:
simplypower-bbj
2023-03-16 19:25:10 +01:00
parent 2b0030c0a8
commit b3d50ea563
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code/selenium_scraper.py
+3
View File
@@ -250,6 +250,8 @@ class Twitter(object):
content['video'] = self._extract_video_url(elem) content['video'] = self._extract_video_url(elem)
# get image(s) # get image(s)
content['images'] = self._extract_post_images(elem) content['images'] = self._extract_post_images(elem)
except KeyboardInterrupt:
return
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
else: else:
@@ -312,6 +314,7 @@ def scrape_account(account: str, until_year: int):
num_posts_found = get_period(account, start_date, end_date) num_posts_found = get_period(account, start_date, end_date)
except KeyboardInterrupt: except KeyboardInterrupt:
logging.warning('KeyboardInterrupt') logging.warning('KeyboardInterrupt')
return
except: except:
date_list.append((start_date, end_date)) date_list.append((start_date, end_date))
logging.warning(f'retrying later due to error getting period {start_date} to {end_date}') logging.warning(f'retrying later due to error getting period {start_date} to {end_date}')