22 KiB
22 KiB
In [3]:
from selenium.webdriver.common.by import By
# import requests
# import pyperclip
from selenium_scraper import Twitter
from datetime import timedelta, datetime
from classes import Tweet
import traceback
import logging
from database import connect_mongoIn [7]:
# reset database
# db = connect_mongo()
# db.drop()
# db.count_documents({'account':'@Maersk'})Out [7]:
0
In [ ]:
# prepare date lists to redo on error
end_date = datetime.utcnow()
start_date = end_date - timedelta(days=10)
end_date_list = [end_date.strftime('%Y-%m-%d')]
start_date_list = [start_date.strftime('%Y-%m-%d')]
while start_date.year > 2010:
end_date = start_date - timedelta(days=1)
end_date_list.append(end_date.strftime('%Y-%m-%d'))
start_date = end_date - timedelta(days=10)
start_date_list.append(start_date.strftime('%Y-%m-%d'))
date_list = list(zip(start_date_list, end_date_list))
date_list[('2023-03-06', '2023-03-16'),
('2023-02-23', '2023-03-05'),
('2023-02-12', '2023-02-22'),
('2023-02-01', '2023-02-11'),
('2023-01-21', '2023-01-31'),
('2023-01-10', '2023-01-20'),
('2022-12-30', '2023-01-09'),
('2022-12-19', '2022-12-29'),
('2022-12-08', '2022-12-18'),
('2022-11-27', '2022-12-07'),
('2022-11-16', '2022-11-26'),
('2022-11-05', '2022-11-15'),
('2022-10-25', '2022-11-04'),
('2022-10-14', '2022-10-24'),
('2022-10-03', '2022-10-13'),
('2022-09-22', '2022-10-02'),
('2022-09-11', '2022-09-21'),
('2022-08-31', '2022-09-10'),
('2022-08-20', '2022-08-30'),
('2022-08-09', '2022-08-19'),
('2022-07-29', '2022-08-08'),
('2022-07-18', '2022-07-28'),
('2022-07-07', '2022-07-17'),
('2022-06-26', '2022-07-06'),
('2022-06-15', '2022-06-25'),
('2022-06-04', '2022-06-14'),
('2022-05-24', '2022-06-03'),
('2022-05-13', '2022-05-23'),
('2022-05-02', '2022-05-12'),
('2022-04-21', '2022-05-01'),
('2022-04-10', '2022-04-20'),
('2022-03-30', '2022-04-09'),
('2022-03-19', '2022-03-29'),
('2022-03-08', '2022-03-18'),
('2022-02-25', '2022-03-07'),
('2022-02-14', '2022-02-24'),
('2022-02-03', '2022-02-13'),
('2022-01-23', '2022-02-02'),
('2022-01-12', '2022-01-22'),
('2022-01-01', '2022-01-11'),
('2021-12-21', '2021-12-31'),
('2021-12-10', '2021-12-20'),
('2021-11-29', '2021-12-09'),
('2021-11-18', '2021-11-28'),
('2021-11-07', '2021-11-17'),
('2021-10-27', '2021-11-06'),
('2021-10-16', '2021-10-26'),
('2021-10-05', '2021-10-15'),
('2021-09-24', '2021-10-04'),
('2021-09-13', '2021-09-23'),
('2021-09-02', '2021-09-12'),
('2021-08-22', '2021-09-01'),
('2021-08-11', '2021-08-21'),
('2021-07-31', '2021-08-10'),
('2021-07-20', '2021-07-30'),
('2021-07-09', '2021-07-19'),
('2021-06-28', '2021-07-08'),
('2021-06-17', '2021-06-27'),
('2021-06-06', '2021-06-16'),
('2021-05-26', '2021-06-05'),
('2021-05-15', '2021-05-25'),
('2021-05-04', '2021-05-14'),
('2021-04-23', '2021-05-03'),
('2021-04-12', '2021-04-22'),
('2021-04-01', '2021-04-11'),
('2021-03-21', '2021-03-31'),
('2021-03-10', '2021-03-20'),
('2021-02-27', '2021-03-09'),
('2021-02-16', '2021-02-26'),
('2021-02-05', '2021-02-15'),
('2021-01-25', '2021-02-04'),
('2021-01-14', '2021-01-24'),
('2021-01-03', '2021-01-13'),
('2020-12-23', '2021-01-02'),
('2020-12-12', '2020-12-22'),
('2020-12-01', '2020-12-11'),
('2020-11-20', '2020-11-30'),
('2020-11-09', '2020-11-19'),
('2020-10-29', '2020-11-08'),
('2020-10-18', '2020-10-28'),
('2020-10-07', '2020-10-17'),
('2020-09-26', '2020-10-06'),
('2020-09-15', '2020-09-25'),
('2020-09-04', '2020-09-14'),
('2020-08-24', '2020-09-03'),
('2020-08-13', '2020-08-23'),
('2020-08-02', '2020-08-12'),
('2020-07-22', '2020-08-01'),
('2020-07-11', '2020-07-21'),
('2020-06-30', '2020-07-10'),
('2020-06-19', '2020-06-29'),
('2020-06-08', '2020-06-18'),
('2020-05-28', '2020-06-07'),
('2020-05-17', '2020-05-27'),
('2020-05-06', '2020-05-16'),
('2020-04-25', '2020-05-05'),
('2020-04-14', '2020-04-24'),
('2020-04-03', '2020-04-13'),
('2020-03-23', '2020-04-02'),
('2020-03-12', '2020-03-22'),
('2020-03-01', '2020-03-11'),
('2020-02-19', '2020-02-29'),
('2020-02-08', '2020-02-18'),
('2020-01-28', '2020-02-07'),
('2020-01-17', '2020-01-27'),
('2020-01-06', '2020-01-16'),
('2019-12-26', '2020-01-05'),
('2019-12-15', '2019-12-25'),
('2019-12-04', '2019-12-14'),
('2019-11-23', '2019-12-03'),
('2019-11-12', '2019-11-22'),
('2019-11-01', '2019-11-11'),
('2019-10-21', '2019-10-31'),
('2019-10-10', '2019-10-20'),
('2019-09-29', '2019-10-09'),
('2019-09-18', '2019-09-28'),
('2019-09-07', '2019-09-17'),
('2019-08-27', '2019-09-06'),
('2019-08-16', '2019-08-26'),
('2019-08-05', '2019-08-15'),
('2019-07-25', '2019-08-04'),
('2019-07-14', '2019-07-24'),
('2019-07-03', '2019-07-13'),
('2019-06-22', '2019-07-02'),
('2019-06-11', '2019-06-21'),
('2019-05-31', '2019-06-10'),
('2019-05-20', '2019-05-30'),
('2019-05-09', '2019-05-19'),
('2019-04-28', '2019-05-08'),
('2019-04-17', '2019-04-27'),
('2019-04-06', '2019-04-16'),
('2019-03-26', '2019-04-05'),
('2019-03-15', '2019-03-25'),
('2019-03-04', '2019-03-14'),
('2019-02-21', '2019-03-03'),
('2019-02-10', '2019-02-20'),
('2019-01-30', '2019-02-09'),
('2019-01-19', '2019-01-29'),
('2019-01-08', '2019-01-18'),
('2018-12-28', '2019-01-07'),
('2018-12-17', '2018-12-27'),
('2018-12-06', '2018-12-16'),
('2018-11-25', '2018-12-05'),
('2018-11-14', '2018-11-24'),
('2018-11-03', '2018-11-13'),
('2018-10-23', '2018-11-02'),
('2018-10-12', '2018-10-22'),
('2018-10-01', '2018-10-11'),
('2018-09-20', '2018-09-30'),
('2018-09-09', '2018-09-19'),
('2018-08-29', '2018-09-08'),
('2018-08-18', '2018-08-28'),
('2018-08-07', '2018-08-17'),
('2018-07-27', '2018-08-06'),
('2018-07-16', '2018-07-26'),
('2018-07-05', '2018-07-15'),
('2018-06-24', '2018-07-04'),
('2018-06-13', '2018-06-23'),
('2018-06-02', '2018-06-12'),
('2018-05-22', '2018-06-01'),
('2018-05-11', '2018-05-21'),
('2018-04-30', '2018-05-10'),
('2018-04-19', '2018-04-29'),
('2018-04-08', '2018-04-18'),
('2018-03-28', '2018-04-07'),
('2018-03-17', '2018-03-27'),
('2018-03-06', '2018-03-16'),
('2018-02-23', '2018-03-05'),
('2018-02-12', '2018-02-22'),
('2018-02-01', '2018-02-11'),
('2018-01-21', '2018-01-31'),
('2018-01-10', '2018-01-20'),
('2017-12-30', '2018-01-09'),
('2017-12-19', '2017-12-29'),
('2017-12-08', '2017-12-18'),
('2017-11-27', '2017-12-07'),
('2017-11-16', '2017-11-26'),
('2017-11-05', '2017-11-15'),
('2017-10-25', '2017-11-04'),
('2017-10-14', '2017-10-24'),
('2017-10-03', '2017-10-13'),
('2017-09-22', '2017-10-02'),
('2017-09-11', '2017-09-21'),
('2017-08-31', '2017-09-10'),
('2017-08-20', '2017-08-30'),
('2017-08-09', '2017-08-19'),
('2017-07-29', '2017-08-08'),
('2017-07-18', '2017-07-28'),
('2017-07-07', '2017-07-17'),
('2017-06-26', '2017-07-06'),
('2017-06-15', '2017-06-25'),
('2017-06-04', '2017-06-14'),
('2017-05-24', '2017-06-03'),
('2017-05-13', '2017-05-23'),
('2017-05-02', '2017-05-12'),
('2017-04-21', '2017-05-01'),
('2017-04-10', '2017-04-20'),
('2017-03-30', '2017-04-09'),
('2017-03-19', '2017-03-29'),
('2017-03-08', '2017-03-18'),
('2017-02-25', '2017-03-07'),
('2017-02-14', '2017-02-24'),
('2017-02-03', '2017-02-13'),
('2017-01-23', '2017-02-02'),
('2017-01-12', '2017-01-22'),
('2017-01-01', '2017-01-11'),
('2016-12-21', '2016-12-31'),
('2016-12-10', '2016-12-20'),
('2016-11-29', '2016-12-09'),
('2016-11-18', '2016-11-28'),
('2016-11-07', '2016-11-17'),
('2016-10-27', '2016-11-06'),
('2016-10-16', '2016-10-26'),
('2016-10-05', '2016-10-15'),
('2016-09-24', '2016-10-04'),
('2016-09-13', '2016-09-23'),
('2016-09-02', '2016-09-12'),
('2016-08-22', '2016-09-01'),
('2016-08-11', '2016-08-21'),
('2016-07-31', '2016-08-10'),
('2016-07-20', '2016-07-30'),
('2016-07-09', '2016-07-19'),
('2016-06-28', '2016-07-08'),
('2016-06-17', '2016-06-27'),
('2016-06-06', '2016-06-16'),
('2016-05-26', '2016-06-05'),
('2016-05-15', '2016-05-25'),
('2016-05-04', '2016-05-14'),
('2016-04-23', '2016-05-03'),
('2016-04-12', '2016-04-22'),
('2016-04-01', '2016-04-11'),
('2016-03-21', '2016-03-31'),
('2016-03-10', '2016-03-20'),
('2016-02-28', '2016-03-09'),
('2016-02-17', '2016-02-27'),
('2016-02-06', '2016-02-16'),
('2016-01-26', '2016-02-05'),
('2016-01-15', '2016-01-25'),
('2016-01-04', '2016-01-14'),
('2015-12-24', '2016-01-03'),
('2015-12-13', '2015-12-23'),
('2015-12-02', '2015-12-12'),
('2015-11-21', '2015-12-01'),
('2015-11-10', '2015-11-20'),
('2015-10-30', '2015-11-09'),
('2015-10-19', '2015-10-29'),
('2015-10-08', '2015-10-18'),
('2015-09-27', '2015-10-07'),
('2015-09-16', '2015-09-26'),
('2015-09-05', '2015-09-15'),
('2015-08-25', '2015-09-04'),
('2015-08-14', '2015-08-24'),
('2015-08-03', '2015-08-13'),
('2015-07-23', '2015-08-02'),
('2015-07-12', '2015-07-22'),
('2015-07-01', '2015-07-11'),
('2015-06-20', '2015-06-30'),
('2015-06-09', '2015-06-19'),
('2015-05-29', '2015-06-08'),
('2015-05-18', '2015-05-28'),
('2015-05-07', '2015-05-17'),
('2015-04-26', '2015-05-06'),
('2015-04-15', '2015-04-25'),
('2015-04-04', '2015-04-14'),
('2015-03-24', '2015-04-03'),
('2015-03-13', '2015-03-23'),
('2015-03-02', '2015-03-12'),
('2015-02-19', '2015-03-01'),
('2015-02-08', '2015-02-18'),
('2015-01-28', '2015-02-07'),
('2015-01-17', '2015-01-27'),
('2015-01-06', '2015-01-16'),
('2014-12-26', '2015-01-05'),
('2014-12-15', '2014-12-25'),
('2014-12-04', '2014-12-14'),
('2014-11-23', '2014-12-03'),
('2014-11-12', '2014-11-22'),
('2014-11-01', '2014-11-11'),
('2014-10-21', '2014-10-31'),
('2014-10-10', '2014-10-20'),
('2014-09-29', '2014-10-09'),
('2014-09-18', '2014-09-28'),
('2014-09-07', '2014-09-17'),
('2014-08-27', '2014-09-06'),
('2014-08-16', '2014-08-26'),
('2014-08-05', '2014-08-15'),
('2014-07-25', '2014-08-04'),
('2014-07-14', '2014-07-24'),
('2014-07-03', '2014-07-13'),
('2014-06-22', '2014-07-02'),
('2014-06-11', '2014-06-21'),
('2014-05-31', '2014-06-10'),
('2014-05-20', '2014-05-30'),
('2014-05-09', '2014-05-19'),
('2014-04-28', '2014-05-08'),
('2014-04-17', '2014-04-27'),
('2014-04-06', '2014-04-16'),
('2014-03-26', '2014-04-05'),
('2014-03-15', '2014-03-25'),
('2014-03-04', '2014-03-14'),
('2014-02-21', '2014-03-03'),
('2014-02-10', '2014-02-20'),
('2014-01-30', '2014-02-09'),
('2014-01-19', '2014-01-29'),
('2014-01-08', '2014-01-18'),
('2013-12-28', '2014-01-07'),
('2013-12-17', '2013-12-27'),
('2013-12-06', '2013-12-16'),
('2013-11-25', '2013-12-05'),
('2013-11-14', '2013-11-24'),
('2013-11-03', '2013-11-13'),
('2013-10-23', '2013-11-02'),
('2013-10-12', '2013-10-22'),
('2013-10-01', '2013-10-11'),
('2013-09-20', '2013-09-30'),
('2013-09-09', '2013-09-19'),
('2013-08-29', '2013-09-08'),
('2013-08-18', '2013-08-28'),
('2013-08-07', '2013-08-17'),
('2013-07-27', '2013-08-06'),
('2013-07-16', '2013-07-26'),
('2013-07-05', '2013-07-15'),
('2013-06-24', '2013-07-04'),
('2013-06-13', '2013-06-23'),
('2013-06-02', '2013-06-12'),
('2013-05-22', '2013-06-01'),
('2013-05-11', '2013-05-21'),
('2013-04-30', '2013-05-10'),
('2013-04-19', '2013-04-29'),
('2013-04-08', '2013-04-18'),
('2013-03-28', '2013-04-07'),
('2013-03-17', '2013-03-27'),
('2013-03-06', '2013-03-16'),
('2013-02-23', '2013-03-05'),
('2013-02-12', '2013-02-22'),
('2013-02-01', '2013-02-11'),
('2013-01-21', '2013-01-31'),
('2013-01-10', '2013-01-20'),
('2012-12-30', '2013-01-09'),
('2012-12-19', '2012-12-29'),
('2012-12-08', '2012-12-18'),
('2012-11-27', '2012-12-07'),
('2012-11-16', '2012-11-26'),
('2012-11-05', '2012-11-15'),
('2012-10-25', '2012-11-04'),
('2012-10-14', '2012-10-24'),
('2012-10-03', '2012-10-13'),
('2012-09-22', '2012-10-02'),
('2012-09-11', '2012-09-21'),
('2012-08-31', '2012-09-10'),
('2012-08-20', '2012-08-30'),
('2012-08-09', '2012-08-19'),
('2012-07-29', '2012-08-08'),
('2012-07-18', '2012-07-28'),
('2012-07-07', '2012-07-17'),
('2012-06-26', '2012-07-06'),
('2012-06-15', '2012-06-25'),
('2012-06-04', '2012-06-14'),
('2012-05-24', '2012-06-03'),
('2012-05-13', '2012-05-23'),
('2012-05-02', '2012-05-12'),
('2012-04-21', '2012-05-01'),
('2012-04-10', '2012-04-20'),
('2012-03-30', '2012-04-09'),
('2012-03-19', '2012-03-29'),
('2012-03-08', '2012-03-18'),
('2012-02-26', '2012-03-07'),
('2012-02-15', '2012-02-25'),
('2012-02-04', '2012-02-14'),
('2012-01-24', '2012-02-03'),
('2012-01-13', '2012-01-23'),
('2012-01-02', '2012-01-12'),
('2011-12-22', '2012-01-01'),
('2011-12-11', '2011-12-21'),
('2011-11-30', '2011-12-10'),
('2011-11-19', '2011-11-29'),
('2011-11-08', '2011-11-18'),
('2011-10-28', '2011-11-07'),
('2011-10-17', '2011-10-27'),
('2011-10-06', '2011-10-16'),
('2011-09-25', '2011-10-05'),
('2011-09-14', '2011-09-24'),
('2011-09-03', '2011-09-13'),
('2011-08-23', '2011-09-02'),
('2011-08-12', '2011-08-22'),
('2011-08-01', '2011-08-11'),
('2011-07-21', '2011-07-31'),
('2011-07-10', '2011-07-20'),
('2011-06-29', '2011-07-09'),
('2011-06-18', '2011-06-28'),
('2011-06-07', '2011-06-17'),
('2011-05-27', '2011-06-06'),
('2011-05-16', '2011-05-26'),
('2011-05-05', '2011-05-15'),
('2011-04-24', '2011-05-04'),
('2011-04-13', '2011-04-23'),
('2011-04-02', '2011-04-12'),
('2011-03-22', '2011-04-01'),
('2011-03-11', '2011-03-21'),
('2011-02-28', '2011-03-10'),
('2011-02-17', '2011-02-27'),
('2011-02-06', '2011-02-16'),
('2011-01-26', '2011-02-05'),
('2011-01-15', '2011-01-25'),
('2011-01-04', '2011-01-14'),
('2010-12-24', '2011-01-03')]In [2]:
def get_period(start_date, end_date):
# setup
account = 'Maersk'
try:
twitter = Twitter()
twitter.setup()
twitter._open_page(account, start_date, end_date)
db = connect_mongo()
# scrape
limit = 500
attempts = 0
while attempts < limit and twitter.unread_posts_available():
try:
content, elem = twitter.get_post()
except Exception:
print('an error occurred')
traceback.print_exc()
break
else:
# print(content)
t = Tweet(**content)
db_id = db.insert_one(t.as_db_dict()).inserted_id
# print(f"""inserted tweet from {t.time.strftime('%Y-%m-%d %H:%M:%S')} with db id: {db_id}""")
attempts += 1
print('number of posts read: ', twitter.number_of_posts_read())
except Exception:
print('error occurred')
finally:
twitter.close()
# now = datetime.utcnow()
# start_date = datetime(2023, 3, 1)
# get_period(start_date, now)ERROR:root:failed extracting post url ERROR:root:failed extracting post url ERROR:root:failed extracting post url ERROR:root:failed extracting post url ERROR:root:failed extracting post url ERROR:root:failed extracting post url ERROR:root:failed extracting post url ERROR:root:failed extracting post url ERROR:root:failed extracting post url ERROR:root:failed extracting post url ERROR:root:failed extracting post url ERROR:root:failed extracting post url
number of posts read: 25
In [ ]:
# keep scraping until all periods have been fetched without error
while len(start_date_list) > 0 and len(end_date_list) > 0:
start_date, end_date = date_list.pop(0)
try:
get_period(start_date, end_date)
except:
date_list.append((start_date, end_date))
print(f'retrying later due to error getting period: [{start_date}:{end_date}]')
else:
print(f'finished getting period: [{start_date}:{end_date}]')In [ ]: