fixed bugs and ran tests
This commit is contained in:
simplypower-bbj
+15
-502
@@ -2,7 +2,7 @@
|
|||||||
"cells": [
|
"cells": [
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 3,
|
"execution_count": 1,
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
@@ -14,12 +14,15 @@
|
|||||||
"from classes import Tweet\n",
|
"from classes import Tweet\n",
|
||||||
"import traceback\n",
|
"import traceback\n",
|
||||||
"import logging\n",
|
"import logging\n",
|
||||||
"from database import connect_mongo"
|
"from database import connect_mongo\n",
|
||||||
|
"\n",
|
||||||
|
"account = 'Maersk'\n",
|
||||||
|
"until_year = 2010"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 7,
|
"execution_count": 2,
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
@@ -28,458 +31,21 @@
|
|||||||
"0"
|
"0"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"execution_count": 7,
|
"execution_count": 2,
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"output_type": "execute_result"
|
"output_type": "execute_result"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"source": [
|
"source": [
|
||||||
"# reset database\n",
|
"# reset database\n",
|
||||||
"# db = connect_mongo()\n",
|
"db = connect_mongo()\n",
|
||||||
"# db.drop()\n",
|
"# db.drop()\n",
|
||||||
"# db.count_documents({'account':'@Maersk'})"
|
"db.count_documents({'account':f'@{account}'})"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": null,
|
"execution_count": 3,
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"[('2023-03-06', '2023-03-16'),\n",
|
|
||||||
" ('2023-02-23', '2023-03-05'),\n",
|
|
||||||
" ('2023-02-12', '2023-02-22'),\n",
|
|
||||||
" ('2023-02-01', '2023-02-11'),\n",
|
|
||||||
" ('2023-01-21', '2023-01-31'),\n",
|
|
||||||
" ('2023-01-10', '2023-01-20'),\n",
|
|
||||||
" ('2022-12-30', '2023-01-09'),\n",
|
|
||||||
" ('2022-12-19', '2022-12-29'),\n",
|
|
||||||
" ('2022-12-08', '2022-12-18'),\n",
|
|
||||||
" ('2022-11-27', '2022-12-07'),\n",
|
|
||||||
" ('2022-11-16', '2022-11-26'),\n",
|
|
||||||
" ('2022-11-05', '2022-11-15'),\n",
|
|
||||||
" ('2022-10-25', '2022-11-04'),\n",
|
|
||||||
" ('2022-10-14', '2022-10-24'),\n",
|
|
||||||
" ('2022-10-03', '2022-10-13'),\n",
|
|
||||||
" ('2022-09-22', '2022-10-02'),\n",
|
|
||||||
" ('2022-09-11', '2022-09-21'),\n",
|
|
||||||
" ('2022-08-31', '2022-09-10'),\n",
|
|
||||||
" ('2022-08-20', '2022-08-30'),\n",
|
|
||||||
" ('2022-08-09', '2022-08-19'),\n",
|
|
||||||
" ('2022-07-29', '2022-08-08'),\n",
|
|
||||||
" ('2022-07-18', '2022-07-28'),\n",
|
|
||||||
" ('2022-07-07', '2022-07-17'),\n",
|
|
||||||
" ('2022-06-26', '2022-07-06'),\n",
|
|
||||||
" ('2022-06-15', '2022-06-25'),\n",
|
|
||||||
" ('2022-06-04', '2022-06-14'),\n",
|
|
||||||
" ('2022-05-24', '2022-06-03'),\n",
|
|
||||||
" ('2022-05-13', '2022-05-23'),\n",
|
|
||||||
" ('2022-05-02', '2022-05-12'),\n",
|
|
||||||
" ('2022-04-21', '2022-05-01'),\n",
|
|
||||||
" ('2022-04-10', '2022-04-20'),\n",
|
|
||||||
" ('2022-03-30', '2022-04-09'),\n",
|
|
||||||
" ('2022-03-19', '2022-03-29'),\n",
|
|
||||||
" ('2022-03-08', '2022-03-18'),\n",
|
|
||||||
" ('2022-02-25', '2022-03-07'),\n",
|
|
||||||
" ('2022-02-14', '2022-02-24'),\n",
|
|
||||||
" ('2022-02-03', '2022-02-13'),\n",
|
|
||||||
" ('2022-01-23', '2022-02-02'),\n",
|
|
||||||
" ('2022-01-12', '2022-01-22'),\n",
|
|
||||||
" ('2022-01-01', '2022-01-11'),\n",
|
|
||||||
" ('2021-12-21', '2021-12-31'),\n",
|
|
||||||
" ('2021-12-10', '2021-12-20'),\n",
|
|
||||||
" ('2021-11-29', '2021-12-09'),\n",
|
|
||||||
" ('2021-11-18', '2021-11-28'),\n",
|
|
||||||
" ('2021-11-07', '2021-11-17'),\n",
|
|
||||||
" ('2021-10-27', '2021-11-06'),\n",
|
|
||||||
" ('2021-10-16', '2021-10-26'),\n",
|
|
||||||
" ('2021-10-05', '2021-10-15'),\n",
|
|
||||||
" ('2021-09-24', '2021-10-04'),\n",
|
|
||||||
" ('2021-09-13', '2021-09-23'),\n",
|
|
||||||
" ('2021-09-02', '2021-09-12'),\n",
|
|
||||||
" ('2021-08-22', '2021-09-01'),\n",
|
|
||||||
" ('2021-08-11', '2021-08-21'),\n",
|
|
||||||
" ('2021-07-31', '2021-08-10'),\n",
|
|
||||||
" ('2021-07-20', '2021-07-30'),\n",
|
|
||||||
" ('2021-07-09', '2021-07-19'),\n",
|
|
||||||
" ('2021-06-28', '2021-07-08'),\n",
|
|
||||||
" ('2021-06-17', '2021-06-27'),\n",
|
|
||||||
" ('2021-06-06', '2021-06-16'),\n",
|
|
||||||
" ('2021-05-26', '2021-06-05'),\n",
|
|
||||||
" ('2021-05-15', '2021-05-25'),\n",
|
|
||||||
" ('2021-05-04', '2021-05-14'),\n",
|
|
||||||
" ('2021-04-23', '2021-05-03'),\n",
|
|
||||||
" ('2021-04-12', '2021-04-22'),\n",
|
|
||||||
" ('2021-04-01', '2021-04-11'),\n",
|
|
||||||
" ('2021-03-21', '2021-03-31'),\n",
|
|
||||||
" ('2021-03-10', '2021-03-20'),\n",
|
|
||||||
" ('2021-02-27', '2021-03-09'),\n",
|
|
||||||
" ('2021-02-16', '2021-02-26'),\n",
|
|
||||||
" ('2021-02-05', '2021-02-15'),\n",
|
|
||||||
" ('2021-01-25', '2021-02-04'),\n",
|
|
||||||
" ('2021-01-14', '2021-01-24'),\n",
|
|
||||||
" ('2021-01-03', '2021-01-13'),\n",
|
|
||||||
" ('2020-12-23', '2021-01-02'),\n",
|
|
||||||
" ('2020-12-12', '2020-12-22'),\n",
|
|
||||||
" ('2020-12-01', '2020-12-11'),\n",
|
|
||||||
" ('2020-11-20', '2020-11-30'),\n",
|
|
||||||
" ('2020-11-09', '2020-11-19'),\n",
|
|
||||||
" ('2020-10-29', '2020-11-08'),\n",
|
|
||||||
" ('2020-10-18', '2020-10-28'),\n",
|
|
||||||
" ('2020-10-07', '2020-10-17'),\n",
|
|
||||||
" ('2020-09-26', '2020-10-06'),\n",
|
|
||||||
" ('2020-09-15', '2020-09-25'),\n",
|
|
||||||
" ('2020-09-04', '2020-09-14'),\n",
|
|
||||||
" ('2020-08-24', '2020-09-03'),\n",
|
|
||||||
" ('2020-08-13', '2020-08-23'),\n",
|
|
||||||
" ('2020-08-02', '2020-08-12'),\n",
|
|
||||||
" ('2020-07-22', '2020-08-01'),\n",
|
|
||||||
" ('2020-07-11', '2020-07-21'),\n",
|
|
||||||
" ('2020-06-30', '2020-07-10'),\n",
|
|
||||||
" ('2020-06-19', '2020-06-29'),\n",
|
|
||||||
" ('2020-06-08', '2020-06-18'),\n",
|
|
||||||
" ('2020-05-28', '2020-06-07'),\n",
|
|
||||||
" ('2020-05-17', '2020-05-27'),\n",
|
|
||||||
" ('2020-05-06', '2020-05-16'),\n",
|
|
||||||
" ('2020-04-25', '2020-05-05'),\n",
|
|
||||||
" ('2020-04-14', '2020-04-24'),\n",
|
|
||||||
" ('2020-04-03', '2020-04-13'),\n",
|
|
||||||
" ('2020-03-23', '2020-04-02'),\n",
|
|
||||||
" ('2020-03-12', '2020-03-22'),\n",
|
|
||||||
" ('2020-03-01', '2020-03-11'),\n",
|
|
||||||
" ('2020-02-19', '2020-02-29'),\n",
|
|
||||||
" ('2020-02-08', '2020-02-18'),\n",
|
|
||||||
" ('2020-01-28', '2020-02-07'),\n",
|
|
||||||
" ('2020-01-17', '2020-01-27'),\n",
|
|
||||||
" ('2020-01-06', '2020-01-16'),\n",
|
|
||||||
" ('2019-12-26', '2020-01-05'),\n",
|
|
||||||
" ('2019-12-15', '2019-12-25'),\n",
|
|
||||||
" ('2019-12-04', '2019-12-14'),\n",
|
|
||||||
" ('2019-11-23', '2019-12-03'),\n",
|
|
||||||
" ('2019-11-12', '2019-11-22'),\n",
|
|
||||||
" ('2019-11-01', '2019-11-11'),\n",
|
|
||||||
" ('2019-10-21', '2019-10-31'),\n",
|
|
||||||
" ('2019-10-10', '2019-10-20'),\n",
|
|
||||||
" ('2019-09-29', '2019-10-09'),\n",
|
|
||||||
" ('2019-09-18', '2019-09-28'),\n",
|
|
||||||
" ('2019-09-07', '2019-09-17'),\n",
|
|
||||||
" ('2019-08-27', '2019-09-06'),\n",
|
|
||||||
" ('2019-08-16', '2019-08-26'),\n",
|
|
||||||
" ('2019-08-05', '2019-08-15'),\n",
|
|
||||||
" ('2019-07-25', '2019-08-04'),\n",
|
|
||||||
" ('2019-07-14', '2019-07-24'),\n",
|
|
||||||
" ('2019-07-03', '2019-07-13'),\n",
|
|
||||||
" ('2019-06-22', '2019-07-02'),\n",
|
|
||||||
" ('2019-06-11', '2019-06-21'),\n",
|
|
||||||
" ('2019-05-31', '2019-06-10'),\n",
|
|
||||||
" ('2019-05-20', '2019-05-30'),\n",
|
|
||||||
" ('2019-05-09', '2019-05-19'),\n",
|
|
||||||
" ('2019-04-28', '2019-05-08'),\n",
|
|
||||||
" ('2019-04-17', '2019-04-27'),\n",
|
|
||||||
" ('2019-04-06', '2019-04-16'),\n",
|
|
||||||
" ('2019-03-26', '2019-04-05'),\n",
|
|
||||||
" ('2019-03-15', '2019-03-25'),\n",
|
|
||||||
" ('2019-03-04', '2019-03-14'),\n",
|
|
||||||
" ('2019-02-21', '2019-03-03'),\n",
|
|
||||||
" ('2019-02-10', '2019-02-20'),\n",
|
|
||||||
" ('2019-01-30', '2019-02-09'),\n",
|
|
||||||
" ('2019-01-19', '2019-01-29'),\n",
|
|
||||||
" ('2019-01-08', '2019-01-18'),\n",
|
|
||||||
" ('2018-12-28', '2019-01-07'),\n",
|
|
||||||
" ('2018-12-17', '2018-12-27'),\n",
|
|
||||||
" ('2018-12-06', '2018-12-16'),\n",
|
|
||||||
" ('2018-11-25', '2018-12-05'),\n",
|
|
||||||
" ('2018-11-14', '2018-11-24'),\n",
|
|
||||||
" ('2018-11-03', '2018-11-13'),\n",
|
|
||||||
" ('2018-10-23', '2018-11-02'),\n",
|
|
||||||
" ('2018-10-12', '2018-10-22'),\n",
|
|
||||||
" ('2018-10-01', '2018-10-11'),\n",
|
|
||||||
" ('2018-09-20', '2018-09-30'),\n",
|
|
||||||
" ('2018-09-09', '2018-09-19'),\n",
|
|
||||||
" ('2018-08-29', '2018-09-08'),\n",
|
|
||||||
" ('2018-08-18', '2018-08-28'),\n",
|
|
||||||
" ('2018-08-07', '2018-08-17'),\n",
|
|
||||||
" ('2018-07-27', '2018-08-06'),\n",
|
|
||||||
" ('2018-07-16', '2018-07-26'),\n",
|
|
||||||
" ('2018-07-05', '2018-07-15'),\n",
|
|
||||||
" ('2018-06-24', '2018-07-04'),\n",
|
|
||||||
" ('2018-06-13', '2018-06-23'),\n",
|
|
||||||
" ('2018-06-02', '2018-06-12'),\n",
|
|
||||||
" ('2018-05-22', '2018-06-01'),\n",
|
|
||||||
" ('2018-05-11', '2018-05-21'),\n",
|
|
||||||
" ('2018-04-30', '2018-05-10'),\n",
|
|
||||||
" ('2018-04-19', '2018-04-29'),\n",
|
|
||||||
" ('2018-04-08', '2018-04-18'),\n",
|
|
||||||
" ('2018-03-28', '2018-04-07'),\n",
|
|
||||||
" ('2018-03-17', '2018-03-27'),\n",
|
|
||||||
" ('2018-03-06', '2018-03-16'),\n",
|
|
||||||
" ('2018-02-23', '2018-03-05'),\n",
|
|
||||||
" ('2018-02-12', '2018-02-22'),\n",
|
|
||||||
" ('2018-02-01', '2018-02-11'),\n",
|
|
||||||
" ('2018-01-21', '2018-01-31'),\n",
|
|
||||||
" ('2018-01-10', '2018-01-20'),\n",
|
|
||||||
" ('2017-12-30', '2018-01-09'),\n",
|
|
||||||
" ('2017-12-19', '2017-12-29'),\n",
|
|
||||||
" ('2017-12-08', '2017-12-18'),\n",
|
|
||||||
" ('2017-11-27', '2017-12-07'),\n",
|
|
||||||
" ('2017-11-16', '2017-11-26'),\n",
|
|
||||||
" ('2017-11-05', '2017-11-15'),\n",
|
|
||||||
" ('2017-10-25', '2017-11-04'),\n",
|
|
||||||
" ('2017-10-14', '2017-10-24'),\n",
|
|
||||||
" ('2017-10-03', '2017-10-13'),\n",
|
|
||||||
" ('2017-09-22', '2017-10-02'),\n",
|
|
||||||
" ('2017-09-11', '2017-09-21'),\n",
|
|
||||||
" ('2017-08-31', '2017-09-10'),\n",
|
|
||||||
" ('2017-08-20', '2017-08-30'),\n",
|
|
||||||
" ('2017-08-09', '2017-08-19'),\n",
|
|
||||||
" ('2017-07-29', '2017-08-08'),\n",
|
|
||||||
" ('2017-07-18', '2017-07-28'),\n",
|
|
||||||
" ('2017-07-07', '2017-07-17'),\n",
|
|
||||||
" ('2017-06-26', '2017-07-06'),\n",
|
|
||||||
" ('2017-06-15', '2017-06-25'),\n",
|
|
||||||
" ('2017-06-04', '2017-06-14'),\n",
|
|
||||||
" ('2017-05-24', '2017-06-03'),\n",
|
|
||||||
" ('2017-05-13', '2017-05-23'),\n",
|
|
||||||
" ('2017-05-02', '2017-05-12'),\n",
|
|
||||||
" ('2017-04-21', '2017-05-01'),\n",
|
|
||||||
" ('2017-04-10', '2017-04-20'),\n",
|
|
||||||
" ('2017-03-30', '2017-04-09'),\n",
|
|
||||||
" ('2017-03-19', '2017-03-29'),\n",
|
|
||||||
" ('2017-03-08', '2017-03-18'),\n",
|
|
||||||
" ('2017-02-25', '2017-03-07'),\n",
|
|
||||||
" ('2017-02-14', '2017-02-24'),\n",
|
|
||||||
" ('2017-02-03', '2017-02-13'),\n",
|
|
||||||
" ('2017-01-23', '2017-02-02'),\n",
|
|
||||||
" ('2017-01-12', '2017-01-22'),\n",
|
|
||||||
" ('2017-01-01', '2017-01-11'),\n",
|
|
||||||
" ('2016-12-21', '2016-12-31'),\n",
|
|
||||||
" ('2016-12-10', '2016-12-20'),\n",
|
|
||||||
" ('2016-11-29', '2016-12-09'),\n",
|
|
||||||
" ('2016-11-18', '2016-11-28'),\n",
|
|
||||||
" ('2016-11-07', '2016-11-17'),\n",
|
|
||||||
" ('2016-10-27', '2016-11-06'),\n",
|
|
||||||
" ('2016-10-16', '2016-10-26'),\n",
|
|
||||||
" ('2016-10-05', '2016-10-15'),\n",
|
|
||||||
" ('2016-09-24', '2016-10-04'),\n",
|
|
||||||
" ('2016-09-13', '2016-09-23'),\n",
|
|
||||||
" ('2016-09-02', '2016-09-12'),\n",
|
|
||||||
" ('2016-08-22', '2016-09-01'),\n",
|
|
||||||
" ('2016-08-11', '2016-08-21'),\n",
|
|
||||||
" ('2016-07-31', '2016-08-10'),\n",
|
|
||||||
" ('2016-07-20', '2016-07-30'),\n",
|
|
||||||
" ('2016-07-09', '2016-07-19'),\n",
|
|
||||||
" ('2016-06-28', '2016-07-08'),\n",
|
|
||||||
" ('2016-06-17', '2016-06-27'),\n",
|
|
||||||
" ('2016-06-06', '2016-06-16'),\n",
|
|
||||||
" ('2016-05-26', '2016-06-05'),\n",
|
|
||||||
" ('2016-05-15', '2016-05-25'),\n",
|
|
||||||
" ('2016-05-04', '2016-05-14'),\n",
|
|
||||||
" ('2016-04-23', '2016-05-03'),\n",
|
|
||||||
" ('2016-04-12', '2016-04-22'),\n",
|
|
||||||
" ('2016-04-01', '2016-04-11'),\n",
|
|
||||||
" ('2016-03-21', '2016-03-31'),\n",
|
|
||||||
" ('2016-03-10', '2016-03-20'),\n",
|
|
||||||
" ('2016-02-28', '2016-03-09'),\n",
|
|
||||||
" ('2016-02-17', '2016-02-27'),\n",
|
|
||||||
" ('2016-02-06', '2016-02-16'),\n",
|
|
||||||
" ('2016-01-26', '2016-02-05'),\n",
|
|
||||||
" ('2016-01-15', '2016-01-25'),\n",
|
|
||||||
" ('2016-01-04', '2016-01-14'),\n",
|
|
||||||
" ('2015-12-24', '2016-01-03'),\n",
|
|
||||||
" ('2015-12-13', '2015-12-23'),\n",
|
|
||||||
" ('2015-12-02', '2015-12-12'),\n",
|
|
||||||
" ('2015-11-21', '2015-12-01'),\n",
|
|
||||||
" ('2015-11-10', '2015-11-20'),\n",
|
|
||||||
" ('2015-10-30', '2015-11-09'),\n",
|
|
||||||
" ('2015-10-19', '2015-10-29'),\n",
|
|
||||||
" ('2015-10-08', '2015-10-18'),\n",
|
|
||||||
" ('2015-09-27', '2015-10-07'),\n",
|
|
||||||
" ('2015-09-16', '2015-09-26'),\n",
|
|
||||||
" ('2015-09-05', '2015-09-15'),\n",
|
|
||||||
" ('2015-08-25', '2015-09-04'),\n",
|
|
||||||
" ('2015-08-14', '2015-08-24'),\n",
|
|
||||||
" ('2015-08-03', '2015-08-13'),\n",
|
|
||||||
" ('2015-07-23', '2015-08-02'),\n",
|
|
||||||
" ('2015-07-12', '2015-07-22'),\n",
|
|
||||||
" ('2015-07-01', '2015-07-11'),\n",
|
|
||||||
" ('2015-06-20', '2015-06-30'),\n",
|
|
||||||
" ('2015-06-09', '2015-06-19'),\n",
|
|
||||||
" ('2015-05-29', '2015-06-08'),\n",
|
|
||||||
" ('2015-05-18', '2015-05-28'),\n",
|
|
||||||
" ('2015-05-07', '2015-05-17'),\n",
|
|
||||||
" ('2015-04-26', '2015-05-06'),\n",
|
|
||||||
" ('2015-04-15', '2015-04-25'),\n",
|
|
||||||
" ('2015-04-04', '2015-04-14'),\n",
|
|
||||||
" ('2015-03-24', '2015-04-03'),\n",
|
|
||||||
" ('2015-03-13', '2015-03-23'),\n",
|
|
||||||
" ('2015-03-02', '2015-03-12'),\n",
|
|
||||||
" ('2015-02-19', '2015-03-01'),\n",
|
|
||||||
" ('2015-02-08', '2015-02-18'),\n",
|
|
||||||
" ('2015-01-28', '2015-02-07'),\n",
|
|
||||||
" ('2015-01-17', '2015-01-27'),\n",
|
|
||||||
" ('2015-01-06', '2015-01-16'),\n",
|
|
||||||
" ('2014-12-26', '2015-01-05'),\n",
|
|
||||||
" ('2014-12-15', '2014-12-25'),\n",
|
|
||||||
" ('2014-12-04', '2014-12-14'),\n",
|
|
||||||
" ('2014-11-23', '2014-12-03'),\n",
|
|
||||||
" ('2014-11-12', '2014-11-22'),\n",
|
|
||||||
" ('2014-11-01', '2014-11-11'),\n",
|
|
||||||
" ('2014-10-21', '2014-10-31'),\n",
|
|
||||||
" ('2014-10-10', '2014-10-20'),\n",
|
|
||||||
" ('2014-09-29', '2014-10-09'),\n",
|
|
||||||
" ('2014-09-18', '2014-09-28'),\n",
|
|
||||||
" ('2014-09-07', '2014-09-17'),\n",
|
|
||||||
" ('2014-08-27', '2014-09-06'),\n",
|
|
||||||
" ('2014-08-16', '2014-08-26'),\n",
|
|
||||||
" ('2014-08-05', '2014-08-15'),\n",
|
|
||||||
" ('2014-07-25', '2014-08-04'),\n",
|
|
||||||
" ('2014-07-14', '2014-07-24'),\n",
|
|
||||||
" ('2014-07-03', '2014-07-13'),\n",
|
|
||||||
" ('2014-06-22', '2014-07-02'),\n",
|
|
||||||
" ('2014-06-11', '2014-06-21'),\n",
|
|
||||||
" ('2014-05-31', '2014-06-10'),\n",
|
|
||||||
" ('2014-05-20', '2014-05-30'),\n",
|
|
||||||
" ('2014-05-09', '2014-05-19'),\n",
|
|
||||||
" ('2014-04-28', '2014-05-08'),\n",
|
|
||||||
" ('2014-04-17', '2014-04-27'),\n",
|
|
||||||
" ('2014-04-06', '2014-04-16'),\n",
|
|
||||||
" ('2014-03-26', '2014-04-05'),\n",
|
|
||||||
" ('2014-03-15', '2014-03-25'),\n",
|
|
||||||
" ('2014-03-04', '2014-03-14'),\n",
|
|
||||||
" ('2014-02-21', '2014-03-03'),\n",
|
|
||||||
" ('2014-02-10', '2014-02-20'),\n",
|
|
||||||
" ('2014-01-30', '2014-02-09'),\n",
|
|
||||||
" ('2014-01-19', '2014-01-29'),\n",
|
|
||||||
" ('2014-01-08', '2014-01-18'),\n",
|
|
||||||
" ('2013-12-28', '2014-01-07'),\n",
|
|
||||||
" ('2013-12-17', '2013-12-27'),\n",
|
|
||||||
" ('2013-12-06', '2013-12-16'),\n",
|
|
||||||
" ('2013-11-25', '2013-12-05'),\n",
|
|
||||||
" ('2013-11-14', '2013-11-24'),\n",
|
|
||||||
" ('2013-11-03', '2013-11-13'),\n",
|
|
||||||
" ('2013-10-23', '2013-11-02'),\n",
|
|
||||||
" ('2013-10-12', '2013-10-22'),\n",
|
|
||||||
" ('2013-10-01', '2013-10-11'),\n",
|
|
||||||
" ('2013-09-20', '2013-09-30'),\n",
|
|
||||||
" ('2013-09-09', '2013-09-19'),\n",
|
|
||||||
" ('2013-08-29', '2013-09-08'),\n",
|
|
||||||
" ('2013-08-18', '2013-08-28'),\n",
|
|
||||||
" ('2013-08-07', '2013-08-17'),\n",
|
|
||||||
" ('2013-07-27', '2013-08-06'),\n",
|
|
||||||
" ('2013-07-16', '2013-07-26'),\n",
|
|
||||||
" ('2013-07-05', '2013-07-15'),\n",
|
|
||||||
" ('2013-06-24', '2013-07-04'),\n",
|
|
||||||
" ('2013-06-13', '2013-06-23'),\n",
|
|
||||||
" ('2013-06-02', '2013-06-12'),\n",
|
|
||||||
" ('2013-05-22', '2013-06-01'),\n",
|
|
||||||
" ('2013-05-11', '2013-05-21'),\n",
|
|
||||||
" ('2013-04-30', '2013-05-10'),\n",
|
|
||||||
" ('2013-04-19', '2013-04-29'),\n",
|
|
||||||
" ('2013-04-08', '2013-04-18'),\n",
|
|
||||||
" ('2013-03-28', '2013-04-07'),\n",
|
|
||||||
" ('2013-03-17', '2013-03-27'),\n",
|
|
||||||
" ('2013-03-06', '2013-03-16'),\n",
|
|
||||||
" ('2013-02-23', '2013-03-05'),\n",
|
|
||||||
" ('2013-02-12', '2013-02-22'),\n",
|
|
||||||
" ('2013-02-01', '2013-02-11'),\n",
|
|
||||||
" ('2013-01-21', '2013-01-31'),\n",
|
|
||||||
" ('2013-01-10', '2013-01-20'),\n",
|
|
||||||
" ('2012-12-30', '2013-01-09'),\n",
|
|
||||||
" ('2012-12-19', '2012-12-29'),\n",
|
|
||||||
" ('2012-12-08', '2012-12-18'),\n",
|
|
||||||
" ('2012-11-27', '2012-12-07'),\n",
|
|
||||||
" ('2012-11-16', '2012-11-26'),\n",
|
|
||||||
" ('2012-11-05', '2012-11-15'),\n",
|
|
||||||
" ('2012-10-25', '2012-11-04'),\n",
|
|
||||||
" ('2012-10-14', '2012-10-24'),\n",
|
|
||||||
" ('2012-10-03', '2012-10-13'),\n",
|
|
||||||
" ('2012-09-22', '2012-10-02'),\n",
|
|
||||||
" ('2012-09-11', '2012-09-21'),\n",
|
|
||||||
" ('2012-08-31', '2012-09-10'),\n",
|
|
||||||
" ('2012-08-20', '2012-08-30'),\n",
|
|
||||||
" ('2012-08-09', '2012-08-19'),\n",
|
|
||||||
" ('2012-07-29', '2012-08-08'),\n",
|
|
||||||
" ('2012-07-18', '2012-07-28'),\n",
|
|
||||||
" ('2012-07-07', '2012-07-17'),\n",
|
|
||||||
" ('2012-06-26', '2012-07-06'),\n",
|
|
||||||
" ('2012-06-15', '2012-06-25'),\n",
|
|
||||||
" ('2012-06-04', '2012-06-14'),\n",
|
|
||||||
" ('2012-05-24', '2012-06-03'),\n",
|
|
||||||
" ('2012-05-13', '2012-05-23'),\n",
|
|
||||||
" ('2012-05-02', '2012-05-12'),\n",
|
|
||||||
" ('2012-04-21', '2012-05-01'),\n",
|
|
||||||
" ('2012-04-10', '2012-04-20'),\n",
|
|
||||||
" ('2012-03-30', '2012-04-09'),\n",
|
|
||||||
" ('2012-03-19', '2012-03-29'),\n",
|
|
||||||
" ('2012-03-08', '2012-03-18'),\n",
|
|
||||||
" ('2012-02-26', '2012-03-07'),\n",
|
|
||||||
" ('2012-02-15', '2012-02-25'),\n",
|
|
||||||
" ('2012-02-04', '2012-02-14'),\n",
|
|
||||||
" ('2012-01-24', '2012-02-03'),\n",
|
|
||||||
" ('2012-01-13', '2012-01-23'),\n",
|
|
||||||
" ('2012-01-02', '2012-01-12'),\n",
|
|
||||||
" ('2011-12-22', '2012-01-01'),\n",
|
|
||||||
" ('2011-12-11', '2011-12-21'),\n",
|
|
||||||
" ('2011-11-30', '2011-12-10'),\n",
|
|
||||||
" ('2011-11-19', '2011-11-29'),\n",
|
|
||||||
" ('2011-11-08', '2011-11-18'),\n",
|
|
||||||
" ('2011-10-28', '2011-11-07'),\n",
|
|
||||||
" ('2011-10-17', '2011-10-27'),\n",
|
|
||||||
" ('2011-10-06', '2011-10-16'),\n",
|
|
||||||
" ('2011-09-25', '2011-10-05'),\n",
|
|
||||||
" ('2011-09-14', '2011-09-24'),\n",
|
|
||||||
" ('2011-09-03', '2011-09-13'),\n",
|
|
||||||
" ('2011-08-23', '2011-09-02'),\n",
|
|
||||||
" ('2011-08-12', '2011-08-22'),\n",
|
|
||||||
" ('2011-08-01', '2011-08-11'),\n",
|
|
||||||
" ('2011-07-21', '2011-07-31'),\n",
|
|
||||||
" ('2011-07-10', '2011-07-20'),\n",
|
|
||||||
" ('2011-06-29', '2011-07-09'),\n",
|
|
||||||
" ('2011-06-18', '2011-06-28'),\n",
|
|
||||||
" ('2011-06-07', '2011-06-17'),\n",
|
|
||||||
" ('2011-05-27', '2011-06-06'),\n",
|
|
||||||
" ('2011-05-16', '2011-05-26'),\n",
|
|
||||||
" ('2011-05-05', '2011-05-15'),\n",
|
|
||||||
" ('2011-04-24', '2011-05-04'),\n",
|
|
||||||
" ('2011-04-13', '2011-04-23'),\n",
|
|
||||||
" ('2011-04-02', '2011-04-12'),\n",
|
|
||||||
" ('2011-03-22', '2011-04-01'),\n",
|
|
||||||
" ('2011-03-11', '2011-03-21'),\n",
|
|
||||||
" ('2011-02-28', '2011-03-10'),\n",
|
|
||||||
" ('2011-02-17', '2011-02-27'),\n",
|
|
||||||
" ('2011-02-06', '2011-02-16'),\n",
|
|
||||||
" ('2011-01-26', '2011-02-05'),\n",
|
|
||||||
" ('2011-01-15', '2011-01-25'),\n",
|
|
||||||
" ('2011-01-04', '2011-01-14'),\n",
|
|
||||||
" ('2010-12-24', '2011-01-03')]"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"metadata": {},
|
|
||||||
"output_type": "display_data"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"# prepare date lists to redo on error\n",
|
|
||||||
"end_date = datetime.utcnow()\n",
|
|
||||||
"start_date = end_date - timedelta(days=10)\n",
|
|
||||||
"\n",
|
|
||||||
"end_date_list = [end_date.strftime('%Y-%m-%d')]\n",
|
|
||||||
"start_date_list = [start_date.strftime('%Y-%m-%d')]\n",
|
|
||||||
"while start_date.year > 2010:\n",
|
|
||||||
" end_date = start_date - timedelta(days=1)\n",
|
|
||||||
" end_date_list.append(end_date.strftime('%Y-%m-%d'))\n",
|
|
||||||
" start_date = end_date - timedelta(days=10)\n",
|
|
||||||
" start_date_list.append(start_date.strftime('%Y-%m-%d'))\n",
|
|
||||||
"\n",
|
|
||||||
"date_list = list(zip(start_date_list, end_date_list))\n",
|
|
||||||
"date_list"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 2,
|
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
@@ -497,69 +63,16 @@
|
|||||||
"ERROR:root:failed extracting post url\n",
|
"ERROR:root:failed extracting post url\n",
|
||||||
"ERROR:root:failed extracting post url\n",
|
"ERROR:root:failed extracting post url\n",
|
||||||
"ERROR:root:failed extracting post url\n",
|
"ERROR:root:failed extracting post url\n",
|
||||||
"ERROR:root:failed extracting post url\n"
|
"ERROR:root:failed extracting post url\n",
|
||||||
]
|
"ERROR:root:failed extracting post url\n",
|
||||||
},
|
"WARNING:root:retrying later due to error getting period: [2023-02-23 00:00:00:2023-03-05 00:00:00]\n"
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"number of posts read: 25\n"
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"source": [
|
"source": [
|
||||||
"def get_period(start_date, end_date):\n",
|
"from selenium_scraper import scrape_account\n",
|
||||||
" # setup\n",
|
|
||||||
" account = 'Maersk'\n",
|
|
||||||
" try:\n",
|
|
||||||
" twitter = Twitter()\n",
|
|
||||||
" twitter.setup()\n",
|
|
||||||
" twitter._open_page(account, start_date, end_date)\n",
|
|
||||||
" db = connect_mongo()\n",
|
|
||||||
" # scrape\n",
|
|
||||||
" limit = 500\n",
|
|
||||||
" attempts = 0\n",
|
|
||||||
" while attempts < limit and twitter.unread_posts_available():\n",
|
|
||||||
" try:\n",
|
|
||||||
" content, elem = twitter.get_post()\n",
|
|
||||||
" except Exception:\n",
|
|
||||||
" print('an error occurred')\n",
|
|
||||||
" traceback.print_exc()\n",
|
|
||||||
" break\n",
|
|
||||||
" else:\n",
|
|
||||||
" # print(content)\n",
|
|
||||||
" t = Tweet(**content)\n",
|
|
||||||
" db_id = db.insert_one(t.as_db_dict()).inserted_id\n",
|
|
||||||
" # print(f\"\"\"inserted tweet from {t.time.strftime('%Y-%m-%d %H:%M:%S')} with db id: {db_id}\"\"\")\n",
|
|
||||||
" attempts += 1\n",
|
|
||||||
" print('number of posts read: ', twitter.number_of_posts_read())\n",
|
|
||||||
" except Exception:\n",
|
|
||||||
" print('error occurred')\n",
|
|
||||||
" finally:\n",
|
|
||||||
" twitter.close()\n",
|
|
||||||
"\n",
|
"\n",
|
||||||
"# now = datetime.utcnow()\n",
|
"scrape_account(account, until_year)"
|
||||||
"# start_date = datetime(2023, 3, 1)\n",
|
|
||||||
"# get_period(start_date, now)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# keep scraping until all periods have been fetched without error\n",
|
|
||||||
"while len(start_date_list) > 0 and len(end_date_list) > 0:\n",
|
|
||||||
" start_date, end_date = date_list.pop(0)\n",
|
|
||||||
" try:\n",
|
|
||||||
" get_period(start_date, end_date)\n",
|
|
||||||
" except:\n",
|
|
||||||
" date_list.append((start_date, end_date))\n",
|
|
||||||
" print(f'retrying later due to error getting period: [{start_date}:{end_date}]')\n",
|
|
||||||
" else:\n",
|
|
||||||
" print(f'finished getting period: [{start_date}:{end_date}]')"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
+66
-36
@@ -8,7 +8,7 @@ import selenium.webdriver.support.expected_conditions as EC
|
|||||||
from selenium.webdriver.remote.webelement import WebElement
|
from selenium.webdriver.remote.webelement import WebElement
|
||||||
from pydantic import BaseModel, validator, parse_obj_as, AnyHttpUrl
|
from pydantic import BaseModel, validator, parse_obj_as, AnyHttpUrl
|
||||||
from typing import Union, List
|
from typing import Union, List
|
||||||
from datetime import datetime
|
from datetime import datetime, timedelta
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
import requests
|
import requests
|
||||||
@@ -17,6 +17,8 @@ import logging
|
|||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
|
from database import connect_mongo
|
||||||
|
from classes import Tweet
|
||||||
|
|
||||||
class Twitter(object):
|
class Twitter(object):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
@@ -256,41 +258,69 @@ class Twitter(object):
|
|||||||
# print('sucessfully read post')
|
# print('sucessfully read post')
|
||||||
return content, elem
|
return content, elem
|
||||||
|
|
||||||
|
def generate_date_list(until_year=2010):
|
||||||
|
end_date = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||||
|
start_date = end_date - timedelta(days=10)
|
||||||
|
end_date_list = [end_date]
|
||||||
|
start_date_list = [start_date]
|
||||||
|
while start_date.year > until_year:
|
||||||
|
end_date = start_date - timedelta(days=1)
|
||||||
|
end_date_list.append(end_date)
|
||||||
|
start_date = end_date - timedelta(days=10)
|
||||||
|
start_date_list.append(start_date)
|
||||||
|
date_list = list(zip(start_date_list, end_date_list))
|
||||||
|
return date_list
|
||||||
|
|
||||||
|
def get_period(account: str, start_date: datetime, end_date: datetime):
|
||||||
|
try:
|
||||||
|
# setup
|
||||||
|
twitter = Twitter()
|
||||||
|
twitter.setup()
|
||||||
|
twitter._open_page(account, start_date, end_date)
|
||||||
|
db = connect_mongo()
|
||||||
|
# scrape
|
||||||
|
limit = 500
|
||||||
|
attempts = 0
|
||||||
|
while attempts < limit and twitter.unread_posts_available():
|
||||||
|
content, elem = twitter.get_post()
|
||||||
|
# print(content)
|
||||||
|
t = Tweet(**content)
|
||||||
|
db_id = db.insert_one(t.as_db_dict()).inserted_id
|
||||||
|
# print(f"""inserted tweet from {t.time.strftime('%Y-%m-%d %H:%M:%S')} with db id: {db_id}""")
|
||||||
|
attempts += 1
|
||||||
|
num_posts_read = twitter.number_of_posts_read()
|
||||||
|
# print('number of posts read: ', num_posts_read)
|
||||||
|
except Exception as e:
|
||||||
|
raise e
|
||||||
|
finally:
|
||||||
|
twitter.close()
|
||||||
|
return num_posts_read
|
||||||
|
|
||||||
# def extract_tweet(webelement: WebElement) -> dict:
|
def scrape_account(account: str, until_year: int):
|
||||||
# post_time = webelement.find_element(By.XPATH, '//time').get_attribute('datetime')
|
# generate date list
|
||||||
# post_text = webelement.find_element(By.CSS_SELECTOR, '[data-testid="tweetText"]').text
|
date_list = generate_date_list(until_year)
|
||||||
# # find images in webelement
|
# start scraping
|
||||||
# img_url_list = [elem.get_attribute('src') for elem in webelement.find_elements(By.TAG_NAME, 'img')]
|
i = 0
|
||||||
# # keep all images with url containing 'media'
|
limit = 1000
|
||||||
# img_url_list = [url for url in img_url_list if 'media' in url]
|
while len(date_list) > 0:
|
||||||
# img_data_list = [requests.get(img_url).content for img_url in img_url_list]
|
if i >= limit:
|
||||||
# tweet = Tweet(
|
logging.error('scraping attempt limit reached!')
|
||||||
# time=post_time,
|
return
|
||||||
# text=post_text,
|
# get period
|
||||||
# images=img_data_list
|
start_date, end_date = date_list.pop(0)
|
||||||
# )
|
try:
|
||||||
# return tweet
|
num_posts_found = get_period(account, start_date, end_date)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
logging.warning('KeyboardInterrupt')
|
||||||
|
except:
|
||||||
|
date_list.append((start_date, end_date))
|
||||||
|
logging.warning(f'retrying later due to error getting period {start_date} to {end_date}')
|
||||||
|
else:
|
||||||
|
logging.info(f'found {num_posts_found} posts between {start_date} and {end_date}')
|
||||||
|
i += 1
|
||||||
|
|
||||||
# def fetch_period(account: str, date_begin: datetime, date_end: datetime) -> List[Tweet]:
|
if __name__ == '__main__':
|
||||||
# # prepare url
|
logging.basicConfig(level=logging.INFO)
|
||||||
# url = base_url(account, date_begin, date_end)
|
account = 'Maersk'
|
||||||
# # setup webdriver
|
until_year = 2010
|
||||||
# driver = webdriver.Chrome('../chromedriver_mac')
|
scrape_account(account, until_year)
|
||||||
# driver.get(str(url))
|
|
||||||
# # wait for popup to appear
|
|
||||||
# elem = WebDriverWait(
|
|
||||||
# driver=driver,
|
|
||||||
# timeout=20
|
|
||||||
# ).until(
|
|
||||||
# EC.presence_of_element_located(
|
|
||||||
# (By.XPATH, "//*[text()='Not now']")
|
|
||||||
# )
|
|
||||||
# )
|
|
||||||
# # turn off notifications - click not now button
|
|
||||||
# elem.click()
|
|
||||||
# # click allow cookies
|
|
||||||
# elem = driver.find_element(By.XPATH, "//*[text()='Accept all cookies']")
|
|
||||||
# elem.click()
|
|
||||||
#
|
|
||||||
Reference in New Issue
Block a user