diff --git a/get_sustainability_tweets.ipynb b/get_sustainability_tweets.ipynb new file mode 100644 index 0000000..b2f375c --- /dev/null +++ b/get_sustainability_tweets.ipynb @@ -0,0 +1,460 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 16, + "id": "f80d3dd8-29a4-4ecf-a1d9-d691ab23c71e", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
rootmean_rankingstd_rankingword_matches
0challeng406248['challenges', 'challenge', 'challenging']
1SDG4230['sdgs']
2ESG6350['esg']
3recycling848171['recycling', 'shiprecycling']
4CSR13450['csr']
5reduc13591199['reduce', 'reducing', 'reduced', 'reduction',...
6planet13900['planet']
7sustainab15161825['sustainability', 'sustainable', 'sustainable...
8future16261540['future', 'futureproofing']
9clean1709859['theoceancleanup', 'cleanup', 'clean', 'clean...
10methanol17551110['methanol', 'emethanol']
11garbage1772511['garbage', 'greatpacificgarbagepatch']
12responsib18401093['responsible', 'responsibility', 'responsibly...
13carb18481396['decarbonisation', 'carbon', 'decarbonization...
14chang20541563['change', 'changing', 'climatechange', 'chang...
15ocean22171657['ocean', 'theoceancleanup', 'oceans', 'oceanp...
16plastic23541587['plastic', 'oceanplastic', 'plasticwaste', 'p...
17neutral23841886['neutral', 'carbonneutral', 'neutrality', 'co...
18environment25371736['environment', 'environmental', 'unenvironmen...
19sulphur25721524['sulphur', 'lowsulphur']
20green25801376['green', 'greenfuels', 'greener', 'greenfuel'...
21emissions29122065['emissions', 'carbonemissions', 'zeroemissions']
22climate29431978['climateaction', 'climate', 'climatechange', ...
23zero29721670['zero', 'netzero', 'zerocarbon', 'zerocarbons...
24eco31281228['ecosystem', 'eco', 'maerskecodelivery', 'eco...
25mission32451937['emissions', 'mission', 'carbonemissions', 'e...
26CO232792165['co2', 'co2emission', 'co2neutral']
27bio35791252['biofuel', 'biofuels', 'biodiversity', 'biohu...
\n", + "
" + ], + "text/plain": [ + " root mean_ranking std_ranking \\\n", + "0 challeng 406 248 \n", + "1 SDG 423 0 \n", + "2 ESG 635 0 \n", + "3 recycling 848 171 \n", + "4 CSR 1345 0 \n", + "5 reduc 1359 1199 \n", + "6 planet 1390 0 \n", + "7 sustainab 1516 1825 \n", + "8 future 1626 1540 \n", + "9 clean 1709 859 \n", + "10 methanol 1755 1110 \n", + "11 garbage 1772 511 \n", + "12 responsib 1840 1093 \n", + "13 carb 1848 1396 \n", + "14 chang 2054 1563 \n", + "15 ocean 2217 1657 \n", + "16 plastic 2354 1587 \n", + "17 neutral 2384 1886 \n", + "18 environment 2537 1736 \n", + "19 sulphur 2572 1524 \n", + "20 green 2580 1376 \n", + "21 emissions 2912 2065 \n", + "22 climate 2943 1978 \n", + "23 zero 2972 1670 \n", + "24 eco 3128 1228 \n", + "25 mission 3245 1937 \n", + "26 CO2 3279 2165 \n", + "27 bio 3579 1252 \n", + "\n", + " word_matches \n", + "0 ['challenges', 'challenge', 'challenging'] \n", + "1 ['sdgs'] \n", + "2 ['esg'] \n", + "3 ['recycling', 'shiprecycling'] \n", + "4 ['csr'] \n", + "5 ['reduce', 'reducing', 'reduced', 'reduction',... \n", + "6 ['planet'] \n", + "7 ['sustainability', 'sustainable', 'sustainable... \n", + "8 ['future', 'futureproofing'] \n", + "9 ['theoceancleanup', 'cleanup', 'clean', 'clean... \n", + "10 ['methanol', 'emethanol'] \n", + "11 ['garbage', 'greatpacificgarbagepatch'] \n", + "12 ['responsible', 'responsibility', 'responsibly... \n", + "13 ['decarbonisation', 'carbon', 'decarbonization... \n", + "14 ['change', 'changing', 'climatechange', 'chang... \n", + "15 ['ocean', 'theoceancleanup', 'oceans', 'oceanp... \n", + "16 ['plastic', 'oceanplastic', 'plasticwaste', 'p... \n", + "17 ['neutral', 'carbonneutral', 'neutrality', 'co... \n", + "18 ['environment', 'environmental', 'unenvironmen... \n", + "19 ['sulphur', 'lowsulphur'] \n", + "20 ['green', 'greenfuels', 'greener', 'greenfuel'... \n", + "21 ['emissions', 'carbonemissions', 'zeroemissions'] \n", + "22 ['climateaction', 'climate', 'climatechange', ... \n", + "23 ['zero', 'netzero', 'zerocarbon', 'zerocarbons... \n", + "24 ['ecosystem', 'eco', 'maerskecodelivery', 'eco... \n", + "25 ['emissions', 'mission', 'carbonemissions', 'e... \n", + "26 ['co2', 'co2emission', 'co2neutral'] \n", + "27 ['biofuel', 'biofuels', 'biodiversity', 'biohu... " + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# load words related to sustainability\n", + "import pandas as pd\n", + "\n", + "path = 'word_root_ranking.csv'\n", + "df = pd.read_csv(path, index_col=0)\n", + "\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "075f141b-16cd-4577-aa5d-97948fda44ee", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['challenges',\n", + " 'challenge',\n", + " 'challenging',\n", + " 'sdgs',\n", + " 'esg',\n", + " 'recycling',\n", + " 'shiprecycling',\n", + " 'csr',\n", + " 'reduce',\n", + " 'reducing']" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# collect words into list\n", + "\n", + "related_words_list = list()\n", + "for row in df['word_matches']:\n", + " word_list = row.strip('][').replace(\"'\", '').split(', ')\n", + " for word in word_list:\n", + " related_words_list.append(word)\n", + " \n", + "related_words_list[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "5f86e61e-8d79-4aa9-871c-f11c62b12293", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "got 1513 sustainability-related tweets\n" + ] + } + ], + "source": [ + "# fetch all sustainability-related tweets\n", + "from classes import Tweet\n", + "from database import connect as connect_db\n", + "\n", + "db = connect_db()\n", + "\n", + "# generate database cursor\n", + "cursor = db.find({'account': '@Maersk'})\n", + "\n", + "# fetch tweets\n", + "tweet_list = list()\n", + "for element in cursor:\n", + " try:\n", + " elem_text = element['text'].lower()\n", + " related_words_present = any([word in elem_text for word in related_words_list])\n", + " if not related_words_present:\n", + " continue\n", + " tweet_list.append(Tweet(**element))\n", + " except:\n", + " print(f\"failed getting {element['_id']}\")\n", + " \n", + "print(f'got {len(tweet_list)} sustainability-related tweets')" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "0fda59a4-9bcd-4f66-9b68-2070b990ba08", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# prepare output folders\n", + "from pathlib import Path\n", + "tweet_dir = Path('tweets').resolve()\n", + "tweet_dir.mkdir(exist_ok=True)\n", + "media_dir = tweet_dir / 'media'\n", + "media_dir.mkdir(exist_ok=True)\n", + "\n", + "# save tweets\n", + "for tweet in tweet_list:\n", + " time_str = tweet.time.strftime('%Y-%m-%d-%H-%M')\n", + " path = tweet_dir / f\"{time_str}.txt\"\n", + " media_name_list = [f'{time_str}_img{i+1}.png' for i in range(len(tweet.images))]\n", + " # save tweet content\n", + " with open(path, 'w') as f:\n", + " f.write(f'account: {tweet.account}\\n')\n", + " f.write(f'url: {tweet.url}\\n')\n", + " f.write(f'text: {tweet.text}\\n')\n", + " f.write(f\"images: {','.join(media_name_list)}\\n\")\n", + " f.write(f'video: {tweet.video}')\n", + " # save images\n", + " for image, filename in zip(tweet.images, media_name_list):\n", + " image.save(media_dir / filename)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2fe8c01-e2fd-4d01-9498-613b8b4d1884", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}