diff --git a/sustainability_word_list.ipynb b/sustainability_word_list.ipynb new file mode 100644 index 0000000..11859e3 --- /dev/null +++ b/sustainability_word_list.ipynb @@ -0,0 +1,918 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 72, + "id": "0c875ce0-80ad-42f3-af90-9bb1c1e53858", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['CO2', 'CSR', 'ESG', 'SDG', 'bio', 'carb', 'challeng', 'chang', 'clean', 'climate', 'eco', 'emissions', 'environment', 'future', 'garbage', 'green', 'methanol', 'mission', 'neutral', 'ocean', 'planet', 'plastic', 'recycling', 'reduc', 'responsib', 'sulphur', 'sustainab', 'zero']\n" + ] + } + ], + "source": [ + "# load list of words related to sustainability\n", + "path = 'sustainability_words.txt'\n", + "word_list = list()\n", + "with open(path, 'r') as f:\n", + " for line in f.readlines():\n", + " word_list.append(line.replace('\\n', ''))\n", + "\n", + "word_list.sort()\n", + "print(word_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "id": "99837ffc-0eb7-4e89-87e5-eedbe35219bc", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
wordcountsratio
0maersk32070.04001
1is10250.01279
2more9890.01234
3here8250.01029
4trade6420.00801
............
10920phase10.00001
10921agrichemicals10.00001
10922polluting10.00001
10923adams10.00001
10924maerskcapitalmarketsday10.00001
\n", + "

10925 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " word counts ratio\n", + "0 maersk 3207 0.04001\n", + "1 is 1025 0.01279\n", + "2 more 989 0.01234\n", + "3 here 825 0.01029\n", + "4 trade 642 0.00801\n", + "... ... ... ...\n", + "10920 phase 1 0.00001\n", + "10921 agrichemicals 1 0.00001\n", + "10922 polluting 1 0.00001\n", + "10923 adams 1 0.00001\n", + "10924 maerskcapitalmarketsday 1 0.00001\n", + "\n", + "[10925 rows x 3 columns]" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# load word counts from tweets\n", + "import pandas as pd\n", + "\n", + "word_df = pd.read_csv('word_count.csv', index_col=0)\n", + "\n", + "word_df" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "id": "57e749a9-3401-4cb5-98b0-ad68098bf5b4", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['become', 'becomes', 'becoming', 'blueeconomy', 'changed', 'chinaeconomy', 'deconsolidation', 'decorated', 'decoupling', 'ecommerce', 'ecommercelogistics', 'economia', 'economic', 'economically', 'economicgrowth', 'economictimes', 'economies', 'economist', 'economistimpact', 'economy', 'environments', 'fairfuture4seafarers', 'mclean', 'portrecord', 'recognise', 'recognised', 'recognising', 'recognition', 'recognized', 'record', 'recorded', 'records', 'recovery', 'second', 'secondbusiest', 'secondgeneration', 'secondlargest', 'seconds', 'shetradesglobal', 'socioeconomic', 'telecoms', 'theeconomist', 'transoceanic', 'unchanged', 'worldrecord']\n" + ] + } + ], + "source": [ + "# load ignore list\n", + "ignore_list = list()\n", + "path = 'ignore_words.txt'\n", + "with open(path, 'r') as f:\n", + " for line in f.readlines():\n", + " ignore_list.append(line.replace('\\n', ''))\n", + " \n", + "ignore_list.sort()\n", + "print(ignore_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "id": "f2f37c6b-cb1f-4c71-aa24-9601d5bfe7b8", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CO2\n", + "['co2', 'co2emission', 'co2neutral']\n", + "3279.0\n", + "2165.0\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "\n", + "match_word_list = list()\n", + "mean_idx_list = list()\n", + "sigma_idx_list = list()\n", + "for i, word in enumerate(word_list):\n", + " # find index of matching words\n", + " contains_word_mask = word_df['word'].str.contains(word.lower())==True\n", + " counts_above_limit_mask = word_df['counts'].gt(1)\n", + " match_mask = contains_word_mask & counts_above_limit_mask\n", + " match_idx_list = word_df[match_mask].index.tolist()\n", + " # remove indices of unwanted words\n", + " remove_idx_list = list()\n", + " for idx in match_idx_list:\n", + " match_word = word_df.loc[idx, 'word']\n", + " if match_word in ignore_list:\n", + " remove_idx_list.append(idx)\n", + " for idx in remove_idx_list:\n", + " match_idx_list.remove(idx)\n", + " # store matched words\n", + " match_word_list.append(list())\n", + " for idx in match_idx_list:\n", + " match_word = word_df.loc[idx, 'word']\n", + " match_word_list[i].append(match_word)\n", + " # calculate mean and std of word ranking from located indices\n", + " mu = np.array(match_idx_list).mean().round()\n", + " mean_idx_list.append(mu)\n", + " sigma = np.array(match_idx_list).std().round()\n", + " sigma_idx_list.append(sigma)\n", + "\n", + "# show example of results\n", + "idx = 0\n", + "print(word_list[idx])\n", + "print(match_word_list[idx])\n", + "print(mean_idx_list[idx])\n", + "print(sigma_idx_list[idx])" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "id": "f825653d-6c43-46bc-bc11-f9b8f4de5414", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
rootmean_rankingstd_rankingword_matches
0CO23279.02165.0[co2, co2emission, co2neutral]
1CSR1345.00.0[csr]
2ESG635.00.0[esg]
3SDG423.00.0[sdgs]
4bio3579.01252.0[biofuel, biofuels, biodiversity, biohuts]
5carb1848.01396.0[decarbonisation, carbon, decarbonization, car...
6challeng406.0248.0[challenges, challenge, challenging]
7chang2054.01563.0[change, changing, climatechange, changes, exc...
8clean1709.0859.0[theoceancleanup, cleanup, clean, cleanseas, o...
9climate2943.01978.0[climateaction, climate, climatechange, climat...
10eco3128.01228.0[ecosystem, eco, maerskecodelivery, ecofriendl...
11emissions2912.02065.0[emissions, carbonemissions, zeroemissions]
12environment2537.01736.0[environment, environmental, unenvironment, en...
13future1626.01540.0[future, futureproofing]
14garbage1772.0511.0[garbage, greatpacificgarbagepatch]
15green2580.01376.0[green, greenfuels, greener, greenfuel, greenl...
16methanol1755.01110.0[methanol, emethanol]
17mission3245.01937.0[emissions, mission, carbonemissions, eucommis...
18neutral2384.01886.0[neutral, carbonneutral, neutrality, co2neutral]
19ocean2217.01657.0[ocean, theoceancleanup, oceans, oceanplastic,...
20planet1390.00.0[planet]
21plastic2354.01587.0[plastic, oceanplastic, plasticwaste, plasticp...
22recycling848.0171.0[recycling, shiprecycling]
23reduc1359.01199.0[reduce, reducing, reduced, reduction, reducti...
24responsib1840.01093.0[responsible, responsibility, responsibly, res...
25sulphur2572.01524.0[sulphur, lowsulphur]
26sustainab1516.01825.0[sustainability, sustainable, sustainableshipp...
27zero2972.01670.0[zero, netzero, zerocarbon, zerocarbonship, ze...
\n", + "
" + ], + "text/plain": [ + " root mean_ranking std_ranking \\\n", + "0 CO2 3279.0 2165.0 \n", + "1 CSR 1345.0 0.0 \n", + "2 ESG 635.0 0.0 \n", + "3 SDG 423.0 0.0 \n", + "4 bio 3579.0 1252.0 \n", + "5 carb 1848.0 1396.0 \n", + "6 challeng 406.0 248.0 \n", + "7 chang 2054.0 1563.0 \n", + "8 clean 1709.0 859.0 \n", + "9 climate 2943.0 1978.0 \n", + "10 eco 3128.0 1228.0 \n", + "11 emissions 2912.0 2065.0 \n", + "12 environment 2537.0 1736.0 \n", + "13 future 1626.0 1540.0 \n", + "14 garbage 1772.0 511.0 \n", + "15 green 2580.0 1376.0 \n", + "16 methanol 1755.0 1110.0 \n", + "17 mission 3245.0 1937.0 \n", + "18 neutral 2384.0 1886.0 \n", + "19 ocean 2217.0 1657.0 \n", + "20 planet 1390.0 0.0 \n", + "21 plastic 2354.0 1587.0 \n", + "22 recycling 848.0 171.0 \n", + "23 reduc 1359.0 1199.0 \n", + "24 responsib 1840.0 1093.0 \n", + "25 sulphur 2572.0 1524.0 \n", + "26 sustainab 1516.0 1825.0 \n", + "27 zero 2972.0 1670.0 \n", + "\n", + " word_matches \n", + "0 [co2, co2emission, co2neutral] \n", + "1 [csr] \n", + "2 [esg] \n", + "3 [sdgs] \n", + "4 [biofuel, biofuels, biodiversity, biohuts] \n", + "5 [decarbonisation, carbon, decarbonization, car... \n", + "6 [challenges, challenge, challenging] \n", + "7 [change, changing, climatechange, changes, exc... \n", + "8 [theoceancleanup, cleanup, clean, cleanseas, o... \n", + "9 [climateaction, climate, climatechange, climat... \n", + "10 [ecosystem, eco, maerskecodelivery, ecofriendl... \n", + "11 [emissions, carbonemissions, zeroemissions] \n", + "12 [environment, environmental, unenvironment, en... \n", + "13 [future, futureproofing] \n", + "14 [garbage, greatpacificgarbagepatch] \n", + "15 [green, greenfuels, greener, greenfuel, greenl... \n", + "16 [methanol, emethanol] \n", + "17 [emissions, mission, carbonemissions, eucommis... \n", + "18 [neutral, carbonneutral, neutrality, co2neutral] \n", + "19 [ocean, theoceancleanup, oceans, oceanplastic,... \n", + "20 [planet] \n", + "21 [plastic, oceanplastic, plasticwaste, plasticp... \n", + "22 [recycling, shiprecycling] \n", + "23 [reduce, reducing, reduced, reduction, reducti... \n", + "24 [responsible, responsibility, responsibly, res... \n", + "25 [sulphur, lowsulphur] \n", + "26 [sustainability, sustainable, sustainableshipp... \n", + "27 [zero, netzero, zerocarbon, zerocarbonship, ze... " + ] + }, + "execution_count": 77, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.DataFrame({\n", + " 'root': word_list,\n", + " 'mean_ranking': mean_idx_list,\n", + " 'std_ranking': sigma_idx_list,\n", + " 'word_matches': match_word_list\n", + "})\n", + "\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "id": "c4ea948f-66f6-405d-8b35-b390b1b3e4a7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
rootmean_rankingstd_rankingword_matches
0challeng406.0248.0[challenges, challenge, challenging]
1SDG423.00.0[sdgs]
2ESG635.00.0[esg]
3recycling848.0171.0[recycling, shiprecycling]
4CSR1345.00.0[csr]
5reduc1359.01199.0[reduce, reducing, reduced, reduction, reducti...
6planet1390.00.0[planet]
7sustainab1516.01825.0[sustainability, sustainable, sustainableshipp...
8future1626.01540.0[future, futureproofing]
9clean1709.0859.0[theoceancleanup, cleanup, clean, cleanseas, o...
10methanol1755.01110.0[methanol, emethanol]
11garbage1772.0511.0[garbage, greatpacificgarbagepatch]
12responsib1840.01093.0[responsible, responsibility, responsibly, res...
13carb1848.01396.0[decarbonisation, carbon, decarbonization, car...
14chang2054.01563.0[change, changing, climatechange, changes, exc...
15ocean2217.01657.0[ocean, theoceancleanup, oceans, oceanplastic,...
16plastic2354.01587.0[plastic, oceanplastic, plasticwaste, plasticp...
17neutral2384.01886.0[neutral, carbonneutral, neutrality, co2neutral]
18environment2537.01736.0[environment, environmental, unenvironment, en...
19sulphur2572.01524.0[sulphur, lowsulphur]
20green2580.01376.0[green, greenfuels, greener, greenfuel, greenl...
21emissions2912.02065.0[emissions, carbonemissions, zeroemissions]
22climate2943.01978.0[climateaction, climate, climatechange, climat...
23zero2972.01670.0[zero, netzero, zerocarbon, zerocarbonship, ze...
24eco3128.01228.0[ecosystem, eco, maerskecodelivery, ecofriendl...
25mission3245.01937.0[emissions, mission, carbonemissions, eucommis...
26CO23279.02165.0[co2, co2emission, co2neutral]
27bio3579.01252.0[biofuel, biofuels, biodiversity, biohuts]
\n", + "
" + ], + "text/plain": [ + " root mean_ranking std_ranking \\\n", + "0 challeng 406.0 248.0 \n", + "1 SDG 423.0 0.0 \n", + "2 ESG 635.0 0.0 \n", + "3 recycling 848.0 171.0 \n", + "4 CSR 1345.0 0.0 \n", + "5 reduc 1359.0 1199.0 \n", + "6 planet 1390.0 0.0 \n", + "7 sustainab 1516.0 1825.0 \n", + "8 future 1626.0 1540.0 \n", + "9 clean 1709.0 859.0 \n", + "10 methanol 1755.0 1110.0 \n", + "11 garbage 1772.0 511.0 \n", + "12 responsib 1840.0 1093.0 \n", + "13 carb 1848.0 1396.0 \n", + "14 chang 2054.0 1563.0 \n", + "15 ocean 2217.0 1657.0 \n", + "16 plastic 2354.0 1587.0 \n", + "17 neutral 2384.0 1886.0 \n", + "18 environment 2537.0 1736.0 \n", + "19 sulphur 2572.0 1524.0 \n", + "20 green 2580.0 1376.0 \n", + "21 emissions 2912.0 2065.0 \n", + "22 climate 2943.0 1978.0 \n", + "23 zero 2972.0 1670.0 \n", + "24 eco 3128.0 1228.0 \n", + "25 mission 3245.0 1937.0 \n", + "26 CO2 3279.0 2165.0 \n", + "27 bio 3579.0 1252.0 \n", + "\n", + " word_matches \n", + "0 [challenges, challenge, challenging] \n", + "1 [sdgs] \n", + "2 [esg] \n", + "3 [recycling, shiprecycling] \n", + "4 [csr] \n", + "5 [reduce, reducing, reduced, reduction, reducti... \n", + "6 [planet] \n", + "7 [sustainability, sustainable, sustainableshipp... \n", + "8 [future, futureproofing] \n", + "9 [theoceancleanup, cleanup, clean, cleanseas, o... \n", + "10 [methanol, emethanol] \n", + "11 [garbage, greatpacificgarbagepatch] \n", + "12 [responsible, responsibility, responsibly, res... \n", + "13 [decarbonisation, carbon, decarbonization, car... \n", + "14 [change, changing, climatechange, changes, exc... \n", + "15 [ocean, theoceancleanup, oceans, oceanplastic,... \n", + "16 [plastic, oceanplastic, plasticwaste, plasticp... \n", + "17 [neutral, carbonneutral, neutrality, co2neutral] \n", + "18 [environment, environmental, unenvironment, en... \n", + "19 [sulphur, lowsulphur] \n", + "20 [green, greenfuels, greener, greenfuel, greenl... \n", + "21 [emissions, carbonemissions, zeroemissions] \n", + "22 [climateaction, climate, climatechange, climat... \n", + "23 [zero, netzero, zerocarbon, zerocarbonship, ze... \n", + "24 [ecosystem, eco, maerskecodelivery, ecofriendl... \n", + "25 [emissions, mission, carbonemissions, eucommis... \n", + "26 [co2, co2emission, co2neutral] \n", + "27 [biofuel, biofuels, biodiversity, biohuts] " + ] + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# sort values by ranking\n", + "df = df.sort_values(by='mean_ranking').reset_index(drop=True)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "id": "7b3dbcf3-2d95-4a5e-b795-cea63da2a34a", + "metadata": {}, + "outputs": [], + "source": [ + "# save results\n", + "df['mean_ranking'] = df['mean_ranking'].astype(int)\n", + "df['std_ranking'] = df['std_ranking'].astype(int)\n", + "df.to_csv('word_root_ranking.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ef515886-a918-49a0-b9df-7f20bfe4cea4", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}