solveig_master/rank_relevant_tweets.ipynb

In [1]:

# load words related to sustainability
import pandas as pd

path = 'word_root_ranking.csv'
df = pd.read_csv(path, index_col=0)

df

Out [1]:

	root	mean_ranking	std_ranking	word_matches
0	challeng	420	264	['challenges', 'challenge', 'challenging']
1	SDG	423	0	['sdgs']
2	ESG	636	0	['esg']
3	recycling	860	154	['recycling', 'shiprecycling']
4	reduc	1242	965	['reduce', 'reducing', 'reduced', 'reduction',...
5	planet	1295	0	['planet']
6	CSR	1371	0	['csr']
7	sustainab	1527	1873	['sustainability', 'sustainable', 'sustainable...
8	clean	1695	837	['theoceancleanup', 'cleanup', 'clean', 'clean...
9	methanol	1700	1044	['methanol', 'emethanol']
10	future	1770	1684	['future', 'futureproofing']
11	garbage	1808	612	['garbage', 'greatpacificgarbagepatch']
12	responsib	1889	1135	['responsible', 'responsibility', 'responsibly...
13	carb	1952	1526	['decarbonisation', 'carbon', 'decarbonization...
14	chang	2148	1667	['change', 'changing', 'climatechange', 'chang...
15	ocean	2169	1629	['ocean', 'theoceancleanup', 'oceans', 'oceanp...
16	plastic	2320	1490	['plastic', 'oceanplastic', 'plasticwaste', 'p...
17	neutral	2382	1798	['neutral', 'carbonneutral', 'neutrality', 'co...
18	environment	2391	1574	['environment', 'environmental', 'unenvironmen...
19	green	2393	1158	['green', 'greenfuels', 'greener', 'greenfuel'...
20	sulphur	2828	1772	['sulphur', 'lowsulphur']
21	emissions	2925	2046	['emissions', 'carbonemissions', 'zeroemissions']
22	zero	2977	1646	['zero', 'netzero', 'zerocarbon', 'zerocarbons...
23	climate	3046	2095	['climateaction', 'climate', 'climatechange', ...
24	eco	3074	1269	['ecosystem', 'eco', 'maerskecodelivery', 'eco...
25	mission	3247	1950	['emissions', 'mission', 'eucommission', 'carb...
26	CO2	3390	2275	['co2', 'co2neutral', 'co2emission']
27	bio	3687	1274	['biofuel', 'biofuels', 'biohuts', 'biodiversi...

In [2]:

# collect words into list

related_words_list = list()
for row in df['word_matches']:
    word_list = row.strip('][').replace("'", '').split(', ')
    for word in word_list:
        related_words_list.append(word)
        
related_words_list[:10]

Out [2]:

['challenges',
 'challenge',
 'challenging',
 'sdgs',
 'esg',
 'recycling',
 'shiprecycling',
 'reduce',
 'reducing',
 'reduced']

In [19]:

# load in all tweet texts
from pathlib import Path
import re

tweet_dir = Path('tweets').resolve()
path_list = list(tweet_dir.glob('*.txt'))
score_list = list()
for path in path_list:
    with open(path, 'r') as f:
        text = f.read()
    text = re.search(
        pattern='(?<=text: )(.|\n)*(?=images: )',
        string=text
    ).group(0)
    score = sum([related_word in text for related_word in related_words_list])
    score_list.append(score)

len(score_list)

Out [19]:

1511

In [20]:

max(score_list)

Out [20]:

15

In [25]:

df = pd.DataFrame(
    {
        'path': path_list,
        'score': score_list
    }
)

df = df.sort_values(by='score', ascending=False)
df.reset_index(drop=True, inplace=True)
df

Out [25]:

	path	score
0	/home/brian/solveig_master/tweets/2021-08-24-0...	15
1	/home/brian/solveig_master/tweets/2021-06-22-1...	14
2	/home/brian/solveig_master/tweets/2020-07-13-1...	14
3	/home/brian/solveig_master/tweets/2022-04-28-1...	13
4	/home/brian/solveig_master/tweets/2022-10-20-1...	13
...	...	...
1506	/home/brian/solveig_master/tweets/2022-08-31-1...	0
1507	/home/brian/solveig_master/tweets/2022-12-17-0...	0
1508	/home/brian/solveig_master/tweets/2021-09-24-0...	0
1509	/home/brian/solveig_master/tweets/2014-02-06-1...	0
1510	/home/brian/solveig_master/tweets/2017-10-12-1...	0

1511 rows × 2 columns

In [27]:

df.to_csv('tweets_relevance_list.csv')

In [ ]: