solveig_master/get_sustainability_tweets.ipynb

In [16]:

# load words related to sustainability
import pandas as pd

path = 'word_root_ranking.csv'
df = pd.read_csv(path, index_col=0)

df

Out [16]:

	root	mean_ranking	std_ranking	word_matches
0	challeng	406	248	['challenges', 'challenge', 'challenging']
1	SDG	423	0	['sdgs']
2	ESG	635	0	['esg']
3	recycling	848	171	['recycling', 'shiprecycling']
4	CSR	1345	0	['csr']
5	reduc	1359	1199	['reduce', 'reducing', 'reduced', 'reduction',...
6	planet	1390	0	['planet']
7	sustainab	1516	1825	['sustainability', 'sustainable', 'sustainable...
8	future	1626	1540	['future', 'futureproofing']
9	clean	1709	859	['theoceancleanup', 'cleanup', 'clean', 'clean...
10	methanol	1755	1110	['methanol', 'emethanol']
11	garbage	1772	511	['garbage', 'greatpacificgarbagepatch']
12	responsib	1840	1093	['responsible', 'responsibility', 'responsibly...
13	carb	1848	1396	['decarbonisation', 'carbon', 'decarbonization...
14	chang	2054	1563	['change', 'changing', 'climatechange', 'chang...
15	ocean	2217	1657	['ocean', 'theoceancleanup', 'oceans', 'oceanp...
16	plastic	2354	1587	['plastic', 'oceanplastic', 'plasticwaste', 'p...
17	neutral	2384	1886	['neutral', 'carbonneutral', 'neutrality', 'co...
18	environment	2537	1736	['environment', 'environmental', 'unenvironmen...
19	sulphur	2572	1524	['sulphur', 'lowsulphur']
20	green	2580	1376	['green', 'greenfuels', 'greener', 'greenfuel'...
21	emissions	2912	2065	['emissions', 'carbonemissions', 'zeroemissions']
22	climate	2943	1978	['climateaction', 'climate', 'climatechange', ...
23	zero	2972	1670	['zero', 'netzero', 'zerocarbon', 'zerocarbons...
24	eco	3128	1228	['ecosystem', 'eco', 'maerskecodelivery', 'eco...
25	mission	3245	1937	['emissions', 'mission', 'carbonemissions', 'e...
26	CO2	3279	2165	['co2', 'co2emission', 'co2neutral']
27	bio	3579	1252	['biofuel', 'biofuels', 'biodiversity', 'biohu...

In [17]:

# collect words into list

related_words_list = list()
for row in df['word_matches']:
    word_list = row.strip('][').replace("'", '').split(', ')
    for word in word_list:
        related_words_list.append(word)
        
related_words_list[:10]

Out [17]:

['challenges',
 'challenge',
 'challenging',
 'sdgs',
 'esg',
 'recycling',
 'shiprecycling',
 'csr',
 'reduce',
 'reducing']

In [31]:

# fetch all sustainability-related tweets
from classes import Tweet
from database import connect as connect_db

db = connect_db()

# generate database cursor
cursor = db.find({'account': '@Maersk'})

# fetch tweets
tweet_list = list()
for element in cursor:
    try:
        elem_text = element['text'].lower()
        related_words_present = any([word in elem_text for word in related_words_list])
        if not related_words_present:
            continue
        tweet_list.append(Tweet(**element))
    except:
        print(f"failed getting {element['_id']}")
        
print(f'got {len(tweet_list)} sustainability-related tweets')

got 1513 sustainability-related tweets

In [55]:

# prepare output folders
from pathlib import Path
tweet_dir = Path('tweets').resolve()
tweet_dir.mkdir(exist_ok=True)
media_dir = tweet_dir / 'media'
media_dir.mkdir(exist_ok=True)

# save tweets
for tweet in tweet_list:
    time_str = tweet.time.strftime('%Y-%m-%d-%H-%M')
    path = tweet_dir / f"{time_str}.txt"
    media_name_list = [f'{time_str}_img{i+1}.png' for i in range(len(tweet.images))]
    # save tweet content
    with open(path, 'w') as f:
        f.write(f'account: {tweet.account}\n')
        f.write(f'url: {tweet.url}\n')
        f.write(f'text: {tweet.text}\n')
        f.write(f"images: {','.join(media_name_list)}\n")
        f.write(f'video: {tweet.video}')
    # save images
    for image, filename in zip(tweet.images, media_name_list):
        image.save(media_dir / filename)

In [ ]: