solveig_master/get_sustainability_tweets.ipynb

In [1]:

# load words related to sustainability
import pandas as pd

path = 'word_root_ranking.csv'
df = pd.read_csv(path, index_col=0)

df

Out [1]:

	root	mean_ranking	std_ranking	word_matches
0	challeng	420	264	['challenges', 'challenge', 'challenging']
1	SDG	423	0	['sdgs']
2	ESG	636	0	['esg']
3	recycling	860	154	['recycling', 'shiprecycling']
4	reduc	1242	965	['reduce', 'reducing', 'reduced', 'reduction',...
5	planet	1295	0	['planet']
6	CSR	1371	0	['csr']
7	sustainab	1527	1873	['sustainability', 'sustainable', 'sustainable...
8	clean	1695	837	['theoceancleanup', 'cleanup', 'clean', 'clean...
9	methanol	1700	1044	['methanol', 'emethanol']
10	future	1770	1684	['future', 'futureproofing']
11	garbage	1808	612	['garbage', 'greatpacificgarbagepatch']
12	responsib	1889	1135	['responsible', 'responsibility', 'responsibly...
13	carb	1952	1526	['decarbonisation', 'carbon', 'decarbonization...
14	chang	2148	1667	['change', 'changing', 'climatechange', 'chang...
15	ocean	2169	1629	['ocean', 'theoceancleanup', 'oceans', 'oceanp...
16	plastic	2320	1490	['plastic', 'oceanplastic', 'plasticwaste', 'p...
17	neutral	2382	1798	['neutral', 'carbonneutral', 'neutrality', 'co...
18	environment	2391	1574	['environment', 'environmental', 'unenvironmen...
19	green	2393	1158	['green', 'greenfuels', 'greener', 'greenfuel'...
20	sulphur	2828	1772	['sulphur', 'lowsulphur']
21	emissions	2925	2046	['emissions', 'carbonemissions', 'zeroemissions']
22	zero	2977	1646	['zero', 'netzero', 'zerocarbon', 'zerocarbons...
23	climate	3046	2095	['climateaction', 'climate', 'climatechange', ...
24	eco	3074	1269	['ecosystem', 'eco', 'maerskecodelivery', 'eco...
25	mission	3247	1950	['emissions', 'mission', 'eucommission', 'carb...
26	CO2	3390	2275	['co2', 'co2neutral', 'co2emission']
27	bio	3687	1274	['biofuel', 'biofuels', 'biohuts', 'biodiversi...

In [2]:

# collect words into list

related_words_list = list()
for row in df['word_matches']:
    word_list = row.strip('][').replace("'", '').split(', ')
    for word in word_list:
        related_words_list.append(word)
        
related_words_list[:10]

Out [2]:

['challenges',
 'challenge',
 'challenging',
 'sdgs',
 'esg',
 'recycling',
 'shiprecycling',
 'reduce',
 'reducing',
 'reduced']

In [3]:

# fetch all sustainability-related tweets
from classes import Tweet
from database import connect as connect_db

db = connect_db()

# generate database cursor
cursor = db.find({'account': '@Maersk'})

# fetch tweets
tweet_list = list()
for element in cursor:
    try:
        elem_text = element['text'].lower()
        related_words_present = any([word in elem_text for word in related_words_list])
        if not related_words_present:
            continue
        tweet_list.append(Tweet(**element))
    except:
        print(f"failed getting {element['_id']}")
        
print(f'got {len(tweet_list)} sustainability-related tweets')

got 1513 sustainability-related tweets

In [4]:

# prepare output folders
from pathlib import Path
tweet_dir = Path('tweets').resolve()
tweet_dir.mkdir(exist_ok=True)
media_dir = tweet_dir / 'media'
media_dir.mkdir(exist_ok=True)

# save tweets
for tweet in tweet_list:
    time_str = tweet.time.strftime('%Y-%m-%d-%H-%M')
    path = tweet_dir / f"{time_str}.txt"
    media_name_list = [f'{time_str}_img{i+1}.png' for i in range(len(tweet.images))]
    # save tweet content
    with open(path, 'w') as f:
        f.write(f'account: {tweet.account}\n')
        f.write(f'url: {tweet.url}\n')
        f.write(f'text: {tweet.text}\n')
        f.write(f"images: {','.join(media_name_list)}\n")
        f.write(f'video: {tweet.video}')
    # save images
    for image, filename in zip(tweet.images, media_name_list):
        image.save(media_dir / filename)

In [ ]: