solveig_master/root_word_frequency_plot.ipynb

In [1]:

# load in data
import pandas as pd
from pathlib import Path

tweet_df = pd.read_csv('tweets_relevance_list.csv', index_col=0)

# remove rows with score of zero
tweet_df = tweet_df[tweet_df['score'].gt(0)]

# remove wrong folder
tweet_df['path'] = tweet_df['path'].str.replace('/home/brian/solveig_master/', '')

# ensure datatype
tweet_df['path'] = tweet_df['path'].apply(Path)

tweet_df

Out [1]:

	path	score
0	tweets/2021-08-24-07-40.txt	15
1	tweets/2021-06-22-13-19.txt	14
2	tweets/2020-07-13-10-03.txt	14
3	tweets/2022-04-28-12-38.txt	13
4	tweets/2022-10-20-14-08.txt	13
...	...	...
1331	tweets/2016-11-08-13-35.txt	1
1332	tweets/2022-06-16-12-15.txt	1
1333	tweets/2016-05-12-12-48.txt	1
1334	tweets/2016-06-01-11-03.txt	1
1335	tweets/2014-01-28-08-50.txt	1

1336 rows × 2 columns

In [2]:

# load in word root list
word_root_df = pd.read_csv('word_root_ranking.csv', index_col=0)

# parse string-lists
word_root_df['word_matches'] = word_root_df['word_matches'].str.strip("[]'").str.split("', '")
# word_list = sum(word_list, []) # flatten nested list

word_root_df

Out [2]:

	root	mean_ranking	std_ranking	word_matches
0	challeng	420	264	[challenges, challenge, challenging]
1	SDG	423	0	[sdgs]
2	ESG	636	0	[esg]
3	recycling	860	154	[recycling, shiprecycling]
4	reduc	1242	965	[reduce, reducing, reduced, reduction, reducti...
5	planet	1295	0	[planet]
6	CSR	1371	0	[csr]
7	sustainab	1527	1873	[sustainability, sustainable, sustainableshipp...
8	clean	1695	837	[theoceancleanup, cleanup, clean, cleanseas, o...
9	methanol	1700	1044	[methanol, emethanol]
10	future	1770	1684	[future, futureproofing]
11	garbage	1808	612	[garbage, greatpacificgarbagepatch]
12	responsib	1889	1135	[responsible, responsibility, responsibly, res...
13	carb	1952	1526	[decarbonisation, carbon, decarbonization, car...
14	chang	2148	1667	[change, changing, climatechange, changes, exc...
15	ocean	2169	1629	[ocean, theoceancleanup, oceans, oceanplastic,...
16	plastic	2320	1490	[plastic, oceanplastic, plasticwaste, plasticp...
17	neutral	2382	1798	[neutral, carbonneutral, neutrality, co2neutral]
18	environment	2391	1574	[environment, environmental, unenvironment, en...
19	green	2393	1158	[green, greenfuels, greener, greenfuel, greent...
20	sulphur	2828	1772	[sulphur, lowsulphur]
21	emissions	2925	2046	[emissions, carbonemissions, zeroemissions]
22	zero	2977	1646	[zero, netzero, zerocarbon, zerocarbonship, ze...
23	climate	3046	2095	[climateaction, climate, climatechange, climat...
24	eco	3074	1269	[ecosystem, eco, maerskecodelivery, ecofriendl...
25	mission	3247	1950	[emissions, mission, eucommission, carbonemiss...
26	CO2	3390	2275	[co2, co2neutral, co2emission]
27	bio	3687	1274	[biofuel, biofuels, biohuts, biodiversity]

In [3]:

# define time bins
period_range = pd.period_range(start='2011-01-01', end='2023-03-31', freq='M')

period_range

Out [3]:

PeriodIndex(['2011-01', '2011-02', '2011-03', '2011-04', '2011-05', '2011-06',
             '2011-07', '2011-08', '2011-09', '2011-10',
             ...
             '2022-06', '2022-07', '2022-08', '2022-09', '2022-10', '2022-11',
             '2022-12', '2023-01', '2023-02', '2023-03'],
            dtype='period[M]', length=147)

In [4]:

# prepare dataframe
df = pd.DataFrame(index=period_range, columns=word_root_df['root'])
df.iloc[:, :] = 0 # set all to 0

df

Out [4]:

root	challeng	SDG	ESG	recycling	reduc	planet	CSR	sustainab	clean	methanol	...	environment	green	sulphur	emissions	zero	climate	eco	mission	CO2	bio
2011-01	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-02	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-03	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-04	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-05	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
2022-11	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2022-12	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2023-01	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2023-02	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2023-03	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0

147 rows × 28 columns

In [5]:

df.index

Out [5]:

PeriodIndex(['2011-01', '2011-02', '2011-03', '2011-04', '2011-05', '2011-06',
             '2011-07', '2011-08', '2011-09', '2011-10',
             ...
             '2022-06', '2022-07', '2022-08', '2022-09', '2022-10', '2022-11',
             '2022-12', '2023-01', '2023-02', '2023-03'],
            dtype='period[M]', length=147)

In [6]:

# count words usage
from datetime import datetime
import re

# path = tweet_df.loc[0, 'path']
# print('path: ', path)
for path in tweet_df['path']:
    # determine period
    ts = datetime.strptime(path.stem, '%Y-%m-%d-%H-%M')
    ts = pd.to_datetime(ts).to_period('M')
    # print('period: ', ts)
    period_mask = ts == df.index
    # load in tweet
    with open(path, 'r') as f:
        text = f.read()
    # extract tweet text
    text = re.search(
        pattern='(?<=text: )(.|\n)*(?=images: )',
        string=text
    ).group(0)
    # determine if words present in text
    for word_root in df.columns[1:]:
        # print('word root: ', word_root)
        word_root_mask = word_root_df['root'].eq(word_root)
        word_matches = word_root_df.loc[word_root_mask, 'word_matches'].to_list()
        word_matches = sum(word_matches, []) # unpack nested list
        # print('word matches: ', word_matches)
        for word in word_matches:
            # print('word: ', word)
            if word in text:
                df.loc[period_mask, word_root] += 1

df

Out [6]:

root	challeng	SDG	ESG	recycling	reduc	planet	CSR	sustainab	clean	methanol	...	environment	green	sulphur	emissions	zero	climate	eco	mission	CO2	bio
2011-01	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-02	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-03	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-04	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-05	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
2022-11	0	0	0	5	0	0	0	5	0	1	...	0	5	0	2	2	5	3	7	0	0
2022-12	0	0	0	0	2	0	0	6	4	3	...	1	3	0	3	0	2	12	11	0	0
2023-01	0	0	1	0	2	0	0	3	0	0	...	0	1	0	2	0	0	3	6	0	0
2023-02	0	0	1	0	0	0	0	4	4	1	...	5	1	0	1	4	0	5	4	0	2
2023-03	0	0	0	1	1	0	0	4	2	1	...	3	5	0	1	0	0	4	3	0	0

147 rows × 28 columns

In [7]:

# plot of combined use of root words over time
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
%matplotlib widget
plt.figure(figsize=(12,8))

time = df.index.to_timestamp()
counts = df.sum(axis=1)
plt.plot(time, counts, label='sustainability-related words')

mean = df.sum(axis=1).rolling(12, min_periods=1).mean()
plt.plot(time, mean, label='12-month mean')
std = df.sum(axis=1).rolling(12, min_periods=1).std()
plt.fill_between(
    x=time,
    y1=mean-std,
    y2=mean+std,
    alpha=0.2,
    label='12-month standard deviation'
)

locator = mdates.MonthLocator(interval=12)
fmt = mdates.DateFormatter('%Y-%m')
x_axis = plt.gca().xaxis
x_axis.set_major_locator(locator)
x_axis.set_major_formatter(fmt)
plt.xticks(rotation=45)
plt.ylabel('count (n)')
plt.legend(loc='upper left')
plt.tight_layout()
sns.despine(left=True)
sns.set_style("ticks",{'axes.grid' : True})
plt.savefig('plots/total_keyword_counts.png')
plt.show()

Above mean + std

In [8]:

# threshold = mean + std
# threshold_mask = counts > threshold
# threshold_mask

# tracked_words_list = list()
# for period in df.index[threshold_mask]:
#     # determine words with counts above mean + std
#     x = df.loc[period, :]
#     x = x[x>0] # ignore words with 0-count
#     threshold = x.mean() + x.std()
#     mask = df.loc[period, :].gt(threshold)
#     label_list = df.columns[mask].tolist()
#     for label in label_list:
#         tracked_words_list.append(label)
# tracked_words_list = list(set(tracked_words_list))
# tracked_words_list

In [9]:

# import seaborn as sns

# #define Seaborn color palette to use
# colors = sns.color_palette('Spectral', n_colors=len(tracked_words_list))
# plt.figure(figsize=(12, 8))

# for i, word in enumerate(tracked_words_list):
#     y = df.loc[:, word]
#     plt.plot(time, y, color=colors[i], label=word, alpha=0.8, linewidth=2)

# # ax = df.loc[:, tracked_words_list].plot(kind='line', layout='tight', figsize=(12,8))

# plt.ylabel('counts (n)')
# plt.xlabel('period')
# plt.legend()
# plt.show()

Below mean - std

In [10]:

# threshold = mean - std
# threshold_mask = counts < threshold
# threshold_mask

# tracked_words_list = list()
# for period in df.index[threshold_mask]:
#     # determine words with counts above mean + std
#     x = df.loc[period, :]
#     x = x[x>0] # ignore words with 0-count
#     threshold = x.mean() + x.std()
#     mask = df.loc[period, :].gt(threshold)
#     label_list = df.columns[mask].tolist()
#     for label in label_list:
#         tracked_words_list.append(label)
# tracked_words_list = list(set(tracked_words_list))
# tracked_words_list

Chosen Intervals

In [11]:

# period_list = [
#     '2015-12',
#     '2016-11',
#     '2018-08',
#     '2018-09',
#     '2021-03',
#     '2021-06',
#     '2021-08',
#     '2022-12'
# ]

# tracked_words_list = list()
# for period in period_list:
#     # determine words with counts above mean + std
#     x = df.loc[period, :]
#     x = x[x>0] # ignore words with 0-count
#     threshold = x.mean() + x.std()
#     mask = df.loc[period, :].gt(threshold)
#     label_list = df.columns[mask].tolist()
#     for label in label_list:
#         tracked_words_list.append(label)
# tracked_words_list = list(set(tracked_words_list))

# print(tracked_words_list)

# import seaborn as sns

# #define Seaborn color palette to use
# colors = sns.color_palette('Spectral', n_colors=len(tracked_words_list))
# plt.figure(figsize=(12, 8))

# for i, word in enumerate(tracked_words_list):
#     y = df.loc[:, word]
#     plt.plot(time, y, color=colors[i], label=word, alpha=0.8, linewidth=2)

# # ax = df.loc[:, tracked_words_list].plot(kind='line', layout='tight', figsize=(12,8))

# plt.ylabel('counts (n)')
# plt.xlabel('period')
# plt.legend()
# plt.show()

Piecharts of distribution from all periods

In [12]:

from tqdm.notebook import tqdm
for period in tqdm(df.index):
    x = df.loc[period, :]
    mask = x > 0
    x = x[mask]
    labels = df.columns[mask]

    #define Seaborn color palette to use
    colors = sns.color_palette('Blues')[0:len(df)]

    #create pie chart
    fig = plt.figure(figsize=(6,6))
    plt.pie(x=x, labels=labels, colors = colors, autopct='%.0f%%', startangle=0)
    plt.tight_layout()
    # plt.title(period)
    plt.savefig(f'word_distribution_pie_charts/total_keywords_{period}.png')
    plt.close()
    # plt.show()

  0%|          | 0/147 [00:00<?, ?it/s]

Determine Importance of individual words

In [13]:

df.sum(axis=0)

Out [13]:

root
challeng         0
SDG              0
ESG              2
recycling       33
reduc          152
planet          11
CSR              2
sustainab      252
clean          169
methanol        25
future         119
garbage         10
responsib       41
carb           305
chang          187
ocean          246
plastic         64
neutral         60
environment     95
green          114
sulphur         10
emissions       84
zero            82
climate        109
eco            414
mission        283
CO2              8
bio             11
dtype: object

In [14]:

import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
%matplotlib widget
counts = df.sum(axis=0)
mask = counts > 0
x = counts[mask]
labels = df.columns[mask]

#define Seaborn color palette to use
colors = sns.color_palette('Blues')[0:len(df)]

#create pie chart
fig = plt.figure(figsize=(6,6))
plt.pie(x=x, labels=labels, colors = colors, autopct='%.0f%%', startangle=0)
plt.tight_layout()
# plt.title(period)
plt.savefig(f'plots/total_keywords_pie.png')
# plt.close()
plt.show()

In [15]:

mean = counts.mean()
std = counts.std()
threshold = mean + std

mask = counts > threshold
x = counts[mask]
labels = df.columns[mask]

#define Seaborn color palette to use
colors = sns.color_palette('Blues')[0:len(df)]

#create pie chart
fig = plt.figure(figsize=(6,6))
plt.pie(x=x, labels=labels, colors = colors, autopct='%.0f%%', startangle=0)
plt.tight_layout()
# plt.title(period)
plt.savefig(f'plots/total_keywords_pie_significant.png')
# plt.close()
plt.show()

In [16]:

tracked_words_list = labels.tolist()

import seaborn as sns
time = df.index.to_timestamp()

#define Seaborn color palette to use
colors = sns.color_palette('colorblind', n_colors=len(tracked_words_list))
plt.figure(figsize=(12, 8))

for i, word in enumerate(tracked_words_list):
    y = df.loc[:, word]
    plt.plot(time, y, color=colors[i], label=word, alpha=0.75, linewidth=2)

# ax = df.loc[:, tracked_words_list].plot(kind='line', layout='tight', figsize=(12,8))

plt.ylabel('counts (n)')
plt.xlabel('period')
plt.legend(loc='upper left')
plt.savefig('plots/total_keyword_counts_significant.png')
plt.show()

In [ ]: