solveig_master/root_word_frequency_plot.ipynb

In [1]:

# load in data
import pandas as pd
from pathlib import Path

tweet_df = pd.read_csv('tweets_relevance_list.csv', index_col=0)

# remove rows with score of zero
tweet_df = tweet_df[tweet_df['score'].gt(0)]

# remove wrong folder
tweet_df['path'] = tweet_df['path'].str.replace('/home/brian/solveig_master/', '')

# ensure datatype
tweet_df['path'] = tweet_df['path'].apply(Path)

tweet_df

Out [1]:

	path	score
0	tweets/2021-08-24-07-40.txt	15
1	tweets/2021-06-22-13-19.txt	14
2	tweets/2020-07-13-10-03.txt	14
3	tweets/2022-04-28-12-38.txt	13
4	tweets/2022-10-20-14-08.txt	13
...	...	...
1331	tweets/2016-11-08-13-35.txt	1
1332	tweets/2022-06-16-12-15.txt	1
1333	tweets/2016-05-12-12-48.txt	1
1334	tweets/2016-06-01-11-03.txt	1
1335	tweets/2014-01-28-08-50.txt	1

1336 rows × 2 columns

In [2]:

# load in word root list
word_root_df = pd.read_csv('word_root_ranking.csv', index_col=0)

# parse string-lists
word_root_df['word_matches'] = word_root_df['word_matches'].str.strip("[]'").str.split("', '")
# word_list = sum(word_list, []) # flatten nested list

word_root_df

Out [2]:

	root	mean_ranking	std_ranking	word_matches
0	challeng	420	264	[challenges, challenge, challenging]
1	SDG	423	0	[sdgs]
2	ESG	636	0	[esg]
3	recycling	860	154	[recycling, shiprecycling]
4	reduc	1242	965	[reduce, reducing, reduced, reduction, reducti...
5	planet	1295	0	[planet]
6	CSR	1371	0	[csr]
7	sustainab	1527	1873	[sustainability, sustainable, sustainableshipp...
8	clean	1695	837	[theoceancleanup, cleanup, clean, cleanseas, o...
9	methanol	1700	1044	[methanol, emethanol]
10	future	1770	1684	[future, futureproofing]
11	garbage	1808	612	[garbage, greatpacificgarbagepatch]
12	responsib	1889	1135	[responsible, responsibility, responsibly, res...
13	carb	1952	1526	[decarbonisation, carbon, decarbonization, car...
14	chang	2148	1667	[change, changing, climatechange, changes, exc...
15	ocean	2169	1629	[ocean, theoceancleanup, oceans, oceanplastic,...
16	plastic	2320	1490	[plastic, oceanplastic, plasticwaste, plasticp...
17	neutral	2382	1798	[neutral, carbonneutral, neutrality, co2neutral]
18	environment	2391	1574	[environment, environmental, unenvironment, en...
19	green	2393	1158	[green, greenfuels, greener, greenfuel, greent...
20	sulphur	2828	1772	[sulphur, lowsulphur]
21	emissions	2925	2046	[emissions, carbonemissions, zeroemissions]
22	zero	2977	1646	[zero, netzero, zerocarbon, zerocarbonship, ze...
23	climate	3046	2095	[climateaction, climate, climatechange, climat...
24	eco	3074	1269	[ecosystem, eco, maerskecodelivery, ecofriendl...
25	mission	3247	1950	[emissions, mission, eucommission, carbonemiss...
26	CO2	3390	2275	[co2, co2neutral, co2emission]
27	bio	3687	1274	[biofuel, biofuels, biohuts, biodiversity]

In [3]:

# define time bins
period_range = pd.period_range(start='2011-01-01', end='2023-03-31', freq='M')

period_range

Out [3]:

PeriodIndex(['2011-01', '2011-02', '2011-03', '2011-04', '2011-05', '2011-06',
             '2011-07', '2011-08', '2011-09', '2011-10',
             ...
             '2022-06', '2022-07', '2022-08', '2022-09', '2022-10', '2022-11',
             '2022-12', '2023-01', '2023-02', '2023-03'],
            dtype='period[M]', length=147)

In [4]:

# prepare dataframe
df = pd.DataFrame(index=period_range, columns=word_root_df['root'])
df.iloc[:, :] = 0 # set all to 0

df

Out [4]:

root	challeng	SDG	ESG	recycling	reduc	planet	CSR	sustainab	clean	methanol	...	environment	green	sulphur	emissions	zero	climate	eco	mission	CO2	bio
2011-01	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-02	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-03	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-04	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-05	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
2022-11	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2022-12	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2023-01	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2023-02	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2023-03	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0

147 rows × 28 columns

In [5]:

df.index

Out [5]:

PeriodIndex(['2011-01', '2011-02', '2011-03', '2011-04', '2011-05', '2011-06',
             '2011-07', '2011-08', '2011-09', '2011-10',
             ...
             '2022-06', '2022-07', '2022-08', '2022-09', '2022-10', '2022-11',
             '2022-12', '2023-01', '2023-02', '2023-03'],
            dtype='period[M]', length=147)

In [6]:

# count words usage
from datetime import datetime
import re

# path = tweet_df.loc[0, 'path']
# print('path: ', path)
for path in tweet_df['path']:
    # determine period
    ts = datetime.strptime(path.stem, '%Y-%m-%d-%H-%M')
    ts = pd.to_datetime(ts).to_period('M')
    # print('period: ', ts)
    period_mask = ts == df.index
    # load in tweet
    with open(path, 'r') as f:
        text = f.read()
    # extract tweet text
    text = re.search(
        pattern='(?<=text: )(.|\n)*(?=images: )',
        string=text
    ).group(0)
    # determine if words present in text
    for word_root in df.columns[1:]:
        # print('word root: ', word_root)
        word_root_mask = word_root_df['root'].eq(word_root)
        word_matches = word_root_df.loc[word_root_mask, 'word_matches'].to_list()
        word_matches = sum(word_matches, []) # unpack nested list
        # print('word matches: ', word_matches)
        for word in word_matches:
            # print('word: ', word)
            if word in text:
                df.loc[period_mask, word_root] += 1

df

Out [6]:

root	challeng	SDG	ESG	recycling	reduc	planet	CSR	sustainab	clean	methanol	...	environment	green	sulphur	emissions	zero	climate	eco	mission	CO2	bio
2011-01	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-02	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-03	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-04	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2011-05	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
2022-11	0	0	0	5	0	0	0	5	0	1	...	0	5	0	2	2	5	3	7	0	0
2022-12	0	0	0	0	2	0	0	6	4	3	...	1	3	0	3	0	2	12	11	0	0
2023-01	0	0	1	0	2	0	0	3	0	0	...	0	1	0	2	0	0	3	6	0	0
2023-02	0	0	1	0	0	0	0	4	4	1	...	5	1	0	1	4	0	5	4	0	2
2023-03	0	0	0	1	1	0	0	4	2	1	...	3	5	0	1	0	0	4	3	0	0

147 rows × 28 columns

In [24]:

import matplotlib.pyplot as plt

for year in range(2011, 2024):
    year_mask = df.index.year == year
    ax = df[year_mask].plot(kind='line', layout='tight', figsize=(18,12))
# fig, ax = plt.subplots(1,1,figsize=(18,12))
# [ax.plot(df.index, df[col]) for col in df.columns]
    ax.set_ylabel('counts (n)')
    ax.set_xlabel('period')
    plt.show()

In [19]:

import matplotlib.pyplot as plt
import matplotlib.dates as mdates

# locator = mdates.MonthLocator()
# fmt = mdates.DateFormatter('%b')
# fmt = mdates.DateFormatter('%Y-%m')

ax = df.plot(kind='line', layout='tight', figsize=(18,12))
# fig, ax = plt.subplots(1,1,figsize=(18,12))
# [ax.plot(df.index, df[col]) for col in df.columns]
ax.set_ylabel('counts (n)')
ax.set_xlabel('period')

# X.set_major_locator(locator)
# X.set_major_formatter(mdates.DateFormatter('%Y'))
plt.show()

In [31]:

# plot only words that have a peak above 5
word_mask = df.max(axis=0) >= 5

df_extract = df.loc[:, word_mask]
df_extract[df_extract<5] = 0

ax = df_extract.plot(kind='line', layout='tight', figsize=(18,12))

ax.set_ylabel('counts (n)')
ax.set_xlabel('period')

plt.show()

/var/folders/g9/p4ywkqyn4pd1t7z3lpmz4s7w0000gn/T/ipykernel_62080/3593302067.py:5: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  df_extract[df_extract<5] = 0
/var/folders/g9/p4ywkqyn4pd1t7z3lpmz4s7w0000gn/T/ipykernel_62080/3593302067.py:5: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  df_extract[df_extract<5] = 0

In [40]:

# plot of combined use of root words over time
ax = df.sum(axis=1).plot(kind='line', layout='tight', figsize=(18,12))
df.sum(axis=1).rolling(12, min_periods=1).mean().plot(ax=ax, kind='line', layout='tight', figsize=(18,12))
ax.set_ylabel('counts (n)')
ax.set_xlabel('period')

plt.show()

In [ ]: