solveig_master/tweet_frequency_count.ipynb

In [1]:

# load in data
import pandas as pd
from pathlib import Path

tweet_df = pd.read_csv('tweets_relevance_list.csv', index_col=0)

# remove rows with score of zero
tweet_df = tweet_df[tweet_df['score'].gt(0)]

# remove wrong folder
tweet_df['path'] = tweet_df['path'].str.replace('/home/brian/solveig_master/', '')

# ensure datatype
tweet_df['path'] = tweet_df['path'].apply(Path)

tweet_df

Out [1]:

	path	score
0	tweets/2021-08-24-07-40.txt	15
1	tweets/2021-06-22-13-19.txt	14
2	tweets/2020-07-13-10-03.txt	14
3	tweets/2022-04-28-12-38.txt	13
4	tweets/2022-10-20-14-08.txt	13
...	...	...
1331	tweets/2016-11-08-13-35.txt	1
1332	tweets/2022-06-16-12-15.txt	1
1333	tweets/2016-05-12-12-48.txt	1
1334	tweets/2016-06-01-11-03.txt	1
1335	tweets/2014-01-28-08-50.txt	1

1336 rows × 2 columns

In [2]:

# define time bins
period_range = pd.period_range(start='2011-01-01', end='2023-03-31', freq='M')

period_range

Out [2]:

PeriodIndex(['2011-01', '2011-02', '2011-03', '2011-04', '2011-05', '2011-06',
             '2011-07', '2011-08', '2011-09', '2011-10',
             ...
             '2022-06', '2022-07', '2022-08', '2022-09', '2022-10', '2022-11',
             '2022-12', '2023-01', '2023-02', '2023-03'],
            dtype='period[M]', length=147)

In [3]:

# prepare dataframe
df = pd.DataFrame(index=period_range, columns=['count'])
df.iloc[:, :] = 0 # set all to 0

df

Out [3]:

	count
2011-01	0
2011-02	0
2011-03	0
2011-04	0
2011-05	0
...	...
2022-11	0
2022-12	0
2023-01	0
2023-02	0
2023-03	0

147 rows × 1 columns

In [4]:

df.index

Out [4]:

PeriodIndex(['2011-01', '2011-02', '2011-03', '2011-04', '2011-05', '2011-06',
             '2011-07', '2011-08', '2011-09', '2011-10',
             ...
             '2022-06', '2022-07', '2022-08', '2022-09', '2022-10', '2022-11',
             '2022-12', '2023-01', '2023-02', '2023-03'],
            dtype='period[M]', length=147)

In [5]:

# count words usage
from datetime import datetime

# path = tweet_df.loc[0, 'path']
# print('path: ', path)
for path in tweet_df['path']:
    # determine period
    ts = datetime.strptime(path.stem, '%Y-%m-%d-%H-%M')
    ts = pd.to_datetime(ts).to_period('M')
    # print('period: ', ts)
    # period_mask = ts == df.index
    # increase count
    df.loc[ts, 'count'] += 1

df

Out [5]:

	count
2011-01	0
2011-02	0
2011-03	0
2011-04	0
2011-05	0
...	...
2022-11	23
2022-12	29
2023-01	19
2023-02	20
2023-03	15

147 rows × 1 columns

In [7]:

# plot of combined use of root words over time
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
%matplotlib widget
plt.figure(figsize=(12,8))

time = df.index.to_timestamp()
counts = df['count']
plt.plot(time, counts, label='aggregated counts per month')

mean = df.sum(axis=1).rolling(12, min_periods=1).mean()
plt.plot(time, mean, label='12-month mean')
std = df.sum(axis=1).rolling(12, min_periods=1).std()
plt.fill_between(
    x=time,
    y1=mean-std,
    y2=mean+std,
    alpha=0.2,
    label='standard deviation'
)

locator = mdates.MonthLocator(interval=12)
fmt = mdates.DateFormatter('%Y-%m')
x_axis = plt.gca().xaxis
x_axis.set_major_locator(locator)
x_axis.set_major_formatter(fmt)
plt.xticks(rotation=45)
plt.ylabel('tweet count (n)')
plt.legend(loc='upper left')
plt.tight_layout()
sns.despine(left=True)
sns.set_style("ticks",{'axes.grid' : True})
plt.savefig('plots/total_tweet_counts.png')
plt.show()

In [ ]: