solveig_master/tweet_sustainability_ratio.ipynb

In [6]:

# load in data
import pandas as pd
from pathlib import Path

tweet_df = pd.read_csv('tweets_relevance_list.csv', index_col=0)

# remove rows with score of zero
tweet_df = tweet_df[tweet_df['score'].gt(0)]

# remove wrong folder
tweet_df['path'] = tweet_df['path'].str.replace('/home/brian/solveig_master/', '')

# ensure datatype
tweet_df['path'] = tweet_df['path'].apply(Path)

tweet_df

Out [6]:

	path	score
0	tweets/2021-08-24-07-40.txt	15
1	tweets/2021-06-22-13-19.txt	14
2	tweets/2020-07-13-10-03.txt	14
3	tweets/2022-04-28-12-38.txt	13
4	tweets/2022-10-20-14-08.txt	13
...	...	...
1331	tweets/2016-11-08-13-35.txt	1
1332	tweets/2022-06-16-12-15.txt	1
1333	tweets/2016-05-12-12-48.txt	1
1334	tweets/2016-06-01-11-03.txt	1
1335	tweets/2014-01-28-08-50.txt	1

1336 rows × 2 columns

In [7]:

# define time bins
period_range = pd.period_range(start='2011-01-01', end='2023-03-31', freq='M')

period_range

Out [7]:

PeriodIndex(['2011-01', '2011-02', '2011-03', '2011-04', '2011-05', '2011-06',
             '2011-07', '2011-08', '2011-09', '2011-10',
             ...
             '2022-06', '2022-07', '2022-08', '2022-09', '2022-10', '2022-11',
             '2022-12', '2023-01', '2023-02', '2023-03'],
            dtype='period[M]', length=147)

In [8]:

# prepare dataframe
df = pd.DataFrame(index=period_range, columns=['count', 'total'])
df.iloc[:, :] = 0 # set all to 0

df

Out [8]:

	count	total
2011-01	0	0
2011-02	0	0
2011-03	0	0
2011-04	0	0
2011-05	0	0
...	...	...
2022-11	0	0
2022-12	0	0
2023-01	0	0
2023-02	0	0
2023-03	0	0

147 rows × 2 columns

In [9]:

# count words usage
from datetime import datetime

for path in tweet_df['path']:
    # determine period
    ts = datetime.strptime(path.stem, '%Y-%m-%d-%H-%M')
    ts = pd.to_datetime(ts).to_period('M')
    # increase count
    df.loc[ts, 'count'] += 1

df

Out [9]:

	count	total
2011-01	0	0
2011-02	0	0
2011-03	0	0
2011-04	0	0
2011-05	0	0
...	...	...
2022-11	23	0
2022-12	29	0
2023-01	19	0
2023-02	20	0
2023-03	15	0

147 rows × 2 columns

In [5]:

from database import connect as connect_db

db = connect_db()

# load in all tweet dates
cursor = db.find({'account': '@Maersk'})
time_list = list()
for element in cursor:
    try:
        time_list.append(element['time'])
    except:
        print(f"failed getting {element['_id']}")
        
print(f'got {len(time_list)} tweets')

with open('tweet_time_list.txt', 'w') as fh:
    fh.write('\n'.join(time_list))

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[5], line 8
      6 cursor = db.find({'account': '@Maersk'})
      7 time_list = list()
----> 8 for element in cursor:
      9     try:
     10         time_list.append(element['time'])

File ~/Desktop/solveig_master/venv/lib/python3.8/site-packages/pymongo/cursor.py:1248, in Cursor.next(self)
   1246 if self.__empty:
   1247     raise StopIteration
-> 1248 if len(self.__data) or self._refresh():
   1249     return self.__data.popleft()
   1250 else:

File ~/Desktop/solveig_master/venv/lib/python3.8/site-packages/pymongo/cursor.py:1139, in Cursor._refresh(self)
   1136     return len(self.__data)
   1138 if not self.__session:
-> 1139     self.__session = self.__collection.database.client._ensure_session()
   1141 if self.__id is None:  # Query
   1142     if (self.__min or self.__max) and not self.__hint:

File ~/Desktop/solveig_master/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1740, in MongoClient._ensure_session(self, session)
   1735     return session
   1737 try:
   1738     # Don't make implicit sessions causally consistent. Applications
   1739     # should always opt-in.
-> 1740     return self.__start_session(True, causal_consistency=False)
   1741 except (ConfigurationError, InvalidOperation):
   1742     # Sessions not supported.
   1743     return None

File ~/Desktop/solveig_master/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1685, in MongoClient.__start_session(self, implicit, **kwargs)
   1682 def __start_session(self, implicit, **kwargs):
   1683     # Raises ConfigurationError if sessions are not supported.
   1684     if implicit:
-> 1685         self._topology._check_implicit_session_support()
   1686         server_session = _EmptyServerSession()
   1687     else:

File ~/Desktop/solveig_master/venv/lib/python3.8/site-packages/pymongo/topology.py:538, in Topology._check_implicit_session_support(self)
    536 def _check_implicit_session_support(self):
    537     with self._lock:
--> 538         self._check_session_support()

File ~/Desktop/solveig_master/venv/lib/python3.8/site-packages/pymongo/topology.py:554, in Topology._check_session_support(self)
    550         self._select_servers_loop(
    551             any_server_selector, self.get_server_selection_timeout(), None
    552         )
    553 elif not self._description.readable_servers:
--> 554     self._select_servers_loop(
    555         readable_server_selector, self.get_server_selection_timeout(), None
    556     )
    558 session_timeout = self._description.logical_session_timeout_minutes
    559 if session_timeout is None:

File ~/Desktop/solveig_master/venv/lib/python3.8/site-packages/pymongo/topology.py:250, in Topology._select_servers_loop(self, selector, timeout, address)
    244 self._request_check_all()
    246 # Release the lock and wait for the topology description to
    247 # change, or for a timeout. We won't miss any changes that
    248 # came after our most recent apply_selector call, since we've
    249 # held the lock until now.
--> 250 self._condition.wait(common.MIN_HEARTBEAT_INTERVAL)
    251 self._description.check_compatible()
    252 now = time.monotonic()

File /usr/local/opt/[email protected]/Frameworks/Python.framework/Versions/3.8/lib/python3.8/threading.py:306, in Condition.wait(self, timeout)
    304 else:
    305     if timeout > 0:
--> 306         gotit = waiter.acquire(True, timeout)
    307     else:
    308         gotit = waiter.acquire(False)

KeyboardInterrupt: 

In [10]:

# load in time list
with open('tweet_time_list.txt', 'r') as fh:
    content = fh.read()

time_list = content.split('\n')
time_list[:5]

Out [10]:

['2023-03-12 10:02:05',
 '2023-03-13 12:05:06',
 '2023-03-10 12:24:49',
 '2023-03-08 08:43:44',
 '2023-03-15 11:10:19']

In [11]:

# count total tweets
from datetime import datetime

for time_str in time_list:
    # determine period
    ts = datetime.strptime(time_str, '%Y-%m-%d %H:%M:%S')
    ts = pd.to_datetime(ts).to_period('M')
    # increase count
    df.loc[ts, 'total'] += 1

df

Out [11]:

	count	total
2011-01	0	0
2011-02	0	0
2011-03	0	0
2011-04	0	0
2011-05	0	0
...	...	...
2022-11	23	57
2022-12	29	65
2023-01	19	51
2023-02	20	63
2023-03	15	32

147 rows × 2 columns

In [12]:

# calculate ratio
df['ratio'] = 0

above_zero_mask = df['total'].gt(0)

df.loc[above_zero_mask, 'ratio'] = df.loc[above_zero_mask, 'count'] / df.loc[above_zero_mask, 'total']

df

Out [12]:

	count	total	ratio
2011-01	0	0	0
2011-02	0	0	0
2011-03	0	0	0
2011-04	0	0	0
2011-05	0	0	0
...	...	...	...
2022-11	23	57	0.403509
2022-12	29	65	0.446154
2023-01	19	51	0.372549
2023-02	20	63	0.31746
2023-03	15	32	0.46875

147 rows × 3 columns

In [13]:

# plot of combined use of root words over time
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
%matplotlib widget
plt.figure(figsize=(12,8))

time = df.index.to_timestamp()
counts = df['ratio'] * 100
plt.plot(time, counts, label='sustainability-related tweets')

mean = counts.rolling(12, min_periods=1).mean()
plt.plot(time, mean, label='12-month mean')
std = counts.rolling(12, min_periods=1).std()
plt.fill_between(
    x=time,
    y1=mean-std,
    y2=mean+std,
    alpha=0.2,
    label='12-month standard deviation'
)

locator = mdates.MonthLocator(interval=12)
fmt = mdates.DateFormatter('%Y-%m')
x_axis = plt.gca().xaxis
x_axis.set_major_locator(locator)
x_axis.set_major_formatter(fmt)
plt.xticks(rotation=45)
plt.ylabel('percent of total tweets (%)')
plt.legend(loc='upper left')
plt.tight_layout()
sns.despine(left=True)
sns.set_style("ticks",{'axes.grid' : True})
plt.savefig('plots/total_tweet_ratio.png')
plt.show()

In [ ]: