solveig_master/count_words.ipynb

In [5]:

import pandas as pd

from classes import Tweet
from database import connect as connect_db

db = connect_db()

In [6]:

# load in all tweet texts
cursor = db.find({'account': '@Maersk'})
text_list = list()
for element in cursor:
    try:
        text_list.append(element['text'])
    except:
        print(f"failed getting {element['_id']}")
        
print(f'got {len(text_list)} tweets')

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[6], line 4
      2 cursor = db.find({'account': '@Maersk'})
      3 text_list = list()
----> 4 for element in cursor:
      5     try:
      6         text_list.append(element['text'])

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1248, in Cursor.next(self)
   1246 if self.__empty:
   1247     raise StopIteration
-> 1248 if len(self.__data) or self._refresh():
   1249     return self.__data.popleft()
   1250 else:

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1188, in Cursor._refresh(self)
   1173     # Exhaust cursors don't send getMore messages.
   1174     g = self._getmore_class(
   1175         self.__dbname,
   1176         self.__collname,
   (...)
   1186         self.__comment,
   1187     )
-> 1188     self.__send_message(g)
   1190 return len(self.__data)

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1052, in Cursor.__send_message(self, operation)
   1049     raise InvalidOperation("exhaust cursors do not support auto encryption")
   1051 try:
-> 1052     response = client._run_operation(
   1053         operation, self._unpack_response, address=self.__address
   1054     )
   1055 except OperationFailure as exc:
   1056     if exc.code in _CURSOR_CLOSED_ERRORS or self.__exhaust:
   1057         # Don't send killCursors because the cursor is already closed.

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/_csot.py:105, in apply.<locals>.csot_wrapper(self, *args, **kwargs)
    103         with _TimeoutContext(timeout):
    104             return func(self, *args, **kwargs)
--> 105 return func(self, *args, **kwargs)

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1330, in MongoClient._run_operation(self, operation, unpack_res, address)
   1325     operation.reset()  # Reset op in case of retry.
   1326     return server.run_operation(
   1327         sock_info, operation, read_preference, self._event_listeners, unpack_res
   1328     )
-> 1330 return self._retryable_read(
   1331     _cmd,
   1332     operation.read_preference,
   1333     operation.session,
   1334     address=address,
   1335     retryable=isinstance(operation, message._Query),
   1336 )

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/_csot.py:105, in apply.<locals>.csot_wrapper(self, *args, **kwargs)
    103         with _TimeoutContext(timeout):
    104             return func(self, *args, **kwargs)
--> 105 return func(self, *args, **kwargs)

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1448, in MongoClient._retryable_read(self, func, read_pref, session, address, retryable)
   1446             assert last_error is not None
   1447             raise last_error
-> 1448         return func(session, server, sock_info, read_pref)
   1449 except ServerSelectionTimeoutError:
   1450     if retrying:
   1451         # The application may think the write was never attempted
   1452         # if we raise ServerSelectionTimeoutError on the retry
   1453         # attempt. Raise the original exception instead.

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1326, in MongoClient._run_operation.<locals>._cmd(session, server, sock_info, read_preference)
   1324 def _cmd(session, server, sock_info, read_preference):
   1325     operation.reset()  # Reset op in case of retry.
-> 1326     return server.run_operation(
   1327         sock_info, operation, read_preference, self._event_listeners, unpack_res
   1328     )

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/server.py:115, in Server.run_operation(self, sock_info, operation, read_preference, listeners, unpack_res)
    113 else:
    114     sock_info.send_message(data, max_doc_size)
--> 115     reply = sock_info.receive_message(request_id)
    117 # Unpack and check for command errors.
    118 if use_cmd:

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/pool.py:821, in SocketInfo.receive_message(self, request_id)
    819     return receive_message(self, request_id, self.max_message_size)
    820 except BaseException as error:
--> 821     self._raise_connection_failure(error)

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/pool.py:819, in SocketInfo.receive_message(self, request_id)
    814 """Receive a raw BSON message or raise ConnectionFailure.
    815 
    816 If any exception is raised, the socket is closed.
    817 """
    818 try:
--> 819     return receive_message(self, request_id, self.max_message_size)
    820 except BaseException as error:
    821     self._raise_connection_failure(error)

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/network.py:238, in receive_message(sock_info, request_id, max_message_size)
    236     data = decompress(_receive_data_on_socket(sock_info, length - 25, deadline), compressor_id)
    237 else:
--> 238     data = _receive_data_on_socket(sock_info, length - 16, deadline)
    240 try:
    241     unpack_reply = _UNPACK_REPLY[op_code]

File ~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/network.py:299, in _receive_data_on_socket(sock_info, length, deadline)
    297     if _csot.get_timeout():
    298         sock_info.set_socket_timeout(max(deadline - time.monotonic(), 0))
--> 299     chunk_length = sock_info.sock.recv_into(mv[bytes_read:])
    300 except BLOCKING_IO_ERRORS:
    301     raise socket.timeout("timed out")

KeyboardInterrupt: 

In [ ]:

Questions:
‘challengessee’
‘challengeor’
‘changetheworld’
‘changeat’
‘oceanswe’
‘planets’
‘futureproofing’
‘leadthefuture’
‘futureready’
‘futureproof’
'futureofshipping' # hashtag

In [48]:

for text in text_list:
    if 'challengessee' in text:
        print(text)

In [20]:

# combine all text into one list
full_text = ' '.join(text_list)
full_text = full_text.replace('\n', ' ')
print('total characters: ', len(full_text))
full_text[:1000]

Out [20]:

total characters:  906246

"A spotless deck before delivery Susan Mærsk was built by Odense Steel Shipyard in 1954 and a final inspection of the ship takes place before delivery. The ship was deployed on The Far East service - the company's first liner service initiated in 1928.  #Maerskheritage #Maersk At #Maersk, we are honoured to partner with customers who share our vision for a cleaner, more sustainable future. By spearheading the movement to provide green shipping solutions, we’re building a brighter tomorrow for our world. More: https://bddy.me/3mSdkug #AllTheWay 14 years cooking for a multi-cultural crew!  For Sandy,  a workday onboard of Mette Maersk starts at 5:30 in the morning, as she gets ready to bake a new batch of fresh bread, to be ready in time for breakfast with great joy from the crew onboard. #Maersk #Lifeatsea #Chiefcook Embracing #Equity on #InternationalWomenDay! The transport and logistics industry has traditionally been male dominated, and this is no exception for Maersk. This needs to c"

In [50]:

import re

len(re.findall('challengeor', full_text))

Out [50]:

1

In [22]:

import re

len(re.findall(' CO2', full_text))

Out [22]:

53

In [23]:

# clean text
from cleantext import clean

full_text = clean(
    text=full_text,
    fix_unicode=True,
    to_ascii=True,
    lower=True,
    normalize_whitespace=True,
    no_line_breaks=True,
    no_urls=True,
    no_emails=True,
    no_phone_numbers=True,
    no_numbers=True,
    no_digits=False,
    no_currency_symbols=True,
    no_punct=True,
    no_emoji=True,
    replace_with_url='',
    replace_with_email='',
    replace_with_phone_number='',
    replace_with_number='',
    #replace_with_digit='', # to be able to handle CO2
    replace_with_currency_symbol='',
    replace_with_punct='',
    lang='en'
)

# handle case from CO2 and remove other digits
#full_text = full_text.replace(' co<digit> ', ' co2 ')
#full_text = full_text.replace('<digit>', '')

In [24]:

import re

re.findall('co[1-9]\w+', full_text)

Out [24]:

['co2neutral',
 'co2emission',
 'co2emission',
 'co1prbwlw',
 'co1idd6st',
 'co1l7bctb',
 'co1lfltua',
 'co1o1ksg7',
 'co1ccchqp',
 'co2emissions',
 'co11bdxbv',
 'co2neutral']

In [25]:

import re

re.findall(' CO2 ', full_text)

Out [25]:

[]

In [26]:

import re

len(re.findall(' co2 ', full_text))

Out [26]:

53

In [27]:

# remove unwanted word classes
import nltk
#nltk.download('punkt')
#nltk.download('averaged_perceptron_tagger')
token_list = nltk.word_tokenize(full_text)
tag_list = nltk.pos_tag(token_list)

unwanted_class_list = [
    'CC', # coordinating conjunction
    'DT', # determiner
    'IN', # preposition/subordinating conjunction
    'TO', # to go ‘to’ the store
    'PRP', # personal pronoun I, he, she
    'PRP$', # possessive pronoun my, his, hers
]

word_list = list()
for word, tag in tag_list:
    if tag not in unwanted_class_list:
        word_list.append(word)
        
print('total words: ', len(word_list))

total words:  80155

In [28]:

# find unique words
unique_word_list = list(set(word_list))
print('total unique words: ', len(unique_word_list))

total unique words:  10925

In [29]:

# count unique words
count_list = list()
for unique_word in unique_word_list:
    count_list.append(word_list.count(unique_word))

In [30]:

# present results
import pandas as pd
word_df = pd.DataFrame.from_dict(
    {
        'word': unique_word_list,
        'counts': count_list
    }
)
word_df = word_df.sort_values(by='counts', ascending=False).reset_index(drop=True)
word_df['ratio'] = word_df['counts'] / len(word_list)

word_df.head(30)

Out [30]:

	word	counts	ratio
0	maersk	3207	0.040010
1	is	1025	0.012788
2	more	989	0.012339
3	here	825	0.010293
4	trade	642	0.008009
5	are	639	0.007972
6	how	634	0.007910
7	logistics	611	0.007623
8	global	467	0.005826
9	new	459	0.005726
10	can	453	0.005652
11	shipping	443	0.005527
12	alltheway	342	0.004267
13	learn	339	0.004229
14	have	314	0.003917
15	be	305	0.003805
16	see	302	0.003768
17	supplychain	296	0.003693
18	supply	295	0.003680
19	will	281	0.003506
20	read	274	0.003418
21	has	271	0.003381
22	what	267	0.003331
23	world	261	0.003256
24	container	248	0.003094
25	business	243	0.003032
26	growth	235	0.002932
27	watch	230	0.002869
28	one	228	0.002844
29	up	221	0.002757

In [31]:

# save results
word_df['ratio'] = word_df['ratio'].map('{:.5f}'.format) # ensure consistent formatting in csv
word_df.to_csv('word_count.csv')

In [ ]: