solveig_master/count_words.ipynb

In [1]:

import pandas as pd

from classes import Tweet
from database import connect as connect_db

db = connect_db()

In [2]:

# load in all tweet texts
cursor = db.find({'account': '@Maersk'})
text_list = list()
for element in cursor:
    try:
        text_list.append(element['text'])
    except:
        print(f"failed getting {element['_id']}")
        
print(f'got {len(text_list)} tweets')

got 5190 tweets

In [3]:

# combine all text into one list
full_text = ' '.join(text_list)
full_text = full_text.replace('\n', ' ')
print('total characters: ', len(full_text))
full_text[:1000]

Out [3]:

total characters:  906246

"A spotless deck before delivery Susan Mærsk was built by Odense Steel Shipyard in 1954 and a final inspection of the ship takes place before delivery. The ship was deployed on The Far East service - the company's first liner service initiated in 1928.  #Maerskheritage #Maersk At #Maersk, we are honoured to partner with customers who share our vision for a cleaner, more sustainable future. By spearheading the movement to provide green shipping solutions, we’re building a brighter tomorrow for our world. More: https://bddy.me/3mSdkug #AllTheWay 14 years cooking for a multi-cultural crew!  For Sandy,  a workday onboard of Mette Maersk starts at 5:30 in the morning, as she gets ready to bake a new batch of fresh bread, to be ready in time for breakfast with great joy from the crew onboard. #Maersk #Lifeatsea #Chiefcook Embracing #Equity on #InternationalWomenDay! The transport and logistics industry has traditionally been male dominated, and this is no exception for Maersk. This needs to c"

In [4]:

# clean text
from cleantext import clean

full_text = clean(
    text=full_text,
    fix_unicode=True,
    to_ascii=True,
    lower=True,
    normalize_whitespace=True,
    no_line_breaks=True,
    no_urls=True,
    no_emails=True,
    no_phone_numbers=True,
    no_numbers=True,
    no_digits=False,
    no_currency_symbols=True,
    no_punct=True,
    no_emoji=True,
    replace_with_url='',
    replace_with_email='',
    replace_with_phone_number='',
    replace_with_number='',
    #replace_with_digit='', # to be able to handle CO2
    replace_with_currency_symbol='',
    replace_with_punct='',
    lang='en'
)

# handle case from CO2 and remove other digits
#full_text = full_text.replace(' co<digit> ', ' co2 ')
#full_text = full_text.replace('<digit>', '')

In [9]:

# remove unwanted word classes
import nltk
#nltk.download('punkt')
#nltk.download('averaged_perceptron_tagger')
token_list = nltk.word_tokenize(full_text)
tag_list = nltk.pos_tag(token_list)

unwanted_class_list = [
    'CC', # coordinating conjunction
    'DT', # determiner
    'IN', # preposition/subordinating conjunction
    'TO', # to go ‘to’ the store
    'PRP', # personal pronoun I, he, she
    'PRP$', # possessive pronoun my, his, hers
]

word_list = list()
for word, tag in tag_list:
    if tag not in unwanted_class_list:
        word_list.append(word)
        
print('total words: ', len(word_list))

total words:  80155

In [10]:

# find unique words
unique_word_list = list(set(word_list))
print('total unique words: ', len(unique_word_list))

total unique words:  10925

In [11]:

# count unique words
count_list = list()
for unique_word in unique_word_list:
    count_list.append(word_list.count(unique_word))

In [12]:

# present results
import pandas as pd
word_df = pd.DataFrame.from_dict(
    {
        'word': unique_word_list,
        'counts': count_list
    }
)
word_df = word_df.sort_values(by='counts', ascending=False).reset_index(drop=True)
word_df['ratio'] = word_df['counts'] / len(word_list)

word_df.head(30)

Out [12]:

	word	counts	ratio
0	maersk	3207	0.040010
1	is	1025	0.012788
2	more	989	0.012339
3	here	825	0.010293
4	trade	642	0.008009
5	are	639	0.007972
6	how	634	0.007910
7	logistics	611	0.007623
8	global	467	0.005826
9	new	459	0.005726
10	can	453	0.005652
11	shipping	443	0.005527
12	alltheway	342	0.004267
13	learn	339	0.004229
14	have	314	0.003917
15	be	305	0.003805
16	see	302	0.003768
17	supplychain	296	0.003693
18	supply	295	0.003680
19	will	281	0.003506
20	read	274	0.003418
21	has	271	0.003381
22	what	267	0.003331
23	world	261	0.003256
24	container	248	0.003094
25	business	243	0.003032
26	growth	235	0.002932
27	watch	230	0.002869
28	one	228	0.002844
29	port	221	0.002757

In [13]:

# save results
word_df['ratio'] = word_df['ratio'].map('{:.5f}'.format) # ensure consistent formatting in csv
word_df.to_csv('word_count.csv')

In [ ]: