{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "29d44422-b6ea-4cdd-8366-7a9daedc8dd4", "metadata": { "tags": [] }, "outputs": [], "source": [ "import pandas as pd\n", "\n", "from classes import Tweet\n", "from database import connect as connect_db\n", "\n", "db = connect_db()" ] }, { "cell_type": "code", "execution_count": 2, "id": "0d043248-c8ad-47bc-90b6-40f02ab279ba", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "got 5190 tweets\n" ] } ], "source": [ "# load in all tweet texts\n", "cursor = db.find({'account': '@Maersk'})\n", "text_list = list()\n", "for element in cursor:\n", " try:\n", " text_list.append(element['text'])\n", " except:\n", " print(f\"failed getting {element['_id']}\")\n", " \n", "print(f'got {len(text_list)} tweets')" ] }, { "cell_type": "code", "execution_count": 3, "id": "d320a5e9-926e-42dd-983e-9cf1b49fdae9", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total characters: 906246\n" ] }, { "data": { "text/plain": [ "\"A spotless deck before delivery Susan Mærsk was built by Odense Steel Shipyard in 1954 and a final inspection of the ship takes place before delivery. The ship was deployed on The Far East service - the company's first liner service initiated in 1928. #Maerskheritage #Maersk At #Maersk, we are honoured to partner with customers who share our vision for a cleaner, more sustainable future. By spearheading the movement to provide green shipping solutions, we’re building a brighter tomorrow for our world. More: https://bddy.me/3mSdkug #AllTheWay 14 years cooking for a multi-cultural crew! For Sandy, a workday onboard of Mette Maersk starts at 5:30 in the morning, as she gets ready to bake a new batch of fresh bread, to be ready in time for breakfast with great joy from the crew onboard. #Maersk #Lifeatsea #Chiefcook Embracing #Equity on #InternationalWomenDay! The transport and logistics industry has traditionally been male dominated, and this is no exception for Maersk. This needs to c\"" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# combine all text into one list\n", "full_text = ' '.join(text_list)\n", "full_text = full_text.replace('\\n', ' ')\n", "print('total characters: ', len(full_text))\n", "full_text[:1000]" ] }, { "cell_type": "code", "execution_count": 4, "id": "2b9f7c25-af69-4862-8ac0-edb87afc0c52", "metadata": { "tags": [] }, "outputs": [], "source": [ "# clean text\n", "from cleantext import clean\n", "\n", "full_text = clean(\n", " text=full_text,\n", " fix_unicode=True,\n", " to_ascii=True,\n", " lower=True,\n", " normalize_whitespace=True,\n", " no_line_breaks=True,\n", " no_urls=True,\n", " no_emails=True,\n", " no_phone_numbers=True,\n", " no_numbers=True,\n", " no_digits=False,\n", " no_currency_symbols=True,\n", " no_punct=True,\n", " no_emoji=True,\n", " replace_with_url='',\n", " replace_with_email='',\n", " replace_with_phone_number='',\n", " replace_with_number='',\n", " #replace_with_digit='', # to be able to handle CO2\n", " replace_with_currency_symbol='',\n", " replace_with_punct='',\n", " lang='en'\n", ")\n", "\n", "# handle case from CO2 and remove other digits\n", "#full_text = full_text.replace(' co ', ' co2 ')\n", "#full_text = full_text.replace('', '')" ] }, { "cell_type": "code", "execution_count": 9, "id": "dabcc3c3-7dad-4d12-8e12-7ce3a80305e0", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total words: 80155\n" ] } ], "source": [ "# remove unwanted word classes\n", "import nltk\n", "#nltk.download('punkt')\n", "#nltk.download('averaged_perceptron_tagger')\n", "token_list = nltk.word_tokenize(full_text)\n", "tag_list = nltk.pos_tag(token_list)\n", "\n", "unwanted_class_list = [\n", " 'CC', # coordinating conjunction\n", " 'DT', # determiner\n", " 'IN', # preposition/subordinating conjunction\n", " 'TO', # to go ‘to’ the store\n", " 'PRP', # personal pronoun I, he, she\n", " 'PRP$', # possessive pronoun my, his, hers\n", "]\n", "\n", "word_list = list()\n", "for word, tag in tag_list:\n", " if tag not in unwanted_class_list:\n", " word_list.append(word)\n", " \n", "print('total words: ', len(word_list))" ] }, { "cell_type": "code", "execution_count": 10, "id": "23862fe3-7cfe-4089-af1c-1eb25b33cfcd", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "total unique words: 10925\n" ] } ], "source": [ "# find unique words\n", "unique_word_list = list(set(word_list))\n", "print('total unique words: ', len(unique_word_list))" ] }, { "cell_type": "code", "execution_count": 11, "id": "5a55242e-82bd-4e1d-acc7-4ded348f5afd", "metadata": { "tags": [] }, "outputs": [], "source": [ "# count unique words\n", "count_list = list()\n", "for unique_word in unique_word_list:\n", " count_list.append(word_list.count(unique_word))" ] }, { "cell_type": "code", "execution_count": 12, "id": "f325ce44-d296-4458-a443-74958f0465ed", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
wordcountsratio
0maersk32070.040010
1is10250.012788
2more9890.012339
3here8250.010293
4trade6420.008009
5are6390.007972
6how6340.007910
7logistics6110.007623
8global4670.005826
9new4590.005726
10can4530.005652
11shipping4430.005527
12alltheway3420.004267
13learn3390.004229
14have3140.003917
15be3050.003805
16see3020.003768
17supplychain2960.003693
18supply2950.003680
19will2810.003506
20read2740.003418
21has2710.003381
22what2670.003331
23world2610.003256
24container2480.003094
25business2430.003032
26growth2350.002932
27watch2300.002869
28one2280.002844
29port2210.002757
\n", "
" ], "text/plain": [ " word counts ratio\n", "0 maersk 3207 0.040010\n", "1 is 1025 0.012788\n", "2 more 989 0.012339\n", "3 here 825 0.010293\n", "4 trade 642 0.008009\n", "5 are 639 0.007972\n", "6 how 634 0.007910\n", "7 logistics 611 0.007623\n", "8 global 467 0.005826\n", "9 new 459 0.005726\n", "10 can 453 0.005652\n", "11 shipping 443 0.005527\n", "12 alltheway 342 0.004267\n", "13 learn 339 0.004229\n", "14 have 314 0.003917\n", "15 be 305 0.003805\n", "16 see 302 0.003768\n", "17 supplychain 296 0.003693\n", "18 supply 295 0.003680\n", "19 will 281 0.003506\n", "20 read 274 0.003418\n", "21 has 271 0.003381\n", "22 what 267 0.003331\n", "23 world 261 0.003256\n", "24 container 248 0.003094\n", "25 business 243 0.003032\n", "26 growth 235 0.002932\n", "27 watch 230 0.002869\n", "28 one 228 0.002844\n", "29 port 221 0.002757" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# present results\n", "import pandas as pd\n", "word_df = pd.DataFrame.from_dict(\n", " {\n", " 'word': unique_word_list,\n", " 'counts': count_list\n", " }\n", ")\n", "word_df = word_df.sort_values(by='counts', ascending=False).reset_index(drop=True)\n", "word_df['ratio'] = word_df['counts'] / len(word_list)\n", "\n", "word_df.head(30)" ] }, { "cell_type": "code", "execution_count": 13, "id": "363cd061-5de6-442f-96e6-acc4ee4cace4", "metadata": { "tags": [] }, "outputs": [], "source": [ "# save results\n", "word_df['ratio'] = word_df['ratio'].map('{:.5f}'.format) # ensure consistent formatting in csv\n", "word_df.to_csv('word_count.csv')" ] }, { "cell_type": "code", "execution_count": null, "id": "d4802935-9152-40a2-b459-2d30e47948b6", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }