521 lines
15 KiB
Plaintext
521 lines
15 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "29d44422-b6ea-4cdd-8366-7a9daedc8dd4",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"\n",
|
||
"from classes import Tweet\n",
|
||
"from database import connect as connect_db\n",
|
||
"\n",
|
||
"db = connect_db()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "0d043248-c8ad-47bc-90b6-40f02ab279ba",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"got 5190 tweets\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# load in all tweet texts\n",
|
||
"cursor = db.find({'account': '@Maersk'})\n",
|
||
"text_list = list()\n",
|
||
"for element in cursor:\n",
|
||
" try:\n",
|
||
" text_list.append(element['text'])\n",
|
||
" except:\n",
|
||
" print(f\"failed getting {element['_id']}\")\n",
|
||
" \n",
|
||
"print(f'got {len(text_list)} tweets')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "d320a5e9-926e-42dd-983e-9cf1b49fdae9",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"total characters: 906246\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"\"A spotless deck before delivery Susan Mærsk was built by Odense Steel Shipyard in 1954 and a final inspection of the ship takes place before delivery. The ship was deployed on The Far East service - the company's first liner service initiated in 1928. #Maerskheritage #Maersk At #Maersk, we are honoured to partner with customers who share our vision for a cleaner, more sustainable future. By spearheading the movement to provide green shipping solutions, we’re building a brighter tomorrow for our world. More: https://bddy.me/3mSdkug #AllTheWay 14 years cooking for a multi-cultural crew! For Sandy, a workday onboard of Mette Maersk starts at 5:30 in the morning, as she gets ready to bake a new batch of fresh bread, to be ready in time for breakfast with great joy from the crew onboard. #Maersk #Lifeatsea #Chiefcook Embracing #Equity on #InternationalWomenDay! The transport and logistics industry has traditionally been male dominated, and this is no exception for Maersk. This needs to c\""
|
||
]
|
||
},
|
||
"execution_count": 3,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"# combine all text into one list\n",
|
||
"full_text = ' '.join(text_list)\n",
|
||
"full_text = full_text.replace('\\n', ' ')\n",
|
||
"print('total characters: ', len(full_text))\n",
|
||
"full_text[:1000]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "2b9f7c25-af69-4862-8ac0-edb87afc0c52",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# clean text\n",
|
||
"from cleantext import clean\n",
|
||
"\n",
|
||
"full_text = clean(\n",
|
||
" text=full_text,\n",
|
||
" fix_unicode=True,\n",
|
||
" to_ascii=True,\n",
|
||
" lower=True,\n",
|
||
" normalize_whitespace=True,\n",
|
||
" no_line_breaks=True,\n",
|
||
" no_urls=True,\n",
|
||
" no_emails=True,\n",
|
||
" no_phone_numbers=True,\n",
|
||
" no_numbers=True,\n",
|
||
" no_digits=False,\n",
|
||
" no_currency_symbols=True,\n",
|
||
" no_punct=True,\n",
|
||
" no_emoji=True,\n",
|
||
" replace_with_url='',\n",
|
||
" replace_with_email='',\n",
|
||
" replace_with_phone_number='',\n",
|
||
" replace_with_number='',\n",
|
||
" #replace_with_digit='', # to be able to handle CO2\n",
|
||
" replace_with_currency_symbol='',\n",
|
||
" replace_with_punct='',\n",
|
||
" lang='en'\n",
|
||
")\n",
|
||
"\n",
|
||
"# handle case from CO2 and remove other digits\n",
|
||
"#full_text = full_text.replace(' co<digit> ', ' co2 ')\n",
|
||
"#full_text = full_text.replace('<digit>', '')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"id": "dabcc3c3-7dad-4d12-8e12-7ce3a80305e0",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"total words: 80155\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# remove unwanted word classes\n",
|
||
"import nltk\n",
|
||
"#nltk.download('punkt')\n",
|
||
"#nltk.download('averaged_perceptron_tagger')\n",
|
||
"token_list = nltk.word_tokenize(full_text)\n",
|
||
"tag_list = nltk.pos_tag(token_list)\n",
|
||
"\n",
|
||
"unwanted_class_list = [\n",
|
||
" 'CC', # coordinating conjunction\n",
|
||
" 'DT', # determiner\n",
|
||
" 'IN', # preposition/subordinating conjunction\n",
|
||
" 'TO', # to go ‘to’ the store\n",
|
||
" 'PRP', # personal pronoun I, he, she\n",
|
||
" 'PRP$', # possessive pronoun my, his, hers\n",
|
||
"]\n",
|
||
"\n",
|
||
"word_list = list()\n",
|
||
"for word, tag in tag_list:\n",
|
||
" if tag not in unwanted_class_list:\n",
|
||
" word_list.append(word)\n",
|
||
" \n",
|
||
"print('total words: ', len(word_list))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 10,
|
||
"id": "23862fe3-7cfe-4089-af1c-1eb25b33cfcd",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"total unique words: 10925\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# find unique words\n",
|
||
"unique_word_list = list(set(word_list))\n",
|
||
"print('total unique words: ', len(unique_word_list))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 11,
|
||
"id": "5a55242e-82bd-4e1d-acc7-4ded348f5afd",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# count unique words\n",
|
||
"count_list = list()\n",
|
||
"for unique_word in unique_word_list:\n",
|
||
" count_list.append(word_list.count(unique_word))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 12,
|
||
"id": "f325ce44-d296-4458-a443-74958f0465ed",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>word</th>\n",
|
||
" <th>counts</th>\n",
|
||
" <th>ratio</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>maersk</td>\n",
|
||
" <td>3207</td>\n",
|
||
" <td>0.040010</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>is</td>\n",
|
||
" <td>1025</td>\n",
|
||
" <td>0.012788</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>more</td>\n",
|
||
" <td>989</td>\n",
|
||
" <td>0.012339</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>here</td>\n",
|
||
" <td>825</td>\n",
|
||
" <td>0.010293</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>trade</td>\n",
|
||
" <td>642</td>\n",
|
||
" <td>0.008009</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>5</th>\n",
|
||
" <td>are</td>\n",
|
||
" <td>639</td>\n",
|
||
" <td>0.007972</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>6</th>\n",
|
||
" <td>how</td>\n",
|
||
" <td>634</td>\n",
|
||
" <td>0.007910</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>7</th>\n",
|
||
" <td>logistics</td>\n",
|
||
" <td>611</td>\n",
|
||
" <td>0.007623</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>8</th>\n",
|
||
" <td>global</td>\n",
|
||
" <td>467</td>\n",
|
||
" <td>0.005826</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>9</th>\n",
|
||
" <td>new</td>\n",
|
||
" <td>459</td>\n",
|
||
" <td>0.005726</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>10</th>\n",
|
||
" <td>can</td>\n",
|
||
" <td>453</td>\n",
|
||
" <td>0.005652</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>11</th>\n",
|
||
" <td>shipping</td>\n",
|
||
" <td>443</td>\n",
|
||
" <td>0.005527</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>12</th>\n",
|
||
" <td>alltheway</td>\n",
|
||
" <td>342</td>\n",
|
||
" <td>0.004267</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>13</th>\n",
|
||
" <td>learn</td>\n",
|
||
" <td>339</td>\n",
|
||
" <td>0.004229</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>14</th>\n",
|
||
" <td>have</td>\n",
|
||
" <td>314</td>\n",
|
||
" <td>0.003917</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>15</th>\n",
|
||
" <td>be</td>\n",
|
||
" <td>305</td>\n",
|
||
" <td>0.003805</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>16</th>\n",
|
||
" <td>see</td>\n",
|
||
" <td>302</td>\n",
|
||
" <td>0.003768</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>17</th>\n",
|
||
" <td>supplychain</td>\n",
|
||
" <td>296</td>\n",
|
||
" <td>0.003693</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>18</th>\n",
|
||
" <td>supply</td>\n",
|
||
" <td>295</td>\n",
|
||
" <td>0.003680</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>19</th>\n",
|
||
" <td>will</td>\n",
|
||
" <td>281</td>\n",
|
||
" <td>0.003506</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>20</th>\n",
|
||
" <td>read</td>\n",
|
||
" <td>274</td>\n",
|
||
" <td>0.003418</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>21</th>\n",
|
||
" <td>has</td>\n",
|
||
" <td>271</td>\n",
|
||
" <td>0.003381</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>22</th>\n",
|
||
" <td>what</td>\n",
|
||
" <td>267</td>\n",
|
||
" <td>0.003331</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>23</th>\n",
|
||
" <td>world</td>\n",
|
||
" <td>261</td>\n",
|
||
" <td>0.003256</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>24</th>\n",
|
||
" <td>container</td>\n",
|
||
" <td>248</td>\n",
|
||
" <td>0.003094</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>25</th>\n",
|
||
" <td>business</td>\n",
|
||
" <td>243</td>\n",
|
||
" <td>0.003032</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>26</th>\n",
|
||
" <td>growth</td>\n",
|
||
" <td>235</td>\n",
|
||
" <td>0.002932</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>27</th>\n",
|
||
" <td>watch</td>\n",
|
||
" <td>230</td>\n",
|
||
" <td>0.002869</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>28</th>\n",
|
||
" <td>one</td>\n",
|
||
" <td>228</td>\n",
|
||
" <td>0.002844</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>29</th>\n",
|
||
" <td>port</td>\n",
|
||
" <td>221</td>\n",
|
||
" <td>0.002757</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" word counts ratio\n",
|
||
"0 maersk 3207 0.040010\n",
|
||
"1 is 1025 0.012788\n",
|
||
"2 more 989 0.012339\n",
|
||
"3 here 825 0.010293\n",
|
||
"4 trade 642 0.008009\n",
|
||
"5 are 639 0.007972\n",
|
||
"6 how 634 0.007910\n",
|
||
"7 logistics 611 0.007623\n",
|
||
"8 global 467 0.005826\n",
|
||
"9 new 459 0.005726\n",
|
||
"10 can 453 0.005652\n",
|
||
"11 shipping 443 0.005527\n",
|
||
"12 alltheway 342 0.004267\n",
|
||
"13 learn 339 0.004229\n",
|
||
"14 have 314 0.003917\n",
|
||
"15 be 305 0.003805\n",
|
||
"16 see 302 0.003768\n",
|
||
"17 supplychain 296 0.003693\n",
|
||
"18 supply 295 0.003680\n",
|
||
"19 will 281 0.003506\n",
|
||
"20 read 274 0.003418\n",
|
||
"21 has 271 0.003381\n",
|
||
"22 what 267 0.003331\n",
|
||
"23 world 261 0.003256\n",
|
||
"24 container 248 0.003094\n",
|
||
"25 business 243 0.003032\n",
|
||
"26 growth 235 0.002932\n",
|
||
"27 watch 230 0.002869\n",
|
||
"28 one 228 0.002844\n",
|
||
"29 port 221 0.002757"
|
||
]
|
||
},
|
||
"execution_count": 12,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"# present results\n",
|
||
"import pandas as pd\n",
|
||
"word_df = pd.DataFrame.from_dict(\n",
|
||
" {\n",
|
||
" 'word': unique_word_list,\n",
|
||
" 'counts': count_list\n",
|
||
" }\n",
|
||
")\n",
|
||
"word_df = word_df.sort_values(by='counts', ascending=False).reset_index(drop=True)\n",
|
||
"word_df['ratio'] = word_df['counts'] / len(word_list)\n",
|
||
"\n",
|
||
"word_df.head(30)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 13,
|
||
"id": "363cd061-5de6-442f-96e6-acc4ee4cace4",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# save results\n",
|
||
"word_df['ratio'] = word_df['ratio'].map('{:.5f}'.format) # ensure consistent formatting in csv\n",
|
||
"word_df.to_csv('word_count.csv')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "d4802935-9152-40a2-b459-2d30e47948b6",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.8.10"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|