diff --git a/count_words.ipynb b/count_words.ipynb new file mode 100644 index 0000000..342706a --- /dev/null +++ b/count_words.ipynb @@ -0,0 +1,708 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "id": "29d44422-b6ea-4cdd-8366-7a9daedc8dd4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from classes import Tweet\n", + "from database import connect as connect_db\n", + "\n", + "db = connect_db()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "0d043248-c8ad-47bc-90b6-40f02ab279ba", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m cursor \u001b[39m=\u001b[39m db\u001b[39m.\u001b[39mfind({\u001b[39m'\u001b[39m\u001b[39maccount\u001b[39m\u001b[39m'\u001b[39m: \u001b[39m'\u001b[39m\u001b[39m@Maersk\u001b[39m\u001b[39m'\u001b[39m})\n\u001b[1;32m 3\u001b[0m text_list \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m()\n\u001b[0;32m----> 4\u001b[0m \u001b[39mfor\u001b[39;00m element \u001b[39min\u001b[39;00m cursor:\n\u001b[1;32m 5\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 6\u001b[0m text_list\u001b[39m.\u001b[39mappend(element[\u001b[39m'\u001b[39m\u001b[39mtext\u001b[39m\u001b[39m'\u001b[39m])\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1248\u001b[0m, in \u001b[0;36mCursor.next\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1246\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__empty:\n\u001b[1;32m 1247\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m\n\u001b[0;32m-> 1248\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__data) \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_refresh():\n\u001b[1;32m 1249\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__data\u001b[39m.\u001b[39mpopleft()\n\u001b[1;32m 1250\u001b[0m \u001b[39melse\u001b[39;00m:\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1188\u001b[0m, in \u001b[0;36mCursor._refresh\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1173\u001b[0m \u001b[39m# Exhaust cursors don't send getMore messages.\u001b[39;00m\n\u001b[1;32m 1174\u001b[0m g \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_getmore_class(\n\u001b[1;32m 1175\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__dbname,\n\u001b[1;32m 1176\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__collname,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1186\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__comment,\n\u001b[1;32m 1187\u001b[0m )\n\u001b[0;32m-> 1188\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m__send_message(g)\n\u001b[1;32m 1190\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__data)\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1052\u001b[0m, in \u001b[0;36mCursor.__send_message\u001b[0;34m(self, operation)\u001b[0m\n\u001b[1;32m 1049\u001b[0m \u001b[39mraise\u001b[39;00m InvalidOperation(\u001b[39m\"\u001b[39m\u001b[39mexhaust cursors do not support auto encryption\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 1051\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 1052\u001b[0m response \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39;49m_run_operation(\n\u001b[1;32m 1053\u001b[0m operation, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_unpack_response, address\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m__address\n\u001b[1;32m 1054\u001b[0m )\n\u001b[1;32m 1055\u001b[0m \u001b[39mexcept\u001b[39;00m OperationFailure \u001b[39mas\u001b[39;00m exc:\n\u001b[1;32m 1056\u001b[0m \u001b[39mif\u001b[39;00m exc\u001b[39m.\u001b[39mcode \u001b[39min\u001b[39;00m _CURSOR_CLOSED_ERRORS \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__exhaust:\n\u001b[1;32m 1057\u001b[0m \u001b[39m# Don't send killCursors because the cursor is already closed.\u001b[39;00m\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/_csot.py:105\u001b[0m, in \u001b[0;36mapply..csot_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[39mwith\u001b[39;00m _TimeoutContext(timeout):\n\u001b[1;32m 104\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m--> 105\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1330\u001b[0m, in \u001b[0;36mMongoClient._run_operation\u001b[0;34m(self, operation, unpack_res, address)\u001b[0m\n\u001b[1;32m 1325\u001b[0m operation\u001b[39m.\u001b[39mreset() \u001b[39m# Reset op in case of retry.\u001b[39;00m\n\u001b[1;32m 1326\u001b[0m \u001b[39mreturn\u001b[39;00m server\u001b[39m.\u001b[39mrun_operation(\n\u001b[1;32m 1327\u001b[0m sock_info, operation, read_preference, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_event_listeners, unpack_res\n\u001b[1;32m 1328\u001b[0m )\n\u001b[0;32m-> 1330\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_retryable_read(\n\u001b[1;32m 1331\u001b[0m _cmd,\n\u001b[1;32m 1332\u001b[0m operation\u001b[39m.\u001b[39;49mread_preference,\n\u001b[1;32m 1333\u001b[0m operation\u001b[39m.\u001b[39;49msession,\n\u001b[1;32m 1334\u001b[0m address\u001b[39m=\u001b[39;49maddress,\n\u001b[1;32m 1335\u001b[0m retryable\u001b[39m=\u001b[39;49m\u001b[39misinstance\u001b[39;49m(operation, message\u001b[39m.\u001b[39;49m_Query),\n\u001b[1;32m 1336\u001b[0m )\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/_csot.py:105\u001b[0m, in \u001b[0;36mapply..csot_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[39mwith\u001b[39;00m _TimeoutContext(timeout):\n\u001b[1;32m 104\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m--> 105\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1448\u001b[0m, in \u001b[0;36mMongoClient._retryable_read\u001b[0;34m(self, func, read_pref, session, address, retryable)\u001b[0m\n\u001b[1;32m 1446\u001b[0m \u001b[39massert\u001b[39;00m last_error \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 1447\u001b[0m \u001b[39mraise\u001b[39;00m last_error\n\u001b[0;32m-> 1448\u001b[0m \u001b[39mreturn\u001b[39;00m func(session, server, sock_info, read_pref)\n\u001b[1;32m 1449\u001b[0m \u001b[39mexcept\u001b[39;00m ServerSelectionTimeoutError:\n\u001b[1;32m 1450\u001b[0m \u001b[39mif\u001b[39;00m retrying:\n\u001b[1;32m 1451\u001b[0m \u001b[39m# The application may think the write was never attempted\u001b[39;00m\n\u001b[1;32m 1452\u001b[0m \u001b[39m# if we raise ServerSelectionTimeoutError on the retry\u001b[39;00m\n\u001b[1;32m 1453\u001b[0m \u001b[39m# attempt. Raise the original exception instead.\u001b[39;00m\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1326\u001b[0m, in \u001b[0;36mMongoClient._run_operation.._cmd\u001b[0;34m(session, server, sock_info, read_preference)\u001b[0m\n\u001b[1;32m 1324\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_cmd\u001b[39m(session, server, sock_info, read_preference):\n\u001b[1;32m 1325\u001b[0m operation\u001b[39m.\u001b[39mreset() \u001b[39m# Reset op in case of retry.\u001b[39;00m\n\u001b[0;32m-> 1326\u001b[0m \u001b[39mreturn\u001b[39;00m server\u001b[39m.\u001b[39;49mrun_operation(\n\u001b[1;32m 1327\u001b[0m sock_info, operation, read_preference, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_event_listeners, unpack_res\n\u001b[1;32m 1328\u001b[0m )\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/server.py:115\u001b[0m, in \u001b[0;36mServer.run_operation\u001b[0;34m(self, sock_info, operation, read_preference, listeners, unpack_res)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 114\u001b[0m sock_info\u001b[39m.\u001b[39msend_message(data, max_doc_size)\n\u001b[0;32m--> 115\u001b[0m reply \u001b[39m=\u001b[39m sock_info\u001b[39m.\u001b[39;49mreceive_message(request_id)\n\u001b[1;32m 117\u001b[0m \u001b[39m# Unpack and check for command errors.\u001b[39;00m\n\u001b[1;32m 118\u001b[0m \u001b[39mif\u001b[39;00m use_cmd:\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/pool.py:821\u001b[0m, in \u001b[0;36mSocketInfo.receive_message\u001b[0;34m(self, request_id)\u001b[0m\n\u001b[1;32m 819\u001b[0m \u001b[39mreturn\u001b[39;00m receive_message(\u001b[39mself\u001b[39m, request_id, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmax_message_size)\n\u001b[1;32m 820\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m error:\n\u001b[0;32m--> 821\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_raise_connection_failure(error)\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/pool.py:819\u001b[0m, in \u001b[0;36mSocketInfo.receive_message\u001b[0;34m(self, request_id)\u001b[0m\n\u001b[1;32m 814\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Receive a raw BSON message or raise ConnectionFailure.\u001b[39;00m\n\u001b[1;32m 815\u001b[0m \n\u001b[1;32m 816\u001b[0m \u001b[39mIf any exception is raised, the socket is closed.\u001b[39;00m\n\u001b[1;32m 817\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 818\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 819\u001b[0m \u001b[39mreturn\u001b[39;00m receive_message(\u001b[39mself\u001b[39;49m, request_id, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_message_size)\n\u001b[1;32m 820\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m error:\n\u001b[1;32m 821\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_raise_connection_failure(error)\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/network.py:238\u001b[0m, in \u001b[0;36mreceive_message\u001b[0;34m(sock_info, request_id, max_message_size)\u001b[0m\n\u001b[1;32m 236\u001b[0m data \u001b[39m=\u001b[39m decompress(_receive_data_on_socket(sock_info, length \u001b[39m-\u001b[39m \u001b[39m25\u001b[39m, deadline), compressor_id)\n\u001b[1;32m 237\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 238\u001b[0m data \u001b[39m=\u001b[39m _receive_data_on_socket(sock_info, length \u001b[39m-\u001b[39;49m \u001b[39m16\u001b[39;49m, deadline)\n\u001b[1;32m 240\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 241\u001b[0m unpack_reply \u001b[39m=\u001b[39m _UNPACK_REPLY[op_code]\n", + "File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/network.py:299\u001b[0m, in \u001b[0;36m_receive_data_on_socket\u001b[0;34m(sock_info, length, deadline)\u001b[0m\n\u001b[1;32m 297\u001b[0m \u001b[39mif\u001b[39;00m _csot\u001b[39m.\u001b[39mget_timeout():\n\u001b[1;32m 298\u001b[0m sock_info\u001b[39m.\u001b[39mset_socket_timeout(\u001b[39mmax\u001b[39m(deadline \u001b[39m-\u001b[39m time\u001b[39m.\u001b[39mmonotonic(), \u001b[39m0\u001b[39m))\n\u001b[0;32m--> 299\u001b[0m chunk_length \u001b[39m=\u001b[39m sock_info\u001b[39m.\u001b[39;49msock\u001b[39m.\u001b[39;49mrecv_into(mv[bytes_read:])\n\u001b[1;32m 300\u001b[0m \u001b[39mexcept\u001b[39;00m BLOCKING_IO_ERRORS:\n\u001b[1;32m 301\u001b[0m \u001b[39mraise\u001b[39;00m socket\u001b[39m.\u001b[39mtimeout(\u001b[39m\"\u001b[39m\u001b[39mtimed out\u001b[39m\u001b[39m\"\u001b[39m)\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "# load in all tweet texts\n", + "cursor = db.find({'account': '@Maersk'})\n", + "text_list = list()\n", + "for element in cursor:\n", + " try:\n", + " text_list.append(element['text'])\n", + " except:\n", + " print(f\"failed getting {element['_id']}\")\n", + " \n", + "print(f'got {len(text_list)} tweets')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9968ba82-9215-4069-be32-b3afa96ebe8e", + "metadata": {}, + "outputs": [], + "source": [ + "Questions:\n", + "‘challengessee’\n", + "‘challengeor’\n", + "‘changetheworld’\n", + "‘changeat’\n", + "‘oceanswe’\n", + "‘planets’\n", + "‘futureproofing’\n", + "‘leadthefuture’\n", + "‘futureready’\n", + "‘futureproof’\n", + "'futureofshipping' # hashtag\n" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "23ba20c5-adf4-4655-a5b0-3662956ff9aa", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "for text in text_list:\n", + " if 'challengessee' in text:\n", + " print(text)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "d320a5e9-926e-42dd-983e-9cf1b49fdae9", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total characters: 906246\n" + ] + }, + { + "data": { + "text/plain": [ + "\"A spotless deck before delivery Susan Mærsk was built by Odense Steel Shipyard in 1954 and a final inspection of the ship takes place before delivery. The ship was deployed on The Far East service - the company's first liner service initiated in 1928. #Maerskheritage #Maersk At #Maersk, we are honoured to partner with customers who share our vision for a cleaner, more sustainable future. By spearheading the movement to provide green shipping solutions, we’re building a brighter tomorrow for our world. More: https://bddy.me/3mSdkug #AllTheWay 14 years cooking for a multi-cultural crew! For Sandy, a workday onboard of Mette Maersk starts at 5:30 in the morning, as she gets ready to bake a new batch of fresh bread, to be ready in time for breakfast with great joy from the crew onboard. #Maersk #Lifeatsea #Chiefcook Embracing #Equity on #InternationalWomenDay! The transport and logistics industry has traditionally been male dominated, and this is no exception for Maersk. This needs to c\"" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# combine all text into one list\n", + "full_text = ' '.join(text_list)\n", + "full_text = full_text.replace('\\n', ' ')\n", + "print('total characters: ', len(full_text))\n", + "full_text[:1000]" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "918694e3-7ef7-4902-9d78-39a23e531c67", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import re\n", + "\n", + "len(re.findall('challengeor', full_text))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "bb70eefd-ee98-43b5-b2f9-5cc4b05ffcd8", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "53" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import re\n", + "\n", + "len(re.findall(' CO2', full_text))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "2b9f7c25-af69-4862-8ac0-edb87afc0c52", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# clean text\n", + "from cleantext import clean\n", + "\n", + "full_text = clean(\n", + " text=full_text,\n", + " fix_unicode=True,\n", + " to_ascii=True,\n", + " lower=True,\n", + " normalize_whitespace=True,\n", + " no_line_breaks=True,\n", + " no_urls=True,\n", + " no_emails=True,\n", + " no_phone_numbers=True,\n", + " no_numbers=True,\n", + " no_digits=False,\n", + " no_currency_symbols=True,\n", + " no_punct=True,\n", + " no_emoji=True,\n", + " replace_with_url='',\n", + " replace_with_email='',\n", + " replace_with_phone_number='',\n", + " replace_with_number='',\n", + " #replace_with_digit='', # to be able to handle CO2\n", + " replace_with_currency_symbol='',\n", + " replace_with_punct='',\n", + " lang='en'\n", + ")\n", + "\n", + "# handle case from CO2 and remove other digits\n", + "#full_text = full_text.replace(' co ', ' co2 ')\n", + "#full_text = full_text.replace('', '')" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "284be43e-fd23-4f12-a88e-9a869ab480ea", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['co2neutral',\n", + " 'co2emission',\n", + " 'co2emission',\n", + " 'co1prbwlw',\n", + " 'co1idd6st',\n", + " 'co1l7bctb',\n", + " 'co1lfltua',\n", + " 'co1o1ksg7',\n", + " 'co1ccchqp',\n", + " 'co2emissions',\n", + " 'co11bdxbv',\n", + " 'co2neutral']" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import re\n", + "\n", + "re.findall('co[1-9]\\w+', full_text)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "f7920c88-6f3c-409c-9f8a-289268c04995", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import re\n", + "\n", + "re.findall(' CO2 ', full_text)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "527cccef-e9c8-4679-b6ef-6ae09d470353", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "53" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import re\n", + "\n", + "len(re.findall(' co2 ', full_text))" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "dabcc3c3-7dad-4d12-8e12-7ce3a80305e0", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total words: 80155\n" + ] + } + ], + "source": [ + "# remove unwanted word classes\n", + "import nltk\n", + "#nltk.download('punkt')\n", + "#nltk.download('averaged_perceptron_tagger')\n", + "token_list = nltk.word_tokenize(full_text)\n", + "tag_list = nltk.pos_tag(token_list)\n", + "\n", + "unwanted_class_list = [\n", + " 'CC', # coordinating conjunction\n", + " 'DT', # determiner\n", + " 'IN', # preposition/subordinating conjunction\n", + " 'TO', # to go ‘to’ the store\n", + " 'PRP', # personal pronoun I, he, she\n", + " 'PRP$', # possessive pronoun my, his, hers\n", + "]\n", + "\n", + "word_list = list()\n", + "for word, tag in tag_list:\n", + " if tag not in unwanted_class_list:\n", + " word_list.append(word)\n", + " \n", + "print('total words: ', len(word_list))" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "23862fe3-7cfe-4089-af1c-1eb25b33cfcd", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total unique words: 10925\n" + ] + } + ], + "source": [ + "# find unique words\n", + "unique_word_list = list(set(word_list))\n", + "print('total unique words: ', len(unique_word_list))" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "5a55242e-82bd-4e1d-acc7-4ded348f5afd", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# count unique words\n", + "count_list = list()\n", + "for unique_word in unique_word_list:\n", + " count_list.append(word_list.count(unique_word))" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "f325ce44-d296-4458-a443-74958f0465ed", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
wordcountsratio
0maersk32070.040010
1is10250.012788
2more9890.012339
3here8250.010293
4trade6420.008009
5are6390.007972
6how6340.007910
7logistics6110.007623
8global4670.005826
9new4590.005726
10can4530.005652
11shipping4430.005527
12alltheway3420.004267
13learn3390.004229
14have3140.003917
15be3050.003805
16see3020.003768
17supplychain2960.003693
18supply2950.003680
19will2810.003506
20read2740.003418
21has2710.003381
22what2670.003331
23world2610.003256
24container2480.003094
25business2430.003032
26growth2350.002932
27watch2300.002869
28one2280.002844
29up2210.002757
\n", + "
" + ], + "text/plain": [ + " word counts ratio\n", + "0 maersk 3207 0.040010\n", + "1 is 1025 0.012788\n", + "2 more 989 0.012339\n", + "3 here 825 0.010293\n", + "4 trade 642 0.008009\n", + "5 are 639 0.007972\n", + "6 how 634 0.007910\n", + "7 logistics 611 0.007623\n", + "8 global 467 0.005826\n", + "9 new 459 0.005726\n", + "10 can 453 0.005652\n", + "11 shipping 443 0.005527\n", + "12 alltheway 342 0.004267\n", + "13 learn 339 0.004229\n", + "14 have 314 0.003917\n", + "15 be 305 0.003805\n", + "16 see 302 0.003768\n", + "17 supplychain 296 0.003693\n", + "18 supply 295 0.003680\n", + "19 will 281 0.003506\n", + "20 read 274 0.003418\n", + "21 has 271 0.003381\n", + "22 what 267 0.003331\n", + "23 world 261 0.003256\n", + "24 container 248 0.003094\n", + "25 business 243 0.003032\n", + "26 growth 235 0.002932\n", + "27 watch 230 0.002869\n", + "28 one 228 0.002844\n", + "29 up 221 0.002757" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# present results\n", + "import pandas as pd\n", + "word_df = pd.DataFrame.from_dict(\n", + " {\n", + " 'word': unique_word_list,\n", + " 'counts': count_list\n", + " }\n", + ")\n", + "word_df = word_df.sort_values(by='counts', ascending=False).reset_index(drop=True)\n", + "word_df['ratio'] = word_df['counts'] / len(word_list)\n", + "\n", + "word_df.head(30)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "363cd061-5de6-442f-96e6-acc4ee4cace4", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# save results\n", + "word_df['ratio'] = word_df['ratio'].map('{:.5f}'.format) # ensure consistent formatting in csv\n", + "word_df.to_csv('word_count.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4802935-9152-40a2-b459-2d30e47948b6", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}