709 lines
30 KiB
Plaintext
709 lines
30 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "29d44422-b6ea-4cdd-8366-7a9daedc8dd4",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"\n",
|
||
"from classes import Tweet\n",
|
||
"from database import connect as connect_db\n",
|
||
"\n",
|
||
"db = connect_db()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"id": "0d043248-c8ad-47bc-90b6-40f02ab279ba",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"ename": "KeyboardInterrupt",
|
||
"evalue": "",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
|
||
"Cell \u001b[0;32mIn[6], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m cursor \u001b[39m=\u001b[39m db\u001b[39m.\u001b[39mfind({\u001b[39m'\u001b[39m\u001b[39maccount\u001b[39m\u001b[39m'\u001b[39m: \u001b[39m'\u001b[39m\u001b[39m@Maersk\u001b[39m\u001b[39m'\u001b[39m})\n\u001b[1;32m 3\u001b[0m text_list \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m()\n\u001b[0;32m----> 4\u001b[0m \u001b[39mfor\u001b[39;00m element \u001b[39min\u001b[39;00m cursor:\n\u001b[1;32m 5\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 6\u001b[0m text_list\u001b[39m.\u001b[39mappend(element[\u001b[39m'\u001b[39m\u001b[39mtext\u001b[39m\u001b[39m'\u001b[39m])\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1248\u001b[0m, in \u001b[0;36mCursor.next\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1246\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__empty:\n\u001b[1;32m 1247\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m\n\u001b[0;32m-> 1248\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__data) \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_refresh():\n\u001b[1;32m 1249\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__data\u001b[39m.\u001b[39mpopleft()\n\u001b[1;32m 1250\u001b[0m \u001b[39melse\u001b[39;00m:\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1188\u001b[0m, in \u001b[0;36mCursor._refresh\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1173\u001b[0m \u001b[39m# Exhaust cursors don't send getMore messages.\u001b[39;00m\n\u001b[1;32m 1174\u001b[0m g \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_getmore_class(\n\u001b[1;32m 1175\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__dbname,\n\u001b[1;32m 1176\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__collname,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1186\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__comment,\n\u001b[1;32m 1187\u001b[0m )\n\u001b[0;32m-> 1188\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m__send_message(g)\n\u001b[1;32m 1190\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__data)\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/cursor.py:1052\u001b[0m, in \u001b[0;36mCursor.__send_message\u001b[0;34m(self, operation)\u001b[0m\n\u001b[1;32m 1049\u001b[0m \u001b[39mraise\u001b[39;00m InvalidOperation(\u001b[39m\"\u001b[39m\u001b[39mexhaust cursors do not support auto encryption\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 1051\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 1052\u001b[0m response \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39;49m_run_operation(\n\u001b[1;32m 1053\u001b[0m operation, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_unpack_response, address\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m__address\n\u001b[1;32m 1054\u001b[0m )\n\u001b[1;32m 1055\u001b[0m \u001b[39mexcept\u001b[39;00m OperationFailure \u001b[39mas\u001b[39;00m exc:\n\u001b[1;32m 1056\u001b[0m \u001b[39mif\u001b[39;00m exc\u001b[39m.\u001b[39mcode \u001b[39min\u001b[39;00m _CURSOR_CLOSED_ERRORS \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__exhaust:\n\u001b[1;32m 1057\u001b[0m \u001b[39m# Don't send killCursors because the cursor is already closed.\u001b[39;00m\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/_csot.py:105\u001b[0m, in \u001b[0;36mapply.<locals>.csot_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[39mwith\u001b[39;00m _TimeoutContext(timeout):\n\u001b[1;32m 104\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m--> 105\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1330\u001b[0m, in \u001b[0;36mMongoClient._run_operation\u001b[0;34m(self, operation, unpack_res, address)\u001b[0m\n\u001b[1;32m 1325\u001b[0m operation\u001b[39m.\u001b[39mreset() \u001b[39m# Reset op in case of retry.\u001b[39;00m\n\u001b[1;32m 1326\u001b[0m \u001b[39mreturn\u001b[39;00m server\u001b[39m.\u001b[39mrun_operation(\n\u001b[1;32m 1327\u001b[0m sock_info, operation, read_preference, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_event_listeners, unpack_res\n\u001b[1;32m 1328\u001b[0m )\n\u001b[0;32m-> 1330\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_retryable_read(\n\u001b[1;32m 1331\u001b[0m _cmd,\n\u001b[1;32m 1332\u001b[0m operation\u001b[39m.\u001b[39;49mread_preference,\n\u001b[1;32m 1333\u001b[0m operation\u001b[39m.\u001b[39;49msession,\n\u001b[1;32m 1334\u001b[0m address\u001b[39m=\u001b[39;49maddress,\n\u001b[1;32m 1335\u001b[0m retryable\u001b[39m=\u001b[39;49m\u001b[39misinstance\u001b[39;49m(operation, message\u001b[39m.\u001b[39;49m_Query),\n\u001b[1;32m 1336\u001b[0m )\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/_csot.py:105\u001b[0m, in \u001b[0;36mapply.<locals>.csot_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[39mwith\u001b[39;00m _TimeoutContext(timeout):\n\u001b[1;32m 104\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m--> 105\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1448\u001b[0m, in \u001b[0;36mMongoClient._retryable_read\u001b[0;34m(self, func, read_pref, session, address, retryable)\u001b[0m\n\u001b[1;32m 1446\u001b[0m \u001b[39massert\u001b[39;00m last_error \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 1447\u001b[0m \u001b[39mraise\u001b[39;00m last_error\n\u001b[0;32m-> 1448\u001b[0m \u001b[39mreturn\u001b[39;00m func(session, server, sock_info, read_pref)\n\u001b[1;32m 1449\u001b[0m \u001b[39mexcept\u001b[39;00m ServerSelectionTimeoutError:\n\u001b[1;32m 1450\u001b[0m \u001b[39mif\u001b[39;00m retrying:\n\u001b[1;32m 1451\u001b[0m \u001b[39m# The application may think the write was never attempted\u001b[39;00m\n\u001b[1;32m 1452\u001b[0m \u001b[39m# if we raise ServerSelectionTimeoutError on the retry\u001b[39;00m\n\u001b[1;32m 1453\u001b[0m \u001b[39m# attempt. Raise the original exception instead.\u001b[39;00m\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/mongo_client.py:1326\u001b[0m, in \u001b[0;36mMongoClient._run_operation.<locals>._cmd\u001b[0;34m(session, server, sock_info, read_preference)\u001b[0m\n\u001b[1;32m 1324\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_cmd\u001b[39m(session, server, sock_info, read_preference):\n\u001b[1;32m 1325\u001b[0m operation\u001b[39m.\u001b[39mreset() \u001b[39m# Reset op in case of retry.\u001b[39;00m\n\u001b[0;32m-> 1326\u001b[0m \u001b[39mreturn\u001b[39;00m server\u001b[39m.\u001b[39;49mrun_operation(\n\u001b[1;32m 1327\u001b[0m sock_info, operation, read_preference, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_event_listeners, unpack_res\n\u001b[1;32m 1328\u001b[0m )\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/server.py:115\u001b[0m, in \u001b[0;36mServer.run_operation\u001b[0;34m(self, sock_info, operation, read_preference, listeners, unpack_res)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 114\u001b[0m sock_info\u001b[39m.\u001b[39msend_message(data, max_doc_size)\n\u001b[0;32m--> 115\u001b[0m reply \u001b[39m=\u001b[39m sock_info\u001b[39m.\u001b[39;49mreceive_message(request_id)\n\u001b[1;32m 117\u001b[0m \u001b[39m# Unpack and check for command errors.\u001b[39;00m\n\u001b[1;32m 118\u001b[0m \u001b[39mif\u001b[39;00m use_cmd:\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/pool.py:821\u001b[0m, in \u001b[0;36mSocketInfo.receive_message\u001b[0;34m(self, request_id)\u001b[0m\n\u001b[1;32m 819\u001b[0m \u001b[39mreturn\u001b[39;00m receive_message(\u001b[39mself\u001b[39m, request_id, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmax_message_size)\n\u001b[1;32m 820\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m error:\n\u001b[0;32m--> 821\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_raise_connection_failure(error)\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/pool.py:819\u001b[0m, in \u001b[0;36mSocketInfo.receive_message\u001b[0;34m(self, request_id)\u001b[0m\n\u001b[1;32m 814\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Receive a raw BSON message or raise ConnectionFailure.\u001b[39;00m\n\u001b[1;32m 815\u001b[0m \n\u001b[1;32m 816\u001b[0m \u001b[39mIf any exception is raised, the socket is closed.\u001b[39;00m\n\u001b[1;32m 817\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 818\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 819\u001b[0m \u001b[39mreturn\u001b[39;00m receive_message(\u001b[39mself\u001b[39;49m, request_id, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_message_size)\n\u001b[1;32m 820\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m error:\n\u001b[1;32m 821\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_raise_connection_failure(error)\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/network.py:238\u001b[0m, in \u001b[0;36mreceive_message\u001b[0;34m(sock_info, request_id, max_message_size)\u001b[0m\n\u001b[1;32m 236\u001b[0m data \u001b[39m=\u001b[39m decompress(_receive_data_on_socket(sock_info, length \u001b[39m-\u001b[39m \u001b[39m25\u001b[39m, deadline), compressor_id)\n\u001b[1;32m 237\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 238\u001b[0m data \u001b[39m=\u001b[39m _receive_data_on_socket(sock_info, length \u001b[39m-\u001b[39;49m \u001b[39m16\u001b[39;49m, deadline)\n\u001b[1;32m 240\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 241\u001b[0m unpack_reply \u001b[39m=\u001b[39m _UNPACK_REPLY[op_code]\n",
|
||
"File \u001b[0;32m~/Desktop/Solveig/venv/lib/python3.8/site-packages/pymongo/network.py:299\u001b[0m, in \u001b[0;36m_receive_data_on_socket\u001b[0;34m(sock_info, length, deadline)\u001b[0m\n\u001b[1;32m 297\u001b[0m \u001b[39mif\u001b[39;00m _csot\u001b[39m.\u001b[39mget_timeout():\n\u001b[1;32m 298\u001b[0m sock_info\u001b[39m.\u001b[39mset_socket_timeout(\u001b[39mmax\u001b[39m(deadline \u001b[39m-\u001b[39m time\u001b[39m.\u001b[39mmonotonic(), \u001b[39m0\u001b[39m))\n\u001b[0;32m--> 299\u001b[0m chunk_length \u001b[39m=\u001b[39m sock_info\u001b[39m.\u001b[39;49msock\u001b[39m.\u001b[39;49mrecv_into(mv[bytes_read:])\n\u001b[1;32m 300\u001b[0m \u001b[39mexcept\u001b[39;00m BLOCKING_IO_ERRORS:\n\u001b[1;32m 301\u001b[0m \u001b[39mraise\u001b[39;00m socket\u001b[39m.\u001b[39mtimeout(\u001b[39m\"\u001b[39m\u001b[39mtimed out\u001b[39m\u001b[39m\"\u001b[39m)\n",
|
||
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# load in all tweet texts\n",
|
||
"cursor = db.find({'account': '@Maersk'})\n",
|
||
"text_list = list()\n",
|
||
"for element in cursor:\n",
|
||
" try:\n",
|
||
" text_list.append(element['text'])\n",
|
||
" except:\n",
|
||
" print(f\"failed getting {element['_id']}\")\n",
|
||
" \n",
|
||
"print(f'got {len(text_list)} tweets')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "9968ba82-9215-4069-be32-b3afa96ebe8e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"Questions:\n",
|
||
"‘challengessee’\n",
|
||
"‘challengeor’\n",
|
||
"‘changetheworld’\n",
|
||
"‘changeat’\n",
|
||
"‘oceanswe’\n",
|
||
"‘planets’\n",
|
||
"‘futureproofing’\n",
|
||
"‘leadthefuture’\n",
|
||
"‘futureready’\n",
|
||
"‘futureproof’\n",
|
||
"'futureofshipping' # hashtag\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 48,
|
||
"id": "23ba20c5-adf4-4655-a5b0-3662956ff9aa",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"for text in text_list:\n",
|
||
" if 'challengessee' in text:\n",
|
||
" print(text)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 20,
|
||
"id": "d320a5e9-926e-42dd-983e-9cf1b49fdae9",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"total characters: 906246\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"\"A spotless deck before delivery Susan Mærsk was built by Odense Steel Shipyard in 1954 and a final inspection of the ship takes place before delivery. The ship was deployed on The Far East service - the company's first liner service initiated in 1928. #Maerskheritage #Maersk At #Maersk, we are honoured to partner with customers who share our vision for a cleaner, more sustainable future. By spearheading the movement to provide green shipping solutions, we’re building a brighter tomorrow for our world. More: https://bddy.me/3mSdkug #AllTheWay 14 years cooking for a multi-cultural crew! For Sandy, a workday onboard of Mette Maersk starts at 5:30 in the morning, as she gets ready to bake a new batch of fresh bread, to be ready in time for breakfast with great joy from the crew onboard. #Maersk #Lifeatsea #Chiefcook Embracing #Equity on #InternationalWomenDay! The transport and logistics industry has traditionally been male dominated, and this is no exception for Maersk. This needs to c\""
|
||
]
|
||
},
|
||
"execution_count": 20,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"# combine all text into one list\n",
|
||
"full_text = ' '.join(text_list)\n",
|
||
"full_text = full_text.replace('\\n', ' ')\n",
|
||
"print('total characters: ', len(full_text))\n",
|
||
"full_text[:1000]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 50,
|
||
"id": "918694e3-7ef7-4902-9d78-39a23e531c67",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"1"
|
||
]
|
||
},
|
||
"execution_count": 50,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"import re\n",
|
||
"\n",
|
||
"len(re.findall('challengeor', full_text))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 22,
|
||
"id": "bb70eefd-ee98-43b5-b2f9-5cc4b05ffcd8",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"53"
|
||
]
|
||
},
|
||
"execution_count": 22,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"import re\n",
|
||
"\n",
|
||
"len(re.findall(' CO2', full_text))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 23,
|
||
"id": "2b9f7c25-af69-4862-8ac0-edb87afc0c52",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# clean text\n",
|
||
"from cleantext import clean\n",
|
||
"\n",
|
||
"full_text = clean(\n",
|
||
" text=full_text,\n",
|
||
" fix_unicode=True,\n",
|
||
" to_ascii=True,\n",
|
||
" lower=True,\n",
|
||
" normalize_whitespace=True,\n",
|
||
" no_line_breaks=True,\n",
|
||
" no_urls=True,\n",
|
||
" no_emails=True,\n",
|
||
" no_phone_numbers=True,\n",
|
||
" no_numbers=True,\n",
|
||
" no_digits=False,\n",
|
||
" no_currency_symbols=True,\n",
|
||
" no_punct=True,\n",
|
||
" no_emoji=True,\n",
|
||
" replace_with_url='',\n",
|
||
" replace_with_email='',\n",
|
||
" replace_with_phone_number='',\n",
|
||
" replace_with_number='',\n",
|
||
" #replace_with_digit='', # to be able to handle CO2\n",
|
||
" replace_with_currency_symbol='',\n",
|
||
" replace_with_punct='',\n",
|
||
" lang='en'\n",
|
||
")\n",
|
||
"\n",
|
||
"# handle case from CO2 and remove other digits\n",
|
||
"#full_text = full_text.replace(' co<digit> ', ' co2 ')\n",
|
||
"#full_text = full_text.replace('<digit>', '')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 24,
|
||
"id": "284be43e-fd23-4f12-a88e-9a869ab480ea",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"['co2neutral',\n",
|
||
" 'co2emission',\n",
|
||
" 'co2emission',\n",
|
||
" 'co1prbwlw',\n",
|
||
" 'co1idd6st',\n",
|
||
" 'co1l7bctb',\n",
|
||
" 'co1lfltua',\n",
|
||
" 'co1o1ksg7',\n",
|
||
" 'co1ccchqp',\n",
|
||
" 'co2emissions',\n",
|
||
" 'co11bdxbv',\n",
|
||
" 'co2neutral']"
|
||
]
|
||
},
|
||
"execution_count": 24,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"import re\n",
|
||
"\n",
|
||
"re.findall('co[1-9]\\w+', full_text)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 25,
|
||
"id": "f7920c88-6f3c-409c-9f8a-289268c04995",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"[]"
|
||
]
|
||
},
|
||
"execution_count": 25,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"import re\n",
|
||
"\n",
|
||
"re.findall(' CO2 ', full_text)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 26,
|
||
"id": "527cccef-e9c8-4679-b6ef-6ae09d470353",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"53"
|
||
]
|
||
},
|
||
"execution_count": 26,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"import re\n",
|
||
"\n",
|
||
"len(re.findall(' co2 ', full_text))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 27,
|
||
"id": "dabcc3c3-7dad-4d12-8e12-7ce3a80305e0",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"total words: 80155\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# remove unwanted word classes\n",
|
||
"import nltk\n",
|
||
"#nltk.download('punkt')\n",
|
||
"#nltk.download('averaged_perceptron_tagger')\n",
|
||
"token_list = nltk.word_tokenize(full_text)\n",
|
||
"tag_list = nltk.pos_tag(token_list)\n",
|
||
"\n",
|
||
"unwanted_class_list = [\n",
|
||
" 'CC', # coordinating conjunction\n",
|
||
" 'DT', # determiner\n",
|
||
" 'IN', # preposition/subordinating conjunction\n",
|
||
" 'TO', # to go ‘to’ the store\n",
|
||
" 'PRP', # personal pronoun I, he, she\n",
|
||
" 'PRP$', # possessive pronoun my, his, hers\n",
|
||
"]\n",
|
||
"\n",
|
||
"word_list = list()\n",
|
||
"for word, tag in tag_list:\n",
|
||
" if tag not in unwanted_class_list:\n",
|
||
" word_list.append(word)\n",
|
||
" \n",
|
||
"print('total words: ', len(word_list))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 28,
|
||
"id": "23862fe3-7cfe-4089-af1c-1eb25b33cfcd",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"total unique words: 10925\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# find unique words\n",
|
||
"unique_word_list = list(set(word_list))\n",
|
||
"print('total unique words: ', len(unique_word_list))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 29,
|
||
"id": "5a55242e-82bd-4e1d-acc7-4ded348f5afd",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# count unique words\n",
|
||
"count_list = list()\n",
|
||
"for unique_word in unique_word_list:\n",
|
||
" count_list.append(word_list.count(unique_word))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 30,
|
||
"id": "f325ce44-d296-4458-a443-74958f0465ed",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>word</th>\n",
|
||
" <th>counts</th>\n",
|
||
" <th>ratio</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>maersk</td>\n",
|
||
" <td>3207</td>\n",
|
||
" <td>0.040010</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>is</td>\n",
|
||
" <td>1025</td>\n",
|
||
" <td>0.012788</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>more</td>\n",
|
||
" <td>989</td>\n",
|
||
" <td>0.012339</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>here</td>\n",
|
||
" <td>825</td>\n",
|
||
" <td>0.010293</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>trade</td>\n",
|
||
" <td>642</td>\n",
|
||
" <td>0.008009</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>5</th>\n",
|
||
" <td>are</td>\n",
|
||
" <td>639</td>\n",
|
||
" <td>0.007972</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>6</th>\n",
|
||
" <td>how</td>\n",
|
||
" <td>634</td>\n",
|
||
" <td>0.007910</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>7</th>\n",
|
||
" <td>logistics</td>\n",
|
||
" <td>611</td>\n",
|
||
" <td>0.007623</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>8</th>\n",
|
||
" <td>global</td>\n",
|
||
" <td>467</td>\n",
|
||
" <td>0.005826</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>9</th>\n",
|
||
" <td>new</td>\n",
|
||
" <td>459</td>\n",
|
||
" <td>0.005726</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>10</th>\n",
|
||
" <td>can</td>\n",
|
||
" <td>453</td>\n",
|
||
" <td>0.005652</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>11</th>\n",
|
||
" <td>shipping</td>\n",
|
||
" <td>443</td>\n",
|
||
" <td>0.005527</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>12</th>\n",
|
||
" <td>alltheway</td>\n",
|
||
" <td>342</td>\n",
|
||
" <td>0.004267</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>13</th>\n",
|
||
" <td>learn</td>\n",
|
||
" <td>339</td>\n",
|
||
" <td>0.004229</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>14</th>\n",
|
||
" <td>have</td>\n",
|
||
" <td>314</td>\n",
|
||
" <td>0.003917</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>15</th>\n",
|
||
" <td>be</td>\n",
|
||
" <td>305</td>\n",
|
||
" <td>0.003805</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>16</th>\n",
|
||
" <td>see</td>\n",
|
||
" <td>302</td>\n",
|
||
" <td>0.003768</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>17</th>\n",
|
||
" <td>supplychain</td>\n",
|
||
" <td>296</td>\n",
|
||
" <td>0.003693</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>18</th>\n",
|
||
" <td>supply</td>\n",
|
||
" <td>295</td>\n",
|
||
" <td>0.003680</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>19</th>\n",
|
||
" <td>will</td>\n",
|
||
" <td>281</td>\n",
|
||
" <td>0.003506</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>20</th>\n",
|
||
" <td>read</td>\n",
|
||
" <td>274</td>\n",
|
||
" <td>0.003418</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>21</th>\n",
|
||
" <td>has</td>\n",
|
||
" <td>271</td>\n",
|
||
" <td>0.003381</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>22</th>\n",
|
||
" <td>what</td>\n",
|
||
" <td>267</td>\n",
|
||
" <td>0.003331</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>23</th>\n",
|
||
" <td>world</td>\n",
|
||
" <td>261</td>\n",
|
||
" <td>0.003256</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>24</th>\n",
|
||
" <td>container</td>\n",
|
||
" <td>248</td>\n",
|
||
" <td>0.003094</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>25</th>\n",
|
||
" <td>business</td>\n",
|
||
" <td>243</td>\n",
|
||
" <td>0.003032</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>26</th>\n",
|
||
" <td>growth</td>\n",
|
||
" <td>235</td>\n",
|
||
" <td>0.002932</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>27</th>\n",
|
||
" <td>watch</td>\n",
|
||
" <td>230</td>\n",
|
||
" <td>0.002869</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>28</th>\n",
|
||
" <td>one</td>\n",
|
||
" <td>228</td>\n",
|
||
" <td>0.002844</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>29</th>\n",
|
||
" <td>up</td>\n",
|
||
" <td>221</td>\n",
|
||
" <td>0.002757</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" word counts ratio\n",
|
||
"0 maersk 3207 0.040010\n",
|
||
"1 is 1025 0.012788\n",
|
||
"2 more 989 0.012339\n",
|
||
"3 here 825 0.010293\n",
|
||
"4 trade 642 0.008009\n",
|
||
"5 are 639 0.007972\n",
|
||
"6 how 634 0.007910\n",
|
||
"7 logistics 611 0.007623\n",
|
||
"8 global 467 0.005826\n",
|
||
"9 new 459 0.005726\n",
|
||
"10 can 453 0.005652\n",
|
||
"11 shipping 443 0.005527\n",
|
||
"12 alltheway 342 0.004267\n",
|
||
"13 learn 339 0.004229\n",
|
||
"14 have 314 0.003917\n",
|
||
"15 be 305 0.003805\n",
|
||
"16 see 302 0.003768\n",
|
||
"17 supplychain 296 0.003693\n",
|
||
"18 supply 295 0.003680\n",
|
||
"19 will 281 0.003506\n",
|
||
"20 read 274 0.003418\n",
|
||
"21 has 271 0.003381\n",
|
||
"22 what 267 0.003331\n",
|
||
"23 world 261 0.003256\n",
|
||
"24 container 248 0.003094\n",
|
||
"25 business 243 0.003032\n",
|
||
"26 growth 235 0.002932\n",
|
||
"27 watch 230 0.002869\n",
|
||
"28 one 228 0.002844\n",
|
||
"29 up 221 0.002757"
|
||
]
|
||
},
|
||
"execution_count": 30,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"# present results\n",
|
||
"import pandas as pd\n",
|
||
"word_df = pd.DataFrame.from_dict(\n",
|
||
" {\n",
|
||
" 'word': unique_word_list,\n",
|
||
" 'counts': count_list\n",
|
||
" }\n",
|
||
")\n",
|
||
"word_df = word_df.sort_values(by='counts', ascending=False).reset_index(drop=True)\n",
|
||
"word_df['ratio'] = word_df['counts'] / len(word_list)\n",
|
||
"\n",
|
||
"word_df.head(30)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 31,
|
||
"id": "363cd061-5de6-442f-96e6-acc4ee4cace4",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# save results\n",
|
||
"word_df['ratio'] = word_df['ratio'].map('{:.5f}'.format) # ensure consistent formatting in csv\n",
|
||
"word_df.to_csv('word_count.csv')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "d4802935-9152-40a2-b459-2d30e47948b6",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.8.16"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|