{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>root</th>\n",
       "      <th>mean_ranking</th>\n",
       "      <th>std_ranking</th>\n",
       "      <th>word_matches</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>challeng</td>\n",
       "      <td>420</td>\n",
       "      <td>264</td>\n",
       "      <td>['challenges', 'challenge', 'challenging']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SDG</td>\n",
       "      <td>423</td>\n",
       "      <td>0</td>\n",
       "      <td>['sdgs']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ESG</td>\n",
       "      <td>636</td>\n",
       "      <td>0</td>\n",
       "      <td>['esg']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>recycling</td>\n",
       "      <td>860</td>\n",
       "      <td>154</td>\n",
       "      <td>['recycling', 'shiprecycling']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>reduc</td>\n",
       "      <td>1242</td>\n",
       "      <td>965</td>\n",
       "      <td>['reduce', 'reducing', 'reduced', 'reduction',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>planet</td>\n",
       "      <td>1295</td>\n",
       "      <td>0</td>\n",
       "      <td>['planet']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CSR</td>\n",
       "      <td>1371</td>\n",
       "      <td>0</td>\n",
       "      <td>['csr']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>sustainab</td>\n",
       "      <td>1527</td>\n",
       "      <td>1873</td>\n",
       "      <td>['sustainability', 'sustainable', 'sustainable...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>clean</td>\n",
       "      <td>1695</td>\n",
       "      <td>837</td>\n",
       "      <td>['theoceancleanup', 'cleanup', 'clean', 'clean...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>methanol</td>\n",
       "      <td>1700</td>\n",
       "      <td>1044</td>\n",
       "      <td>['methanol', 'emethanol']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>future</td>\n",
       "      <td>1770</td>\n",
       "      <td>1684</td>\n",
       "      <td>['future', 'futureproofing']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>garbage</td>\n",
       "      <td>1808</td>\n",
       "      <td>612</td>\n",
       "      <td>['garbage', 'greatpacificgarbagepatch']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>responsib</td>\n",
       "      <td>1889</td>\n",
       "      <td>1135</td>\n",
       "      <td>['responsible', 'responsibility', 'responsibly...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>carb</td>\n",
       "      <td>1952</td>\n",
       "      <td>1526</td>\n",
       "      <td>['decarbonisation', 'carbon', 'decarbonization...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>chang</td>\n",
       "      <td>2148</td>\n",
       "      <td>1667</td>\n",
       "      <td>['change', 'changing', 'climatechange', 'chang...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ocean</td>\n",
       "      <td>2169</td>\n",
       "      <td>1629</td>\n",
       "      <td>['ocean', 'theoceancleanup', 'oceans', 'oceanp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>plastic</td>\n",
       "      <td>2320</td>\n",
       "      <td>1490</td>\n",
       "      <td>['plastic', 'oceanplastic', 'plasticwaste', 'p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>neutral</td>\n",
       "      <td>2382</td>\n",
       "      <td>1798</td>\n",
       "      <td>['neutral', 'carbonneutral', 'neutrality', 'co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>environment</td>\n",
       "      <td>2391</td>\n",
       "      <td>1574</td>\n",
       "      <td>['environment', 'environmental', 'unenvironmen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>green</td>\n",
       "      <td>2393</td>\n",
       "      <td>1158</td>\n",
       "      <td>['green', 'greenfuels', 'greener', 'greenfuel'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>sulphur</td>\n",
       "      <td>2828</td>\n",
       "      <td>1772</td>\n",
       "      <td>['sulphur', 'lowsulphur']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>emissions</td>\n",
       "      <td>2925</td>\n",
       "      <td>2046</td>\n",
       "      <td>['emissions', 'carbonemissions', 'zeroemissions']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>zero</td>\n",
       "      <td>2977</td>\n",
       "      <td>1646</td>\n",
       "      <td>['zero', 'netzero', 'zerocarbon', 'zerocarbons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>climate</td>\n",
       "      <td>3046</td>\n",
       "      <td>2095</td>\n",
       "      <td>['climateaction', 'climate', 'climatechange', ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>eco</td>\n",
       "      <td>3074</td>\n",
       "      <td>1269</td>\n",
       "      <td>['ecosystem', 'eco', 'maerskecodelivery', 'eco...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>mission</td>\n",
       "      <td>3247</td>\n",
       "      <td>1950</td>\n",
       "      <td>['emissions', 'mission', 'eucommission', 'carb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>CO2</td>\n",
       "      <td>3390</td>\n",
       "      <td>2275</td>\n",
       "      <td>['co2', 'co2neutral', 'co2emission']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>bio</td>\n",
       "      <td>3687</td>\n",
       "      <td>1274</td>\n",
       "      <td>['biofuel', 'biofuels', 'biohuts', 'biodiversi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           root  mean_ranking  std_ranking   \n",
       "0      challeng           420          264  \\\n",
       "1           SDG           423            0   \n",
       "2           ESG           636            0   \n",
       "3     recycling           860          154   \n",
       "4         reduc          1242          965   \n",
       "5        planet          1295            0   \n",
       "6           CSR          1371            0   \n",
       "7     sustainab          1527         1873   \n",
       "8         clean          1695          837   \n",
       "9      methanol          1700         1044   \n",
       "10       future          1770         1684   \n",
       "11      garbage          1808          612   \n",
       "12    responsib          1889         1135   \n",
       "13         carb          1952         1526   \n",
       "14        chang          2148         1667   \n",
       "15        ocean          2169         1629   \n",
       "16      plastic          2320         1490   \n",
       "17      neutral          2382         1798   \n",
       "18  environment          2391         1574   \n",
       "19        green          2393         1158   \n",
       "20      sulphur          2828         1772   \n",
       "21    emissions          2925         2046   \n",
       "22         zero          2977         1646   \n",
       "23      climate          3046         2095   \n",
       "24          eco          3074         1269   \n",
       "25      mission          3247         1950   \n",
       "26          CO2          3390         2275   \n",
       "27          bio          3687         1274   \n",
       "\n",
       "                                         word_matches  \n",
       "0          ['challenges', 'challenge', 'challenging']  \n",
       "1                                            ['sdgs']  \n",
       "2                                             ['esg']  \n",
       "3                      ['recycling', 'shiprecycling']  \n",
       "4   ['reduce', 'reducing', 'reduced', 'reduction',...  \n",
       "5                                          ['planet']  \n",
       "6                                             ['csr']  \n",
       "7   ['sustainability', 'sustainable', 'sustainable...  \n",
       "8   ['theoceancleanup', 'cleanup', 'clean', 'clean...  \n",
       "9                           ['methanol', 'emethanol']  \n",
       "10                       ['future', 'futureproofing']  \n",
       "11            ['garbage', 'greatpacificgarbagepatch']  \n",
       "12  ['responsible', 'responsibility', 'responsibly...  \n",
       "13  ['decarbonisation', 'carbon', 'decarbonization...  \n",
       "14  ['change', 'changing', 'climatechange', 'chang...  \n",
       "15  ['ocean', 'theoceancleanup', 'oceans', 'oceanp...  \n",
       "16  ['plastic', 'oceanplastic', 'plasticwaste', 'p...  \n",
       "17  ['neutral', 'carbonneutral', 'neutrality', 'co...  \n",
       "18  ['environment', 'environmental', 'unenvironmen...  \n",
       "19  ['green', 'greenfuels', 'greener', 'greenfuel'...  \n",
       "20                          ['sulphur', 'lowsulphur']  \n",
       "21  ['emissions', 'carbonemissions', 'zeroemissions']  \n",
       "22  ['zero', 'netzero', 'zerocarbon', 'zerocarbons...  \n",
       "23  ['climateaction', 'climate', 'climatechange', ...  \n",
       "24  ['ecosystem', 'eco', 'maerskecodelivery', 'eco...  \n",
       "25  ['emissions', 'mission', 'eucommission', 'carb...  \n",
       "26               ['co2', 'co2neutral', 'co2emission']  \n",
       "27  ['biofuel', 'biofuels', 'biohuts', 'biodiversi...  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load words related to sustainability\n",
    "import pandas as pd\n",
    "\n",
    "path = 'word_root_ranking.csv'\n",
    "df = pd.read_csv(path, index_col=0)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['challenges',\n",
       " 'challenge',\n",
       " 'challenging',\n",
       " 'sdgs',\n",
       " 'esg',\n",
       " 'recycling',\n",
       " 'shiprecycling',\n",
       " 'reduce',\n",
       " 'reducing',\n",
       " 'reduced']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# collect words into list\n",
    "\n",
    "related_words_list = list()\n",
    "for row in df['word_matches']:\n",
    "    word_list = row.strip('][').replace(\"'\", '').split(', ')\n",
    "    for word in word_list:\n",
    "        related_words_list.append(word)\n",
    "        \n",
    "related_words_list[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1511"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load in all tweet texts\n",
    "from pathlib import Path\n",
    "import re\n",
    "\n",
    "tweet_dir = Path('tweets').resolve()\n",
    "path_list = list(tweet_dir.glob('*.txt'))\n",
    "score_list = list()\n",
    "for path in path_list:\n",
    "    with open(path, 'r') as f:\n",
    "        text = f.read()\n",
    "    text = re.search(\n",
    "        pattern='(?<=text: )(.|\\n)*(?=images: )',\n",
    "        string=text\n",
    "    ).group(0)\n",
    "    score = sum([related_word in text for related_word in related_words_list])\n",
    "    score_list.append(score)\n",
    "\n",
    "len(score_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(score_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/home/brian/solveig_master/tweets/2021-08-24-0...</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/home/brian/solveig_master/tweets/2021-06-22-1...</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/home/brian/solveig_master/tweets/2020-07-13-1...</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/home/brian/solveig_master/tweets/2022-04-28-1...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/home/brian/solveig_master/tweets/2022-10-20-1...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1506</th>\n",
       "      <td>/home/brian/solveig_master/tweets/2022-08-31-1...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1507</th>\n",
       "      <td>/home/brian/solveig_master/tweets/2022-12-17-0...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1508</th>\n",
       "      <td>/home/brian/solveig_master/tweets/2021-09-24-0...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1509</th>\n",
       "      <td>/home/brian/solveig_master/tweets/2014-02-06-1...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1510</th>\n",
       "      <td>/home/brian/solveig_master/tweets/2017-10-12-1...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1511 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   path  score\n",
       "0     /home/brian/solveig_master/tweets/2021-08-24-0...     15\n",
       "1     /home/brian/solveig_master/tweets/2021-06-22-1...     14\n",
       "2     /home/brian/solveig_master/tweets/2020-07-13-1...     14\n",
       "3     /home/brian/solveig_master/tweets/2022-04-28-1...     13\n",
       "4     /home/brian/solveig_master/tweets/2022-10-20-1...     13\n",
       "...                                                 ...    ...\n",
       "1506  /home/brian/solveig_master/tweets/2022-08-31-1...      0\n",
       "1507  /home/brian/solveig_master/tweets/2022-12-17-0...      0\n",
       "1508  /home/brian/solveig_master/tweets/2021-09-24-0...      0\n",
       "1509  /home/brian/solveig_master/tweets/2014-02-06-1...      0\n",
       "1510  /home/brian/solveig_master/tweets/2017-10-12-1...      0\n",
       "\n",
       "[1511 rows x 2 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(\n",
    "    {\n",
    "        'path': path_list,\n",
    "        'score': score_list\n",
    "    }\n",
    ")\n",
    "\n",
    "df = df.sort_values(by='score', ascending=False)\n",
    "df.reset_index(drop=True, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('tweets_relevance_list.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}