{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of tweets found: 154115\n",
      "[2022 2021 2020 2019 2018 2017]\n",
      "2022: odense university hospital seeking department manager in odense read more at <url> jobdk jobiodense ...\n",
      "2021: happy new year everyone odense denmark <url> basketball legend stephen curry and yao ming simply hol...\n",
      "2020: happy new year i have a great view of the fireworks from my balcony newyearseve fireworks odense den...\n",
      "2019: happy new year happynewyear happy0000 newyearseve fireworks odense denmark <url> glow wishing you an...\n",
      "2018: happy new year happynewyear newyearseve <number> odense denmark <url> someone from odense denmark or...\n",
      "2017: happy new year goody year dk odense danmark almost midnight odense denmark <url> denmarks largest se...\n"
     ]
    }
   ],
   "source": [
    "from database import get_all\n",
    "from clean_data import clean_content_en\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "# load data from database\n",
    "df = get_all()\n",
    "\n",
    "# remove rows without english translation\n",
    "mask = df['content_en'].notna()\n",
    "df = df[mask]\n",
    "print(f'number of tweets found: {len(df)}')\n",
    "\n",
    "# determine year of creation\n",
    "df['year'] = pd.to_datetime(df['time'], utc=True).dt.year\n",
    "year_list = df['year'].unique()\n",
    "print(year_list)\n",
    "\n",
    "year_text_map = dict()\n",
    "for year in year_list:\n",
    "    mask = df['year'].eq(year)\n",
    "    content_list = df.loc[mask, 'content_en'].to_list()\n",
    "    content_list = clean_content_en(content_list)\n",
    "    year_text_map[str(year)] = ' '.join(text for text in content_list)\n",
    "\n",
    "# content_list = df['content_en'].to_list()\n",
    "# content_list = clean_content_en(content_list)\n",
    "# wordcloud_text = ' '.join(text for text in content_list)\n",
    "\n",
    "# wordcloud_text[:250]\n",
    "for year, text in year_text_map.items():\n",
    "    print(f'{year}: {text[:100]}...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generated wordcloud for year 2022\n",
      "generated wordcloud for year 2021\n",
      "generated wordcloud for year 2020\n",
      "generated wordcloud for year 2019\n",
      "generated wordcloud for year 2018\n",
      "generated wordcloud for year 2017\n"
     ]
    }
   ],
   "source": [
    "from wordcloud import WordCloud\n",
    "\n",
    "excluded_word_list = ['<url>', '<number>', 'number', 'odense', 'more', 'here', 'read', 'have', 'been', 'this', 'year', 'would', 'that', 'absolutely', 'over', 'other', 'were', 'amomg', 'until', 'from', 'taken', 'just', 'dating', 'hottest', 'flirt', 'chat']\n",
    "wc = WordCloud(\n",
    "    width=1600,\n",
    "    height=900,\n",
    "    stopwords=excluded_word_list,\n",
    "    min_font_size=5,\n",
    "    scale=2.5,\n",
    "    background_color='#F9F9FA',\n",
    "    collocations=True,\n",
    "    regexp=r'[a-zA-Z#&]+',\n",
    "    max_words=50,\n",
    "    min_word_length=4,\n",
    "    collocation_threshold=3,\n",
    "    colormap='plasma'\n",
    ")\n",
    "\n",
    "img_year_map = dict()\n",
    "for year, text in year_text_map.items():\n",
    "    wc_img = wc.generate(text)\n",
    "    img_arr = wc_img.to_array()\n",
    "    img_year_map[year] = img_arr\n",
    "    print(f'generated wordcloud for year {year}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 344536 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 313724 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 352212 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 343856 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 358488 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 339320 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import plotly.express as px\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "# def show_wordcloud(img_arr, year):\n",
    "#     fig = px.imshow(img_arr, title=str(year))\n",
    "#     fig.update_layout(\n",
    "#         title_x=0.5,\n",
    "#         xaxis={'visible': False},\n",
    "#         yaxis={'visible': False},\n",
    "#         margin={\n",
    "#             't': 40,\n",
    "#             'b': 0,\n",
    "#             'l': 0,\n",
    "#             'r': 0\n",
    "#         },\n",
    "#         hovermode=False,\n",
    "#         paper_bgcolor='#F9F9FA'\n",
    "#     )\n",
    "#     fig.show()\n",
    "\n",
    "def show_wordcloud(img_arr, year):\n",
    "    fig = plt.figure(dpi=100, figsize=(12, 8))\n",
    "    plt.imshow(img_arr)\n",
    "    plt.title(year)\n",
    "    # plt.tight_layout()\n",
    "    plt.axis('off')\n",
    "    root_path = os.path.dirname(os.path.abspath(''))\n",
    "    img_path = os.path.join(root_path, 'img', f'wordcloud_{year}.png')\n",
    "    plt.savefig(img_path)\n",
    "    plt.show()\n",
    "\n",
    "for year, img_arr in img_year_map.items():\n",
    "    show_wordcloud(img_arr, year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
