twitter_scraper/code/plot.ipynb

In [1]:

from database import get_all
from clean_data import clean_content_en

import pandas as pd

# load data from database
df = get_all()

# remove rows without english translation
mask = df['content_en'].notna()
df = df[mask]
print(f'number of tweets found: {len(df)}')

# determine year of creation
df['year'] = pd.to_datetime(df['time'], utc=True).dt.year
year_list = df['year'].unique()
print(year_list)

year_text_map = dict()
for year in year_list:
    mask = df['year'].eq(year)
    content_list = df.loc[mask, 'content_en'].to_list()
    content_list = clean_content_en(content_list)
    year_text_map[str(year)] = ' '.join(text for text in content_list)

# content_list = df['content_en'].to_list()
# content_list = clean_content_en(content_list)
# wordcloud_text = ' '.join(text for text in content_list)

# wordcloud_text[:250]
for year, text in year_text_map.items():
    print(f'{year}: {text[:100]}...')

number of tweets found: 154115
[2022 2021 2020 2019 2018 2017]
2022: odense university hospital seeking department manager in odense read more at <url> jobdk jobiodense ...
2021: happy new year everyone odense denmark <url> basketball legend stephen curry and yao ming simply hol...
2020: happy new year i have a great view of the fireworks from my balcony newyearseve fireworks odense den...
2019: happy new year happynewyear happy0000 newyearseve fireworks odense denmark <url> glow wishing you an...
2018: happy new year happynewyear newyearseve <number> odense denmark <url> someone from odense denmark or...
2017: happy new year goody year dk odense danmark almost midnight odense denmark <url> denmarks largest se...

In [4]:

from wordcloud import WordCloud

excluded_word_list = ['<url>', '<number>', 'number', 'odense', 'more', 'here', 'read', 'have', 'been', 'this', 'year', 'would', 'that', 'absolutely', 'over', 'other', 'were', 'amomg', 'until', 'from', 'taken', 'just', 'dating', 'hottest', 'flirt', 'chat']
wc = WordCloud(
    width=1600,
    height=900,
    stopwords=excluded_word_list,
    min_font_size=5,
    scale=2.5,
    background_color='#F9F9FA',
    collocations=True,
    regexp=r'[a-zA-Z#&]+',
    max_words=50,
    min_word_length=4,
    collocation_threshold=3,
    colormap='plasma'
)

img_year_map = dict()
for year, text in year_text_map.items():
    wc_img = wc.generate(text)
    img_arr = wc_img.to_array()
    img_year_map[year] = img_arr
    print(f'generated wordcloud for year {year}')

generated wordcloud for year 2022
generated wordcloud for year 2021
generated wordcloud for year 2020
generated wordcloud for year 2019
generated wordcloud for year 2018
generated wordcloud for year 2017

In [5]:

import plotly.express as px
import matplotlib.pyplot as plt
import os

# def show_wordcloud(img_arr, year):
#     fig = px.imshow(img_arr, title=str(year))
#     fig.update_layout(
#         title_x=0.5,
#         xaxis={'visible': False},
#         yaxis={'visible': False},
#         margin={
#             't': 40,
#             'b': 0,
#             'l': 0,
#             'r': 0
#         },
#         hovermode=False,
#         paper_bgcolor='#F9F9FA'
#     )
#     fig.show()

def show_wordcloud(img_arr, year):
    fig = plt.figure(dpi=100, figsize=(12, 8))
    plt.imshow(img_arr)
    plt.title(year)
    # plt.tight_layout()
    plt.axis('off')
    root_path = os.path.dirname(os.path.abspath(''))
    img_path = os.path.join(root_path, 'img', f'wordcloud_{year}.png')
    plt.savefig(img_path)
    plt.show()

for year, img_arr in img_year_map.items():
    show_wordcloud(img_arr, year)

In [ ]: