twitter_scraper/code/plot.ipynb

In [1]:

from database import get_all
from clean_data import clean_content_en

import pandas as pd

# load data from database
df = get_all()

# remove rows without english translation
mask = df['content_en'].notna()
df = df[mask]
print(f'number of tweets found: {len(df)}')

# determine year of creation
df['year'] = pd.to_datetime(df['time'], utc=True).dt.year
year_list = df['year'].unique()
print(year_list)

year_text_map = dict()
for year in year_list:
    mask = df['year'].eq(year)
    content_list = df.loc[mask, 'content_en'].to_list()
    content_list = clean_content_en(content_list)
    year_text_map[str(year)] = ' '.join(text for text in content_list)

# content_list = df['content_en'].to_list()
# content_list = clean_content_en(content_list)
# wordcloud_text = ' '.join(text for text in content_list)

# wordcloud_text[:250]
for year, text in year_text_map.items():
    print(f'{year}: {text[:100]}...')

number of tweets found: 58805
[2022 2021 2020 2019 2018 2017]
2022: odense university hospital seeking department manager in odense read more at <url> jobdk jobiodense ...
2021: basketball legend stephen curry and yao ming simply hold new years in odense tv0newsdk dkmedier <url...
2020: happy new year i have a great view of the fireworks from my balcony newyearseve fireworks odense den...
2019: glow wishing you and your loved ones a very happy new year we hope your <number> will be filled with...
2018: dreaming us back to the hot summer and langelandsfestival0000 dittedegner which start do you think b...
2017: new post buy gaffa prisen <number> tickets <number><number><number> <number><number><number> odeon o...

In [4]:

from wordcloud import WordCloud

excluded_word_list = ['<url>', '<number>', 'number', 'odense', 'more', 'here', 'read', 'have', 'been', 'this', 'year', 'would', 'that', 'absolutely', 'over', 'among other', 'were']
wc = WordCloud(
    width=1600,
    height=900,
    stopwords=excluded_word_list,
    min_font_size=5,
    scale=2.5,
    background_color='#F9F9FA',
    collocations=True,
    regexp=r'[a-zA-Z#&]+',
    max_words=50,
    min_word_length=4,
    collocation_threshold=3,
    colormap='plasma'
)

img_year_map = dict()
for year, text in year_text_map.items():
    wc_img = wc.generate(text)
    img_arr = wc_img.to_array()
    img_year_map[year] = img_arr
    print(f'generated wordcloud for year {year}')

generated wordcloud for year 2022
generated wordcloud for year 2021
generated wordcloud for year 2020
generated wordcloud for year 2019
generated wordcloud for year 2018
generated wordcloud for year 2017

In [18]:

import plotly.express as px
import matplotlib.pyplot as plt
import os

# def show_wordcloud(img_arr, year):
#     fig = px.imshow(img_arr, title=str(year))
#     fig.update_layout(
#         title_x=0.5,
#         xaxis={'visible': False},
#         yaxis={'visible': False},
#         margin={
#             't': 40,
#             'b': 0,
#             'l': 0,
#             'r': 0
#         },
#         hovermode=False,
#         paper_bgcolor='#F9F9FA'
#     )
#     fig.show()

def show_wordcloud(img_arr, year):
    fig = plt.figure(dpi=100, figsize=(12, 8))
    plt.imshow(img_arr)
    plt.title(year)
    # plt.tight_layout()
    plt.axis('off')
    root_path = os.path.dirname(os.path.abspath(''))
    img_path = os.path.join(root_path, 'img', f'wordcloud_{year}.png')
    plt.savefig(img_path)
    plt.show()

for year, img_arr in img_year_map.items():
    show_wordcloud(img_arr, year)

In [ ]: