twitter_scraper/code/plot.ipynb

In [1]:

from database import get_all
from clean_data import clean_content_en

import pandas as pd

# load data from database
df = get_all()

# remove rows without english translation
mask = df['content_en'].notna()
df = df[mask]
print(f'number of tweets found: {len(df)}')

# determine year of creation
df['year'] = pd.to_datetime(df['time'], utc=True).dt.year
year_list = df['year'].unique()
print(year_list)

year_text_map = dict()
for year in year_list:
    mask = df['year'].eq(year)
    content_list = df.loc[mask, 'content_en'].to_list()
    content_list = clean_content_en(content_list)
    year_text_map[str(year)] = ' '.join(text for text in content_list)

# content_list = df['content_en'].to_list()
# content_list = clean_content_en(content_list)
# wordcloud_text = ' '.join(text for text in content_list)

# wordcloud_text[:250]
for year, text in year_text_map.items():
    print(f'{year}: {text[:100]}...')

number of tweets found: 6263
[2022]
2022: odense university hospital seeking department manager in odense read more at <url> jobdk jobiodense ...

In [2]:

from wordcloud import WordCloud

excluded_word_list = ['<url>', '<number>', 'number', 'odense', 'more', 'here', 'read', 'have', 'been', 'this', 'year']
wc = WordCloud(
    width=1600,
    height=900,
    stopwords=excluded_word_list,
    min_font_size=5,
    scale=2.5,
    background_color='#F9F9FA',
    collocations=True,
    regexp=r'[a-zA-Z#&]+',
    max_words=50,
    min_word_length=4,
    collocation_threshold=3,
    colormap='plasma'
)

img_year_map = dict()
for year, text in year_text_map.items():
    wc_img = wc.generate(text)
    img_arr = wc_img.to_array()
    img_year_map[year] = img_arr
    print(f'generated wordcloud for year {year}')

generated wordcloud for year 2022

In [15]:

import plotly.express as px

def show_wordcloud(img_arr, year):
    fig = px.imshow(img_arr, title=str(year))
    fig.update_layout(
        title_x=0.5,
        xaxis={'visible': False},
        yaxis={'visible': False},
        margin={
            't': 40,
            'b': 0,
            'l': 0,
            'r': 0
        },
        hovermode=False,
        paper_bgcolor='#F9F9FA'
    )
    fig.show()

for year, img_arr in img_year_map.items():
    show_wordcloud(img_arr, year)

In [ ]: