76 lines
2.6 KiB
Python
76 lines
2.6 KiB
Python
# public packages
|
|
import snscrape.modules.twitter as sntwitter
|
|
from tqdm import tqdm
|
|
import time
|
|
# database-related
|
|
from database import Data_table, insert_list, get_all
|
|
from dotenv import load_dotenv
|
|
# own code
|
|
from clean_data import clean_content_df
|
|
from translate import translate
|
|
|
|
def run_snscrape():
|
|
# prepare lists to hold information
|
|
username_list = list()
|
|
timestamp_list = list()
|
|
content_list = list()
|
|
# begin scraping
|
|
query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
|
|
for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
|
|
# store information in lists
|
|
username_list.append(tweet.user.username)
|
|
timestamp_list.append(tweet.date)
|
|
content_list.append(tweet.content)
|
|
# when 100 tweets have been collected
|
|
if len(content_list) == 100:
|
|
# send data to database
|
|
insert_list(username_list, timestamp_list, content_list)
|
|
# empty lists
|
|
username_list = list()
|
|
timestamp_list = list()
|
|
content_list = list()
|
|
# check in on progress
|
|
if i % 100 == 0:
|
|
print(f'found {i} tweets')
|
|
|
|
def translate_content():
|
|
# load all content from database
|
|
print('loading data from database...')
|
|
df = get_all()
|
|
# remove already translated content
|
|
df = df[df['content_en'].isna()]
|
|
# clean content
|
|
print('cleaning content...')
|
|
content_clean_list, id_list = clean_content_df(df)
|
|
# translate content
|
|
print('translating content...')
|
|
with Data_table() as dt:
|
|
for id, content in tqdm(zip(id_list, content_clean_list), ascii=True, total=len(id_list)):
|
|
# translate content
|
|
try:
|
|
content_trans = translate(content)
|
|
except Exception:
|
|
print(f'error when translating content with id={id}')
|
|
continue
|
|
# transfer to database
|
|
try:
|
|
val_dict = {
|
|
'id': id,
|
|
'content_en': content_trans
|
|
}
|
|
query = (
|
|
f'UPDATE {dt._name} '
|
|
' SET content_en = (%(content_en)s) '
|
|
'WHERE id = (%(id)s) '
|
|
)
|
|
dt.execute(query, val_dict)
|
|
except Exception:
|
|
print(f'error when sending translated content to database: id={id}')
|
|
# take a break to avoid being banned by Google
|
|
time.sleep(2)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
load_dotenv()
|
|
# run_snscrape()
|
|
translate_content() |