112 lines
3.6 KiB
Python
112 lines
3.6 KiB
Python
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
|
|
from setup_logging import setup_logging
|
|
from selenium_scraper import scrape
|
|
from database import insert_list
|
|
from dotenv import load_dotenv
|
|
import logging
|
|
from datetime import datetime
|
|
import time
|
|
import os
|
|
from tqdm import tqdm
|
|
|
|
import snscrape.modules.twitter as sntwitter
|
|
import pandas as pd
|
|
|
|
from database import get_all
|
|
from clean_data import clean_content_df
|
|
from translate import translate, get_untranslated
|
|
from database import Data_table
|
|
|
|
# def update():
|
|
# topic = 'Odense'
|
|
# # scrape latest data
|
|
# username_list, timestamp_list, content_list = scrape(topic, limit=10, headless=False)
|
|
# # send data to database
|
|
# insert_list(username_list, timestamp_list, content_list)
|
|
|
|
# if __name__ == '__main__':
|
|
# # setup
|
|
# load_dotenv()
|
|
# setup_logging()
|
|
# logging.info('finished setup')
|
|
# # main loop
|
|
# trigger_minute = datetime.now().minute
|
|
# while True:
|
|
# try:
|
|
# now = datetime.now()
|
|
# if now.minute == trigger_minute:
|
|
# try:
|
|
# update()
|
|
# except Exception as e:
|
|
# logging.warning(e)
|
|
# else:
|
|
# logging.info('finished cycle')
|
|
# time.sleep(60)
|
|
# time.sleep(0.1)
|
|
# except Exception as e:
|
|
# logging.critical(e)
|
|
# break
|
|
# except KeyboardInterrupt:
|
|
# logging.info('KeyboardInterrupt')
|
|
# break
|
|
|
|
def run_snscrape():
|
|
# prepare variables
|
|
username_list = list()
|
|
timestamp_list = list()
|
|
content_list = list()
|
|
# begin scraping
|
|
query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
|
|
for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
|
|
username_list.append(tweet.user.username)
|
|
timestamp_list.append(tweet.date)
|
|
content_list.append(tweet.content)
|
|
if len(content_list) == 100:
|
|
# send data to datbase
|
|
insert_list(username_list, timestamp_list, content_list)
|
|
username_list = list()
|
|
timestamp_list = list()
|
|
content_list = list()
|
|
if i % 100 == 0:
|
|
print(f'found {i} tweets')
|
|
|
|
def translate_content():
|
|
# load all content from database
|
|
print('loading data from database...')
|
|
df = get_all()
|
|
# remove already translated content
|
|
df = df[df['content_en'].isna()]
|
|
# clean content
|
|
print('cleaning content...')
|
|
content_clean_list, id_list = clean_content_df(df)
|
|
# translate content
|
|
print('translating content...')
|
|
with Data_table() as dt:
|
|
for id, content in tqdm(zip(id_list, content_clean_list), ascii=True, total=len(id_list)):
|
|
# translate content
|
|
try:
|
|
content_trans = translate(content)
|
|
except Exception as e:
|
|
print(f'error when translating content with id={id}')
|
|
continue
|
|
# transfer to database
|
|
try:
|
|
val_dict = {
|
|
'id': id,
|
|
'content_en': content_trans
|
|
}
|
|
query = (
|
|
f'UPDATE {dt._name} '
|
|
' SET content_en = (%(content_en)s) '
|
|
'WHERE id = (%(id)s) '
|
|
)
|
|
dt.execute(query, val_dict)
|
|
except Exception as e:
|
|
print(f'error when sending translated content to database: id={id}')
|
|
time.sleep(2)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
load_dotenv()
|
|
# run_snscrape()
|
|
translate_content() |