diff --git a/.env b/.env index 42a1ed8..5c67e03 100644 --- a/.env +++ b/.env @@ -3,7 +3,4 @@ DATABASE_USERNAME=twitter_scraper DATABASE_PASSWORD='EqQKZrowAJXJ' DATABASE_HOST=192.168.1.2 DATABASE_PORT=5432 -DATABASE_DATABASE=twitter_scraper -LOGGER_LEVEL=info -DISCORD_URL=https://discord.com/api/webhooks/1053450686093873223/DIy33Epm7lnUqAs-YtsushI1H6sfnI76anpYcDeezd1LoXYwA5ODwRjvTlwVVI6-_2Wt -DISCORD_SERVICE_NAME=event_logger \ No newline at end of file +DATABASE_DATABASE=twitter_scraper \ No newline at end of file diff --git a/README.md b/README.md index ea324c5..04a0a37 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # twitter_scraper -Code to scrape twitter for a given hashtag. \ No newline at end of file +Code to scrape twitter for a given hashtag. diff --git a/code/clean_data.py b/code/clean_data.py index 29504af..c151af7 100644 --- a/code/clean_data.py +++ b/code/clean_data.py @@ -1,12 +1,13 @@ +# public packages from cleantext import clean def clean_content(text): - return clean(text, to_ascii=False) + return clean(text, to_ascii=False) # avoid ascii, as it does not have æ,ø,å def clean_content_df(df): - content_clean = df['content'].apply(clean_content).to_list() + content_clean_list = df['content'].apply(clean_content).to_list() id_list = df['id'].to_list() - return content_clean, id_list + return content_clean_list, id_list def clean_content_en(text_list): text_list_clean = list() diff --git a/code/database.py b/code/database.py index 71282bb..3a3f935 100644 --- a/code/database.py +++ b/code/database.py @@ -1,4 +1,3 @@ -#!/Users/brian/Projects/twitter_scraper/venv/bin/python import logging import os import traceback @@ -171,17 +170,6 @@ def insert_list(username_list, timestamp_list, content_list): except Exception as e: logging.warning(e) -def insert_translated(val_dict): - assert 'id' in val_dict.keys() - assert 'content_en' in val_dict.keys() - with Data_table() as dt: - query = ( - f'UPDATE {dt._name} ' - ' SET content_en = (%(content_en)s) ' - 'WHERE id = (%(id)s) ' - ) - dt.execute(query, val_dict) - def insert_translated_list(content_list, id_list): with Data_table() as dt: for id, content in zip(id_list, content_list): @@ -209,4 +197,29 @@ def get_all(): with Data_table() as dt: query = f'SELECT * FROM {dt._name} ORDER BY time DESC ' df = pd.read_sql_query(query, dt.con) - return df \ No newline at end of file + return df + +def insert_translated(dt, val_dict): + assert 'id' in val_dict.keys() + assert 'content_en' in val_dict.keys() + query = ( + f'UPDATE {dt._name} ' + ' SET content_en = (%(content_en)s) ' + 'WHERE id = (%(id)s) ' + ) + dt.execute(query, val_dict) + +def get_untranslated(dt): + query = ( + f'SELECT * FROM {dt._name} ' + 'WHERE content_en IS NULL ' + 'ORDER BY random() ' + 'LIMIT 1 ' + ) + df = pd.read_sql_query(query, dt.con) + if len(df)==0: return None + val_dict = { + 'id': int(df['id'][0]), + 'content': str(df['content'][0]) + } + return val_dict \ No newline at end of file diff --git a/code/main.py b/code/main.py deleted file mode 100644 index f85ff63..0000000 --- a/code/main.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/Users/brian/Projects/twitter_scraper/venv/bin/python -from setup_logging import setup_logging -from selenium_scraper import scrape -from database import insert_list -from dotenv import load_dotenv -import logging -from datetime import datetime -import time -import os -from tqdm import tqdm - -import snscrape.modules.twitter as sntwitter -import pandas as pd - -from database import get_all -from clean_data import clean_content_df -from translate import translate, get_untranslated -from database import Data_table - -# def update(): -# topic = 'Odense' -# # scrape latest data -# username_list, timestamp_list, content_list = scrape(topic, limit=10, headless=False) -# # send data to database -# insert_list(username_list, timestamp_list, content_list) - -# if __name__ == '__main__': -# # setup -# load_dotenv() -# setup_logging() -# logging.info('finished setup') -# # main loop -# trigger_minute = datetime.now().minute -# while True: -# try: -# now = datetime.now() -# if now.minute == trigger_minute: -# try: -# update() -# except Exception as e: -# logging.warning(e) -# else: -# logging.info('finished cycle') -# time.sleep(60) -# time.sleep(0.1) -# except Exception as e: -# logging.critical(e) -# break -# except KeyboardInterrupt: -# logging.info('KeyboardInterrupt') -# break - -def run_snscrape(): - # prepare variables - username_list = list() - timestamp_list = list() - content_list = list() - # begin scraping - query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies' - for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()): - username_list.append(tweet.user.username) - timestamp_list.append(tweet.date) - content_list.append(tweet.content) - if len(content_list) == 100: - # send data to datbase - insert_list(username_list, timestamp_list, content_list) - username_list = list() - timestamp_list = list() - content_list = list() - if i % 100 == 0: - print(f'found {i} tweets') - -def translate_content(): - # load all content from database - print('loading data from database...') - df = get_all() - # remove already translated content - df = df[df['content_en'].isna()] - # clean content - print('cleaning content...') - content_clean_list, id_list = clean_content_df(df) - # translate content - print('translating content...') - with Data_table() as dt: - for id, content in tqdm(zip(id_list, content_clean_list), ascii=True, total=len(id_list)): - # translate content - try: - content_trans = translate(content) - except Exception as e: - print(f'error when translating content with id={id}') - continue - # transfer to database - try: - val_dict = { - 'id': id, - 'content_en': content_trans - } - query = ( - f'UPDATE {dt._name} ' - ' SET content_en = (%(content_en)s) ' - 'WHERE id = (%(id)s) ' - ) - dt.execute(query, val_dict) - except Exception as e: - print(f'error when sending translated content to database: id={id}') - time.sleep(2) - - -if __name__ == '__main__': - load_dotenv() - # run_snscrape() - translate_content() \ No newline at end of file diff --git a/code/plot.ipynb b/code/plot.ipynb index 907e4b1..29da567 100644 --- a/code/plot.ipynb +++ b/code/plot.ipynb @@ -227,7 +227,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.6" + "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]" }, "orig_nbformat": 4, "vscode": { diff --git a/code/selenium_scraper.ipynb b/code/selenium_scraper.ipynb deleted file mode 100644 index eec414c..0000000 --- a/code/selenium_scraper.ipynb +++ /dev/null @@ -1,92 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "12\n" - ] - } - ], - "source": [ - "from selenium.webdriver.common.action_chains import ActionChains\n", - "from dateutil.parser import parse\n", - "from selenium import webdriver\n", - "import os\n", - "import importlib\n", - "import scraper\n", - "\n", - "importlib.reload(scraper)\n", - "topic = 'Odense'\n", - "limit = 10\n", - "username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)\n", - "\n", - "print(len(username_list))" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "289" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "max([len(c) for c in content_list])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# headless mode - has proven difficult...\n", - "# database access\n", - "# logging with warning to discord\n", - "# main loop - go once every hour" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/code/selenium_scraper.py b/code/selenium_scraper.py deleted file mode 100644 index ce1d2d5..0000000 --- a/code/selenium_scraper.py +++ /dev/null @@ -1,88 +0,0 @@ -from selenium.webdriver.common.action_chains import ActionChains -from dateutil.parser import parse -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.chrome.options import Options -from fake_useragent import UserAgent - -def load_topic(topic, headless=True): - # instantiate driver - options = Options() - if headless: - options.headless = True - # fake a user agent to not be denied service - ua = UserAgent() - userAgent = ua.random - options.add_argument(f'user-agent={userAgent}') - driver = webdriver.Chrome(options=options) - # open webpage - url = f'https://twitter.com/search?q=%23{topic}&src=typed_query&f=live' - driver.get(url) - # wait for popup and click it - timeout = 30 # seconds - xpath_str = "//*[text()='Not now']" - element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str))) - element.click() - return driver - -def list_tweets(driver): - timeout = 30 # seconds - _ = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-testid="tweet"]'))) - return driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]') - -def scroll_to(driver, tweet): - action = ActionChains(driver) - action.move_to_element(tweet).perform() - -def get_id(tweet): - return tweet.id - -def get_timestamp(tweet): - timestamp = tweet.find_element(By.TAG_NAME, "time").get_attribute("datetime") - return parse(timestamp) - -def get_content(tweet): - return tweet.find_element(By.CSS_SELECTOR, 'div[lang]').text - -def get_username(tweet): - return tweet.find_element(By.XPATH, ".//span[contains(text(), '@')]").text - -def extract(driver, limit=100): - id_list = list() - username_list = list() - timestamp_list = list() - content_list = list() - while len(id_list) < limit: - tweet_list = list_tweets(driver) - for tweet in tweet_list: - id = get_id(tweet) - if id in id_list: continue - scroll_to(driver, tweet) - username_list.append(get_username(tweet)) - timestamp_list.append(get_timestamp(tweet)) - content_list.append(get_content(tweet)) - id_list.append(id) - return username_list, timestamp_list, content_list - -def scrape(topic, limit=100, headless=True): - try: - # prepare driver - driver = load_topic(topic, headless) - except Exception as e: - print('failed instantiating webdriver') - print(e) - return [], [], [] - try: - # extract information - username_list, timestamp_list, content_list = extract(driver, limit) - except Exception as e: - print('error when extracting tweets') - print(e) - return [], [], [] - finally: - # close driver - driver.close() - driver.quit() - return username_list, timestamp_list, content_list diff --git a/code/setup_logging.py b/code/setup_logging.py deleted file mode 100644 index c2e778d..0000000 --- a/code/setup_logging.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/Users/brian/Projects/twitter_scraper/venv/bin/python -from discord_logging.handler import DiscordHandler -from dotenv import load_dotenv -import logging -import os - -def setup_logging(): - # setup stream handler - fmt = '[%(asctime)s] %(levelname)s:%(filename)s:%(funcName)s:%(message)s' - datefmt = '%Y-%m-%d %H:%M:%S' - logger_level = os.getenv('LOGGER_LEVEL') - level = getattr(logging, logger_level.upper()) - logging.basicConfig(format=fmt, datefmt=datefmt, level=level) - logger = logging.getLogger() - # add discord handler - discord_handler = DiscordHandler( - service_name = os.getenv('DISCORD_SERVICE_NAME'), - webhook_url = os.getenv('DISCORD_URL'), - ) - discord_handler.setFormatter(logging.Formatter('%(message)s')) - discord_handler.setLevel(logging.WARNING) - logger.addHandler(discord_handler) - logging.debug('finished') - -if __name__ == '__main__': - load_dotenv() - setup_logging() - logging.warning('test') diff --git a/code/show_trans_progress.py b/code/show_trans_progress.py deleted file mode 100644 index d3dc96d..0000000 --- a/code/show_trans_progress.py +++ /dev/null @@ -1,32 +0,0 @@ -from database import get_all, Data_table -from tqdm import tqdm -import time -from dotenv import load_dotenv -import pandas as pd - -def get_progress(dt): - query = 'SELECT count(*) from data_table' - res = pd.read_sql_query(query, dt.con) - total = res['count'][0] - query = 'SELECT count(*) from data_table where content_en is not null' - res = pd.read_sql_query(query, dt.con) - progress = res['count'][0] - return progress, total - -if __name__ == '__main__': - load_dotenv() - try: - with Data_table() as dt: - # continously update progress bar - progress, total = get_progress(dt) - with tqdm(total=total, initial=progress) as pbar: - while True: - pbar.n = progress - pbar.refresh() - time.sleep(1) - progress, total = get_progress(dt) - except Exception as e: - print(f'error occurred: {e}') - except KeyboardInterrupt: - print('KeyboardInterrupt') - \ No newline at end of file diff --git a/code/sntwitter_scraper.ipynb b/code/sntwitter_scraper.ipynb index d026aa5..ae449c2 100644 --- a/code/sntwitter_scraper.ipynb +++ b/code/sntwitter_scraper.ipynb @@ -26,15 +26,21 @@ "source": [ "import snscrape.modules.twitter as sntwitter\n", "import pandas as pd\n", - "tweet_list = list()\n", + "\n", + "# prepare variables\n", "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n", "limit = 1000\n", - "\n", + "tweet_list = list()\n", + "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n", + "# begin scraping\n", "for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):\n", - " if i % 100 == 0: print(f'found {i} tweets')\n", - " if i > limit: break\n", + " # if limit is reached, break out of loop\n", + " if limit > 0 and i > limit: break\n", + " # if another 100 tweets found, tell it\n", + " if i % 100 == 0: \n", + " print(f'found {i} tweets')\n", " tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])\n", - "\n", + "# create dataframe\n", "df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])" ] }, @@ -216,7 +222,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.6" + "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]" }, "orig_nbformat": 4, "vscode": { diff --git a/code/sntwitter_scraper.py b/code/sntwitter_scraper.py index efca04a..7269e2e 100644 --- a/code/sntwitter_scraper.py +++ b/code/sntwitter_scraper.py @@ -1,18 +1,34 @@ +# public packages import snscrape.modules.twitter as sntwitter import pandas as pd +# database-related +from database import insert_list +from dotenv import load_dotenv -def scrape(limit=0): - # prepare variables - tweet_list = list() - query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies' +def scrape(): + # prepare lists to hold information + username_list = list() + timestamp_list = list() + content_list = list() # begin scraping + query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies' for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()): - # if limit is reached, break out of loop - if limit > 0 and i > limit: break - # if another 100 tweets found, tell it - if i % 100 == 0: + # store information in lists + username_list.append(tweet.user.username) + timestamp_list.append(tweet.date) + content_list.append(tweet.content) + # when 100 tweets have been collected + if len(content_list) == 100: + # send data to database + insert_list(username_list, timestamp_list, content_list) + # empty lists + username_list = list() + timestamp_list = list() + content_list = list() + # check in on progress + if i % 100 == 0: print(f'found {i} tweets') - tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username]) - # create dataframe - df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username']) - return df + +if __name__ == '__main__': + load_dotenv() + scrape() \ No newline at end of file diff --git a/code/translate.py b/code/translate.py index 8450738..6381322 100644 --- a/code/translate.py +++ b/code/translate.py @@ -1,15 +1,11 @@ +# public packages from langdetect import detect -from google_trans_new import google_translator -import pandas as pd -import multiprocessing as mp -import numpy as np +from google_trans_new import google_translator # had to download package to fix bug and be able to run code +# database-related +from database import Data_table, get_untranslated, insert_translated from dotenv import load_dotenv -from tqdm import tqdm -import logging - -from database import Data_table +# own scripts from clean_data import clean_content -from setup_logging import setup_logging def translate(text, target_lang='en'): try: @@ -26,65 +22,39 @@ def translate(text, target_lang='en'): return '' return text_trans -def get_untranslated(dt): - query = ( - f'SELECT * FROM {dt._name} ' - 'WHERE content_en IS NULL ' - 'ORDER BY random() ' - 'LIMIT 1 ' - ) - df = pd.read_sql_query(query, dt.con) - if len(df)==0: return None - val_dict = { - 'id': int(df['id'][0]), - 'content': str(df['content'][0]) - } - return val_dict - -def insert_translated(dt, val_dict): - assert 'id' in val_dict.keys() - assert 'content_en' in val_dict.keys() - query = ( - f'UPDATE {dt._name} ' - ' SET content_en = (%(content_en)s) ' - 'WHERE id = (%(id)s) ' - ) - dt.execute(query, val_dict) - def continously_translate(): with Data_table() as dt: - logging.info('began translating') - while True: + print('began translating') + while True: # keep going forever try: val_dict = get_untranslated(dt) except Exception as e: - logging.warning(f'error when getting untranslated content from database: {e}') + print(f'error when getting untranslated content from database: {e}') continue + # if nothing left untranslated, stop if val_dict is None: break - # clean + # clean text try: content_clean = clean_content(val_dict['content']) except Exception as e: - logging.warning(f'error when cleaning content: {e}') + print(f'error when cleaning content: {e}') continue - # translate + # translate text try: content_en = translate(content_clean) except Exception as e: - logging.warning(f'error when translating content: {e}') + print(f'error when translating content: {e}') continue # upload to database val_dict['content_en'] = content_en try: insert_translated(dt, val_dict) except Exception as e: - logging.warning(f'error when sending translated content to database: {e}') + print(f'error when sending translated content to database: {e}') continue - logging.info('translated content') - logging.warning('stopped translating') + print('translated content') + print('stopped translating') if __name__ == '__main__': load_dotenv() - setup_logging() continously_translate() - # later try to set all empty strings back to null and retranslate failed...