diff --git a/code/selenium_scraper.ipynb b/code/selenium_scraper.ipynb deleted file mode 100644 index eec414c..0000000 --- a/code/selenium_scraper.ipynb +++ /dev/null @@ -1,92 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "12\n" - ] - } - ], - "source": [ - "from selenium.webdriver.common.action_chains import ActionChains\n", - "from dateutil.parser import parse\n", - "from selenium import webdriver\n", - "import os\n", - "import importlib\n", - "import scraper\n", - "\n", - "importlib.reload(scraper)\n", - "topic = 'Odense'\n", - "limit = 10\n", - "username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)\n", - "\n", - "print(len(username_list))" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "289" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "max([len(c) for c in content_list])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# headless mode - has proven difficult...\n", - "# database access\n", - "# logging with warning to discord\n", - "# main loop - go once every hour" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/code/selenium_scraper.py b/code/selenium_scraper.py deleted file mode 100644 index ce1d2d5..0000000 --- a/code/selenium_scraper.py +++ /dev/null @@ -1,88 +0,0 @@ -from selenium.webdriver.common.action_chains import ActionChains -from dateutil.parser import parse -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.chrome.options import Options -from fake_useragent import UserAgent - -def load_topic(topic, headless=True): - # instantiate driver - options = Options() - if headless: - options.headless = True - # fake a user agent to not be denied service - ua = UserAgent() - userAgent = ua.random - options.add_argument(f'user-agent={userAgent}') - driver = webdriver.Chrome(options=options) - # open webpage - url = f'https://twitter.com/search?q=%23{topic}&src=typed_query&f=live' - driver.get(url) - # wait for popup and click it - timeout = 30 # seconds - xpath_str = "//*[text()='Not now']" - element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str))) - element.click() - return driver - -def list_tweets(driver): - timeout = 30 # seconds - _ = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-testid="tweet"]'))) - return driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]') - -def scroll_to(driver, tweet): - action = ActionChains(driver) - action.move_to_element(tweet).perform() - -def get_id(tweet): - return tweet.id - -def get_timestamp(tweet): - timestamp = tweet.find_element(By.TAG_NAME, "time").get_attribute("datetime") - return parse(timestamp) - -def get_content(tweet): - return tweet.find_element(By.CSS_SELECTOR, 'div[lang]').text - -def get_username(tweet): - return tweet.find_element(By.XPATH, ".//span[contains(text(), '@')]").text - -def extract(driver, limit=100): - id_list = list() - username_list = list() - timestamp_list = list() - content_list = list() - while len(id_list) < limit: - tweet_list = list_tweets(driver) - for tweet in tweet_list: - id = get_id(tweet) - if id in id_list: continue - scroll_to(driver, tweet) - username_list.append(get_username(tweet)) - timestamp_list.append(get_timestamp(tweet)) - content_list.append(get_content(tweet)) - id_list.append(id) - return username_list, timestamp_list, content_list - -def scrape(topic, limit=100, headless=True): - try: - # prepare driver - driver = load_topic(topic, headless) - except Exception as e: - print('failed instantiating webdriver') - print(e) - return [], [], [] - try: - # extract information - username_list, timestamp_list, content_list = extract(driver, limit) - except Exception as e: - print('error when extracting tweets') - print(e) - return [], [], [] - finally: - # close driver - driver.close() - driver.quit() - return username_list, timestamp_list, content_list diff --git a/code/setup_logging.py b/code/setup_logging.py deleted file mode 100644 index c2e778d..0000000 --- a/code/setup_logging.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/Users/brian/Projects/twitter_scraper/venv/bin/python -from discord_logging.handler import DiscordHandler -from dotenv import load_dotenv -import logging -import os - -def setup_logging(): - # setup stream handler - fmt = '[%(asctime)s] %(levelname)s:%(filename)s:%(funcName)s:%(message)s' - datefmt = '%Y-%m-%d %H:%M:%S' - logger_level = os.getenv('LOGGER_LEVEL') - level = getattr(logging, logger_level.upper()) - logging.basicConfig(format=fmt, datefmt=datefmt, level=level) - logger = logging.getLogger() - # add discord handler - discord_handler = DiscordHandler( - service_name = os.getenv('DISCORD_SERVICE_NAME'), - webhook_url = os.getenv('DISCORD_URL'), - ) - discord_handler.setFormatter(logging.Formatter('%(message)s')) - discord_handler.setLevel(logging.WARNING) - logger.addHandler(discord_handler) - logging.debug('finished') - -if __name__ == '__main__': - load_dotenv() - setup_logging() - logging.warning('test') diff --git a/code/show_trans_progress.py b/code/show_trans_progress.py deleted file mode 100644 index d992abf..0000000 --- a/code/show_trans_progress.py +++ /dev/null @@ -1,32 +0,0 @@ -from database import Data_table -from tqdm import tqdm -import time -from dotenv import load_dotenv -import pandas as pd - -def get_progress(dt): - query = 'SELECT count(*) from data_table' - res = pd.read_sql_query(query, dt.con) - total = res['count'][0] - query = 'SELECT count(*) from data_table where content_en is not null' - res = pd.read_sql_query(query, dt.con) - progress = res['count'][0] - return progress, total - -if __name__ == '__main__': - load_dotenv() - try: - with Data_table() as dt: - # continously update progress bar - progress, total = get_progress(dt) - with tqdm(total=total, initial=progress) as pbar: - while True: - pbar.n = progress - pbar.refresh() - time.sleep(1) - progress, total = get_progress(dt) - except Exception as e: - print(f'error occurred: {e}') - except KeyboardInterrupt: - print('KeyboardInterrupt') - \ No newline at end of file diff --git a/code/translate.py b/code/translate.py index f27f02f..fbb04e2 100644 --- a/code/translate.py +++ b/code/translate.py @@ -1,6 +1,6 @@ # public packages from langdetect import detect -from google_trans_new import google_translator +from google_trans_new import google_translator # had to download package to fix bug and be able to run code # database-related from database import Data_table, get_untranslated, insert_translated from dotenv import load_dotenv