added comment

code/selenium_scraper.ipynb

@@ -1,92 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "12\n"
-     ]
-    }
-   ],
-   "source": [
-    "from selenium.webdriver.common.action_chains import ActionChains\n",
-    "from dateutil.parser import parse\n",
-    "from selenium import webdriver\n",
-    "import os\n",
-    "import importlib\n",
-    "import scraper\n",
-    "\n",
-    "importlib.reload(scraper)\n",
-    "topic = 'Odense'\n",
-    "limit = 10\n",
-    "username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)\n",
-    "\n",
-    "print(len(username_list))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "289"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "max([len(c) for c in content_list])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# headless mode - has proven difficult...\n",
-    "# database access\n",
-    "# logging with warning to discord\n",
-    "# main loop - go once every hour"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "venv",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
-  },
-  "orig_nbformat": 4,
-  "vscode": {
-   "interpreter": {
-    "hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

code/selenium_scraper.py

@@ -1,88 +0,0 @@
-from selenium.webdriver.common.action_chains import ActionChains
-from dateutil.parser import parse
-from selenium import webdriver
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.chrome.options import Options
-from fake_useragent import UserAgent
-
-def load_topic(topic, headless=True):
-    # instantiate driver
-    options = Options()
-    if headless:
-        options.headless = True
-        # fake a user agent to not be denied service
-        ua = UserAgent()
-        userAgent = ua.random
-        options.add_argument(f'user-agent={userAgent}')
-    driver = webdriver.Chrome(options=options)
-    # open webpage
-    url = f'https://twitter.com/search?q=%23{topic}&src=typed_query&f=live'
-    driver.get(url)
-    # wait for popup and click it
-    timeout = 30 # seconds
-    xpath_str = "//*[text()='Not now']"
-    element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str)))
-    element.click()
-    return driver
-
-def list_tweets(driver):
-    timeout = 30 # seconds
-    _ = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-testid="tweet"]')))
-    return driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]')
-
-def scroll_to(driver, tweet):
-    action = ActionChains(driver)
-    action.move_to_element(tweet).perform()
-
-def get_id(tweet):
-    return tweet.id
-
-def get_timestamp(tweet):
-    timestamp = tweet.find_element(By.TAG_NAME, "time").get_attribute("datetime")
-    return parse(timestamp)
-
-def get_content(tweet):
-    return tweet.find_element(By.CSS_SELECTOR, 'div[lang]').text
-
-def get_username(tweet):
-    return tweet.find_element(By.XPATH, ".//span[contains(text(), '@')]").text
-
-def extract(driver, limit=100):
-    id_list = list()
-    username_list = list()
-    timestamp_list = list()
-    content_list = list()
-    while len(id_list) < limit:
-        tweet_list = list_tweets(driver)
-        for tweet in tweet_list:
-            id = get_id(tweet)
-            if id in id_list: continue
-            scroll_to(driver, tweet)
-            username_list.append(get_username(tweet))
-            timestamp_list.append(get_timestamp(tweet))
-            content_list.append(get_content(tweet))
-            id_list.append(id)
-    return username_list, timestamp_list, content_list
-    
-def scrape(topic, limit=100, headless=True):
-    try:
-        # prepare driver
-        driver = load_topic(topic, headless)
-    except Exception as e:
-        print('failed instantiating webdriver')
-        print(e)
-        return [], [], []
-    try:
-        # extract information
-        username_list, timestamp_list, content_list = extract(driver, limit)
-    except Exception as e:
-        print('error when extracting tweets')
-        print(e)
-        return [], [], []
-    finally:
-        # close driver
-        driver.close()
-        driver.quit()
-    return username_list, timestamp_list, content_list

code/setup_logging.py

@@ -1,28 +0,0 @@
-#!/Users/brian/Projects/twitter_scraper/venv/bin/python
-from discord_logging.handler import DiscordHandler
-from dotenv import load_dotenv
-import logging
-import os
-
-def setup_logging():
-    # setup stream handler
-    fmt = '[%(asctime)s] %(levelname)s:%(filename)s:%(funcName)s:%(message)s'
-    datefmt = '%Y-%m-%d %H:%M:%S'
-    logger_level = os.getenv('LOGGER_LEVEL')
-    level = getattr(logging, logger_level.upper())
-    logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
-    logger = logging.getLogger()
-    # add discord handler
-    discord_handler = DiscordHandler(
-        service_name = os.getenv('DISCORD_SERVICE_NAME'),
-        webhook_url = os.getenv('DISCORD_URL'),
-    )
-    discord_handler.setFormatter(logging.Formatter('%(message)s'))
-    discord_handler.setLevel(logging.WARNING)
-    logger.addHandler(discord_handler)
-    logging.debug('finished')
-
-if __name__ == '__main__':
-    load_dotenv()
-    setup_logging()
-    logging.warning('test')

code/show_trans_progress.py

@@ -1,32 +0,0 @@
-from database import Data_table
-from tqdm import tqdm
-import time
-from dotenv import load_dotenv
-import pandas as pd
-
-def get_progress(dt):
-    query = 'SELECT count(*) from data_table'
-    res = pd.read_sql_query(query, dt.con)
-    total = res['count'][0]
-    query = 'SELECT count(*) from data_table where content_en is not null'
-    res = pd.read_sql_query(query, dt.con)
-    progress = res['count'][0]
-    return progress, total
-
-if __name__ == '__main__':
-    load_dotenv()
-    try:
-        with Data_table() as dt:
-            # continously update progress bar
-            progress, total = get_progress(dt)
-            with tqdm(total=total, initial=progress) as pbar:
-                while True:
-                    pbar.n = progress
-                    pbar.refresh()
-                    time.sleep(1)
-                    progress, total = get_progress(dt)
-    except Exception as e:
-        print(f'error occurred: {e}')
-    except KeyboardInterrupt:
-        print('KeyboardInterrupt')
-    

code/translate.py

@@ -1,6 +1,6 @@
 # public packages
 from langdetect import detect
-from google_trans_new import google_translator
+from google_trans_new import google_translator # had to download package to fix bug and be able to run code
 # database-related
 from database import Data_table, get_untranslated, insert_translated
 from dotenv import load_dotenv