Merge branch 'main' of 192.168.1.2:brian/twitter_scraper into main

.env

@@ -3,7 +3,4 @@ DATABASE_USERNAME=twitter_scraper
 DATABASE_PASSWORD='EqQKZrowAJXJ'
 DATABASE_HOST=192.168.1.2
 DATABASE_PORT=5432
-DATABASE_DATABASE=twitter_scraper
+DATABASE_DATABASE=twitter_scraper
-LOGGER_LEVEL=info
-DISCORD_URL=https://discord.com/api/webhooks/1053450686093873223/DIy33Epm7lnUqAs-YtsushI1H6sfnI76anpYcDeezd1LoXYwA5ODwRjvTlwVVI6-_2Wt
-DISCORD_SERVICE_NAME=event_logger

README.md

@@ -1,3 +1,3 @@
 # twitter_scraper
 
-Code to scrape twitter for a given hashtag.
+Code to scrape twitter for a given hashtag.

code/clean_data.py

@@ -1,12 +1,13 @@
+# public packages
 from cleantext import clean
 
 def clean_content(text):
-    return clean(text, to_ascii=False)
+    return clean(text, to_ascii=False) # avoid ascii, as it does not have æ,ø,å
 
 def clean_content_df(df):
-    content_clean = df['content'].apply(clean_content).to_list()
+    content_clean_list = df['content'].apply(clean_content).to_list()
     id_list = df['id'].to_list()
-    return content_clean, id_list
+    return content_clean_list, id_list
 
 def clean_content_en(text_list):
     text_list_clean = list()

code/database.py

@@ -1,4 +1,3 @@
-#!/Users/brian/Projects/twitter_scraper/venv/bin/python
 import logging
 import os
 import traceback
@@ -171,17 +170,6 @@ def insert_list(username_list, timestamp_list, content_list):
             except Exception as e:
                 logging.warning(e)
 
-def insert_translated(val_dict):
-    assert 'id' in val_dict.keys()
-    assert 'content_en' in val_dict.keys()
-    with Data_table() as dt:
-        query = (
-            f'UPDATE {dt._name} ' 
-            ' SET content_en = (%(content_en)s) '
-            'WHERE id = (%(id)s) '
-        )
-        dt.execute(query, val_dict)
-
 def insert_translated_list(content_list, id_list):
     with Data_table() as dt:
         for id, content in zip(id_list, content_list):
@@ -209,4 +197,29 @@ def get_all():
     with Data_table() as dt:
         query = f'SELECT * FROM {dt._name} ORDER BY time DESC '
         df = pd.read_sql_query(query, dt.con)
-    return df
+    return df
+
+def insert_translated(dt, val_dict):
+    assert 'id' in val_dict.keys()
+    assert 'content_en' in val_dict.keys()
+    query = (
+        f'UPDATE {dt._name} ' 
+        ' SET content_en = (%(content_en)s) '
+        'WHERE id = (%(id)s) '
+    )
+    dt.execute(query, val_dict)
+
+def get_untranslated(dt):
+    query = (
+        f'SELECT * FROM {dt._name} '
+        'WHERE content_en IS NULL '
+        'ORDER BY random() '
+        'LIMIT 1 '
+    )
+    df = pd.read_sql_query(query, dt.con)
+    if len(df)==0: return None
+    val_dict = {
+        'id': int(df['id'][0]),
+        'content': str(df['content'][0])
+    }
+    return val_dict

code/main.py

@@ -1,112 +0,0 @@
-#!/Users/brian/Projects/twitter_scraper/venv/bin/python
-from setup_logging import setup_logging
-from selenium_scraper import scrape
-from database import insert_list
-from dotenv import load_dotenv
-import logging
-from datetime import datetime
-import time
-import os
-from tqdm import tqdm
-
-import snscrape.modules.twitter as sntwitter
-import pandas as pd
-
-from database import get_all
-from clean_data import clean_content_df
-from translate import translate, get_untranslated
-from database import Data_table
-
-# def update():
-#     topic = 'Odense'
-#     # scrape latest data
-#     username_list, timestamp_list, content_list = scrape(topic, limit=10, headless=False)
-#     # send data to database
-#     insert_list(username_list, timestamp_list, content_list)
-
-# if __name__ == '__main__':
-#     # setup
-#     load_dotenv()
-#     setup_logging()
-#     logging.info('finished setup')
-#     # main loop
-#     trigger_minute = datetime.now().minute
-#     while True:
-#         try:
-#             now = datetime.now()
-#             if now.minute == trigger_minute:
-#                 try:
-#                     update()
-#                 except Exception as e:
-#                     logging.warning(e)
-#                 else:
-#                     logging.info('finished cycle')
-#                 time.sleep(60)
-#             time.sleep(0.1)
-#         except Exception as e:
-#             logging.critical(e)
-#             break
-#         except KeyboardInterrupt:
-#             logging.info('KeyboardInterrupt')
-#             break
-
-def run_snscrape():
-    # prepare variables
-    username_list = list()
-    timestamp_list = list()
-    content_list = list()
-    # begin scraping
-    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
-    for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
-        username_list.append(tweet.user.username)
-        timestamp_list.append(tweet.date)
-        content_list.append(tweet.content)
-        if len(content_list) == 100:
-            # send data to datbase
-            insert_list(username_list, timestamp_list, content_list)
-            username_list = list()
-            timestamp_list = list()
-            content_list = list()
-        if i % 100 == 0: 
-            print(f'found {i} tweets')
-
-def translate_content():
-    # load all content from database
-    print('loading data from database...')
-    df = get_all()
-    # remove already translated content
-    df = df[df['content_en'].isna()]
-    # clean content
-    print('cleaning content...')
-    content_clean_list, id_list = clean_content_df(df)
-    # translate content
-    print('translating content...')
-    with Data_table() as dt:
-        for id, content in tqdm(zip(id_list, content_clean_list), ascii=True, total=len(id_list)):
-            # translate content
-            try:
-                content_trans = translate(content)
-            except Exception as e:
-                print(f'error when translating content with id={id}')
-                continue
-            # transfer to database
-            try:
-                val_dict = {
-                    'id': id,
-                    'content_en': content_trans
-                }
-                query = (
-                    f'UPDATE {dt._name} ' 
-                    ' SET content_en = (%(content_en)s) '
-                    'WHERE id = (%(id)s) '
-                )
-                dt.execute(query, val_dict)
-            except Exception as e:
-                print(f'error when sending translated content to database: id={id}')
-            time.sleep(2)
-
-
-if __name__ == '__main__':
-    load_dotenv()
-    # run_snscrape()
-    translate_content()

code/plot.ipynb

@@ -227,7 +227,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.6"
+   "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
   },
   "orig_nbformat": 4,
   "vscode": {

code/selenium_scraper.ipynb

@@ -1,92 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "12\n"
-     ]
-    }
-   ],
-   "source": [
-    "from selenium.webdriver.common.action_chains import ActionChains\n",
-    "from dateutil.parser import parse\n",
-    "from selenium import webdriver\n",
-    "import os\n",
-    "import importlib\n",
-    "import scraper\n",
-    "\n",
-    "importlib.reload(scraper)\n",
-    "topic = 'Odense'\n",
-    "limit = 10\n",
-    "username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)\n",
-    "\n",
-    "print(len(username_list))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "289"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "max([len(c) for c in content_list])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# headless mode - has proven difficult...\n",
-    "# database access\n",
-    "# logging with warning to discord\n",
-    "# main loop - go once every hour"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "venv",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
-  },
-  "orig_nbformat": 4,
-  "vscode": {
-   "interpreter": {
-    "hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

code/selenium_scraper.py

@@ -1,88 +0,0 @@
-from selenium.webdriver.common.action_chains import ActionChains
-from dateutil.parser import parse
-from selenium import webdriver
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.chrome.options import Options
-from fake_useragent import UserAgent
-
-def load_topic(topic, headless=True):
-    # instantiate driver
-    options = Options()
-    if headless:
-        options.headless = True
-        # fake a user agent to not be denied service
-        ua = UserAgent()
-        userAgent = ua.random
-        options.add_argument(f'user-agent={userAgent}')
-    driver = webdriver.Chrome(options=options)
-    # open webpage
-    url = f'https://twitter.com/search?q=%23{topic}&src=typed_query&f=live'
-    driver.get(url)
-    # wait for popup and click it
-    timeout = 30 # seconds
-    xpath_str = "//*[text()='Not now']"
-    element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str)))
-    element.click()
-    return driver
-
-def list_tweets(driver):
-    timeout = 30 # seconds
-    _ = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-testid="tweet"]')))
-    return driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]')
-
-def scroll_to(driver, tweet):
-    action = ActionChains(driver)
-    action.move_to_element(tweet).perform()
-
-def get_id(tweet):
-    return tweet.id
-
-def get_timestamp(tweet):
-    timestamp = tweet.find_element(By.TAG_NAME, "time").get_attribute("datetime")
-    return parse(timestamp)
-
-def get_content(tweet):
-    return tweet.find_element(By.CSS_SELECTOR, 'div[lang]').text
-
-def get_username(tweet):
-    return tweet.find_element(By.XPATH, ".//span[contains(text(), '@')]").text
-
-def extract(driver, limit=100):
-    id_list = list()
-    username_list = list()
-    timestamp_list = list()
-    content_list = list()
-    while len(id_list) < limit:
-        tweet_list = list_tweets(driver)
-        for tweet in tweet_list:
-            id = get_id(tweet)
-            if id in id_list: continue
-            scroll_to(driver, tweet)
-            username_list.append(get_username(tweet))
-            timestamp_list.append(get_timestamp(tweet))
-            content_list.append(get_content(tweet))
-            id_list.append(id)
-    return username_list, timestamp_list, content_list
-    
-def scrape(topic, limit=100, headless=True):
-    try:
-        # prepare driver
-        driver = load_topic(topic, headless)
-    except Exception as e:
-        print('failed instantiating webdriver')
-        print(e)
-        return [], [], []
-    try:
-        # extract information
-        username_list, timestamp_list, content_list = extract(driver, limit)
-    except Exception as e:
-        print('error when extracting tweets')
-        print(e)
-        return [], [], []
-    finally:
-        # close driver
-        driver.close()
-        driver.quit()
-    return username_list, timestamp_list, content_list

code/setup_logging.py

@@ -1,28 +0,0 @@
-#!/Users/brian/Projects/twitter_scraper/venv/bin/python
-from discord_logging.handler import DiscordHandler
-from dotenv import load_dotenv
-import logging
-import os
-
-def setup_logging():
-    # setup stream handler
-    fmt = '[%(asctime)s] %(levelname)s:%(filename)s:%(funcName)s:%(message)s'
-    datefmt = '%Y-%m-%d %H:%M:%S'
-    logger_level = os.getenv('LOGGER_LEVEL')
-    level = getattr(logging, logger_level.upper())
-    logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
-    logger = logging.getLogger()
-    # add discord handler
-    discord_handler = DiscordHandler(
-        service_name = os.getenv('DISCORD_SERVICE_NAME'),
-        webhook_url = os.getenv('DISCORD_URL'),
-    )
-    discord_handler.setFormatter(logging.Formatter('%(message)s'))
-    discord_handler.setLevel(logging.WARNING)
-    logger.addHandler(discord_handler)
-    logging.debug('finished')
-
-if __name__ == '__main__':
-    load_dotenv()
-    setup_logging()
-    logging.warning('test')

code/show_trans_progress.py

@@ -1,32 +0,0 @@
-from database import get_all, Data_table
-from tqdm import tqdm
-import time
-from dotenv import load_dotenv
-import pandas as pd
-
-def get_progress(dt):
-    query = 'SELECT count(*) from data_table'
-    res = pd.read_sql_query(query, dt.con)
-    total = res['count'][0]
-    query = 'SELECT count(*) from data_table where content_en is not null'
-    res = pd.read_sql_query(query, dt.con)
-    progress = res['count'][0]
-    return progress, total
-
-if __name__ == '__main__':
-    load_dotenv()
-    try:
-        with Data_table() as dt:
-            # continously update progress bar
-            progress, total = get_progress(dt)
-            with tqdm(total=total, initial=progress) as pbar:
-                while True:
-                    pbar.n = progress
-                    pbar.refresh()
-                    time.sleep(1)
-                    progress, total = get_progress(dt)
-    except Exception as e:
-        print(f'error occurred: {e}')
-    except KeyboardInterrupt:
-        print('KeyboardInterrupt')
-    

code/sntwitter_scraper.ipynb

@@ -26,15 +26,21 @@
    "source": [
     "import snscrape.modules.twitter as sntwitter\n",
     "import pandas as pd\n",
-    "tweet_list = list()\n",
+    "\n",
+    "# prepare variables\n",
     "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
     "limit = 1000\n",
-    "\n",
+    "tweet_list = list()\n",
+    "query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
+    "# begin scraping\n",
     "for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):\n",
-    "    if i % 100 == 0: print(f'found {i} tweets')\n",
+    "    # if limit is reached, break out of loop\n",
-    "    if i > limit: break\n",
+    "    if limit > 0 and i > limit: break\n",
+    "    # if another 100 tweets found, tell it\n",
+    "    if i % 100 == 0: \n",
+    "        print(f'found {i} tweets')\n",
     "    tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])\n",
-    "\n",
+    "# create dataframe\n",
     "df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])"
    ]
   },
@@ -216,7 +222,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.6"
+   "version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
   },
   "orig_nbformat": 4,
   "vscode": {

code/sntwitter_scraper.py

@@ -1,18 +1,34 @@
+# public packages
 import snscrape.modules.twitter as sntwitter
 import pandas as pd
+# database-related
+from database import insert_list
+from dotenv import load_dotenv
 
-def scrape(limit=0):
+def scrape():
-    # prepare variables
+    # prepare lists to hold information
-    tweet_list = list()
+    username_list = list()
-    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
+    timestamp_list = list()
+    content_list = list()
     # begin scraping
+    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
     for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
-        # if limit is reached, break out of loop
+        # store information in lists
-        if limit > 0 and i > limit: break
+        username_list.append(tweet.user.username)
-        # if another 100 tweets found, tell it
+        timestamp_list.append(tweet.date)
-        if i % 100 == 0: 
+        content_list.append(tweet.content)
+        # when 100 tweets have been collected
+        if len(content_list) == 100:
+            # send data to database
+            insert_list(username_list, timestamp_list, content_list)
+            # empty lists
+            username_list = list()
+            timestamp_list = list()
+            content_list = list()
+        # check in on progress
+        if i % 100 == 0:
             print(f'found {i} tweets')
-        tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])
+
-    # create dataframe
+if __name__ == '__main__':
-    df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])
+    load_dotenv()
-    return df
+    scrape()

code/translate.py

@@ -1,15 +1,11 @@
+# public packages
 from langdetect import detect
-from google_trans_new import google_translator
+from google_trans_new import google_translator # had to download package to fix bug and be able to run code
-import pandas as pd
+# database-related
-import multiprocessing as mp
+from database import Data_table, get_untranslated, insert_translated
-import numpy as np
 from dotenv import load_dotenv
-from tqdm import tqdm
+# own scripts
-import logging
-
-from database import Data_table
 from clean_data import clean_content
-from setup_logging import setup_logging
 
 def translate(text, target_lang='en'):
     try:
@@ -26,65 +22,39 @@ def translate(text, target_lang='en'):
         return ''
     return text_trans
 
-def get_untranslated(dt):
-    query = (
-        f'SELECT * FROM {dt._name} '
-        'WHERE content_en IS NULL '
-        'ORDER BY random() '
-        'LIMIT 1 '
-    )
-    df = pd.read_sql_query(query, dt.con)
-    if len(df)==0: return None
-    val_dict = {
-        'id': int(df['id'][0]),
-        'content': str(df['content'][0])
-    }
-    return val_dict
-
-def insert_translated(dt, val_dict):
-    assert 'id' in val_dict.keys()
-    assert 'content_en' in val_dict.keys()
-    query = (
-        f'UPDATE {dt._name} ' 
-        ' SET content_en = (%(content_en)s) '
-        'WHERE id = (%(id)s) '
-    )
-    dt.execute(query, val_dict)
-
 def continously_translate():
     with Data_table() as dt:
-        logging.info('began translating')
+        print('began translating')
-        while True:
+        while True: # keep going forever
             try:
                 val_dict = get_untranslated(dt)
             except Exception as e:
-                logging.warning(f'error when getting untranslated content from database: {e}')
+                print(f'error when getting untranslated content from database: {e}')
                 continue
+            # if nothing left untranslated, stop
             if val_dict is None: break
-            # clean
+            # clean text
             try:
                 content_clean = clean_content(val_dict['content'])
             except Exception as e:
-                logging.warning(f'error when cleaning content: {e}')
+                print(f'error when cleaning content: {e}')
                 continue
-            # translate
+            # translate text
             try:
                 content_en = translate(content_clean)
             except Exception as e:
-                logging.warning(f'error when translating content: {e}')
+                print(f'error when translating content: {e}')
                 continue
             # upload to database
             val_dict['content_en'] = content_en
             try:
                 insert_translated(dt, val_dict)
             except Exception as e:
-                logging.warning(f'error when sending translated content to database: {e}')
+                print(f'error when sending translated content to database: {e}')
                 continue
-            logging.info('translated content')
+            print('translated content')
-        logging.warning('stopped translating')
+        print('stopped translating')
 
 if __name__ == '__main__':
     load_dotenv()
-    setup_logging()
     continously_translate()
-    # later try to set all empty strings back to null and retranslate failed...