brian/twitter_scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added comment

Browse Source
This commit is contained in:
simplypower-bbj
2023-01-09 19:09:00 +01:00
parent f17f3a708d
commit 7aa3b2a191
5 changed files with 1 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code/selenium_scraper.ipynb
-92
View File
@@ -1,92 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"12\n"
]
}
],
"source": [
"from selenium.webdriver.common.action_chains import ActionChains\n",
"from dateutil.parser import parse\n",
"from selenium import webdriver\n",
"import os\n",
"import importlib\n",
"import scraper\n",
"\n",
"importlib.reload(scraper)\n",
"topic = 'Odense'\n",
"limit = 10\n",
"username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)\n",
"\n",
"print(len(username_list))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"289"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"max([len(c) for c in content_list])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# headless mode - has proven difficult...\n",
"# database access\n",
"# logging with warning to discord\n",
"# main loop - go once every hour"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}
code/selenium_scraper.py
-88
View File
@@ -1,88 +0,0 @@
from selenium.webdriver.common.action_chains import ActionChains
from dateutil.parser import parse
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.chrome.options import Options
from fake_useragent import UserAgent
def load_topic(topic, headless=True):
# instantiate driver
options = Options()
if headless:
options.headless = True
# fake a user agent to not be denied service
ua = UserAgent()
userAgent = ua.random
options.add_argument(f'user-agent={userAgent}')
driver = webdriver.Chrome(options=options)
# open webpage
url = f'https://twitter.com/search?q=%23{topic}&src=typed_query&f=live'
driver.get(url)
# wait for popup and click it
timeout = 30 # seconds
xpath_str = "//*[text()='Not now']"
element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str)))
element.click()
return driver
def list_tweets(driver):
timeout = 30 # seconds
_ = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-testid="tweet"]')))
return driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]')
def scroll_to(driver, tweet):
action = ActionChains(driver)
action.move_to_element(tweet).perform()
def get_id(tweet):
return tweet.id
def get_timestamp(tweet):
timestamp = tweet.find_element(By.TAG_NAME, "time").get_attribute("datetime")
return parse(timestamp)
def get_content(tweet):
return tweet.find_element(By.CSS_SELECTOR, 'div[lang]').text
def get_username(tweet):
return tweet.find_element(By.XPATH, ".//span[contains(text(), '@')]").text
def extract(driver, limit=100):
id_list = list()
username_list = list()
timestamp_list = list()
content_list = list()
while len(id_list) < limit:
tweet_list = list_tweets(driver)
for tweet in tweet_list:
id = get_id(tweet)
if id in id_list: continue
scroll_to(driver, tweet)
username_list.append(get_username(tweet))
timestamp_list.append(get_timestamp(tweet))
content_list.append(get_content(tweet))
id_list.append(id)
return username_list, timestamp_list, content_list
def scrape(topic, limit=100, headless=True):
try:
# prepare driver
driver = load_topic(topic, headless)
except Exception as e:
print('failed instantiating webdriver')
print(e)
return [], [], []
try:
# extract information
username_list, timestamp_list, content_list = extract(driver, limit)
except Exception as e:
print('error when extracting tweets')
print(e)
return [], [], []
finally:
# close driver
driver.close()
driver.quit()
return username_list, timestamp_list, content_list
code/setup_logging.py
-28
View File
@@ -1,28 +0,0 @@
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
from discord_logging.handler import DiscordHandler
from dotenv import load_dotenv
import logging
import os
def setup_logging():
# setup stream handler
fmt = '[%(asctime)s] %(levelname)s:%(filename)s:%(funcName)s:%(message)s'
datefmt = '%Y-%m-%d %H:%M:%S'
logger_level = os.getenv('LOGGER_LEVEL')
level = getattr(logging, logger_level.upper())
logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
logger = logging.getLogger()
# add discord handler
discord_handler = DiscordHandler(
service_name = os.getenv('DISCORD_SERVICE_NAME'),
webhook_url = os.getenv('DISCORD_URL'),
)
discord_handler.setFormatter(logging.Formatter('%(message)s'))
discord_handler.setLevel(logging.WARNING)
logger.addHandler(discord_handler)
logging.debug('finished')
if __name__ == '__main__':
load_dotenv()
setup_logging()
logging.warning('test')
code/show_trans_progress.py
-32
View File
@@ -1,32 +0,0 @@
from database import Data_table
from tqdm import tqdm
import time
from dotenv import load_dotenv
import pandas as pd
def get_progress(dt):
query = 'SELECT count(*) from data_table'
res = pd.read_sql_query(query, dt.con)
total = res['count'][0]
query = 'SELECT count(*) from data_table where content_en is not null'
res = pd.read_sql_query(query, dt.con)
progress = res['count'][0]
return progress, total
if __name__ == '__main__':
load_dotenv()
try:
with Data_table() as dt:
# continously update progress bar
progress, total = get_progress(dt)
with tqdm(total=total, initial=progress) as pbar:
while True:
pbar.n = progress
pbar.refresh()
time.sleep(1)
progress, total = get_progress(dt)
except Exception as e:
print(f'error occurred: {e}')
except KeyboardInterrupt:
print('KeyboardInterrupt')
code/translate.py
+1 -1
View File
@@ -1,6 +1,6 @@
# public packages
from langdetect import detect
from google_trans_new import google_translator
from google_trans_new import google_translator # had to download package to fix bug and be able to run code
# database-related
from database import Data_table, get_untranslated, insert_translated
from dotenv import load_dotenv