moved files to code folder
This commit is contained in:
simplypower-bbj
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'username': '@otwndk',\n",
|
||||
" 'time': datetime.datetime(2022, 12, 16, 21, 19, 9, tzinfo=tzutc()),\n",
|
||||
" 'content': 'Sponsoreret: Verdensmestre, europamestre og store danske medaljetagere rammer Odense til et prestigefyldt stævne\\n#odense #fyn #cykling'}"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from database import Data_table\n",
|
||||
"from scraper import scrape\n",
|
||||
"\n",
|
||||
"topic = 'Odense'\n",
|
||||
"username_list, timestamp_list, content_list = scrape(topic, limit=3, headless=False)\n",
|
||||
"\n",
|
||||
"val_dict = {\n",
|
||||
" 'username': username_list[0],\n",
|
||||
" 'time': timestamp_list[0],\n",
|
||||
" 'content': content_list[0]\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(val_dict)\n",
|
||||
"\n",
|
||||
"with Data_table() as dt:\n",
|
||||
" dt.insert(val_dict)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.6"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
|
||||
import logging
|
||||
import os
|
||||
import traceback
|
||||
import urllib.parse
|
||||
|
||||
import pandas as pd
|
||||
import psycopg2
|
||||
from dotenv import load_dotenv
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
|
||||
class Database(object):
|
||||
def __init__(self):
|
||||
# generate uri
|
||||
engine = os.getenv('DATABASE_ENGINE')
|
||||
username = os.getenv('DATABASE_USERNAME')
|
||||
password = os.getenv('DATABASE_PASSWORD')
|
||||
password = urllib.parse.quote_plus(password) # escape special characters
|
||||
host = os.getenv('DATABASE_HOST')
|
||||
port = os.getenv('DATABASE_PORT')
|
||||
database = os.getenv('DATABASE_DATABASE')
|
||||
self._uri = f'{engine}://{username}:{password}@{host}:{port}/{database}'
|
||||
# prepare connection to database
|
||||
self._engine = create_engine(self._uri)
|
||||
self.con = None
|
||||
|
||||
def __enter__(self):
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, tb):
|
||||
if not exc_type is None:
|
||||
traceback.print_exception(exc_type, exc_value, tb)
|
||||
self.disconnect()
|
||||
|
||||
def connect(self):
|
||||
try:
|
||||
con = self._engine.connect()
|
||||
except psycopg2.OperationalError as e:
|
||||
logging.warning(e)
|
||||
else:
|
||||
self.con = con
|
||||
|
||||
def disconnect(self):
|
||||
self.con.close()
|
||||
|
||||
def execute(self, query, var=None):
|
||||
assert self.con is not None, 'connection to database could not be established'
|
||||
try:
|
||||
if var is None:
|
||||
self.con.execute(query)
|
||||
else:
|
||||
self.con.execute(query, var)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise
|
||||
|
||||
def get(self, query):
|
||||
df = pd.read_sql_query(query, self.con)
|
||||
return df
|
||||
|
||||
class Schema(Database):
|
||||
def __init__(self, name):
|
||||
super(Schema, self).__init__()
|
||||
self._name = name
|
||||
|
||||
def __enter__(self):
|
||||
self.connect()
|
||||
if not self.exists():
|
||||
self.create()
|
||||
return self
|
||||
|
||||
def exists(self):
|
||||
query = (
|
||||
'SELECT schema_name '
|
||||
'FROM information_schema.schemata '
|
||||
'ORDER BY schema_name '
|
||||
';'
|
||||
)
|
||||
df = self.get(query)
|
||||
schema_list = df['schema_name'].tolist()
|
||||
return self._name in schema_list
|
||||
|
||||
def create(self):
|
||||
query = f'CREATE SCHEMA {self._name};'
|
||||
self.execute(query)
|
||||
|
||||
def purge(self):
|
||||
query = f'DROP SCHEMA IF EXISTS {self._name} CASCADE;'
|
||||
self.execute(query)
|
||||
self.create()
|
||||
|
||||
class Table(Database):
|
||||
def __init__(self, name):
|
||||
super(Table, self).__init__()
|
||||
self._name = name
|
||||
|
||||
def __enter__(self):
|
||||
self.connect()
|
||||
if not self.exists():
|
||||
self.create()
|
||||
return self
|
||||
|
||||
def exists(self):
|
||||
query = (
|
||||
'SELECT table_name '
|
||||
'FROM information_schema.tables '
|
||||
"WHERE table_schema='public' "
|
||||
'ORDER BY table_name '
|
||||
)
|
||||
df = self.get(query)
|
||||
table_list = df['table_name'].tolist()
|
||||
return self._name in table_list
|
||||
|
||||
def create(self):
|
||||
# placeholder for childrens' methods
|
||||
pass
|
||||
|
||||
def purge(self):
|
||||
query = f'DROP TABLE IF EXISTS {self._name} CASCADE;'
|
||||
self.execute(query)
|
||||
self.create()
|
||||
|
||||
class Data_table(Table):
|
||||
def __init__(self):
|
||||
super(Data_table, self).__init__('data_table')
|
||||
|
||||
def create(self):
|
||||
query = (
|
||||
f'CREATE TABLE {self._name} '
|
||||
'('
|
||||
'id SERIAL PRIMARY KEY, '
|
||||
'username VARCHAR(15) NOT NULL, '
|
||||
'time TIMESTAMPTZ NOT NULL, '
|
||||
'content TEXT NOT NULL, '
|
||||
'UNIQUE (username, time, content) '
|
||||
')'
|
||||
';'
|
||||
)
|
||||
self.execute(query)
|
||||
|
||||
def insert(self, val_dict):
|
||||
query = (
|
||||
f'INSERT INTO {self._name} (username, time, content) '
|
||||
'VALUES (%(username)s, %(time)s, %(content)s) '
|
||||
'ON CONFLICT (username, time, content) DO NOTHING '
|
||||
';'
|
||||
)
|
||||
self.execute(query, val_dict)
|
||||
|
||||
def insert(username, timestamp, content):
|
||||
val_dict = {
|
||||
'username': username,
|
||||
'time': timestamp,
|
||||
'content': content
|
||||
}
|
||||
with Data_table() as dt:
|
||||
dt.insert(val_dict)
|
||||
@@ -0,0 +1,93 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"12\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from selenium.webdriver.common.action_chains import ActionChains\n",
|
||||
"from dateutil.parser import parse\n",
|
||||
"from selenium import webdriver\n",
|
||||
"import os\n",
|
||||
"import importlib\n",
|
||||
"import scraper\n",
|
||||
"\n",
|
||||
"importlib.reload(scraper)\n",
|
||||
"topic = 'Odense'\n",
|
||||
"limit = 10\n",
|
||||
"username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)\n",
|
||||
"\n",
|
||||
"print(len(username_list))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"289"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"max([len(c) for c in content_list])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# headless mode - has proven difficult...\n",
|
||||
"# database access\n",
|
||||
"# logging with warning to discord\n",
|
||||
"# main loop - go once every hour\n",
|
||||
"# containerize"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.6"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
from selenium.webdriver.common.action_chains import ActionChains
|
||||
from dateutil.parser import parse
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from fake_useragent import UserAgent
|
||||
|
||||
def load_topic(topic, headless=True):
|
||||
# instantiate driver
|
||||
options = Options()
|
||||
if headless:
|
||||
options.headless = True
|
||||
# fake a user agent to not be denied service
|
||||
ua = UserAgent()
|
||||
userAgent = ua.random
|
||||
options.add_argument(f'user-agent={userAgent}')
|
||||
driver = webdriver.Chrome(options=options)
|
||||
# open webpage
|
||||
url = f'https://twitter.com/search?q=%23{topic}&src=typed_query&f=live'
|
||||
driver.get(url)
|
||||
# wait for popup and click it
|
||||
timeout = 30 # seconds
|
||||
xpath_str = "//*[text()='Not now']"
|
||||
element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str)))
|
||||
element.click()
|
||||
return driver
|
||||
|
||||
def list_tweets(driver):
|
||||
timeout = 30 # seconds
|
||||
_ = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-testid="tweet"]')))
|
||||
return driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]')
|
||||
|
||||
def scroll_to(driver, tweet):
|
||||
action = ActionChains(driver)
|
||||
action.move_to_element(tweet).perform()
|
||||
|
||||
def get_id(tweet):
|
||||
return tweet.id
|
||||
|
||||
def get_timestamp(tweet):
|
||||
timestamp = tweet.find_element(By.TAG_NAME, "time").get_attribute("datetime")
|
||||
return parse(timestamp)
|
||||
|
||||
def get_content(tweet):
|
||||
return tweet.find_element(By.CSS_SELECTOR, 'div[lang]').text
|
||||
|
||||
def get_username(tweet):
|
||||
return tweet.find_element(By.XPATH, ".//span[contains(text(), '@')]").text
|
||||
|
||||
def extract(driver, limit=100):
|
||||
id_list = list()
|
||||
username_list = list()
|
||||
timestamp_list = list()
|
||||
content_list = list()
|
||||
while len(id_list) < limit:
|
||||
tweet_list = list_tweets(driver)
|
||||
for tweet in tweet_list:
|
||||
id = get_id(tweet)
|
||||
if id in id_list: continue
|
||||
scroll_to(driver, tweet)
|
||||
username_list.append(get_username(tweet))
|
||||
timestamp_list.append(get_timestamp(tweet))
|
||||
content_list.append(get_content(tweet))
|
||||
id_list.append(id)
|
||||
return username_list, timestamp_list, content_list
|
||||
|
||||
def scrape(topic, limit=100, headless=True):
|
||||
try:
|
||||
# prepare driver
|
||||
driver = load_topic(topic, headless)
|
||||
except Exception as e:
|
||||
print('failed instantiating webdriver')
|
||||
print(e)
|
||||
return [], [], []
|
||||
try:
|
||||
# extract information
|
||||
username_list, timestamp_list, content_list = extract(driver, limit)
|
||||
except Exception as e:
|
||||
print('error when extracting tweets')
|
||||
print(e)
|
||||
finally:
|
||||
# close driver
|
||||
driver.close()
|
||||
driver.quit()
|
||||
return username_list, timestamp_list, content_list
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
|
||||
from discord_logging.handler import DiscordHandler
|
||||
from dotenv import load_dotenv
|
||||
import logging
|
||||
import os
|
||||
|
||||
def setup_logging():
|
||||
# setup stream handler
|
||||
fmt = '[%(asctime)s] %(levelname)s:%(filename)s:%(funcName)s:%(message)s'
|
||||
datefmt = '%Y-%m-%d %H:%M:%S'
|
||||
logger_level = os.getenv('LOGGER_LEVEL')
|
||||
level = getattr(logging, logger_level.upper())
|
||||
logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
|
||||
logger = logging.getLogger()
|
||||
# add discord handler
|
||||
discord_handler = DiscordHandler(
|
||||
service_name = 'data_collection',
|
||||
webhook_url = os.getenv('DISCORD_URL'),
|
||||
)
|
||||
discord_handler.setFormatter(logging.Formatter('%(message)s'))
|
||||
discord_handler.setLevel(logging.WARNING)
|
||||
logger.addHandler(discord_handler)
|
||||
logging.debug('finished')
|
||||
|
||||
if __name__ == '__main__':
|
||||
load_dotenv()
|
||||
setup_logging()
|
||||
logging.warning('test')
|
||||
Reference in New Issue
Block a user