brian/twitter_scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' of 192.168.1.2:brian/twitter_scraper into main

Browse Source
This commit is contained in:
brian
2023-01-11 21:10:06 +00:00
parent 5072951822 91ccc1cfed
commit 4d82b2172e
13 changed files with 89 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
-3
View File
@@ -4,6 +4,3 @@ DATABASE_PASSWORD='EqQKZrowAJXJ'
DATABASE_HOST=192.168.1.2
DATABASE_PORT=5432
DATABASE_DATABASE=twitter_scraper
LOGGER_LEVEL=info
DISCORD_URL=https://discord.com/api/webhooks/1053450686093873223/DIy33Epm7lnUqAs-YtsushI1H6sfnI76anpYcDeezd1LoXYwA5ODwRjvTlwVVI6-_2Wt
DISCORD_SERVICE_NAME=event_logger
code/clean_data.py
+4 -3
View File
@@ -1,12 +1,13 @@
# public packages
from cleantext import clean
def clean_content(text):
return clean(text, to_ascii=False)
return clean(text, to_ascii=False) # avoid ascii, as it does not have æ,ø,å
def clean_content_df(df):
content_clean = df['content'].apply(clean_content).to_list()
content_clean_list = df['content'].apply(clean_content).to_list()
id_list = df['id'].to_list()
return content_clean, id_list
return content_clean_list, id_list
def clean_content_en(text_list):
text_list_clean = list()
code/database.py
+25 -12
View File
@@ -1,4 +1,3 @@
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
import logging
import os
import traceback
@@ -171,17 +170,6 @@ def insert_list(username_list, timestamp_list, content_list):
except Exception as e:
logging.warning(e)
def insert_translated(val_dict):
assert 'id' in val_dict.keys()
assert 'content_en' in val_dict.keys()
with Data_table() as dt:
query = (
f'UPDATE {dt._name} '
' SET content_en = (%(content_en)s) '
'WHERE id = (%(id)s) '
)
dt.execute(query, val_dict)
def insert_translated_list(content_list, id_list):
with Data_table() as dt:
for id, content in zip(id_list, content_list):
@@ -210,3 +198,28 @@ def get_all():
query = f'SELECT * FROM {dt._name} ORDER BY time DESC '
df = pd.read_sql_query(query, dt.con)
return df
def insert_translated(dt, val_dict):
assert 'id' in val_dict.keys()
assert 'content_en' in val_dict.keys()
query = (
f'UPDATE {dt._name} '
' SET content_en = (%(content_en)s) '
'WHERE id = (%(id)s) '
)
dt.execute(query, val_dict)
def get_untranslated(dt):
query = (
f'SELECT * FROM {dt._name} '
'WHERE content_en IS NULL '
'ORDER BY random() '
'LIMIT 1 '
)
df = pd.read_sql_query(query, dt.con)
if len(df)==0: return None
val_dict = {
'id': int(df['id'][0]),
'content': str(df['content'][0])
}
return val_dict
code/main.py
-112
View File
@@ -1,112 +0,0 @@
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
from setup_logging import setup_logging
from selenium_scraper import scrape
from database import insert_list
from dotenv import load_dotenv
import logging
from datetime import datetime
import time
import os
from tqdm import tqdm
import snscrape.modules.twitter as sntwitter
import pandas as pd
from database import get_all
from clean_data import clean_content_df
from translate import translate, get_untranslated
from database import Data_table
# def update():
# topic = 'Odense'
# # scrape latest data
# username_list, timestamp_list, content_list = scrape(topic, limit=10, headless=False)
# # send data to database
# insert_list(username_list, timestamp_list, content_list)
# if __name__ == '__main__':
# # setup
# load_dotenv()
# setup_logging()
# logging.info('finished setup')
# # main loop
# trigger_minute = datetime.now().minute
# while True:
# try:
# now = datetime.now()
# if now.minute == trigger_minute:
# try:
# update()
# except Exception as e:
# logging.warning(e)
# else:
# logging.info('finished cycle')
# time.sleep(60)
# time.sleep(0.1)
# except Exception as e:
# logging.critical(e)
# break
# except KeyboardInterrupt:
# logging.info('KeyboardInterrupt')
# break
def run_snscrape():
# prepare variables
username_list = list()
timestamp_list = list()
content_list = list()
# begin scraping
query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
username_list.append(tweet.user.username)
timestamp_list.append(tweet.date)
content_list.append(tweet.content)
if len(content_list) == 100:
# send data to datbase
insert_list(username_list, timestamp_list, content_list)
username_list = list()
timestamp_list = list()
content_list = list()
if i % 100 == 0:
print(f'found {i} tweets')
def translate_content():
# load all content from database
print('loading data from database...')
df = get_all()
# remove already translated content
df = df[df['content_en'].isna()]
# clean content
print('cleaning content...')
content_clean_list, id_list = clean_content_df(df)
# translate content
print('translating content...')
with Data_table() as dt:
for id, content in tqdm(zip(id_list, content_clean_list), ascii=True, total=len(id_list)):
# translate content
try:
content_trans = translate(content)
except Exception as e:
print(f'error when translating content with id={id}')
continue
# transfer to database
try:
val_dict = {
'id': id,
'content_en': content_trans
}
query = (
f'UPDATE {dt._name} '
' SET content_en = (%(content_en)s) '
'WHERE id = (%(id)s) '
)
dt.execute(query, val_dict)
except Exception as e:
print(f'error when sending translated content to database: id={id}')
time.sleep(2)
if __name__ == '__main__':
load_dotenv()
# run_snscrape()
translate_content()
code/plot.ipynb
+1 -1
View File
@@ -227,7 +227,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
"version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
},
"orig_nbformat": 4,
"vscode": {
code/selenium_scraper.ipynb
-92
View File
@@ -1,92 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"12\n"
]
}
],
"source": [
"from selenium.webdriver.common.action_chains import ActionChains\n",
"from dateutil.parser import parse\n",
"from selenium import webdriver\n",
"import os\n",
"import importlib\n",
"import scraper\n",
"\n",
"importlib.reload(scraper)\n",
"topic = 'Odense'\n",
"limit = 10\n",
"username_list, timestamp_list, content_list = scraper.scrape(topic, limit, headless=False)\n",
"\n",
"print(len(username_list))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"289"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"max([len(c) for c in content_list])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# headless mode - has proven difficult...\n",
"# database access\n",
"# logging with warning to discord\n",
"# main loop - go once every hour"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "1d532f3642617da00cbdbdc5ef98bd8d4ca226c95ac593ade79d8cbc880a2afe"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}
code/selenium_scraper.py
-88
View File
@@ -1,88 +0,0 @@
from selenium.webdriver.common.action_chains import ActionChains
from dateutil.parser import parse
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.chrome.options import Options
from fake_useragent import UserAgent
def load_topic(topic, headless=True):
# instantiate driver
options = Options()
if headless:
options.headless = True
# fake a user agent to not be denied service
ua = UserAgent()
userAgent = ua.random
options.add_argument(f'user-agent={userAgent}')
driver = webdriver.Chrome(options=options)
# open webpage
url = f'https://twitter.com/search?q=%23{topic}&src=typed_query&f=live'
driver.get(url)
# wait for popup and click it
timeout = 30 # seconds
xpath_str = "//*[text()='Not now']"
element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str)))
element.click()
return driver
def list_tweets(driver):
timeout = 30 # seconds
_ = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-testid="tweet"]')))
return driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]')
def scroll_to(driver, tweet):
action = ActionChains(driver)
action.move_to_element(tweet).perform()
def get_id(tweet):
return tweet.id
def get_timestamp(tweet):
timestamp = tweet.find_element(By.TAG_NAME, "time").get_attribute("datetime")
return parse(timestamp)
def get_content(tweet):
return tweet.find_element(By.CSS_SELECTOR, 'div[lang]').text
def get_username(tweet):
return tweet.find_element(By.XPATH, ".//span[contains(text(), '@')]").text
def extract(driver, limit=100):
id_list = list()
username_list = list()
timestamp_list = list()
content_list = list()
while len(id_list) < limit:
tweet_list = list_tweets(driver)
for tweet in tweet_list:
id = get_id(tweet)
if id in id_list: continue
scroll_to(driver, tweet)
username_list.append(get_username(tweet))
timestamp_list.append(get_timestamp(tweet))
content_list.append(get_content(tweet))
id_list.append(id)
return username_list, timestamp_list, content_list
def scrape(topic, limit=100, headless=True):
try:
# prepare driver
driver = load_topic(topic, headless)
except Exception as e:
print('failed instantiating webdriver')
print(e)
return [], [], []
try:
# extract information
username_list, timestamp_list, content_list = extract(driver, limit)
except Exception as e:
print('error when extracting tweets')
print(e)
return [], [], []
finally:
# close driver
driver.close()
driver.quit()
return username_list, timestamp_list, content_list
code/setup_logging.py
-28
View File
@@ -1,28 +0,0 @@
#!/Users/brian/Projects/twitter_scraper/venv/bin/python
from discord_logging.handler import DiscordHandler
from dotenv import load_dotenv
import logging
import os
def setup_logging():
# setup stream handler
fmt = '[%(asctime)s] %(levelname)s:%(filename)s:%(funcName)s:%(message)s'
datefmt = '%Y-%m-%d %H:%M:%S'
logger_level = os.getenv('LOGGER_LEVEL')
level = getattr(logging, logger_level.upper())
logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
logger = logging.getLogger()
# add discord handler
discord_handler = DiscordHandler(
service_name = os.getenv('DISCORD_SERVICE_NAME'),
webhook_url = os.getenv('DISCORD_URL'),
)
discord_handler.setFormatter(logging.Formatter('%(message)s'))
discord_handler.setLevel(logging.WARNING)
logger.addHandler(discord_handler)
logging.debug('finished')
if __name__ == '__main__':
load_dotenv()
setup_logging()
logging.warning('test')
code/show_trans_progress.py
-32
View File
@@ -1,32 +0,0 @@
from database import get_all, Data_table
from tqdm import tqdm
import time
from dotenv import load_dotenv
import pandas as pd
def get_progress(dt):
query = 'SELECT count(*) from data_table'
res = pd.read_sql_query(query, dt.con)
total = res['count'][0]
query = 'SELECT count(*) from data_table where content_en is not null'
res = pd.read_sql_query(query, dt.con)
progress = res['count'][0]
return progress, total
if __name__ == '__main__':
load_dotenv()
try:
with Data_table() as dt:
# continously update progress bar
progress, total = get_progress(dt)
with tqdm(total=total, initial=progress) as pbar:
while True:
pbar.n = progress
pbar.refresh()
time.sleep(1)
progress, total = get_progress(dt)
except Exception as e:
print(f'error occurred: {e}')
except KeyboardInterrupt:
print('KeyboardInterrupt')
code/sntwitter_scraper.ipynb
+12 -6
View File
@@ -26,15 +26,21 @@
"source": [
"import snscrape.modules.twitter as sntwitter\n",
"import pandas as pd\n",
"tweet_list = list()\n",
"\n",
"# prepare variables\n",
"query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
"limit = 1000\n",
"\n",
"tweet_list = list()\n",
"query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'\n",
"# begin scraping\n",
"for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):\n",
" if i % 100 == 0: print(f'found {i} tweets')\n",
" if i > limit: break\n",
" # if limit is reached, break out of loop\n",
" if limit > 0 and i > limit: break\n",
" # if another 100 tweets found, tell it\n",
" if i % 100 == 0: \n",
" print(f'found {i} tweets')\n",
" tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])\n",
"\n",
"# create dataframe\n",
"df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])"
]
},
@@ -216,7 +222,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
"version": "3.10.6 (main, Aug 11 2022, 13:49:25) [Clang 13.1.6 (clang-1316.0.21.2.5)]"
},
"orig_nbformat": 4,
"vscode": {
code/sntwitter_scraper.py
+27 -11
View File
@@ -1,18 +1,34 @@
# public packages
import snscrape.modules.twitter as sntwitter
import pandas as pd
# database-related
from database import insert_list
from dotenv import load_dotenv
def scrape(limit=0):
# prepare variables
tweet_list = list()
query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
def scrape():
# prepare lists to hold information
username_list = list()
timestamp_list = list()
content_list = list()
# begin scraping
query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
# if limit is reached, break out of loop
if limit > 0 and i > limit: break
# if another 100 tweets found, tell it
# store information in lists
username_list.append(tweet.user.username)
timestamp_list.append(tweet.date)
content_list.append(tweet.content)
# when 100 tweets have been collected
if len(content_list) == 100:
# send data to database
insert_list(username_list, timestamp_list, content_list)
# empty lists
username_list = list()
timestamp_list = list()
content_list = list()
# check in on progress
if i % 100 == 0:
print(f'found {i} tweets')
tweet_list.append([tweet.date, tweet.id, tweet.content, tweet.user.username])
# create dataframe
df = pd.DataFrame(tweet_list, columns=['Datetime', 'Id', 'Text', 'Username'])
return df
if __name__ == '__main__':
load_dotenv()
scrape()
code/translate.py
+16 -46
View File
@@ -1,15 +1,11 @@
# public packages
from langdetect import detect
from google_trans_new import google_translator
import pandas as pd
import multiprocessing as mp
import numpy as np
from google_trans_new import google_translator # had to download package to fix bug and be able to run code
# database-related
from database import Data_table, get_untranslated, insert_translated
from dotenv import load_dotenv
from tqdm import tqdm
import logging
from database import Data_table
# own scripts
from clean_data import clean_content
from setup_logging import setup_logging
def translate(text, target_lang='en'):
try:
@@ -26,65 +22,39 @@ def translate(text, target_lang='en'):
return ''
return text_trans
def get_untranslated(dt):
query = (
f'SELECT * FROM {dt._name} '
'WHERE content_en IS NULL '
'ORDER BY random() '
'LIMIT 1 '
)
df = pd.read_sql_query(query, dt.con)
if len(df)==0: return None
val_dict = {
'id': int(df['id'][0]),
'content': str(df['content'][0])
}
return val_dict
def insert_translated(dt, val_dict):
assert 'id' in val_dict.keys()
assert 'content_en' in val_dict.keys()
query = (
f'UPDATE {dt._name} '
' SET content_en = (%(content_en)s) '
'WHERE id = (%(id)s) '
)
dt.execute(query, val_dict)
def continously_translate():
with Data_table() as dt:
logging.info('began translating')
while True:
print('began translating')
while True: # keep going forever
try:
val_dict = get_untranslated(dt)
except Exception as e:
logging.warning(f'error when getting untranslated content from database: {e}')
print(f'error when getting untranslated content from database: {e}')
continue
# if nothing left untranslated, stop
if val_dict is None: break
# clean
# clean text
try:
content_clean = clean_content(val_dict['content'])
except Exception as e:
logging.warning(f'error when cleaning content: {e}')
print(f'error when cleaning content: {e}')
continue
# translate
# translate text
try:
content_en = translate(content_clean)
except Exception as e:
logging.warning(f'error when translating content: {e}')
print(f'error when translating content: {e}')
continue
# upload to database
val_dict['content_en'] = content_en
try:
insert_translated(dt, val_dict)
except Exception as e:
logging.warning(f'error when sending translated content to database: {e}')
print(f'error when sending translated content to database: {e}')
continue
logging.info('translated content')
logging.warning('stopped translating')
print('translated content')
print('stopped translating')
if __name__ == '__main__':
load_dotenv()
setup_logging()
continously_translate()
# later try to set all empty strings back to null and retranslate failed...