88 lines
2.9 KiB
Python
88 lines
2.9 KiB
Python
from selenium.webdriver.common.action_chains import ActionChains
|
|
from dateutil.parser import parse
|
|
from selenium import webdriver
|
|
from selenium.webdriver.common.by import By
|
|
from selenium.webdriver.support import expected_conditions as EC
|
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
from selenium.webdriver.chrome.options import Options
|
|
from fake_useragent import UserAgent
|
|
|
|
def load_topic(topic, headless=True):
|
|
# instantiate driver
|
|
options = Options()
|
|
if headless:
|
|
options.headless = True
|
|
# fake a user agent to not be denied service
|
|
ua = UserAgent()
|
|
userAgent = ua.random
|
|
options.add_argument(f'user-agent={userAgent}')
|
|
driver = webdriver.Chrome(options=options)
|
|
# open webpage
|
|
url = f'https://twitter.com/search?q=%23{topic}&src=typed_query&f=live'
|
|
driver.get(url)
|
|
# wait for popup and click it
|
|
timeout = 30 # seconds
|
|
xpath_str = "//*[text()='Not now']"
|
|
element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str)))
|
|
element.click()
|
|
return driver
|
|
|
|
def list_tweets(driver):
|
|
timeout = 30 # seconds
|
|
_ = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-testid="tweet"]')))
|
|
return driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]')
|
|
|
|
def scroll_to(driver, tweet):
|
|
action = ActionChains(driver)
|
|
action.move_to_element(tweet).perform()
|
|
|
|
def get_id(tweet):
|
|
return tweet.id
|
|
|
|
def get_timestamp(tweet):
|
|
timestamp = tweet.find_element(By.TAG_NAME, "time").get_attribute("datetime")
|
|
return parse(timestamp)
|
|
|
|
def get_content(tweet):
|
|
return tweet.find_element(By.CSS_SELECTOR, 'div[lang]').text
|
|
|
|
def get_username(tweet):
|
|
return tweet.find_element(By.XPATH, ".//span[contains(text(), '@')]").text
|
|
|
|
def extract(driver, limit=100):
|
|
id_list = list()
|
|
username_list = list()
|
|
timestamp_list = list()
|
|
content_list = list()
|
|
while len(id_list) < limit:
|
|
tweet_list = list_tweets(driver)
|
|
for tweet in tweet_list:
|
|
id = get_id(tweet)
|
|
if id in id_list: continue
|
|
scroll_to(driver, tweet)
|
|
username_list.append(get_username(tweet))
|
|
timestamp_list.append(get_timestamp(tweet))
|
|
content_list.append(get_content(tweet))
|
|
id_list.append(id)
|
|
return username_list, timestamp_list, content_list
|
|
|
|
def scrape(topic, limit=100, headless=True):
|
|
try:
|
|
# prepare driver
|
|
driver = load_topic(topic, headless)
|
|
except Exception as e:
|
|
print('failed instantiating webdriver')
|
|
print(e)
|
|
return [], [], []
|
|
try:
|
|
# extract information
|
|
username_list, timestamp_list, content_list = extract(driver, limit)
|
|
except Exception as e:
|
|
print('error when extracting tweets')
|
|
print(e)
|
|
finally:
|
|
# close driver
|
|
driver.close()
|
|
driver.quit()
|
|
return username_list, timestamp_list, content_list
|