from selenium.webdriver.common.action_chains import ActionChains
from dateutil.parser import parse
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.chrome.options import Options
from fake_useragent import UserAgent

def load_topic(topic, headless=True):
    # instantiate driver
    options = Options()
    if headless:
        options.headless = True
        # fake a user agent to not be denied service
        ua = UserAgent()
        userAgent = ua.random
        options.add_argument(f'user-agent={userAgent}')
    driver = webdriver.Chrome(options=options)
    # open webpage
    url = f'https://twitter.com/search?q=%23{topic}&src=typed_query&f=live'
    driver.get(url)
    # wait for popup and click it
    timeout = 30 # seconds
    xpath_str = "//*[text()='Not now']"
    element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str)))
    element.click()
    return driver

def list_tweets(driver):
    timeout = 30 # seconds
    _ = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-testid="tweet"]')))
    return driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]')

def scroll_to(driver, tweet):
    action = ActionChains(driver)
    action.move_to_element(tweet).perform()

def get_id(tweet):
    return tweet.id

def get_timestamp(tweet):
    timestamp = tweet.find_element(By.TAG_NAME, "time").get_attribute("datetime")
    return parse(timestamp)

def get_content(tweet):
    return tweet.find_element(By.CSS_SELECTOR, 'div[lang]').text

def get_username(tweet):
    return tweet.find_element(By.XPATH, ".//span[contains(text(), '@')]").text

def extract(driver, limit=100):
    id_list = list()
    username_list = list()
    timestamp_list = list()
    content_list = list()
    while len(id_list) < limit:
        tweet_list = list_tweets(driver)
        for tweet in tweet_list:
            id = get_id(tweet)
            if id in id_list: continue
            scroll_to(driver, tweet)
            username_list.append(get_username(tweet))
            timestamp_list.append(get_timestamp(tweet))
            content_list.append(get_content(tweet))
            id_list.append(id)
    return username_list, timestamp_list, content_list
    
def scrape(topic, limit=100, headless=True):
    try:
        # prepare driver
        driver = load_topic(topic, headless)
    except Exception as e:
        print('failed instantiating webdriver')
        print(e)
        return [], [], []
    try:
        # extract information
        username_list, timestamp_list, content_list = extract(driver, limit)
    except Exception as e:
        print('error when extracting tweets')
        print(e)
        return [], [], []
    finally:
        # close driver
        driver.close()
        driver.quit()
    return username_list, timestamp_list, content_list