made and tested code to scrape twitter for a specific topic

scraper.py

@@ -0,0 +1,87 @@
+from selenium.webdriver.common.action_chains import ActionChains
+from dateutil.parser import parse
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.chrome.options import Options
+from fake_useragent import UserAgent
+
+def load_topic(topic, headless=True):
+    # instantiate driver
+    options = Options()
+    if headless:
+        options.headless = True
+        # fake a user agent to not be denied service
+        ua = UserAgent()
+        userAgent = ua.random
+        options.add_argument(f'user-agent={userAgent}')
+    driver = webdriver.Chrome(options=options)
+    # open webpage
+    url = f'https://twitter.com/search?q=%23{topic}&src=typed_query&f=live'
+    driver.get(url)
+    # wait for popup and click it
+    timeout = 30 # seconds
+    xpath_str = "//*[text()='Not now']"
+    element = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.XPATH, xpath_str)))
+    element.click()
+    return driver
+
+def list_tweets(driver):
+    timeout = 30 # seconds
+    _ = WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-testid="tweet"]')))
+    return driver.find_elements(By.CSS_SELECTOR, '[data-testid="tweet"]')
+
+def scroll_to(driver, tweet):
+    action = ActionChains(driver)
+    action.move_to_element(tweet).perform()
+
+def get_id(tweet):
+    return tweet.id
+
+def get_timestamp(tweet):
+    timestamp = tweet.find_element(By.TAG_NAME, "time").get_attribute("datetime")
+    return parse(timestamp)
+
+def get_content(tweet):
+    return tweet.find_element(By.CSS_SELECTOR, 'div[lang]').text
+
+def get_username(tweet):
+    return tweet.find_element(By.XPATH, ".//span[contains(text(), '@')]").text
+
+def extract(driver, limit=100):
+    id_list = list()
+    username_list = list()
+    timestamp_list = list()
+    content_list = list()
+    while len(id_list) < limit:
+        tweet_list = list_tweets(driver)
+        for tweet in tweet_list:
+            id = get_id(tweet)
+            if id in id_list: continue
+            scroll_to(driver, tweet)
+            username_list.append(get_username(tweet))
+            timestamp_list.append(get_timestamp(tweet))
+            content_list.append(get_content(tweet))
+            id_list.append(id)
+    return username_list, timestamp_list, content_list
+    
+def scrape(topic, limit=100, headless=True):
+    try:
+        # prepare driver
+        driver = load_topic(topic, headless)
+    except Exception as e:
+        print('failed instantiating webdriver')
+        print(e)
+        return [], [], []
+    try:
+        # extract information
+        username_list, timestamp_list, content_list = extract(driver, limit)
+    except Exception as e:
+        print('error when extracting tweets')
+        print(e)
+    finally:
+        # close driver
+        driver.close()
+        driver.quit()
+    return username_list, timestamp_list, content_list