35 lines
1.2 KiB
Python
35 lines
1.2 KiB
Python
# public packages
|
|
import snscrape.modules.twitter as sntwitter
|
|
import pandas as pd
|
|
# database-related
|
|
from database import insert_list
|
|
from dotenv import load_dotenv
|
|
|
|
def scrape(query=''):
|
|
assert len(query) > 0, 'query string must be provided'
|
|
# prepare lists to hold information
|
|
username_list = list()
|
|
timestamp_list = list()
|
|
content_list = list()
|
|
# begin scraping
|
|
# query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
|
|
for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
|
|
# store information in lists
|
|
username_list.append(tweet.user.username)
|
|
timestamp_list.append(tweet.date)
|
|
content_list.append(tweet.content)
|
|
# when 100 tweets have been collected
|
|
if len(content_list) == 100:
|
|
# send data to database
|
|
insert_list(username_list, timestamp_list, content_list)
|
|
# empty lists
|
|
username_list = list()
|
|
timestamp_list = list()
|
|
content_list = list()
|
|
# check in on progress
|
|
if i % 100 == 0:
|
|
print(f'found {i} tweets')
|
|
|
|
if __name__ == '__main__':
|
|
load_dotenv()
|
|
scrape() |