added script to download images
This commit is contained in:
@@ -0,0 +1,69 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
import requests
|
||||||
|
from retry import retry
|
||||||
|
|
||||||
|
from image_download.classes import Instagram
|
||||||
|
|
||||||
|
|
||||||
|
def get_sources() -> pd.DataFrame:
|
||||||
|
"""Get sources dateframe."""
|
||||||
|
input_path = Path(__file__).parent / 'sources.csv'
|
||||||
|
assert input_path.exists()
|
||||||
|
# load in csv
|
||||||
|
df = pd.read_csv(input_path)
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
@retry(
|
||||||
|
exceptions=AssertionError,
|
||||||
|
tries=3,
|
||||||
|
delay=5,
|
||||||
|
backoff=1.2,
|
||||||
|
jitter=(1, 10),
|
||||||
|
)
|
||||||
|
def download_images(ig_inst: Instagram, filename: str, url: str) -> None:
|
||||||
|
"""
|
||||||
|
Download images from instagram post found on url
|
||||||
|
|
||||||
|
to files with base name from filename.
|
||||||
|
"""
|
||||||
|
# get image urls
|
||||||
|
url_list = ig_inst.get_image_url_list(url)
|
||||||
|
assert len(url_list) > 0
|
||||||
|
# generate filenames
|
||||||
|
chars = 'abcdefghij'
|
||||||
|
filename_list = [name + chars[i] for i in range(len(url_list))]
|
||||||
|
# download images
|
||||||
|
download_dir = Path(__file__).parent / 'imgs'
|
||||||
|
download_dir.mkdir(exist_ok=True)
|
||||||
|
for filename, url in zip(filename_list, url_list):
|
||||||
|
output_path = download_dir / f"{filename}.jpg"
|
||||||
|
response = requests.get(url, stream=True)
|
||||||
|
with open(output_path, 'wb') as fh:
|
||||||
|
shutil.copyfileobj(response.raw, fh)
|
||||||
|
logging.info('downloaded %s', output_path)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
# setup logging
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
# get sources df
|
||||||
|
df = get_sources()
|
||||||
|
# setup Instagram interface
|
||||||
|
igi = Instagram()
|
||||||
|
igi.setup()
|
||||||
|
# begin downloading
|
||||||
|
for idx in range(len(df)):
|
||||||
|
# get info from source dataframe
|
||||||
|
name = df.loc[idx, 'name']
|
||||||
|
url = df.loc[idx, 'url']
|
||||||
|
# download post images
|
||||||
|
download_images(ig_inst=igi, filename=name, url=url)
|
||||||
|
# clean up
|
||||||
|
igi.close()
|
||||||
Reference in New Issue
Block a user