From ef11c336756420c2915ce4f2d05cdcf5914e66c5 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Mon, 22 Jul 2024 23:45:19 +0200 Subject: [PATCH] used buffered download --- shared/data_store/get.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/shared/data_store/get.py b/shared/data_store/get.py index 312cad6..d0b03a7 100644 --- a/shared/data_store/get.py +++ b/shared/data_store/get.py @@ -1,7 +1,9 @@ """Definition of get function.""" +import logging import os from io import BytesIO +from traceback import print_exc from minio import Minio @@ -16,10 +18,22 @@ def get( bucket_name = os.getenv('MINIO_BUCKET_NAME') # get buffer try: - response = client.get_object(bucket_name, object_name) - buffer = BytesIO(response.data) + response = client.get_object( + bucket_name=bucket_name, + object_name=object_name, + ) + assert response.status == 200 + buffer = BytesIO() + chunk_size = 2**14 + while chunk := response.read(chunk_size): + buffer.write(chunk) + buffer.seek(0) + logging.debug('got %s', object_name) + return buffer + except Exception as exc: + logging.error('failed getting data from MinIO') + print_exc() + raise exc finally: response.close() response.release_conn() - buffer.seek(0) - return buffer