implemented base functions

shared/datastore/src/get_image.py

@@ -2,13 +2,14 @@
 
 import logging
 import os
-from io import BytesIO
 
 from minio import Minio
 from PIL import Image
 
 from shared.utils import check_env
 
+from .get import get
+
 
 def get_image(
     client: Minio,
@@ -23,19 +24,11 @@ def get_image(
     bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
     object_name = f'images/{object_name}'
     # get object from bucket
-    try:
-        response = client.get_object(
+    buffer = get(
+        client=client,
         bucket_name=bucket_name,
         object_name=object_name,
     )
-        buffer = BytesIO(response.data)
-    except Exception as exc:
-        logging.error('failed getting data from MinIO')
-        raise exc
-    finally:
-        response.close()
-        response.release_conn()
-    buffer.seek(0)
     # convert data to image
     image = Image.open(buffer)
     logging.debug('got data from %s', object_name)

shared/datastore/src/get_model.py

@@ -3,13 +3,14 @@
 import logging
 import os
 from collections import OrderedDict
-from io import BytesIO
 
 import torch
 from minio import Minio
 
 from shared.utils import check_env
 
+from .get import get
+
 
 def get_model(
     client: Minio,
@@ -20,24 +21,16 @@ def get_model(
     assert isinstance(object_name, str)
     assert len(object_name) > 0
     # prepare arguments
-    check_env({'MINIO_BUCKET_NAME_MODELS'})
-    bucket_name = str(os.getenv('MINIO_BUCKET_NAME_MODELS'))
+    check_env({'MINIO_BUCKET_NAME'})
+    bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
     object_name = f'models/{object_name}'
     # get object from bucket
-    try:
-        response = client.get_object(
+    buffer = get(
+        client=client,
         bucket_name=bucket_name,
         object_name=object_name,
     )
-        buffer = BytesIO(response.data)
-    except Exception as exc:
-        logging.error('failed getting data from MinIO')
-        raise exc
-    finally:
-        response.close()
-        response.release_conn()
     # convert data to model checkpoint
-    buffer.seek(0)
     model_content = torch.load(buffer)
     logging.debug('finished')
     return model_content

shared/datastore/src/put_image.py

@@ -22,14 +22,13 @@ def put_image(
     # get bucket name from env
     bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
     assert len(bucket_name) > 0
-    # save image to buffer
+    # save data to buffer
     buffer = BytesIO()
     image.save(buffer, 'png')
     # get md5 of buffer
     checksum = md5(buffer.getbuffer()).hexdigest()
-    # determine object name
-    subfolder = 'images'
-    object_name = f'{subfolder}/{checksum}'
+    # set object name
+    object_name = f'images/{checksum}'
     # send data to bucket
     put(
         client=client,

shared/datastore/src/put_model.py

@@ -9,6 +9,10 @@ import torch
 from minio import Minio
 from torch.nn import Module
 
+from shared.utils import check_env
+
+from .put import put
+
 
 def put_model(
     client: Minio,
@@ -18,28 +22,22 @@ def put_model(
     used as object name."""
     assert isinstance(client, Minio)
     assert isinstance(model, Module)
-    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
-    assert len(bucket_name) > 0
-    subfolder = 'models'
-    # save image to buffer
+    # get bucket name from env
+    check_env({'MINIO_BUCKET_NAME'})
+    bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
+    # save data to buffer
     buffer = BytesIO()
     torch.save(model.state_dict(), buffer)
     # get md5 of image
     checksum = md5(buffer.getbuffer()).hexdigest()
-    # prepare for saving
-    num_bytes = buffer.tell()
-    buffer.seek(0)
+    # set object name
+    object_name = f'models/{checksum}'
     # send data to bucket
-    object_name = f'{subfolder}/{checksum}'
-    try:
-        client.put_object(
+    put(
+        client=client,
+        buffer=buffer,
         bucket_name=bucket_name,
         object_name=object_name,
-            length=num_bytes,
-            data=buffer,
     )
-    except Exception as exc:
-        logging.error('failed saving data to MinIO')
-        raise exc
-    logging.debug('saved data to %s', object_name)
+    logging.debug('finished')
     return checksum