fixed types

misc/generate_random_prediction.py

@@ -4,9 +4,9 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.datastore import connect_minio
+from shared.datastore import Datastore
 from shared.mongodb import connect_mongodb
-from shared.mongodb.classes import VisualCommunication
+from shared.mongodb.src.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 
@@ -20,7 +20,9 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minIO
-    minio_client = connect_minio()
+    datastore = Datastore()
+    datastore.connect()
+    assert datastore._client is not None
     # connect to MongoDB
     collection, db, client = connect_mongodb()
     # get list of image paths
@@ -30,7 +32,7 @@ if __name__ == '__main__':
     print(img_path_list)
     # instantiate data object
     vis_com_list = [
-        VisualCommunication.from_file(path, minio_client=minio_client)
+        VisualCommunication.from_file(path, minio_client=datastore._client)
         for path in img_path_list
     ]
     # generate random predictions

misc/get_visual_communication.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.datastore import connect_minio
+from shared.datastore import Datastore
 from shared.mongodb import connect_mongodb, get_visual_communication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
@@ -19,11 +19,13 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minIO
-    minio_client = connect_minio()
+    datastore = Datastore()
+    datastore.connect()
+    assert datastore._client is not None
     # connect to MongoDB
     collection, db, client = connect_mongodb()
     # get visual communication
     vis_com = get_visual_communication(collection)
     print(repr(vis_com))
-    image = vis_com.get_image(minio_client=minio_client)
+    image = vis_com.get_image(minio_client=datastore._client)
     image.show()

misc/image_upload.py

@@ -5,9 +5,9 @@ from pathlib import Path
 from dotenv import load_dotenv
 from pymongo.errors import DuplicateKeyError
 
-from shared.datastore import connect_minio
+from shared.datastore import Datastore
 from shared.mongodb import connect_mongodb
-from shared.mongodb.classes import VisualCommunication
+from shared.mongodb.src.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 
@@ -21,7 +21,9 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minIO
-    minio_client = connect_minio()
+    datastore = Datastore()
+    datastore.connect()
+    assert datastore._client is not None
     # connect to MongoDB
     collection, db, client = connect_mongodb()
     # get list of image paths
@@ -31,7 +33,7 @@ if __name__ == '__main__':
     print(img_path_list)
     # instantiate data object
     vis_com_list = [
-        VisualCommunication.from_file(path, minio_client=minio_client)
+        VisualCommunication.from_file(path, minio_client=datastore._client)
         for path in img_path_list
     ]
     for vis_com in vis_com_list:

misc/image_upload_to_server.py

@@ -5,9 +5,9 @@ from pathlib import Path
 from dotenv import load_dotenv
 from pymongo.errors import DuplicateKeyError
 
-from shared.datastore import connect_minio
+from shared.datastore import Datastore
 from shared.mongodb import connect_mongodb
-from shared.mongodb.classes import VisualCommunication
+from shared.mongodb.src.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 
@@ -21,7 +21,9 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minIO
-    minio_client = connect_minio()
+    datastore = Datastore()
+    datastore.connect()
+    assert datastore._client is not None
     # connect to MongoDB
     collection, db, client = connect_mongodb()
     # get list of image paths
@@ -37,7 +39,7 @@ if __name__ == '__main__':
     print(f"found {len(img_path_list)} images")
     # create visual communication objects
     vis_com_list = [
-        VisualCommunication.from_file(path, minio_client=minio_client)
+        VisualCommunication.from_file(path, minio_client=datastore._client)
         for path in img_path_list
     ]
     print(f"created {len(vis_com_list)} visual communication objects")

misc/model_io.py

@@ -7,7 +7,7 @@ from dotenv import load_dotenv
 from torchinfo import summary
 
 from model.src.models import VisualCommunicationModel
-from shared.datastore import connect_minio, put_model
+from shared.datastore import Datastore
 from shared.utils import setup_logging
 
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
@@ -20,14 +20,14 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minio
-    client = connect_minio()
+    datastore = Datastore()
+    datastore.connect()
     # instantiate model
     model = VisualCommunicationModel().to(DEVICE)
     # show model weights
     summary(model)
     # put buffer in minio bucket
-    hash_str = put_model(
+    hash_str = datastore.put_model(
-        client=client,
         model=model,
     )
     print(f"hash string: {hash_str}")

misc/new_model_to_minio.py

@@ -7,7 +7,7 @@ from dotenv import load_dotenv
 from torchinfo import summary
 
 from model.src.models import VisualCommunicationModel
-from shared.datastore import connect_minio, put_model
+from shared.datastore import Datastore
 from shared.utils import setup_logging
 
 if __name__ == '__main__':
@@ -18,14 +18,14 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minio
-    client = connect_minio()
+    datastore = Datastore()
+    datastore.connect()
     # instantiate model
     model = VisualCommunicationModel(download_resnet_weights=True)
     # show model weights
     summary(model)
     # put buffer in minio bucket
-    hash_str = put_model(
+    hash_str = datastore.put_model(
-        client=client,
         model=model,
     )
     print(f"hash string: {hash_str}")

misc/prediction_upload.py

@@ -4,9 +4,9 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.datastore import connect_minio
+from shared.datastore import Datastore
 from shared.mongodb import connect_mongodb, upsert_prediction
-from shared.mongodb.classes import VisualCommunication
+from shared.mongodb.src.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 
@@ -20,7 +20,9 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minIO
-    minio_client = connect_minio()
+    datastore = Datastore()
+    datastore.connect()
+    assert datastore._client is not None
     # connect to MongoDB
     collection, db, client = connect_mongodb()
     # get list of image paths
@@ -29,7 +31,7 @@ if __name__ == '__main__':
     img_path_list = [path for path in img_dir.glob('*.jpeg') if path.is_file()]
     # instantiate data object
     vis_com_list = [
-        VisualCommunication.from_file(path, minio_client=minio_client)
+        VisualCommunication.from_file(path, minio_client=datastore._client)
         for path in img_path_list
     ]
     # generate random predictions

model/src/main.py

@@ -9,7 +9,7 @@ from models import VisualCommunicationModel
 from tqdm import tqdm
 from utils import DEVICE, VCDADataset, load_model
 
-from shared.datastore import connect_minio
+from shared.datastore import Datastore
 from shared.mongodb.classes import ModelData
 from shared.utils import setup_logging
 
@@ -17,13 +17,14 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minio
-    minio_client = connect_minio()
+    datastore = Datastore()
+    datastore.connect()
     # instantiate model
-    model: VisualCommunicationModel = load_model(client=minio_client)
+    model: VisualCommunicationModel = load_model(client=datastore._client)
     model.eval()
     # setup dataset
     dataset = VCDADataset(
-        minio_client=minio_client,
+        minio_client=datastore._client,
         data_name_list=[
             '02dbaf48d713e4e6d3a6b98fd2dc866e',
         ],

model/src/utils/load_model.py

@@ -4,10 +4,9 @@ import logging
 from pathlib import Path
 
 import torch
-from minio import Minio
 
 from model.src.models import VisualCommunicationModel
-from shared.datastore import get_model
+from shared.datastore import Datastore
 
 from .get_model_name import get_model_name
 
@@ -15,11 +14,11 @@ DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
 
 def load_model(
-    client: Minio,
+    datastore: Datastore,
 ) -> VisualCommunicationModel:
     """Instantiate model with weights loaded from latest model saved in
     MinIO."""
-    assert isinstance(client, Minio)
+    assert isinstance(datastore, Datastore)
     # instantiate model
     model = VisualCommunicationModel()
     # get model object name
@@ -27,8 +26,7 @@ def load_model(
     model_object_name = get_model_name(path=model_name_path)
     logging.info('using model: %s', model_object_name)
     # load model from minio
-    model_checkpoint = get_model(
+    model_checkpoint = datastore.get_model(
-        client=client,
         object_name=model_object_name,
     )
     model.load_state_dict(model_checkpoint)

model/src/utils/vcda_dataset.py

@@ -2,7 +2,6 @@
 
 import random
 
-from minio import Minio
 from PIL import Image
 from torch import Tensor
 from torch.utils.data import Dataset
@@ -16,7 +15,7 @@ from torchvision.transforms.functional import (
     to_tensor,
 )
 
-from shared.datastore import get_image
+from shared.datastore import Datastore
 
 # resnet18 original normalization values
 RESNET_NORMALIZE_MEAN = [0.485, 0.456, 0.406]
@@ -28,13 +27,13 @@ class VCDADataset(Dataset):
 
     def __init__(
         self,
-        minio_client: Minio,
+        datastore: Datastore,
         data_name_list: list[str],
         do_augment: bool = False,
         random_annotations: bool = False,
     ):
         super().__init__()
-        self.minio_client = minio_client
+        self.datastore = datastore
         self.data_name_list = data_name_list
         self.do_augment = do_augment
         self.random_annotations = random_annotations
@@ -57,8 +56,7 @@ class VCDADataset(Dataset):
     def __getitem__(self, idx):
         # get image from database
         object_name = self.data_name_list[idx]
-        image = get_image(
+        image = self.datastore.get_image(
-            client=self.minio_client,
             object_name=object_name,
         )
         tensor = self.image_to_tensor(image)

model/train.py

@@ -18,7 +18,7 @@ from torch.optim.lr_scheduler import ExponentialLR
 from torch.utils.data import DataLoader
 
 from model.src.utils import VCDADataset, get_class
-from shared.datastore import connect_minio
+from shared.datastore import Datastore
 
 
 def parse_arguments():
@@ -57,18 +57,19 @@ optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
 loss_fn = nn.CrossEntropyLoss()
 
 # create datasets and loaders
-minio_client = connect_minio()
+datastore = Datastore()
+datastore.connect()
 with open('model/src/dataset/train.csv', encoding='utf-8') as fh:
     train_data_name_list = fh.read().split('\n')
 train_dataset = VCDADataset(
-    minio_client=minio_client,
+    datastore=datastore,
     data_name_list=train_data_name_list,
 )
 train_loader = DataLoader(dataset=train_dataset, num_workers=args.loader_workers)
 
 with open('model/src/dataset/val.csv', encoding='utf-8') as fh:
     val_data_name_list = fh.read().split('\n')
-val_dataset = VCDADataset(minio_client=minio_client, data_name_list=val_data_name_list)
+val_dataset = VCDADataset(datastore=datastore, data_name_list=val_data_name_list)
 val_loader = DataLoader(dataset=val_dataset, num_workers=args.loader_workers)
 
 # create trainer and evaluator

other/transfer_images_minio_subfolder.py

@@ -1,11 +1,12 @@
 """Script to move minio images to subfolder."""
 
+import os
 from pathlib import Path
 
 from dotenv import load_dotenv
 from PIL import Image
 
-from shared.datastore import connect_minio, get, put_image
+from shared.datastore import Datastore
 from shared.utils import setup_logging
 
 if __name__ == '__main__':
@@ -16,24 +17,26 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minio
-    minio_client = connect_minio()
+    datastore = Datastore()
+    datastore.connect()
+    assert datastore._client is not None
     # list images in bucket
-    BUCKET_NAME = 'visual-critical-discourse-analysis'
+    BUCKET_NAME = os.getenv(
-    obj_list = minio_client.list_objects(
+        'MINIO_BUCKET_NAME',
+        default='visual-critical-discourse-analysis',
+    )
+    obj_list = datastore._client.list_objects(
         bucket_name=BUCKET_NAME,
     )
     # begin moving images
     for obj in obj_list:
         # get image from minio
-        buffer = get(
+        buffer = datastore._get(
-            client=minio_client,
-            bucket_name=BUCKET_NAME,
             object_name=obj.object_name,
         )
         # convert data to image
         image = Image.open(buffer)
         # put image into minio subfolder
-        put_image(
+        _ = datastore.put_image(
-            client=minio_client,
             image=image,
         )

other/transfer_images_mongo_minio.py

@@ -9,7 +9,7 @@ from bson import ObjectId
 from dotenv import load_dotenv
 from pymongo.collection import Collection
 
-from shared.datastore import connect_minio, put_image
+from shared.datastore import Datastore
 from shared.mongodb import connect_mongodb
 from shared.mongodb.src.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
@@ -81,7 +81,9 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minIO
-    minio_client = connect_minio()
+    datastore = Datastore()
+    datastore.connect()
+    assert datastore._client is not None
     # connect to MongoDB
     collection, db, client = connect_mongodb()
     # list documents in mongoDB
@@ -97,11 +99,10 @@ if __name__ == '__main__':
         try:
             # get image
             image = vis_com.get_image(
-                minio_client=minio_client,
+                minio_client=datastore._client,
             )
             # put buffer in minio
-            object_name = put_image(
+            object_name = datastore.put_image(
-                client=minio_client,
                 image=image,
             )
         except Exception as exc:

shared/utils/tests/unit/check_env_test.py

@@ -39,7 +39,7 @@ class TestFunctionCheckEnv(unittest.TestCase):
         variable that is not set."""
         var_list = {self.not_set_env_var}
         msg = f'environment variable not set: {self.not_set_env_var}'
-        with self.assertRaises(AssertionError, msg=msg):
+        with self.assertRaises(OSError, msg=msg):
             check_env(var_list)
 
     def test_env_vars_set(self):

web_ui/src/app/init_app.py

@@ -8,11 +8,10 @@ import os
 import dash_bootstrap_components as dbc
 from dash import ALL, Dash, Input, Output, State
 from dash_auth import BasicAuth
-from minio import Minio
 from pydantic import ValidationError
 from pymongo.collection import Collection
 
-from shared.datastore import delete as delete_from_minio
+from shared.datastore import Datastore
 from shared.mongodb import count_documents, get_visual_communication, upsert_annotation
 from shared.mongodb.src.classes import ModelData, VisualCommunication
 from shared.mongodb.src.exceptions import NoDocumentFoundException
@@ -22,9 +21,12 @@ from .layout import app_layout
 
 def init_app(
     mongo_collection: Collection,
-    minio_client: Minio,
+    datastore: Datastore,
 ) -> Dash:
     """Initialise web UI application."""
+    assert isinstance(mongo_collection, Collection)
+    assert isinstance(datastore, Datastore)
+    assert datastore._client is not None
     # setup app
     app = Dash(
         name='visual_critical_discourse_analysis_web_ui',
@@ -129,11 +131,12 @@ def init_app(
                     failed_filename_list.append(filename)
                     continue
                 try:
+                    assert datastore._client is not None
                     # instantiate to upload image to minio
                     vis_com = VisualCommunication.from_name_and_image(
                         name=filename,
                         image=image,
-                        minio_client=minio_client,
+                        minio_client=datastore._client,
                     )
                 except Exception as exc:
                     logging.debug(exc)
@@ -146,10 +149,7 @@ def init_app(
                     logging.debug(exc)
                     failed_filename_list.append(filename)
                     # remove document from minio
-                    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
+                    datastore._delete(
-                    delete_from_minio(
-                        client=minio_client,
-                        bucket_name=bucket_name,
                         object_name=vis_com.object_name,
                     )
             assert (
@@ -240,7 +240,8 @@ def init_app(
             )
             # set variables
             vis_com_name = vis_com.name
-            image_src = vis_com.webencoded_image(minio_client=minio_client)
+            assert datastore._client is not None
+            image_src = vis_com.webencoded_image(minio_client=datastore._client)
             if vis_com.prediction is not None:
                 # TODO: update to use optional predictions
                 pass

web_ui/src/main.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import os
 
-from shared.datastore import connect_minio
+from shared.datastore import Datastore
 from shared.mongodb import connect_mongodb
 from shared.utils import check_env, setup_logging
 
@@ -34,12 +34,13 @@ setup_logging()
 collection, db, client = connect_mongodb()
 
 # connect to minio
-minio_client = connect_minio()
+datastore = Datastore()
+datastore.connect()
 
 # initialise application
 app = init_app(
     mongo_collection=collection,
-    minio_client=minio_client,
+    datastore=datastore,
 )
 server = app.server
 server.config.update(SECRET_KEY=os.urandom(24))