diff --git a/misc/generate_random_prediction.py b/misc/generate_random_prediction.py index aea921a..3670fe9 100644 --- a/misc/generate_random_prediction.py +++ b/misc/generate_random_prediction.py @@ -4,9 +4,9 @@ from pathlib import Path from dotenv import load_dotenv -from shared.datastore import connect_minio +from shared.datastore import Datastore from shared.mongodb import connect_mongodb -from shared.mongodb.classes import VisualCommunication +from shared.mongodb.src.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST @@ -20,7 +20,9 @@ if __name__ == '__main__': # setup logging setup_logging() # connect to minIO - minio_client = connect_minio() + datastore = Datastore() + datastore.connect() + assert datastore._client is not None # connect to MongoDB collection, db, client = connect_mongodb() # get list of image paths @@ -30,7 +32,7 @@ if __name__ == '__main__': print(img_path_list) # instantiate data object vis_com_list = [ - VisualCommunication.from_file(path, minio_client=minio_client) + VisualCommunication.from_file(path, minio_client=datastore._client) for path in img_path_list ] # generate random predictions diff --git a/misc/get_visual_communication.py b/misc/get_visual_communication.py index b8885c5..9643a6c 100644 --- a/misc/get_visual_communication.py +++ b/misc/get_visual_communication.py @@ -4,7 +4,7 @@ from pathlib import Path from dotenv import load_dotenv -from shared.datastore import connect_minio +from shared.datastore import Datastore from shared.mongodb import connect_mongodb, get_visual_communication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST @@ -19,11 +19,13 @@ if __name__ == '__main__': # setup logging setup_logging() # connect to minIO - minio_client = connect_minio() + datastore = Datastore() + datastore.connect() + assert datastore._client is not None # connect to MongoDB collection, db, client = connect_mongodb() # get visual communication vis_com = get_visual_communication(collection) print(repr(vis_com)) - image = vis_com.get_image(minio_client=minio_client) + image = vis_com.get_image(minio_client=datastore._client) image.show() diff --git a/misc/image_upload.py b/misc/image_upload.py index 2a85e40..9db6652 100644 --- a/misc/image_upload.py +++ b/misc/image_upload.py @@ -5,9 +5,9 @@ from pathlib import Path from dotenv import load_dotenv from pymongo.errors import DuplicateKeyError -from shared.datastore import connect_minio +from shared.datastore import Datastore from shared.mongodb import connect_mongodb -from shared.mongodb.classes import VisualCommunication +from shared.mongodb.src.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST @@ -21,7 +21,9 @@ if __name__ == '__main__': # setup logging setup_logging() # connect to minIO - minio_client = connect_minio() + datastore = Datastore() + datastore.connect() + assert datastore._client is not None # connect to MongoDB collection, db, client = connect_mongodb() # get list of image paths @@ -31,7 +33,7 @@ if __name__ == '__main__': print(img_path_list) # instantiate data object vis_com_list = [ - VisualCommunication.from_file(path, minio_client=minio_client) + VisualCommunication.from_file(path, minio_client=datastore._client) for path in img_path_list ] for vis_com in vis_com_list: diff --git a/misc/image_upload_to_server.py b/misc/image_upload_to_server.py index 4c8f02e..2333c27 100644 --- a/misc/image_upload_to_server.py +++ b/misc/image_upload_to_server.py @@ -5,9 +5,9 @@ from pathlib import Path from dotenv import load_dotenv from pymongo.errors import DuplicateKeyError -from shared.datastore import connect_minio +from shared.datastore import Datastore from shared.mongodb import connect_mongodb -from shared.mongodb.classes import VisualCommunication +from shared.mongodb.src.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST @@ -21,7 +21,9 @@ if __name__ == '__main__': # setup logging setup_logging() # connect to minIO - minio_client = connect_minio() + datastore = Datastore() + datastore.connect() + assert datastore._client is not None # connect to MongoDB collection, db, client = connect_mongodb() # get list of image paths @@ -37,7 +39,7 @@ if __name__ == '__main__': print(f"found {len(img_path_list)} images") # create visual communication objects vis_com_list = [ - VisualCommunication.from_file(path, minio_client=minio_client) + VisualCommunication.from_file(path, minio_client=datastore._client) for path in img_path_list ] print(f"created {len(vis_com_list)} visual communication objects") diff --git a/misc/model_io.py b/misc/model_io.py index d6d7635..b5bec6d 100644 --- a/misc/model_io.py +++ b/misc/model_io.py @@ -7,7 +7,7 @@ from dotenv import load_dotenv from torchinfo import summary from model.src.models import VisualCommunicationModel -from shared.datastore import connect_minio, put_model +from shared.datastore import Datastore from shared.utils import setup_logging DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') @@ -20,14 +20,14 @@ if __name__ == '__main__': # setup logging setup_logging() # connect to minio - client = connect_minio() + datastore = Datastore() + datastore.connect() # instantiate model model = VisualCommunicationModel().to(DEVICE) # show model weights summary(model) # put buffer in minio bucket - hash_str = put_model( - client=client, + hash_str = datastore.put_model( model=model, ) print(f"hash string: {hash_str}") diff --git a/misc/new_model_to_minio.py b/misc/new_model_to_minio.py index 2f516a0..416dd25 100644 --- a/misc/new_model_to_minio.py +++ b/misc/new_model_to_minio.py @@ -7,7 +7,7 @@ from dotenv import load_dotenv from torchinfo import summary from model.src.models import VisualCommunicationModel -from shared.datastore import connect_minio, put_model +from shared.datastore import Datastore from shared.utils import setup_logging if __name__ == '__main__': @@ -18,14 +18,14 @@ if __name__ == '__main__': # setup logging setup_logging() # connect to minio - client = connect_minio() + datastore = Datastore() + datastore.connect() # instantiate model model = VisualCommunicationModel(download_resnet_weights=True) # show model weights summary(model) # put buffer in minio bucket - hash_str = put_model( - client=client, + hash_str = datastore.put_model( model=model, ) print(f"hash string: {hash_str}") diff --git a/misc/prediction_upload.py b/misc/prediction_upload.py index 6c27211..2506fe7 100644 --- a/misc/prediction_upload.py +++ b/misc/prediction_upload.py @@ -4,9 +4,9 @@ from pathlib import Path from dotenv import load_dotenv -from shared.datastore import connect_minio +from shared.datastore import Datastore from shared.mongodb import connect_mongodb, upsert_prediction -from shared.mongodb.classes import VisualCommunication +from shared.mongodb.src.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST @@ -20,7 +20,9 @@ if __name__ == '__main__': # setup logging setup_logging() # connect to minIO - minio_client = connect_minio() + datastore = Datastore() + datastore.connect() + assert datastore._client is not None # connect to MongoDB collection, db, client = connect_mongodb() # get list of image paths @@ -29,7 +31,7 @@ if __name__ == '__main__': img_path_list = [path for path in img_dir.glob('*.jpeg') if path.is_file()] # instantiate data object vis_com_list = [ - VisualCommunication.from_file(path, minio_client=minio_client) + VisualCommunication.from_file(path, minio_client=datastore._client) for path in img_path_list ] # generate random predictions diff --git a/model/src/main.py b/model/src/main.py index 45cd7eb..91d22c2 100644 --- a/model/src/main.py +++ b/model/src/main.py @@ -9,7 +9,7 @@ from models import VisualCommunicationModel from tqdm import tqdm from utils import DEVICE, VCDADataset, load_model -from shared.datastore import connect_minio +from shared.datastore import Datastore from shared.mongodb.classes import ModelData from shared.utils import setup_logging @@ -17,13 +17,14 @@ if __name__ == '__main__': # setup logging setup_logging() # connect to minio - minio_client = connect_minio() + datastore = Datastore() + datastore.connect() # instantiate model - model: VisualCommunicationModel = load_model(client=minio_client) + model: VisualCommunicationModel = load_model(client=datastore._client) model.eval() # setup dataset dataset = VCDADataset( - minio_client=minio_client, + minio_client=datastore._client, data_name_list=[ '02dbaf48d713e4e6d3a6b98fd2dc866e', ], diff --git a/model/src/utils/load_model.py b/model/src/utils/load_model.py index 311539f..992f498 100644 --- a/model/src/utils/load_model.py +++ b/model/src/utils/load_model.py @@ -4,10 +4,9 @@ import logging from pathlib import Path import torch -from minio import Minio from model.src.models import VisualCommunicationModel -from shared.datastore import get_model +from shared.datastore import Datastore from .get_model_name import get_model_name @@ -15,11 +14,11 @@ DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') def load_model( - client: Minio, + datastore: Datastore, ) -> VisualCommunicationModel: """Instantiate model with weights loaded from latest model saved in MinIO.""" - assert isinstance(client, Minio) + assert isinstance(datastore, Datastore) # instantiate model model = VisualCommunicationModel() # get model object name @@ -27,8 +26,7 @@ def load_model( model_object_name = get_model_name(path=model_name_path) logging.info('using model: %s', model_object_name) # load model from minio - model_checkpoint = get_model( - client=client, + model_checkpoint = datastore.get_model( object_name=model_object_name, ) model.load_state_dict(model_checkpoint) diff --git a/model/src/utils/vcda_dataset.py b/model/src/utils/vcda_dataset.py index 8f97a4b..43f57e7 100644 --- a/model/src/utils/vcda_dataset.py +++ b/model/src/utils/vcda_dataset.py @@ -2,7 +2,6 @@ import random -from minio import Minio from PIL import Image from torch import Tensor from torch.utils.data import Dataset @@ -16,7 +15,7 @@ from torchvision.transforms.functional import ( to_tensor, ) -from shared.datastore import get_image +from shared.datastore import Datastore # resnet18 original normalization values RESNET_NORMALIZE_MEAN = [0.485, 0.456, 0.406] @@ -28,13 +27,13 @@ class VCDADataset(Dataset): def __init__( self, - minio_client: Minio, + datastore: Datastore, data_name_list: list[str], do_augment: bool = False, random_annotations: bool = False, ): super().__init__() - self.minio_client = minio_client + self.datastore = datastore self.data_name_list = data_name_list self.do_augment = do_augment self.random_annotations = random_annotations @@ -57,8 +56,7 @@ class VCDADataset(Dataset): def __getitem__(self, idx): # get image from database object_name = self.data_name_list[idx] - image = get_image( - client=self.minio_client, + image = self.datastore.get_image( object_name=object_name, ) tensor = self.image_to_tensor(image) diff --git a/model/train.py b/model/train.py index d75904e..0446285 100644 --- a/model/train.py +++ b/model/train.py @@ -18,7 +18,7 @@ from torch.optim.lr_scheduler import ExponentialLR from torch.utils.data import DataLoader from model.src.utils import VCDADataset, get_class -from shared.datastore import connect_minio +from shared.datastore import Datastore def parse_arguments(): @@ -57,18 +57,19 @@ optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) loss_fn = nn.CrossEntropyLoss() # create datasets and loaders -minio_client = connect_minio() +datastore = Datastore() +datastore.connect() with open('model/src/dataset/train.csv', encoding='utf-8') as fh: train_data_name_list = fh.read().split('\n') train_dataset = VCDADataset( - minio_client=minio_client, + datastore=datastore, data_name_list=train_data_name_list, ) train_loader = DataLoader(dataset=train_dataset, num_workers=args.loader_workers) with open('model/src/dataset/val.csv', encoding='utf-8') as fh: val_data_name_list = fh.read().split('\n') -val_dataset = VCDADataset(minio_client=minio_client, data_name_list=val_data_name_list) +val_dataset = VCDADataset(datastore=datastore, data_name_list=val_data_name_list) val_loader = DataLoader(dataset=val_dataset, num_workers=args.loader_workers) # create trainer and evaluator diff --git a/other/transfer_images_minio_subfolder.py b/other/transfer_images_minio_subfolder.py index cb5305e..4c960a8 100644 --- a/other/transfer_images_minio_subfolder.py +++ b/other/transfer_images_minio_subfolder.py @@ -1,11 +1,12 @@ """Script to move minio images to subfolder.""" +import os from pathlib import Path from dotenv import load_dotenv from PIL import Image -from shared.datastore import connect_minio, get, put_image +from shared.datastore import Datastore from shared.utils import setup_logging if __name__ == '__main__': @@ -16,24 +17,26 @@ if __name__ == '__main__': # setup logging setup_logging() # connect to minio - minio_client = connect_minio() + datastore = Datastore() + datastore.connect() + assert datastore._client is not None # list images in bucket - BUCKET_NAME = 'visual-critical-discourse-analysis' - obj_list = minio_client.list_objects( + BUCKET_NAME = os.getenv( + 'MINIO_BUCKET_NAME', + default='visual-critical-discourse-analysis', + ) + obj_list = datastore._client.list_objects( bucket_name=BUCKET_NAME, ) # begin moving images for obj in obj_list: # get image from minio - buffer = get( - client=minio_client, - bucket_name=BUCKET_NAME, + buffer = datastore._get( object_name=obj.object_name, ) # convert data to image image = Image.open(buffer) # put image into minio subfolder - put_image( - client=minio_client, + _ = datastore.put_image( image=image, ) diff --git a/other/transfer_images_mongo_minio.py b/other/transfer_images_mongo_minio.py index 7e8f5cc..8b8033c 100644 --- a/other/transfer_images_mongo_minio.py +++ b/other/transfer_images_mongo_minio.py @@ -9,7 +9,7 @@ from bson import ObjectId from dotenv import load_dotenv from pymongo.collection import Collection -from shared.datastore import connect_minio, put_image +from shared.datastore import Datastore from shared.mongodb import connect_mongodb from shared.mongodb.src.classes import VisualCommunication from shared.utils import check_env, setup_logging @@ -81,7 +81,9 @@ if __name__ == '__main__': # setup logging setup_logging() # connect to minIO - minio_client = connect_minio() + datastore = Datastore() + datastore.connect() + assert datastore._client is not None # connect to MongoDB collection, db, client = connect_mongodb() # list documents in mongoDB @@ -97,11 +99,10 @@ if __name__ == '__main__': try: # get image image = vis_com.get_image( - minio_client=minio_client, + minio_client=datastore._client, ) # put buffer in minio - object_name = put_image( - client=minio_client, + object_name = datastore.put_image( image=image, ) except Exception as exc: diff --git a/shared/utils/tests/unit/check_env_test.py b/shared/utils/tests/unit/check_env_test.py index 32ec322..7bc4bc3 100644 --- a/shared/utils/tests/unit/check_env_test.py +++ b/shared/utils/tests/unit/check_env_test.py @@ -39,7 +39,7 @@ class TestFunctionCheckEnv(unittest.TestCase): variable that is not set.""" var_list = {self.not_set_env_var} msg = f'environment variable not set: {self.not_set_env_var}' - with self.assertRaises(AssertionError, msg=msg): + with self.assertRaises(OSError, msg=msg): check_env(var_list) def test_env_vars_set(self): diff --git a/web_ui/src/app/init_app.py b/web_ui/src/app/init_app.py index 897aa69..e479e74 100644 --- a/web_ui/src/app/init_app.py +++ b/web_ui/src/app/init_app.py @@ -8,11 +8,10 @@ import os import dash_bootstrap_components as dbc from dash import ALL, Dash, Input, Output, State from dash_auth import BasicAuth -from minio import Minio from pydantic import ValidationError from pymongo.collection import Collection -from shared.datastore import delete as delete_from_minio +from shared.datastore import Datastore from shared.mongodb import count_documents, get_visual_communication, upsert_annotation from shared.mongodb.src.classes import ModelData, VisualCommunication from shared.mongodb.src.exceptions import NoDocumentFoundException @@ -22,9 +21,12 @@ from .layout import app_layout def init_app( mongo_collection: Collection, - minio_client: Minio, + datastore: Datastore, ) -> Dash: """Initialise web UI application.""" + assert isinstance(mongo_collection, Collection) + assert isinstance(datastore, Datastore) + assert datastore._client is not None # setup app app = Dash( name='visual_critical_discourse_analysis_web_ui', @@ -129,11 +131,12 @@ def init_app( failed_filename_list.append(filename) continue try: + assert datastore._client is not None # instantiate to upload image to minio vis_com = VisualCommunication.from_name_and_image( name=filename, image=image, - minio_client=minio_client, + minio_client=datastore._client, ) except Exception as exc: logging.debug(exc) @@ -146,10 +149,7 @@ def init_app( logging.debug(exc) failed_filename_list.append(filename) # remove document from minio - bucket_name = os.getenv('MINIO_BUCKET_NAME', default='') - delete_from_minio( - client=minio_client, - bucket_name=bucket_name, + datastore._delete( object_name=vis_com.object_name, ) assert ( @@ -240,7 +240,8 @@ def init_app( ) # set variables vis_com_name = vis_com.name - image_src = vis_com.webencoded_image(minio_client=minio_client) + assert datastore._client is not None + image_src = vis_com.webencoded_image(minio_client=datastore._client) if vis_com.prediction is not None: # TODO: update to use optional predictions pass diff --git a/web_ui/src/main.py b/web_ui/src/main.py index d5a6de7..5546b47 100644 --- a/web_ui/src/main.py +++ b/web_ui/src/main.py @@ -4,7 +4,7 @@ from __future__ import annotations import os -from shared.datastore import connect_minio +from shared.datastore import Datastore from shared.mongodb import connect_mongodb from shared.utils import check_env, setup_logging @@ -34,12 +34,13 @@ setup_logging() collection, db, client = connect_mongodb() # connect to minio -minio_client = connect_minio() +datastore = Datastore() +datastore.connect() # initialise application app = init_app( mongo_collection=collection, - minio_client=minio_client, + datastore=datastore, ) server = app.server server.config.update(SECRET_KEY=os.urandom(24))