removed unused packages

misc/generate_random_prediction.py

@@ -1,42 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-from dotenv import load_dotenv
-
-from shared.datastore import Datastore
-from shared.docstore import connect_mongodb
-from shared.docstore.src.classes import VisualCommunication
-from shared.utils import check_env, setup_logging
-from web_ui.src.main import NECESSARY_ENV_VAR_LIST
-
-if __name__ == '__main__':
-    # load in env file
-    env_path = Path(__file__).parent.parent / 'server.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    # ensure env vars set
-    check_env(NECESSARY_ENV_VAR_LIST)
-    # setup logging
-    setup_logging()
-    # connect to minIO
-    datastore = Datastore()
-    datastore.connect()
-    assert datastore._client is not None
-    # connect to MongoDB
-    collection, db, client = connect_mongodb()
-    # get list of image paths
-    test_dir = Path(__file__).parent
-    img_dir = test_dir / 'imgs'
-    img_path_list = [path for path in img_dir.glob('*.jpeg') if path.is_file()]
-    print(img_path_list)
-    # instantiate data object
-    vis_com_list = [
-        VisualCommunication.from_file(path, minio_client=datastore._client)
-        for path in img_path_list
-    ]
-    # generate random predictions
-    for vis_com in vis_com_list:
-        vis_com.generate_random_prediction()
-    for vis_com in vis_com_list:
-        print(vis_com)

misc/get_visual_communication.py

@@ -1,31 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-from dotenv import load_dotenv
-
-from shared.datastore import Datastore
-from shared.docstore import connect_mongodb, get_visual_communication
-from shared.utils import check_env, setup_logging
-from web_ui.src.main import NECESSARY_ENV_VAR_LIST
-
-if __name__ == '__main__':
-    # load in env file
-    env_path = Path(__file__).parent.parent / 'server.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    # ensure env vars set
-    check_env(NECESSARY_ENV_VAR_LIST)
-    # setup logging
-    setup_logging()
-    # connect to minIO
-    datastore = Datastore()
-    datastore.connect()
-    assert datastore._client is not None
-    # connect to MongoDB
-    collection, db, client = connect_mongodb()
-    # get visual communication
-    vis_com = get_visual_communication(collection)
-    print(repr(vis_com))
-    image = vis_com.get_image(minio_client=datastore._client)
-    image.show()

misc/image_upload.py

@@ -1,48 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-from dotenv import load_dotenv
-from pymongo.errors import DuplicateKeyError
-
-from shared.datastore import Datastore
-from shared.docstore import connect_mongodb
-from shared.docstore.src.classes import VisualCommunication
-from shared.utils import check_env, setup_logging
-from web_ui.src.main import NECESSARY_ENV_VAR_LIST
-
-if __name__ == '__main__':
-    # load in env file
-    env_path = Path(__file__).parent.parent / 'server.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    # ensure env vars set
-    check_env(NECESSARY_ENV_VAR_LIST)
-    # setup logging
-    setup_logging()
-    # connect to minIO
-    datastore = Datastore()
-    datastore.connect()
-    assert datastore._client is not None
-    # connect to MongoDB
-    collection, db, client = connect_mongodb()
-    # get list of image paths
-    test_dir = Path(__file__).parent
-    img_dir = test_dir / 'imgs'
-    img_path_list = [path for path in img_dir.glob('*.jpeg') if path.is_file()]
-    print(img_path_list)
-    # instantiate data object
-    vis_com_list = [
-        VisualCommunication.from_file(path, minio_client=datastore._client)
-        for path in img_path_list
-    ]
-    for vis_com in vis_com_list:
-        print(repr(vis_com))
-    # upload images
-    for vis_com in vis_com_list:
-        try:
-            result = collection.insert_one(vis_com.model_dump())
-        except DuplicateKeyError as exc:
-            print('ignoring:\n', exc)
-        else:
-            print(f"inserted document: {result}")

misc/image_upload_to_server.py

@@ -1,53 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-from dotenv import load_dotenv
-from pymongo.errors import DuplicateKeyError
-
-from shared.datastore import Datastore
-from shared.docstore import connect_mongodb
-from shared.docstore.src.classes import VisualCommunication
-from shared.utils import check_env, setup_logging
-from web_ui.src.main import NECESSARY_ENV_VAR_LIST
-
-if __name__ == '__main__':
-    # load in env file
-    env_path = Path(__file__).parent.parent / 'server.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    # ensure env vars set
-    check_env(NECESSARY_ENV_VAR_LIST)
-    # setup logging
-    setup_logging()
-    # connect to minIO
-    datastore = Datastore()
-    datastore.connect()
-    assert datastore._client is not None
-    # connect to MongoDB
-    collection, db, client = connect_mongodb()
-    # get list of image paths
-    ext_img_dir = Path('/Volumes/BW-PSSD/Mixed Methods/')
-    assert ext_img_dir.exists()
-    img_path_list = [
-        path
-        for path in ext_img_dir.glob(
-            '*.jpg',
-        )
-        if path.is_file()
-    ]
-    print(f"found {len(img_path_list)} images")
-    # create visual communication objects
-    vis_com_list = [
-        VisualCommunication.from_file(path, minio_client=datastore._client)
-        for path in img_path_list
-    ]
-    print(f"created {len(vis_com_list)} visual communication objects")
-    # upload images
-    for vis_com in vis_com_list:
-        try:
-            result = collection.insert_one(vis_com.model_dump())
-        except DuplicateKeyError as exc:
-            print('ignoring:\n', exc)
-        else:
-            print(f"inserted document: {result}")

misc/model_io.py

@@ -1,34 +0,0 @@
-"""Test that a model can be saved and loaded again."""
-
-from pathlib import Path
-
-import torch
-from dotenv import load_dotenv
-from torchinfo import summary
-
-from model.src.models import VisualCommunicationModel
-from shared.datastore import Datastore
-from shared.utils import setup_logging
-
-DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-
-if __name__ == '__main__':
-    # load in env file
-    env_path = Path(__file__).parent.parent / 'server.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    # setup logging
-    setup_logging()
-    # connect to minio
-    datastore = Datastore()
-    datastore.connect()
-    # instantiate model
-    model = VisualCommunicationModel().to(DEVICE)
-    # show model weights
-    summary(model)
-    # put buffer in minio bucket
-    hash_str = datastore.put_model(
-        model=model,
-    )
-    print(f"hash string: {hash_str}")
-    print('saved to Minio')

misc/new_model_to_minio.py

@@ -1,32 +0,0 @@
-"""Definition of function to generate a new randomly initialized model and save
-it in Minio datastore."""
-
-from pathlib import Path
-
-from dotenv import load_dotenv
-from torchinfo import summary
-
-from model.src.models import VisualCommunicationModel
-from shared.datastore import Datastore
-from shared.utils import setup_logging
-
-if __name__ == '__main__':
-    # load in env file
-    env_path = Path(__file__).parent.parent / 'server.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    # setup logging
-    setup_logging()
-    # connect to minio
-    datastore = Datastore()
-    datastore.connect()
-    # instantiate model
-    model = VisualCommunicationModel(download_resnet_weights=True)
-    # show model weights
-    summary(model)
-    # put buffer in minio bucket
-    hash_str = datastore.put_model(
-        model=model,
-    )
-    print(f"hash string: {hash_str}")
-    print('saved to Minio')

misc/prediction_upload.py

@@ -1,48 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-from dotenv import load_dotenv
-
-from shared.datastore import Datastore
-from shared.docstore import connect_mongodb, upsert_prediction
-from shared.docstore.src.classes import VisualCommunication
-from shared.utils import check_env, setup_logging
-from web_ui.src.main import NECESSARY_ENV_VAR_LIST
-
-if __name__ == '__main__':
-    # load in env file
-    env_path = Path(__file__).parent.parent / 'server.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    # ensure env vars set
-    check_env(NECESSARY_ENV_VAR_LIST)
-    # setup logging
-    setup_logging()
-    # connect to minIO
-    datastore = Datastore()
-    datastore.connect()
-    assert datastore._client is not None
-    # connect to MongoDB
-    collection, db, client = connect_mongodb()
-    # get list of image paths
-    test_dir = Path(__file__).parent
-    img_dir = test_dir / 'imgs'
-    img_path_list = [path for path in img_dir.glob('*.jpeg') if path.is_file()]
-    # instantiate data object
-    vis_com_list = [
-        VisualCommunication.from_file(path, minio_client=datastore._client)
-        for path in img_path_list
-    ]
-    # generate random predictions
-    for vis_com in vis_com_list:
-        vis_com.generate_random_prediction()
-    # upload visual communication
-    for vis_com in vis_com_list:
-        if vis_com.prediction is None:
-            continue
-        upsert_prediction(
-            collection=collection,
-            vis_com_name=vis_com.name,
-            predictions=vis_com.prediction,
-        )

model/src/main.py

@@ -9,22 +9,16 @@ from models import VisualCommunicationModel
 from tqdm import tqdm
 from utils import DEVICE, VCDADataset, load_model
 
-from shared.datastore import Datastore
-from shared.docstore.classes import ModelData
 from shared.utils import setup_logging
 
 if __name__ == '__main__':
     # setup logging
     setup_logging()
-    # connect to minio
-    datastore = Datastore()
-    datastore.connect()
     # instantiate model
-    model: VisualCommunicationModel = load_model(client=datastore._client)
+    model: VisualCommunicationModel = load_model()
     model.eval()
     # setup dataset
     dataset = VCDADataset(
-        minio_client=datastore._client,
         data_name_list=[
             '02dbaf48d713e4e6d3a6b98fd2dc866e',
         ],
@@ -40,10 +34,10 @@ if __name__ == '__main__':
                 image = torch.unsqueeze(image, 0)  # add artificial batch dimension
                 image = image.to(DEVICE)
                 # make prediction
-                pred: ModelData = model(image)
+                pred: dict = model(image)
             except Exception:
                 print_exc()
                 continue
             else:
-                print(json.dumps(pred.model_dump(), indent=4))
+                print(json.dumps(pred, indent=4))
     logging.debug('finished')

model/src/utils/load_model.py

@@ -6,29 +6,28 @@ from pathlib import Path
 import torch
 
 from model.src.models import VisualCommunicationModel
-from shared.datastore import Datastore
+from shared.repositories import ModelRepository
 
 from .get_model_name import get_model_name
 
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
 
-def load_model(
+def load_model() -> VisualCommunicationModel:
-    datastore: Datastore,
-) -> VisualCommunicationModel:
     """Instantiate model with weights loaded from latest model saved in
     MinIO."""
-    assert isinstance(datastore, Datastore)
     # instantiate model
     model = VisualCommunicationModel()
     # get model object name
     model_name_path = Path('model_name.txt')
     model_object_name = get_model_name(path=model_name_path)
     logging.info('using model: %s', model_object_name)
-    # load model from minio
+    # load model data
-    model_checkpoint = datastore.get_model(
+    with ModelRepository() as repo:
-        object_name=model_object_name,
+        model_data = repo.get_data(model_object_name)
-    )
+    if model_data is None:
+        raise FileNotFoundError(f'model {model_object_name} not found')
+    model_checkpoint = torch.load(model_data.buffer)
     model.load_state_dict(model_checkpoint)
     # clean memory
     model_checkpoint.clear()

model/src/utils/vcda_dataset.py

@@ -15,7 +15,7 @@ from torchvision.transforms.functional import (
     to_tensor,
 )
 
-from shared.datastore import Datastore
+from shared.repositories import ImageRepository
 
 # resnet18 original normalization values
 RESNET_NORMALIZE_MEAN = [0.485, 0.456, 0.406]
@@ -27,13 +27,11 @@ class VCDADataset(Dataset):
 
     def __init__(
         self,
-        datastore: Datastore,
         data_name_list: list[str],
         do_augment: bool = False,
         random_annotations: bool = False,
     ):
         super().__init__()
-        self.datastore = datastore
         self.data_name_list = data_name_list
         self.do_augment = do_augment
         self.random_annotations = random_annotations
@@ -55,11 +53,11 @@ class VCDADataset(Dataset):
 
     def __getitem__(self, idx):
         # get image from database
-        object_name = self.data_name_list[idx]
+        image_name = self.data_name_list[idx]
-        image = self.datastore.get_image(
+        with ImageRepository() as repo:
-            object_name=object_name,
+            image_data = repo.get_data(image_name)
-        )
+        assert image_data is not None
-        tensor = self.image_to_tensor(image)
+        tensor = self.image_to_tensor(image_data.image)
         if self.do_augment:
             tensor = self.augment(tensor)
         return tensor

model/train.py

@@ -18,7 +18,6 @@ from torch.optim.lr_scheduler import ExponentialLR
 from torch.utils.data import DataLoader
 
 from model.src.utils import VCDADataset, get_class
-from shared.datastore import Datastore
 
 
 def parse_arguments():
@@ -57,19 +56,16 @@ optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
 loss_fn = nn.CrossEntropyLoss()
 
 # create datasets and loaders
-datastore = Datastore()
-datastore.connect()
 with open('model/src/dataset/train.csv', encoding='utf-8') as fh:
     train_data_name_list = fh.read().split('\n')
 train_dataset = VCDADataset(
-    datastore=datastore,
     data_name_list=train_data_name_list,
 )
 train_loader = DataLoader(dataset=train_dataset, num_workers=args.loader_workers)
 
 with open('model/src/dataset/val.csv', encoding='utf-8') as fh:
     val_data_name_list = fh.read().split('\n')
-val_dataset = VCDADataset(datastore=datastore, data_name_list=val_data_name_list)
+val_dataset = VCDADataset(data_name_list=val_data_name_list)
 val_loader = DataLoader(dataset=val_dataset, num_workers=args.loader_workers)
 
 # create trainer and evaluator
@@ -141,7 +137,7 @@ to_save = {
 }
 checkpoint_handler = Checkpoint(
     to_save,
-    f"runs/checkpoints/{run_name}",
+    f'runs/checkpoints/{run_name}',
     n_saved=3,
     filename_prefix='best',
     score_function=lambda engine: -engine.state.metrics['loss'],
@@ -155,7 +151,7 @@ if args.checkpoint:
 
 # save model config
 os.makedirs('runs/configs/', exist_ok=True)
-with open(f"runs/configs/{run_name}.json", 'w', encoding='utf-8') as fh:
+with open(f'runs/configs/{run_name}.json', 'w', encoding='utf-8') as fh:
     json.dump(config, fh)
 
 # start training

other/transfer_images_minio_subfolder.py

@@ -1,42 +0,0 @@
-"""Script to move minio images to subfolder."""
-
-import os
-from pathlib import Path
-
-from dotenv import load_dotenv
-from PIL import Image
-
-from shared.datastore import Datastore
-from shared.utils import setup_logging
-
-if __name__ == '__main__':
-    # load in env file
-    env_path = Path(__file__).parent.parent / 'server.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    # setup logging
-    setup_logging()
-    # connect to minio
-    datastore = Datastore()
-    datastore.connect()
-    assert datastore._client is not None
-    # list images in bucket
-    BUCKET_NAME = os.getenv(
-        'MINIO_BUCKET_NAME',
-        default='visual-critical-discourse-analysis',
-    )
-    obj_list = datastore._client.list_objects(
-        bucket_name=BUCKET_NAME,
-    )
-    # begin moving images
-    for obj in obj_list:
-        # get image from minio
-        buffer = datastore._get(
-            object_name=obj.object_name,
-        )
-        # convert data to image
-        image = Image.open(buffer)
-        # put image into minio subfolder
-        _ = datastore.put_image(
-            image=image,
-        )

web_ui/src/app/init_app.py

@@ -4,29 +4,28 @@ from __future__ import annotations
 
 import logging
 import os
+import random
+from base64 import b64decode, b64encode
+from io import BytesIO
 
 import dash_bootstrap_components as dbc
 from dash import ALL, Dash, Input, Output, State
 from dash_auth import BasicAuth
+from PIL import Image
 from pydantic import ValidationError
-from pymongo.collection import Collection
 
-from shared.datastore import Datastore
+from shared.repositories import (
-from shared.docstore import count_documents, get_visual_communication, upsert_annotation
+    ImageData,
-from shared.docstore.src.classes import ModelData, VisualCommunication
+    ImageRepository,
-from shared.docstore.src.exceptions import NoDocumentFoundException
+    VisualCommunicationData,
+    VisualCommunicationRepository,
+)
 
 from .layout import app_layout
 
 
-def init_app(
+def init_app() -> Dash:
-    mongo_collection: Collection,
-    datastore: Datastore,
-) -> Dash:
     """Initialise web UI application."""
-    assert isinstance(mongo_collection, Collection)
-    assert isinstance(datastore, Datastore)
-    assert datastore._client is not None
     # setup app
     app = Dash(
         name='visual_critical_discourse_analysis_web_ui',
@@ -90,14 +89,10 @@ def init_app(
         if filename_list is not None:
             assert isinstance(filename_list, list)
         # get values from database
-        num_total = count_documents(
+        with VisualCommunicationRepository() as repo:
-            collection=mongo_collection,
+            name_list = repo.list_names()
-            only_with_annotation=False,
+        num_total = len(name_list)
-        )
+        num_handled = 0
-        num_handled = count_documents(
-            collection=mongo_collection,
-            only_with_annotation=True,
-        )
         limit = int(num_total / 20)
         label_str = f'{num_handled}/{num_total}' if num_handled >= limit else ''
         return num_handled, num_total, label_str
@@ -125,38 +120,37 @@ def init_app(
             for content, filename in zip(content_list, filename_list):
                 try:
                     # decode image content
-                    image = VisualCommunication.decode_image(content=content)
+                    content_data = content.split(',')[-1]
+                    image = Image.open(BytesIO(b64decode(content_data)))
+                    # instantiate ImageData object
+                    image_data = ImageData(
+                        name=filename,
+                        image=image,
+                    )
+                    # instantiate VisualCommunicationData object
+                    vis_com_data = VisualCommunicationData(name=filename)
                 except Exception:
                     logging.debug('failed decoding %s', filename)
                     failed_filename_list.append(filename)
                     continue
                 try:
-                    assert datastore._client is not None
+                    # save ImageData
-                    # instantiate to upload image to minio
+                    with ImageRepository() as repo:
-                    vis_com = VisualCommunication.from_name_and_image(
+                        repo.put_data(image_data)
-                        name=filename,
+                    # save VisualCommunicationData
-                        image=image,
+                    with VisualCommunicationRepository() as repo:
-                        minio_client=datastore._client,
+                        repo.put_data(vis_com_data)
-                    )
                 except Exception as exc:
                     logging.debug(exc)
                     failed_filename_list.append(filename)
-                    continue
+                    # cleanup failed uploads
-                try:
+                    with ImageRepository() as repo:
-                    # save to mongodb
+                        repo.remove_data(image_data.name)
-                    vis_com.save_to_mongo(collection=mongo_collection)
+                    with VisualCommunicationRepository() as repo:
-                except Exception as exc:
+                        repo.remove_data(vis_com_data.name)
-                    logging.debug(exc)
+            if len(failed_filename_list) == 0:
-                    failed_filename_list.append(filename)
+                err_msg = 'failed uploading:' + '\n'.join(failed_filename_list)
-                    # remove document from minio
+                raise FileNotFoundError(err_msg)
-                    datastore._delete(
-                        object_name=vis_com.object_name,
-                    )
-            assert (
-                len(failed_filename_list) == 0
-            ), f"failed uploading:{
-                '\n'.join(failed_filename_list)
-            }"
         except Exception as exc:
             logging.debug(exc)
             return None, str(exc).title()
@@ -208,7 +202,7 @@ def init_app(
                 logging.info(annotation_keys)
                 for option, value in zip(annotation_keys, annotation_values):
                     if value is None:
-                        raise ValueError(f"{option} is not set")
+                        raise ValueError(f'{option} is not set')
                 # prepare data to save
                 annotation_keys = [elem.replace(' ', '_') for elem in annotation_keys]
                 annotation_values = [
@@ -217,16 +211,21 @@ def init_app(
                 annotation_map = {
                     key: value for key, value in zip(annotation_keys, annotation_values)
                 }
-                # instantiate ModelOutputs object
+                # get data
-                annotations = ModelData.from_annotations(**annotation_map)
+                with VisualCommunicationRepository() as repo:
-                # save data to database
+                    vis_com_data = repo.get_data(vis_com_name)
-                upsert_annotation(
+                    # data already present, as it was loaded earlier to get here
-                    collection=mongo_collection,
+                    assert vis_com_data is not None
-                    vis_com_name=vis_com_name,
+                    # update annotations
-                    annotations=annotations,
+                    vis_com_data_updated = vis_com_data.model_copy(
+                        update={
+                            'annotations': annotation_map,
+                        },
                     )
+                    # save updated data
+                    repo.put_data(vis_com_data_updated)
             except (ValueError, ValidationError) as exc:
-                msg = f"failed saving annotation: {exc}"
+                msg = f'failed saving annotation: {exc}'
                 logging.warning(msg)
                 response[0] = msg
                 return tuple(response)
@@ -234,21 +233,30 @@ def init_app(
         logging.info('trying to get new visual communication')
         try:
             # get data
-            vis_com = get_visual_communication(
+            with VisualCommunicationRepository() as repo:
-                collection=mongo_collection,
+                name_list = repo.list_names()
-                with_annotation=False,
+                random_name = random.choice(name_list)
-            )
+                vis_com = repo.get_data(random_name)
+            # could not get here if data doesn't exist
+            assert vis_com is not None
             # set variables
             vis_com_name = vis_com.name
-            assert datastore._client is not None
+            with ImageRepository() as repo:
-            image_src = vis_com.webencoded_image(minio_client=datastore._client)
+                image_data = repo.get_data(vis_com_name)
+            # could not get here if data doesn't exist
+            assert image_data is not None
+            buffer = BytesIO()
+            image_data.image.save(buffer, format='PNG')
+            image_src = 'data:image/png;base64,' + b64encode(buffer.getvalue()).decode(
+                'utf-8',
+            )
             if vis_com.prediction is not None:
                 # TODO: update to use optional predictions
                 pass
             else:
                 # reset annotations
                 annotation_values = [None for elem in annotation_values]
-        except NoDocumentFoundException:
+        except Exception:
             msg = 'no unannotated data in database'
             logging.warning(msg)
             response[0] = msg

web_ui/src/main.py

@@ -4,15 +4,13 @@ from __future__ import annotations
 
 import os
 
-from shared.datastore import Datastore
-from shared.docstore import connect_mongodb
 from shared.utils import check_env, setup_logging
 
 from .app import init_app
 
 # ensure env vars set
 NECESSARY_ENV_VAR_LIST = {
-    'MONGO_HOST',
+    'MONGO_ENDPOINT',
     'MONGO_DB',
     'MONGO_COLLECTION',
     'MONGO_USER',
@@ -23,24 +21,13 @@ NECESSARY_ENV_VAR_LIST = {
     'MINIO_ACCESS_KEY',
     'MINIO_SECRET_KEY',
     'MINIO_BUCKET_NAME',
-    'MINIO_BUCKET_NAME_MODELS',
 }
 check_env(NECESSARY_ENV_VAR_LIST)
 
 # setup logging stream handler
 setup_logging()
 
-# connect to database
-collection, db, client = connect_mongodb()
-
-# connect to minio
-datastore = Datastore()
-datastore.connect()
-
 # initialise application
-app = init_app(
+app = init_app()
-    mongo_collection=collection,
-    datastore=datastore,
-)
 server = app.server
 server.config.update(SECRET_KEY=os.urandom(24))