renamed module

shared/mongodb/__init__.py

@@ -0,0 +1,14 @@
+"""Database module content."""
+
+from __future__ import annotations
+
+from .classes.dataset import Dataset
+from .classes.exceptions import NoDocumentFoundException
+from .classes.visual_communication import VisualCommunication
+from .utils.connect_mongodb import connect_mongodb
+from .utils.count_documents import count_documents
+from .utils.get_visual_communication import get_visual_communication
+from .utils.list_names import list_names
+from .utils.upsert_annotation import upsert_annotation
+from .utils.upsert_prediction import upsert_prediction
+from .utils.upsert_visual_communication import upsert_visual_communication

shared/mongodb/classes/__init__.py

@@ -0,0 +1,7 @@
+"""Database classes module content."""
+
+from __future__ import annotations
+
+from .dataset import Dataset
+from .exceptions import NoDocumentFoundException
+from .visual_communication import VisualCommunication

shared/mongodb/classes/dataset.py

@@ -0,0 +1,67 @@
+"""Definition of database Dataset class."""
+
+from __future__ import annotations
+
+import logging
+import random
+from datetime import UTC, datetime
+
+from pydantic import BaseModel, Field
+from pymongo.collection import Collection
+
+
+class Dataset(BaseModel):
+    """Database Dataset model."""
+
+    create_time: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    train_names: list[str]
+    test_names: list[str]
+    validation_names: list[str]
+
+    @classmethod
+    def fraction_map(cls) -> dict[str, float]:
+        """Dict with train, test and validation fractions."""
+        # define map
+        split_map = {
+            'train': 0.7,
+            'test': 0.2,
+            'validation': 0.1,
+        }
+        # sanity check
+        assert sum(split_map.values()) == 1.0
+        return split_map
+
+    @classmethod
+    def new_from_name_list(
+        cls,
+        name_list: list[str],
+    ) -> Dataset:
+        """Generate new dataset from list of filenames."""
+        # calculate split fractions
+        fraction_map = cls.fraction_map()
+        num_total = len(name_list)
+        num_validation = round(num_total * fraction_map['validation'])
+        num_test = round(num_total * fraction_map['test'])
+        # split data
+        validation_name_list = random.choices(name_list, k=num_validation)
+        name_list = [name for name in name_list if name not in validation_name_list]
+        test_name_list = random.choices(name_list, k=num_test)
+        train_name_list = [name for name in name_list if name not in test_name_list]
+        # instantiate object
+        dataset = Dataset(
+            train_names=train_name_list,
+            test_names=test_name_list,
+            validation_names=validation_name_list,
+        )
+        logging.debug('finished')
+        return dataset
+
+    def save(
+        self,
+        collection: Collection,
+    ) -> None:
+        """Save dataset to database."""
+        res = collection.insert_one(
+            document=self.model_dump(),
+        )
+        logging.debug('inserted document: %s', res)

shared/mongodb/classes/exceptions.py

@@ -0,0 +1,7 @@
+"""Definition of database exception."""
+
+from __future__ import annotations
+
+
+class NoDocumentFoundException(Exception):
+    """Database exception for when no documents are found."""

shared/mongodb/classes/visual_communication.py

@@ -0,0 +1,126 @@
+"""Definition of VisualCommunication model."""
+
+from __future__ import annotations
+
+import logging
+from base64 import b64decode, b64encode
+from io import BytesIO
+from pathlib import Path
+
+from minio import Minio
+from PIL import Image
+from pydantic import BaseModel, ConfigDict
+from pymongo.collection import Collection
+
+from shared.data_store import get, put
+from shared.dto import ModelData
+
+
+class VisualCommunication(BaseModel):
+    """Visual communication model."""
+
+    name: str
+    object_name: str
+    annotation: ModelData | None = None
+    prediction: ModelData | None = None
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @classmethod
+    def classname(cls) -> str:
+        """Return classname."""
+        return cls.__name__
+
+    @classmethod
+    def upload_image_to_minio(
+        cls,
+        image: Image.Image,
+        minio_client: Minio,
+    ) -> str:
+        """Upload image to MinIO and return MD5 checksum of hashed image."""
+        assert isinstance(image, Image.Image)
+        assert isinstance(minio_client, Minio)
+        buffer = BytesIO()
+        image.save(buffer, 'png')
+        object_name = put(
+            client=minio_client,
+            buffer=buffer,
+        )
+        return object_name
+
+    @classmethod
+    def from_name_and_image(
+        cls,
+        name: str,
+        image: Image.Image,
+        minio_client: Minio,
+    ) -> VisualCommunication:
+        """Instantiate from filename and image that is automatically uploaded
+        to MinIO."""
+        assert isinstance(name, str)
+        assert isinstance(image, Image.Image)
+        assert isinstance(minio_client, Minio)
+        # upload file to minio
+        object_name = VisualCommunication.upload_image_to_minio(
+            image=image,
+            minio_client=minio_client,
+        )
+        return VisualCommunication(name=name, object_name=object_name)
+
+    @classmethod
+    def from_file(cls, path: Path, minio_client: Minio) -> VisualCommunication:
+        """Instantiate from file."""
+        assert isinstance(path, Path)
+        assert isinstance(minio_client, Minio)
+        # determine name
+        name = path.stem
+        # open image
+        image = Image.open(path)
+        image.load()
+        # instantiate object
+        return VisualCommunication.from_name_and_image(
+            name=name,
+            image=image,
+            minio_client=minio_client,
+        )
+
+    @classmethod
+    def decode_image(cls, content: str) -> Image.Image:
+        """Extract image from webencoded content."""
+        _, content_data = content.split(',')
+        return Image.open(BytesIO(b64decode(content_data)))
+
+    def get_image(self, minio_client: Minio) -> Image.Image:
+        """Load image data from minio."""
+        assert isinstance(minio_client, Minio)
+        # get buffer from minio
+        buffer = get(
+            client=minio_client,
+            object_name=self.object_name,
+        )
+        # convert data to image
+        im = Image.open(buffer)
+        return im
+
+    def save_to_mongo(self, collection: Collection) -> None:
+        """Save self as document in MongoDB."""
+        res = collection.insert_one(
+            document=self.model_dump(),
+        )
+        assert res.acknowledged
+
+    def webencoded_image(self, minio_client: Minio) -> str:
+        """Convert image to be displayed on webpage."""
+        assert isinstance(minio_client, Minio)
+        # get image from minio
+        image = self.get_image(minio_client)
+        # convert images to bytes string
+        buffer = BytesIO()
+        image.save(buffer, format='png')
+        img_enc = b64encode(buffer.getvalue()).decode('utf-8')
+        return f"data:image/png;base64, {img_enc}"
+
+    def generate_random_prediction(self, force: bool = False) -> None:
+        """Generate random prediction values."""
+        if not force and self.prediction is not None:
+            logging.warning('set force=True to overwrite existing values.')
+        self.prediction = ModelData.from_random()

shared/mongodb/utils/__init__.py

@@ -0,0 +1,3 @@
+"""Database utils module content."""
+
+from .connect_mongodb import connect_mongodb

shared/mongodb/utils/connect_mongodb.py

@@ -0,0 +1,30 @@
+"""Definition of function to connect to database using environment
+variables."""
+
+import logging
+import os
+
+from dotenv import load_dotenv
+from pymongo import MongoClient
+
+
+def connect_mongodb():
+    """Connect to MongoDB using env vars."""
+    # load env vars
+    load_dotenv()
+    necessary_env_vars = [
+        'MONGO_HOST',
+        'MONGO_DB',
+        'MONGO_COLLECTION',
+    ]
+    for env_var in necessary_env_vars:
+        assert env_var in os.environ, f"{env_var} not found"
+    # connect to database
+    client = MongoClient(os.getenv('MONGO_HOST'))
+    db = client[os.getenv('MONGO_DB')]
+    # extract collection
+    collection = db[os.getenv('MONGO_COLLECTION')]
+    # set unique index on "name"
+    collection.create_index('name', unique=True)
+    logging.debug('finished')
+    return collection, db, client

shared/mongodb/utils/count_documents.py

@@ -0,0 +1,20 @@
+"""Definition of function to count documents in database."""
+
+from __future__ import annotations
+
+from pymongo.collection import Collection
+
+
+def count_documents(
+    collection: Collection,
+    only_with_annotation: bool = False,
+) -> int:
+    """Get the total number of documents in database that matches the
+    filters."""
+    assert isinstance(collection, Collection)
+    assert isinstance(only_with_annotation, bool)
+    # build query
+    query = {}
+    if only_with_annotation:
+        query['annotation'] = {'$ne': None}
+    return collection.count_documents(filter=query)

shared/mongodb/utils/get_dataset.py

@@ -0,0 +1,16 @@
+"""Definition of get_dataset function."""
+
+from typing import Literal
+
+from pymongo.collection import Collection
+
+
+def get_dataset(
+    collection: Collection,
+    type: Literal['train', 'test', 'validation'],
+) -> list[str]:
+    """Get list of data names for the corresponding type."""
+    assert isinstance(collection, Collection)
+    assert isinstance(type, str)
+    assert type in ['train', 'test', 'validation']
+    return []

shared/mongodb/utils/get_visual_communication.py

@@ -0,0 +1,39 @@
+"""Definition of function to get visual communication from database."""
+
+from __future__ import annotations
+
+import logging
+
+from pymongo.collection import Collection
+
+from shared.mongodb import NoDocumentFoundException, VisualCommunication
+
+
+def get_visual_communication(
+    collection: Collection,
+    with_annotation: bool = False,
+) -> VisualCommunication:
+    """Get a random visual communication from the database."""
+    query = {}
+    if with_annotation:
+        query['annotation'] = {'$ne': None}
+    else:
+        query['annotation'] = {'$eq': None}
+    data = collection.aggregate(
+        pipeline=[
+            {
+                '$match': query,  # find using filters
+            },
+            {
+                '$sample': {
+                    'size': 1,  # get one random
+                },
+            },
+        ],
+    )
+    data_list = list(data)  # read data from cursor object
+    if len(data_list) == 0:
+        raise NoDocumentFoundException()
+    vis_com = VisualCommunication.model_validate(data_list[0])
+    logging.debug('finished')
+    return vis_com

shared/mongodb/utils/list_names.py

@@ -0,0 +1,30 @@
+"""Definition of function to list names of all visual communication documents
+in database."""
+
+from __future__ import annotations
+
+from pymongo.collection import Collection
+
+
+def list_names(
+    collection: Collection,
+    only_with_annotation: bool = True,
+) -> list[str]:
+    """List the names of entries that match the filters."""
+    assert isinstance(collection, Collection)
+    assert isinstance(only_with_annotation, bool)
+    # build query
+    query = {}
+    if only_with_annotation:
+        query['annotation'] = {'$ne': None}
+    # execute query
+    res_list = collection.find(
+        filter=query,
+        projection={
+            '_id': False,
+            'name': True,
+        },
+    )
+    # extract information
+    name_list = [elem['name'] for elem in res_list]
+    return name_list

shared/mongodb/utils/save_dataset.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import logging
+
+from pymongo.collection import Collection
+
+from shared.mongodb import Dataset
+
+
+def save_dataset(
+    collection: Collection,
+    dataset: Dataset,
+) -> None:
+    """Save dataset to database."""
+    res = collection.insert_one(
+        document=dataset.model_dump(),
+    )
+    logging.debug('inserted document: %s', res)
+
+
+if __name__ == '__main__':
+    from dotenv import load_dotenv
+
+    load_dotenv('local.env')
+    from shared.mongodb import connect_mongodb, list_names
+
+    # connect to database
+    collection, db, client = connect_mongodb()
+    print(client.server_info())
+
+    name_list = list_names(collection=collection, only_with_annotation=True)
+    ds = Dataset.new_from_name_list(name_list=name_list)
+
+    print(ds)
+    # save_dataset(
+    #     collection=collection,
+    #     dataset=ds
+    # )

shared/mongodb/utils/upsert_annotation.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import logging
+
+from pymongo.collection import Collection
+
+from shared.dto import ModelData
+
+
+def upsert_annotation(
+    collection: Collection,
+    vis_com_name: str,
+    annotations: ModelData,
+) -> None:
+    """Upserts annotation data in the database."""
+    query = {
+        'name': vis_com_name,
+    }
+    update = {
+        '$set': {
+            'annotation': annotations.model_dump(),
+        },
+    }
+    res = collection.update_one(
+        filter=query,
+        update=update,
+        upsert=True,
+    )
+    logging.info('upserted document: %s', res)
+    logging.info('finished')

shared/mongodb/utils/upsert_prediction.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import logging
+
+from pymongo.collection import Collection
+
+from shared.dto import ModelData
+
+
+def upsert_prediction(
+    collection: Collection,
+    vis_com_name: str,
+    predictions: ModelData,
+) -> None:
+    """Upsert prediction data in the database."""
+    query = {
+        'name': vis_com_name,
+    }
+    update = {
+        '$set': {
+            'prediction': predictions.model_dump(),
+        },
+    }
+    res = collection.update_one(
+        filter=query,
+        update=update,
+        upsert=True,
+    )
+    logging.debug('upserted document: %s', res)
+    logging.info('finished')

shared/mongodb/utils/upsert_visual_communication.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from pymongo.collection import Collection
+
+from shared.mongodb import VisualCommunication
+
+
+def upsert_visual_communication(
+    collection: Collection,
+    visual_communication_list: list[VisualCommunication],
+) -> bool:
+    """Upsert VisualCommunication object in the database.
+
+    Returns bool stating success.
+    """
+    response = collection.insert_many(
+        [vis_com.model_dump() for vis_com in visual_communication_list],
+    )
+    return response.acknowledged