Merge pull request 'rename_shared_module_database' (#57) from rename_shared_module_database into main

Reviewed-on: #57

misc/generate_random_prediction.py

@@ -5,8 +5,8 @@ from pathlib import Path
 from dotenv import load_dotenv
 
 from shared.data_store import connect_minio
-from shared.database import connect_mongodb
+from shared.mongodb import connect_mongodb
-from shared.database.classes import VisualCommunication
+from shared.mongodb.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 

misc/get_visual_communication.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from dotenv import load_dotenv
 
 from shared.data_store import connect_minio
-from shared.database import connect_mongodb, get_visual_communication
+from shared.mongodb import connect_mongodb, get_visual_communication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 

misc/image_download.py

@@ -5,8 +5,8 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.database import connect_mongodb
+from shared.mongodb import connect_mongodb
-from shared.database.classes import VisualCommunication
+from shared.mongodb.classes import VisualCommunication
 
 if __name__ == '__main__':
     # prepare env vars

misc/image_upload.py

@@ -6,8 +6,8 @@ from dotenv import load_dotenv
 from pymongo.errors import DuplicateKeyError
 
 from shared.data_store import connect_minio
-from shared.database import connect_mongodb
+from shared.mongodb import connect_mongodb
-from shared.database.classes import VisualCommunication
+from shared.mongodb.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 

misc/image_upload_to_server.py

@@ -6,7 +6,8 @@ from dotenv import load_dotenv
 from pymongo.errors import DuplicateKeyError
 
 from shared.data_store import connect_minio
-from shared.database import VisualCommunication, connect_mongodb
+from shared.mongodb import connect_mongodb
+from shared.mongodb.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 

misc/model_outputs_from_annotation.py

@@ -1,6 +1,4 @@
-from __future__ import annotations
+from shared.mongodb.classes import ModelData
-
-from shared.dto import ModelData
 
 if __name__ == '__main__':
     # instantiate data object

misc/prediction_upload.py

@@ -5,8 +5,8 @@ from pathlib import Path
 from dotenv import load_dotenv
 
 from shared.data_store import connect_minio
-from shared.database import connect_mongodb, upsert_prediction
+from shared.mongodb import connect_mongodb, upsert_prediction
-from shared.database.classes import VisualCommunication
+from shared.mongodb.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 

misc/total_annotated.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.database import connect_mongodb, count_documents
+from shared.mongodb import connect_mongodb, count_documents
 
 if __name__ == '__main__':
     # prepare env vars

misc/total_documents.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.database import connect_mongodb, count_documents
+from shared.mongodb import connect_mongodb, count_documents
 
 if __name__ == '__main__':
     # prepare env vars

model/src/main.py

@@ -10,7 +10,7 @@ from tqdm import tqdm
 from utils import DEVICE, VCDADataset, load_model
 
 from shared.data_store import connect_minio
-from shared.dto import ModelData
+from shared.mongodb.classes import ModelData
 from shared.utils import setup_logging
 
 if __name__ == '__main__':

model/src/models/visual_communication.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from torch import nn
 
-from shared.dto import ModelData
+from shared.mongodb.classes import ModelData
 
 from .angle import AngleTail
 from .contact import ContactTail

other/transfer_images_mongo_minio.py

@@ -11,7 +11,8 @@ from dotenv import load_dotenv
 from pymongo.collection import Collection
 
 from shared.data_store import connect_minio, put
-from shared.database import VisualCommunication, connect_mongodb
+from shared.mongodb import connect_mongodb
+from shared.mongodb.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
 from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 

pyproject.toml

@@ -114,3 +114,7 @@ ignore_missing_imports = true
 [[tool.mypy.overrides]]
 module = "models.*"
 ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "shared.mongodb.*"
+ignore_missing_imports = true

shared/database/__init__.py

@@ -1,14 +0,0 @@
-"""Database module content."""
-
-from __future__ import annotations
-
-from .classes.dataset import Dataset
-from .classes.exceptions import NoDocumentFoundException
-from .classes.visual_communication import VisualCommunication
-from .utils.connect_mongodb import connect_mongodb
-from .utils.count_documents import count_documents
-from .utils.get_visual_communication import get_visual_communication
-from .utils.list_names import list_names
-from .utils.upsert_annotation import upsert_annotation
-from .utils.upsert_prediction import upsert_prediction
-from .utils.upsert_visual_communication import upsert_visual_communication

shared/database/classes/__init__.py

@@ -1,6 +0,0 @@
-"""Database classes module content."""
-from __future__ import annotations
-
-from .dataset import Dataset
-from .exceptions import NoDocumentFoundException
-from .visual_communication import VisualCommunication

shared/database/classes/dataset.py

@@ -1,71 +0,0 @@
-"""Definition of database Dataset class."""
-from __future__ import annotations
-
-import logging
-import random
-from datetime import datetime
-from datetime import UTC
-
-from pydantic import BaseModel
-from pydantic import Field
-from pymongo.collection import Collection
-
-
-class Dataset(BaseModel):
-    """Database Dataset model."""
-    create_time: datetime = Field(default_factory=lambda: datetime.now(UTC))
-    train_names: list[str]
-    test_names: list[str]
-    validation_names: list[str]
-
-    @classmethod
-    def fraction_map(cls) -> dict[str, float]:
-        """Dict with train, test and validation fractions."""
-        # define map
-        split_map = {
-            'train': 0.7,
-            'test': 0.2,
-            'validation': 0.1,
-        }
-        # sanity check
-        assert sum(split_map.values()) == 1.0
-        return split_map
-
-    @classmethod
-    def new_from_name_list(
-        cls,
-        name_list: list[str],
-    ) -> Dataset:
-        """Generate new dataset from list of filenames."""
-        # calculate split fractions
-        fraction_map = cls.fraction_map()
-        num_total = len(name_list)
-        num_validation = round(num_total * fraction_map['validation'])
-        num_test = round(num_total * fraction_map['test'])
-        # split data
-        validation_name_list = random.choices(name_list, k=num_validation)
-        name_list = [
-            name for name in name_list if name not in validation_name_list
-        ]
-        test_name_list = random.choices(name_list, k=num_test)
-        train_name_list = [
-            name for name in name_list if name not in test_name_list
-        ]
-        # instantiate object
-        dataset = Dataset(
-            train_names=train_name_list,
-            test_names=test_name_list,
-            validation_names=validation_name_list,
-        )
-        logging.debug('finished')
-        return dataset
-
-    def save(
-        self,
-        collection: Collection,
-    ) -> None:
-        """Save dataset to database."""
-        res = collection.insert_one(
-            document=self.model_dump(),
-        )
-        logging.debug('inserted document: %s', res)

shared/database/utils/save_dataset.py

@@ -1,38 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from pymongo.collection import Collection
-
-from shared.database import Dataset
-
-
-def save_dataset(
-    collection: Collection,
-    dataset: Dataset,
-) -> None:
-    """Save dataset to database."""
-    res = collection.insert_one(
-        document=dataset.model_dump(),
-    )
-    logging.debug('inserted document: %s', res)
-
-
-if __name__ == '__main__':
-    from dotenv import load_dotenv
-
-    load_dotenv('local.env')
-    from shared.database import connect_mongodb, list_names
-
-    # connect to database
-    collection, db, client = connect_mongodb()
-    print(client.server_info())
-
-    name_list = list_names(collection=collection, only_with_annotation=True)
-    ds = Dataset.new_from_name_list(name_list=name_list)
-
-    print(ds)
-    # save_dataset(
-    #     collection=collection,
-    #     dataset=ds
-    # )

shared/mongodb/__init__.py

@@ -0,0 +1,8 @@
+from .src import classes, exceptions
+from .src.connect_mongodb import connect_mongodb
+from .src.count_documents import count_documents
+from .src.get_visual_communication import get_visual_communication
+from .src.list_names import list_names
+from .src.upsert_annotation import upsert_annotation
+from .src.upsert_prediction import upsert_prediction
+from .src.upsert_visual_communication import upsert_visual_communication

shared/mongodb/src/classes/__init__.py

@@ -1,6 +1,3 @@
-"""Data transfer objects module content."""
-from __future__ import annotations
-
 from .angle import AngleData
 from .contact import ContactData
 from .distance import DistanceData
@@ -12,4 +9,5 @@ from .modality_lighting import ModalityLightingData
 from .model_data import ModelData
 from .point_of_view import PointOfViewData
 from .salience import SalienceData
+from .visual_communication import VisualCommunication
 from .visual_syntax import VisualSyntaxData

shared/mongodb/src/classes/angle.py

@@ -1,4 +1,5 @@
 """Definition of Angle data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel
@@ -6,6 +7,7 @@ from .data_model import DataModel
 
 class AngleData(DataModel):
     """Angle data model."""
+
     high: float
     eye_level: float
     low: float

shared/mongodb/src/classes/contact.py

@@ -1,4 +1,5 @@
 """Definition of ContactData data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel

shared/mongodb/src/classes/distance.py

@@ -1,4 +1,5 @@
 """Definition of DistanceData data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel

shared/mongodb/src/classes/framing.py

@@ -1,4 +1,5 @@
 """Definition of FramingData data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel

shared/mongodb/src/classes/information_value.py

@@ -1,4 +1,5 @@
 """Definition of InformationValueData data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel

shared/mongodb/src/classes/modality_color.py

@@ -1,4 +1,5 @@
 """Definition of ModalityColorData data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel

shared/mongodb/src/classes/modality_depth.py

@@ -1,4 +1,5 @@
 """Definition of ModalityDepthData data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel

shared/mongodb/src/classes/modality_lighting.py

@@ -1,4 +1,5 @@
 """Definition of ModalityLightingData data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel

shared/mongodb/src/classes/point_of_view.py

@@ -1,4 +1,5 @@
 """Definition of PointOfViewData data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel

shared/mongodb/src/classes/salience.py

@@ -1,4 +1,5 @@
 """Definition of SalienceData data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel

shared/mongodb/src/classes/visual_communication.py

@@ -13,7 +13,7 @@ from pydantic import BaseModel, ConfigDict
 from pymongo.collection import Collection
 
 from shared.data_store import get, put
-from shared.dto import ModelData
+from shared.mongodb.classes import ModelData
 
 
 class VisualCommunication(BaseModel):

shared/mongodb/src/classes/visual_syntax.py

@@ -1,4 +1,5 @@
 """Definition of VisualSyntaxData data model."""
+
 from __future__ import annotations
 
 from .data_model import DataModel

shared/mongodb/src/count_documents.py

@@ -1,4 +1,5 @@
 """Definition of function to count documents in database."""
+
 from __future__ import annotations
 
 from pymongo.collection import Collection
@@ -8,10 +9,8 @@ def count_documents(
     collection: Collection,
     only_with_annotation: bool = False,
 ) -> int:
-    """
+    """Get the total number of documents in database that matches the
-    Get the total number of documents
+    filters."""
-    in database that matches the filters.
-    """
     assert isinstance(collection, Collection)
     assert isinstance(only_with_annotation, bool)
     # build query

shared/mongodb/src/exceptions/__init__.py

@@ -0,0 +1 @@
+from .no_document_found import NoDocumentFoundException

shared/mongodb/src/exceptions/no_document_found.py

@@ -1,5 +1,4 @@
 """Definition of database exception."""
-from __future__ import annotations
 
 
 class NoDocumentFoundException(Exception):

shared/mongodb/src/get_visual_communication.py

@@ -1,12 +1,13 @@
 """Definition of function to get visual communication from database."""
+
 from __future__ import annotations
 
 import logging
 
 from pymongo.collection import Collection
 
-from shared.database import NoDocumentFoundException
+from shared.mongodb.classes import VisualCommunication
-from shared.database import VisualCommunication
+from shared.mongodb.exceptions import NoDocumentFoundException
 
 
 def get_visual_communication(
@@ -22,16 +23,16 @@ def get_visual_communication(
     data = collection.aggregate(
         pipeline=[
             {
-                '$match': query,     # find using filters
+                '$match': query,  # find using filters
             },
             {
                 '$sample': {
-                    'size': 1,   # get one random
+                    'size': 1,  # get one random
                 },
             },
         ],
     )
-    data_list = list(data)   # read data from cursor object
+    data_list = list(data)  # read data from cursor object
     if len(data_list) == 0:
         raise NoDocumentFoundException()
     vis_com = VisualCommunication.model_validate(data_list[0])

shared/mongodb/src/list_names.py

@@ -1,7 +1,6 @@
-"""
+"""Definition of function to list names of all visual communication documents
-Definition of function to list names
+in database."""
-of all visual communication documents in database.
+
-"""
 from __future__ import annotations
 
 from pymongo.collection import Collection

shared/mongodb/src/upsert_annotation.py

@@ -4,7 +4,7 @@ import logging
 
 from pymongo.collection import Collection
 
-from shared.dto import ModelData
+from shared.mongodb.classes import ModelData
 
 
 def upsert_annotation(

shared/mongodb/src/upsert_prediction.py

@@ -4,7 +4,7 @@ import logging
 
 from pymongo.collection import Collection
 
-from shared.dto import ModelData
+from shared.mongodb.classes import ModelData
 
 
 def upsert_prediction(

shared/mongodb/src/upsert_visual_communication.py

@@ -2,22 +2,18 @@ from __future__ import annotations
 
 from pymongo.collection import Collection
 
-from shared.database import VisualCommunication
+from shared.mongodb.classes import VisualCommunication
 
 
 def upsert_visual_communication(
     collection: Collection,
     visual_communication_list: list[VisualCommunication],
 ) -> bool:
-    """
+    """Upsert VisualCommunication object in the database.
-    Upsert VisualCommunication object in the database.
+
     Returns bool stating success.
     """
     response = collection.insert_many(
-        [
+        [vis_com.model_dump() for vis_com in visual_communication_list],
-            vis_com.model_dump()
-            for vis_com
-            in visual_communication_list
-        ],
     )
     return response.acknowledged

web_ui/src/app/init_app.py

@@ -1,28 +1,23 @@
 """Definition of init_app function."""
+
 from __future__ import annotations
 
 import logging
 import os
 
 import dash_bootstrap_components as dbc
-from dash import ALL
+from dash import ALL, Dash, Input, Output, State
-from dash import Dash
-from dash import Input
-from dash import Output
-from dash import State
 from dash_auth import BasicAuth
 from minio import Minio
 from pydantic import ValidationError
 from pymongo.collection import Collection
 
-from .layout import app_layout
 from shared.data_store import delete as delete_from_minio
-from shared.database import count_documents
+from shared.mongodb import count_documents, get_visual_communication, upsert_annotation
-from shared.database import get_visual_communication
+from shared.mongodb.classes import ModelData, VisualCommunication
-from shared.database import NoDocumentFoundException
+from shared.mongodb.exceptions import NoDocumentFoundException
-from shared.database import upsert_annotation
+
-from shared.database import VisualCommunication
+from .layout import app_layout
-from shared.dto import ModelData
 
 
 def init_app(
@@ -101,10 +96,8 @@ def init_app(
             collection=mongo_collection,
             only_with_annotation=True,
         )
-        limit = int(num_total/20)
+        limit = int(num_total / 20)
-        label_str = f"{
+        label_str = f'{num_handled}/{num_total}' if num_handled >= limit else ''
-            num_handled
-        }/{num_total}" if num_handled >= limit else ''
         return num_handled, num_total, label_str
 
     # define callback: upload image
@@ -122,9 +115,7 @@ def init_app(
         """Upload image to database through web ui."""
         try:
             # stop if no input
-            assert (
+            assert (content_list is not None) and (
-                content_list is not None
-            ) and (
                 filename_list is not None
             ), 'nothing to upload'
             # handle input
@@ -159,7 +150,9 @@ def init_app(
                         client=minio_client,
                         object_name=vis_com.object_name,
                     )
-            assert len(failed_filename_list) == 0, f"failed uploading:{
+            assert (
+                len(failed_filename_list) == 0
+            ), f"failed uploading:{
                 '\n'.join(failed_filename_list)
             }"
         except Exception as exc:
@@ -208,30 +201,19 @@ def init_app(
             logging.info('saving annotations to database: %s', vis_com_name)
             try:
                 # extract option keys
-                annotation_keys = [
+                annotation_keys = [elem['index'] for elem in annotation_keys]
-                    elem['index']
-                    for elem in annotation_keys
-                ]
                 # ensure all options are set
                 logging.info(annotation_keys)
                 for option, value in zip(annotation_keys, annotation_values):
                     if value is None:
                         raise ValueError(f"{option} is not set")
                 # prepare data to save
-                annotation_keys = [
+                annotation_keys = [elem.replace(' ', '_') for elem in annotation_keys]
-                    elem.replace(' ', '_')
-                    for elem
-                    in annotation_keys
-                ]
                 annotation_values = [
-                    elem.replace(' ', '_').lower()
+                    elem.replace(' ', '_').lower() for elem in annotation_values
-                    for elem
-                    in annotation_values
                 ]
                 annotation_map = {
-                    key: value
+                    key: value for key, value in zip(annotation_keys, annotation_values)
-                    for key, value
-                    in zip(annotation_keys, annotation_values)
                 }
                 # instantiate ModelOutputs object
                 annotations = ModelData.from_annotations(**annotation_map)
@@ -273,8 +255,10 @@ def init_app(
             response[2] = image_src
             response[3] = annotation_values
             logging.info(
-                'finished getting visual communication: %s', vis_com_name,
+                'finished getting visual communication: %s',
+                vis_com_name,
             )
         return tuple(response)
+
     logging.info('initialised app')
     return app

web_ui/src/app/layout/labels.py

@@ -1,20 +1,21 @@
 from __future__ import annotations
 
 import dash_mantine_components as dmc
-from dash import dcc
+from dash import dcc, html
-from dash import html
 
-from shared.dto import AngleData
+from shared.mongodb.classes import (
-from shared.dto import ContactData
+    AngleData,
-from shared.dto import DistanceData
+    ContactData,
-from shared.dto import FramingData
+    DistanceData,
-from shared.dto import InformationValueData
+    FramingData,
-from shared.dto import ModalityColorData
+    InformationValueData,
-from shared.dto import ModalityDepthData
+    ModalityColorData,
-from shared.dto import ModalityLightingData
+    ModalityDepthData,
-from shared.dto import PointOfViewData
+    ModalityLightingData,
-from shared.dto import SalienceData
+    PointOfViewData,
-from shared.dto import VisualSyntaxData
+    SalienceData,
+    VisualSyntaxData,
+)
 
 
 def generate_visual_syntax_options_map():
@@ -54,54 +55,63 @@ experiential_map = generate_visual_syntax_options_map()
 experiential_container = dmc.Col(
     children=[
         html.H4('experiential'.title()),
-    ], span=5,
+    ],
+    span=5,
 )
 for title, options in experiential_map.items():
     id_dict = {'type': 'annotation', 'index': title.replace('_', '-')}
     experiential_container.children.append(
-        dmc.Container([
+        dmc.Container(
-            html.B(title.title()),
+            [
-            dcc.RadioItems(
+                html.B(title.title()),
-                options=[text.replace('_', ' ') for text in options],
+                dcc.RadioItems(
-                id=id_dict,
+                    options=[text.replace('_', ' ') for text in options],
-            ),
+                    id=id_dict,
-        ]),
+                ),
+            ],
+        ),
     )
 # prepare interpersonal container
 interpersonal_map = generate_interpersonal_options_map()
 interpersonal_container = dmc.Col(
     children=[
         html.H4('interpersonal'.title()),
-    ], span=3,
+    ],
+    span=3,
 )
 for title, options in interpersonal_map.items():
     id_dict = {'type': 'annotation', 'index': title.replace('_', '-')}
     interpersonal_container.children.append(
-        dmc.Container([
+        dmc.Container(
-            html.B(title.title()),
+            [
-            dcc.RadioItems(
+                html.B(title.title()),
-                options=[text.replace('_', ' ') for text in options],
+                dcc.RadioItems(
-                id=id_dict,
+                    options=[text.replace('_', ' ') for text in options],
-            ),
+                    id=id_dict,
-        ]),
+                ),
+            ],
+        ),
     )
 # prepare textual container
 textual_map = generate_textual_options_map()
 textual_container = dmc.Col(
     children=[
         html.H4('textual'.title()),
-    ], span=4,
+    ],
+    span=4,
 )
 for title, options in textual_map.items():
     id_dict = {'type': 'annotation', 'index': title.replace('_', '-')}
     textual_container.children.append(
-        dmc.Container([
+        dmc.Container(
-            html.B(title.title()),
+            [
-            dcc.RadioItems(
+                html.B(title.title()),
-                options=[text.replace('_', ' ') for text in options],
+                dcc.RadioItems(
-                id=id_dict,
+                    options=[text.replace('_', ' ') for text in options],
-            ),
+                    id=id_dict,
-        ]),
+                ),
+            ],
+        ),
     )
 
 labels_element = dmc.Grid(

web_ui/src/main.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import os
 
 from shared.data_store import connect_minio
-from shared.database import connect_mongodb
+from shared.mongodb import connect_mongodb
 from shared.utils import check_env, setup_logging
 
 from .app import init_app