From e2338ad7101f20941a2d3a5202b9ca5d9292e819 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:00:13 +0000 Subject: [PATCH 01/10] renamed module --- misc/generate_random_prediction.py | 4 +- misc/get_visual_communication.py | 2 +- misc/image_download.py | 4 +- misc/image_upload.py | 4 +- misc/image_upload_to_server.py | 2 +- misc/prediction_upload.py | 4 +- misc/total_annotated.py | 2 +- misc/total_documents.py | 2 +- other/transfer_images_mongo_minio.py | 2 +- shared/{database => mongodb}/__init__.py | 0 .../{database => mongodb}/classes/__init__.py | 1 + .../{database => mongodb}/classes/dataset.py | 16 ++--- .../classes/exceptions.py | 1 + .../classes/visual_communication.py | 0 .../{database => mongodb}/utils/__init__.py | 0 .../utils/connect_mongodb.py | 0 .../utils/count_documents.py | 7 +-- .../utils/get_dataset.py | 0 .../utils/get_visual_communication.py | 10 ++-- .../{database => mongodb}/utils/list_names.py | 7 +-- .../utils/save_dataset.py | 4 +- .../utils/upsert_annotation.py | 0 .../utils/upsert_prediction.py | 0 .../utils/upsert_visual_communication.py | 12 ++-- web_ui/src/app/init_app.py | 59 ++++++++----------- web_ui/src/main.py | 2 +- 26 files changed, 63 insertions(+), 82 deletions(-) rename shared/{database => mongodb}/__init__.py (100%) rename shared/{database => mongodb}/classes/__init__.py (99%) rename shared/{database => mongodb}/classes/dataset.py (84%) rename shared/{database => mongodb}/classes/exceptions.py (99%) rename shared/{database => mongodb}/classes/visual_communication.py (100%) rename shared/{database => mongodb}/utils/__init__.py (100%) rename shared/{database => mongodb}/utils/connect_mongodb.py (100%) rename shared/{database => mongodb}/utils/count_documents.py (83%) rename shared/{database => mongodb}/utils/get_dataset.py (100%) rename shared/{database => mongodb}/utils/get_visual_communication.py (75%) rename shared/{database => mongodb}/utils/list_names.py (88%) rename shared/{database => mongodb}/utils/save_dataset.py (88%) rename shared/{database => mongodb}/utils/upsert_annotation.py (100%) rename shared/{database => mongodb}/utils/upsert_prediction.py (100%) rename shared/{database => mongodb}/utils/upsert_visual_communication.py (59%) diff --git a/misc/generate_random_prediction.py b/misc/generate_random_prediction.py index edb87f4..61376fd 100644 --- a/misc/generate_random_prediction.py +++ b/misc/generate_random_prediction.py @@ -5,8 +5,8 @@ from pathlib import Path from dotenv import load_dotenv from shared.data_store import connect_minio -from shared.database import connect_mongodb -from shared.database.classes import VisualCommunication +from shared.mongodb import connect_mongodb +from shared.mongodb.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/misc/get_visual_communication.py b/misc/get_visual_communication.py index 725390e..7b62f8d 100644 --- a/misc/get_visual_communication.py +++ b/misc/get_visual_communication.py @@ -5,7 +5,7 @@ from pathlib import Path from dotenv import load_dotenv from shared.data_store import connect_minio -from shared.database import connect_mongodb, get_visual_communication +from shared.mongodb import connect_mongodb, get_visual_communication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/misc/image_download.py b/misc/image_download.py index f84cd25..cc8f28f 100644 --- a/misc/image_download.py +++ b/misc/image_download.py @@ -5,8 +5,8 @@ from pathlib import Path from dotenv import load_dotenv -from shared.database import connect_mongodb -from shared.database.classes import VisualCommunication +from shared.mongodb import connect_mongodb +from shared.mongodb.classes import VisualCommunication if __name__ == '__main__': # prepare env vars diff --git a/misc/image_upload.py b/misc/image_upload.py index 11fa16a..7dcf67b 100644 --- a/misc/image_upload.py +++ b/misc/image_upload.py @@ -6,8 +6,8 @@ from dotenv import load_dotenv from pymongo.errors import DuplicateKeyError from shared.data_store import connect_minio -from shared.database import connect_mongodb -from shared.database.classes import VisualCommunication +from shared.mongodb import connect_mongodb +from shared.mongodb.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/misc/image_upload_to_server.py b/misc/image_upload_to_server.py index 797e6cb..6b0d57f 100644 --- a/misc/image_upload_to_server.py +++ b/misc/image_upload_to_server.py @@ -6,7 +6,7 @@ from dotenv import load_dotenv from pymongo.errors import DuplicateKeyError from shared.data_store import connect_minio -from shared.database import VisualCommunication, connect_mongodb +from shared.mongodb import VisualCommunication, connect_mongodb from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/misc/prediction_upload.py b/misc/prediction_upload.py index a6793ae..4590ede 100644 --- a/misc/prediction_upload.py +++ b/misc/prediction_upload.py @@ -5,8 +5,8 @@ from pathlib import Path from dotenv import load_dotenv from shared.data_store import connect_minio -from shared.database import connect_mongodb, upsert_prediction -from shared.database.classes import VisualCommunication +from shared.mongodb import connect_mongodb, upsert_prediction +from shared.mongodb.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/misc/total_annotated.py b/misc/total_annotated.py index 12c41ad..66f6b04 100644 --- a/misc/total_annotated.py +++ b/misc/total_annotated.py @@ -5,7 +5,7 @@ from pathlib import Path from dotenv import load_dotenv -from shared.database import connect_mongodb, count_documents +from shared.mongodb import connect_mongodb, count_documents if __name__ == '__main__': # prepare env vars diff --git a/misc/total_documents.py b/misc/total_documents.py index fffe1f4..fa8b1c8 100644 --- a/misc/total_documents.py +++ b/misc/total_documents.py @@ -5,7 +5,7 @@ from pathlib import Path from dotenv import load_dotenv -from shared.database import connect_mongodb, count_documents +from shared.mongodb import connect_mongodb, count_documents if __name__ == '__main__': # prepare env vars diff --git a/other/transfer_images_mongo_minio.py b/other/transfer_images_mongo_minio.py index aae76a1..4d4d819 100644 --- a/other/transfer_images_mongo_minio.py +++ b/other/transfer_images_mongo_minio.py @@ -11,7 +11,7 @@ from dotenv import load_dotenv from pymongo.collection import Collection from shared.data_store import connect_minio, put -from shared.database import VisualCommunication, connect_mongodb +from shared.mongodb import VisualCommunication, connect_mongodb from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/shared/database/__init__.py b/shared/mongodb/__init__.py similarity index 100% rename from shared/database/__init__.py rename to shared/mongodb/__init__.py diff --git a/shared/database/classes/__init__.py b/shared/mongodb/classes/__init__.py similarity index 99% rename from shared/database/classes/__init__.py rename to shared/mongodb/classes/__init__.py index 9e260cd..e58720b 100755 --- a/shared/database/classes/__init__.py +++ b/shared/mongodb/classes/__init__.py @@ -1,4 +1,5 @@ """Database classes module content.""" + from __future__ import annotations from .dataset import Dataset diff --git a/shared/database/classes/dataset.py b/shared/mongodb/classes/dataset.py similarity index 84% rename from shared/database/classes/dataset.py rename to shared/mongodb/classes/dataset.py index 0c3fa1a..633c533 100755 --- a/shared/database/classes/dataset.py +++ b/shared/mongodb/classes/dataset.py @@ -1,18 +1,18 @@ """Definition of database Dataset class.""" + from __future__ import annotations import logging import random -from datetime import datetime -from datetime import UTC +from datetime import UTC, datetime -from pydantic import BaseModel -from pydantic import Field +from pydantic import BaseModel, Field from pymongo.collection import Collection class Dataset(BaseModel): """Database Dataset model.""" + create_time: datetime = Field(default_factory=lambda: datetime.now(UTC)) train_names: list[str] test_names: list[str] @@ -44,13 +44,9 @@ class Dataset(BaseModel): num_test = round(num_total * fraction_map['test']) # split data validation_name_list = random.choices(name_list, k=num_validation) - name_list = [ - name for name in name_list if name not in validation_name_list - ] + name_list = [name for name in name_list if name not in validation_name_list] test_name_list = random.choices(name_list, k=num_test) - train_name_list = [ - name for name in name_list if name not in test_name_list - ] + train_name_list = [name for name in name_list if name not in test_name_list] # instantiate object dataset = Dataset( train_names=train_name_list, diff --git a/shared/database/classes/exceptions.py b/shared/mongodb/classes/exceptions.py similarity index 99% rename from shared/database/classes/exceptions.py rename to shared/mongodb/classes/exceptions.py index 5372031..1df8a78 100755 --- a/shared/database/classes/exceptions.py +++ b/shared/mongodb/classes/exceptions.py @@ -1,4 +1,5 @@ """Definition of database exception.""" + from __future__ import annotations diff --git a/shared/database/classes/visual_communication.py b/shared/mongodb/classes/visual_communication.py similarity index 100% rename from shared/database/classes/visual_communication.py rename to shared/mongodb/classes/visual_communication.py diff --git a/shared/database/utils/__init__.py b/shared/mongodb/utils/__init__.py similarity index 100% rename from shared/database/utils/__init__.py rename to shared/mongodb/utils/__init__.py diff --git a/shared/database/utils/connect_mongodb.py b/shared/mongodb/utils/connect_mongodb.py similarity index 100% rename from shared/database/utils/connect_mongodb.py rename to shared/mongodb/utils/connect_mongodb.py diff --git a/shared/database/utils/count_documents.py b/shared/mongodb/utils/count_documents.py similarity index 83% rename from shared/database/utils/count_documents.py rename to shared/mongodb/utils/count_documents.py index 53a4a1c..eb9af6b 100755 --- a/shared/database/utils/count_documents.py +++ b/shared/mongodb/utils/count_documents.py @@ -1,4 +1,5 @@ """Definition of function to count documents in database.""" + from __future__ import annotations from pymongo.collection import Collection @@ -8,10 +9,8 @@ def count_documents( collection: Collection, only_with_annotation: bool = False, ) -> int: - """ - Get the total number of documents - in database that matches the filters. - """ + """Get the total number of documents in database that matches the + filters.""" assert isinstance(collection, Collection) assert isinstance(only_with_annotation, bool) # build query diff --git a/shared/database/utils/get_dataset.py b/shared/mongodb/utils/get_dataset.py similarity index 100% rename from shared/database/utils/get_dataset.py rename to shared/mongodb/utils/get_dataset.py diff --git a/shared/database/utils/get_visual_communication.py b/shared/mongodb/utils/get_visual_communication.py similarity index 75% rename from shared/database/utils/get_visual_communication.py rename to shared/mongodb/utils/get_visual_communication.py index 739d1df..0392215 100755 --- a/shared/database/utils/get_visual_communication.py +++ b/shared/mongodb/utils/get_visual_communication.py @@ -1,12 +1,12 @@ """Definition of function to get visual communication from database.""" + from __future__ import annotations import logging from pymongo.collection import Collection -from shared.database import NoDocumentFoundException -from shared.database import VisualCommunication +from shared.mongodb import NoDocumentFoundException, VisualCommunication def get_visual_communication( @@ -22,16 +22,16 @@ def get_visual_communication( data = collection.aggregate( pipeline=[ { - '$match': query, # find using filters + '$match': query, # find using filters }, { '$sample': { - 'size': 1, # get one random + 'size': 1, # get one random }, }, ], ) - data_list = list(data) # read data from cursor object + data_list = list(data) # read data from cursor object if len(data_list) == 0: raise NoDocumentFoundException() vis_com = VisualCommunication.model_validate(data_list[0]) diff --git a/shared/database/utils/list_names.py b/shared/mongodb/utils/list_names.py similarity index 88% rename from shared/database/utils/list_names.py rename to shared/mongodb/utils/list_names.py index 6184166..86e09dc 100755 --- a/shared/database/utils/list_names.py +++ b/shared/mongodb/utils/list_names.py @@ -1,7 +1,6 @@ -""" -Definition of function to list names -of all visual communication documents in database. -""" +"""Definition of function to list names of all visual communication documents +in database.""" + from __future__ import annotations from pymongo.collection import Collection diff --git a/shared/database/utils/save_dataset.py b/shared/mongodb/utils/save_dataset.py similarity index 88% rename from shared/database/utils/save_dataset.py rename to shared/mongodb/utils/save_dataset.py index 0759152..c9bb39e 100755 --- a/shared/database/utils/save_dataset.py +++ b/shared/mongodb/utils/save_dataset.py @@ -4,7 +4,7 @@ import logging from pymongo.collection import Collection -from shared.database import Dataset +from shared.mongodb import Dataset def save_dataset( @@ -22,7 +22,7 @@ if __name__ == '__main__': from dotenv import load_dotenv load_dotenv('local.env') - from shared.database import connect_mongodb, list_names + from shared.mongodb import connect_mongodb, list_names # connect to database collection, db, client = connect_mongodb() diff --git a/shared/database/utils/upsert_annotation.py b/shared/mongodb/utils/upsert_annotation.py similarity index 100% rename from shared/database/utils/upsert_annotation.py rename to shared/mongodb/utils/upsert_annotation.py diff --git a/shared/database/utils/upsert_prediction.py b/shared/mongodb/utils/upsert_prediction.py similarity index 100% rename from shared/database/utils/upsert_prediction.py rename to shared/mongodb/utils/upsert_prediction.py diff --git a/shared/database/utils/upsert_visual_communication.py b/shared/mongodb/utils/upsert_visual_communication.py similarity index 59% rename from shared/database/utils/upsert_visual_communication.py rename to shared/mongodb/utils/upsert_visual_communication.py index 30c2e7e..b7c9462 100755 --- a/shared/database/utils/upsert_visual_communication.py +++ b/shared/mongodb/utils/upsert_visual_communication.py @@ -2,22 +2,18 @@ from __future__ import annotations from pymongo.collection import Collection -from shared.database import VisualCommunication +from shared.mongodb import VisualCommunication def upsert_visual_communication( collection: Collection, visual_communication_list: list[VisualCommunication], ) -> bool: - """ - Upsert VisualCommunication object in the database. + """Upsert VisualCommunication object in the database. + Returns bool stating success. """ response = collection.insert_many( - [ - vis_com.model_dump() - for vis_com - in visual_communication_list - ], + [vis_com.model_dump() for vis_com in visual_communication_list], ) return response.acknowledged diff --git a/web_ui/src/app/init_app.py b/web_ui/src/app/init_app.py index 481b9b7..9a6c3e5 100644 --- a/web_ui/src/app/init_app.py +++ b/web_ui/src/app/init_app.py @@ -1,28 +1,28 @@ """Definition of init_app function.""" + from __future__ import annotations import logging import os import dash_bootstrap_components as dbc -from dash import ALL -from dash import Dash -from dash import Input -from dash import Output -from dash import State +from dash import ALL, Dash, Input, Output, State from dash_auth import BasicAuth from minio import Minio from pydantic import ValidationError from pymongo.collection import Collection -from .layout import app_layout from shared.data_store import delete as delete_from_minio -from shared.database import count_documents -from shared.database import get_visual_communication -from shared.database import NoDocumentFoundException -from shared.database import upsert_annotation -from shared.database import VisualCommunication from shared.dto import ModelData +from shared.mongodb import ( + NoDocumentFoundException, + VisualCommunication, + count_documents, + get_visual_communication, + upsert_annotation, +) + +from .layout import app_layout def init_app( @@ -101,10 +101,8 @@ def init_app( collection=mongo_collection, only_with_annotation=True, ) - limit = int(num_total/20) - label_str = f"{ - num_handled - }/{num_total}" if num_handled >= limit else '' + limit = int(num_total / 20) + label_str = f'{num_handled}/{num_total}' if num_handled >= limit else '' return num_handled, num_total, label_str # define callback: upload image @@ -122,9 +120,7 @@ def init_app( """Upload image to database through web ui.""" try: # stop if no input - assert ( - content_list is not None - ) and ( + assert (content_list is not None) and ( filename_list is not None ), 'nothing to upload' # handle input @@ -159,7 +155,9 @@ def init_app( client=minio_client, object_name=vis_com.object_name, ) - assert len(failed_filename_list) == 0, f"failed uploading:{ + assert ( + len(failed_filename_list) == 0 + ), f"failed uploading:{ '\n'.join(failed_filename_list) }" except Exception as exc: @@ -208,30 +206,19 @@ def init_app( logging.info('saving annotations to database: %s', vis_com_name) try: # extract option keys - annotation_keys = [ - elem['index'] - for elem in annotation_keys - ] + annotation_keys = [elem['index'] for elem in annotation_keys] # ensure all options are set logging.info(annotation_keys) for option, value in zip(annotation_keys, annotation_values): if value is None: raise ValueError(f"{option} is not set") # prepare data to save - annotation_keys = [ - elem.replace(' ', '_') - for elem - in annotation_keys - ] + annotation_keys = [elem.replace(' ', '_') for elem in annotation_keys] annotation_values = [ - elem.replace(' ', '_').lower() - for elem - in annotation_values + elem.replace(' ', '_').lower() for elem in annotation_values ] annotation_map = { - key: value - for key, value - in zip(annotation_keys, annotation_values) + key: value for key, value in zip(annotation_keys, annotation_values) } # instantiate ModelOutputs object annotations = ModelData.from_annotations(**annotation_map) @@ -273,8 +260,10 @@ def init_app( response[2] = image_src response[3] = annotation_values logging.info( - 'finished getting visual communication: %s', vis_com_name, + 'finished getting visual communication: %s', + vis_com_name, ) return tuple(response) + logging.info('initialised app') return app diff --git a/web_ui/src/main.py b/web_ui/src/main.py index ea48325..6d6fbd5 100644 --- a/web_ui/src/main.py +++ b/web_ui/src/main.py @@ -5,7 +5,7 @@ from __future__ import annotations import os from shared.data_store import connect_minio -from shared.database import connect_mongodb +from shared.mongodb import connect_mongodb from shared.utils import check_env, setup_logging from .app import init_app From 20831c9bc329299d2b98db8c170403224aa1dd63 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:02:36 +0000 Subject: [PATCH 02/10] renamed file --- shared/mongodb/classes/{exceptions.py => no_document_found.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename shared/mongodb/classes/{exceptions.py => no_document_found.py} (100%) diff --git a/shared/mongodb/classes/exceptions.py b/shared/mongodb/classes/no_document_found.py similarity index 100% rename from shared/mongodb/classes/exceptions.py rename to shared/mongodb/classes/no_document_found.py From 6dba06704c6c77514127b51285266e2afe7de4c9 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:04:48 +0000 Subject: [PATCH 03/10] moved definition of exception to separate folder --- shared/mongodb/__init__.py | 1 - shared/mongodb/classes/__init__.py | 1 - shared/mongodb/exceptions/__init__.py | 1 + shared/mongodb/{classes => exceptions}/no_document_found.py | 2 -- shared/mongodb/utils/get_visual_communication.py | 3 ++- 5 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 shared/mongodb/exceptions/__init__.py rename shared/mongodb/{classes => exceptions}/no_document_found.py (80%) diff --git a/shared/mongodb/__init__.py b/shared/mongodb/__init__.py index ed187bd..6ffadda 100644 --- a/shared/mongodb/__init__.py +++ b/shared/mongodb/__init__.py @@ -3,7 +3,6 @@ from __future__ import annotations from .classes.dataset import Dataset -from .classes.exceptions import NoDocumentFoundException from .classes.visual_communication import VisualCommunication from .utils.connect_mongodb import connect_mongodb from .utils.count_documents import count_documents diff --git a/shared/mongodb/classes/__init__.py b/shared/mongodb/classes/__init__.py index e58720b..129df79 100755 --- a/shared/mongodb/classes/__init__.py +++ b/shared/mongodb/classes/__init__.py @@ -3,5 +3,4 @@ from __future__ import annotations from .dataset import Dataset -from .exceptions import NoDocumentFoundException from .visual_communication import VisualCommunication diff --git a/shared/mongodb/exceptions/__init__.py b/shared/mongodb/exceptions/__init__.py new file mode 100644 index 0000000..6daba63 --- /dev/null +++ b/shared/mongodb/exceptions/__init__.py @@ -0,0 +1 @@ +from .no_document_found import NoDocumentFoundException diff --git a/shared/mongodb/classes/no_document_found.py b/shared/mongodb/exceptions/no_document_found.py similarity index 80% rename from shared/mongodb/classes/no_document_found.py rename to shared/mongodb/exceptions/no_document_found.py index 1df8a78..dfe1c99 100755 --- a/shared/mongodb/classes/no_document_found.py +++ b/shared/mongodb/exceptions/no_document_found.py @@ -1,7 +1,5 @@ """Definition of database exception.""" -from __future__ import annotations - class NoDocumentFoundException(Exception): """Database exception for when no documents are found.""" diff --git a/shared/mongodb/utils/get_visual_communication.py b/shared/mongodb/utils/get_visual_communication.py index 0392215..edf9d5c 100755 --- a/shared/mongodb/utils/get_visual_communication.py +++ b/shared/mongodb/utils/get_visual_communication.py @@ -6,7 +6,8 @@ import logging from pymongo.collection import Collection -from shared.mongodb import NoDocumentFoundException, VisualCommunication +from shared.mongodb import VisualCommunication +from shared.mongodb.exceptions import NoDocumentFoundException def get_visual_communication( From ba7a7299c2ecb1b7af65b081d13850add39c34f3 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:05:47 +0000 Subject: [PATCH 04/10] removed unused class --- shared/mongodb/classes/__init__.py | 5 --- shared/mongodb/classes/dataset.py | 67 ------------------------------ 2 files changed, 72 deletions(-) delete mode 100755 shared/mongodb/classes/dataset.py diff --git a/shared/mongodb/classes/__init__.py b/shared/mongodb/classes/__init__.py index 129df79..af91d95 100755 --- a/shared/mongodb/classes/__init__.py +++ b/shared/mongodb/classes/__init__.py @@ -1,6 +1 @@ -"""Database classes module content.""" - -from __future__ import annotations - -from .dataset import Dataset from .visual_communication import VisualCommunication diff --git a/shared/mongodb/classes/dataset.py b/shared/mongodb/classes/dataset.py deleted file mode 100755 index 633c533..0000000 --- a/shared/mongodb/classes/dataset.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Definition of database Dataset class.""" - -from __future__ import annotations - -import logging -import random -from datetime import UTC, datetime - -from pydantic import BaseModel, Field -from pymongo.collection import Collection - - -class Dataset(BaseModel): - """Database Dataset model.""" - - create_time: datetime = Field(default_factory=lambda: datetime.now(UTC)) - train_names: list[str] - test_names: list[str] - validation_names: list[str] - - @classmethod - def fraction_map(cls) -> dict[str, float]: - """Dict with train, test and validation fractions.""" - # define map - split_map = { - 'train': 0.7, - 'test': 0.2, - 'validation': 0.1, - } - # sanity check - assert sum(split_map.values()) == 1.0 - return split_map - - @classmethod - def new_from_name_list( - cls, - name_list: list[str], - ) -> Dataset: - """Generate new dataset from list of filenames.""" - # calculate split fractions - fraction_map = cls.fraction_map() - num_total = len(name_list) - num_validation = round(num_total * fraction_map['validation']) - num_test = round(num_total * fraction_map['test']) - # split data - validation_name_list = random.choices(name_list, k=num_validation) - name_list = [name for name in name_list if name not in validation_name_list] - test_name_list = random.choices(name_list, k=num_test) - train_name_list = [name for name in name_list if name not in test_name_list] - # instantiate object - dataset = Dataset( - train_names=train_name_list, - test_names=test_name_list, - validation_names=validation_name_list, - ) - logging.debug('finished') - return dataset - - def save( - self, - collection: Collection, - ) -> None: - """Save dataset to database.""" - res = collection.insert_one( - document=self.model_dump(), - ) - logging.debug('inserted document: %s', res) From 4ba68d056b88b082bb99f1009515cdb699072613 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:08:37 +0000 Subject: [PATCH 05/10] moved definitions of data classes into module where used --- shared/dto/__init__.py | 15 --------------- shared/mongodb/classes/__init__.py | 12 ++++++++++++ shared/{dto => mongodb/classes}/angle.py | 2 ++ shared/{dto => mongodb/classes}/contact.py | 1 + shared/{dto => mongodb/classes}/data_model.py | 0 shared/{dto => mongodb/classes}/distance.py | 1 + shared/{dto => mongodb/classes}/framing.py | 1 + .../{dto => mongodb/classes}/information_value.py | 1 + shared/{dto => mongodb/classes}/modality_color.py | 1 + shared/{dto => mongodb/classes}/modality_depth.py | 1 + .../{dto => mongodb/classes}/modality_lighting.py | 1 + shared/{dto => mongodb/classes}/model_data.py | 0 shared/{dto => mongodb/classes}/point_of_view.py | 1 + shared/{dto => mongodb/classes}/salience.py | 1 + shared/{dto => mongodb/classes}/visual_syntax.py | 1 + 15 files changed, 24 insertions(+), 15 deletions(-) delete mode 100644 shared/dto/__init__.py rename shared/{dto => mongodb/classes}/angle.py (99%) rename shared/{dto => mongodb/classes}/contact.py (99%) rename shared/{dto => mongodb/classes}/data_model.py (100%) rename shared/{dto => mongodb/classes}/distance.py (99%) rename shared/{dto => mongodb/classes}/framing.py (99%) rename shared/{dto => mongodb/classes}/information_value.py (99%) rename shared/{dto => mongodb/classes}/modality_color.py (99%) rename shared/{dto => mongodb/classes}/modality_depth.py (99%) rename shared/{dto => mongodb/classes}/modality_lighting.py (99%) rename shared/{dto => mongodb/classes}/model_data.py (100%) rename shared/{dto => mongodb/classes}/point_of_view.py (99%) rename shared/{dto => mongodb/classes}/salience.py (99%) rename shared/{dto => mongodb/classes}/visual_syntax.py (99%) diff --git a/shared/dto/__init__.py b/shared/dto/__init__.py deleted file mode 100644 index be1cd38..0000000 --- a/shared/dto/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Data transfer objects module content.""" -from __future__ import annotations - -from .angle import AngleData -from .contact import ContactData -from .distance import DistanceData -from .framing import FramingData -from .information_value import InformationValueData -from .modality_color import ModalityColorData -from .modality_depth import ModalityDepthData -from .modality_lighting import ModalityLightingData -from .model_data import ModelData -from .point_of_view import PointOfViewData -from .salience import SalienceData -from .visual_syntax import VisualSyntaxData diff --git a/shared/mongodb/classes/__init__.py b/shared/mongodb/classes/__init__.py index af91d95..84090da 100755 --- a/shared/mongodb/classes/__init__.py +++ b/shared/mongodb/classes/__init__.py @@ -1 +1,13 @@ +from .angle import AngleData +from .contact import ContactData +from .distance import DistanceData +from .framing import FramingData +from .information_value import InformationValueData +from .modality_color import ModalityColorData +from .modality_depth import ModalityDepthData +from .modality_lighting import ModalityLightingData +from .model_data import ModelData +from .point_of_view import PointOfViewData +from .salience import SalienceData from .visual_communication import VisualCommunication +from .visual_syntax import VisualSyntaxData diff --git a/shared/dto/angle.py b/shared/mongodb/classes/angle.py similarity index 99% rename from shared/dto/angle.py rename to shared/mongodb/classes/angle.py index 5ce2ea0..12491da 100644 --- a/shared/dto/angle.py +++ b/shared/mongodb/classes/angle.py @@ -1,4 +1,5 @@ """Definition of Angle data model.""" + from __future__ import annotations from .data_model import DataModel @@ -6,6 +7,7 @@ from .data_model import DataModel class AngleData(DataModel): """Angle data model.""" + high: float eye_level: float low: float diff --git a/shared/dto/contact.py b/shared/mongodb/classes/contact.py similarity index 99% rename from shared/dto/contact.py rename to shared/mongodb/classes/contact.py index 9a5f1cd..fc08a0a 100644 --- a/shared/dto/contact.py +++ b/shared/mongodb/classes/contact.py @@ -1,4 +1,5 @@ """Definition of ContactData data model.""" + from __future__ import annotations from .data_model import DataModel diff --git a/shared/dto/data_model.py b/shared/mongodb/classes/data_model.py similarity index 100% rename from shared/dto/data_model.py rename to shared/mongodb/classes/data_model.py diff --git a/shared/dto/distance.py b/shared/mongodb/classes/distance.py similarity index 99% rename from shared/dto/distance.py rename to shared/mongodb/classes/distance.py index efd9461..0aebbe6 100644 --- a/shared/dto/distance.py +++ b/shared/mongodb/classes/distance.py @@ -1,4 +1,5 @@ """Definition of DistanceData data model.""" + from __future__ import annotations from .data_model import DataModel diff --git a/shared/dto/framing.py b/shared/mongodb/classes/framing.py similarity index 99% rename from shared/dto/framing.py rename to shared/mongodb/classes/framing.py index 2381ebc..eca22e3 100644 --- a/shared/dto/framing.py +++ b/shared/mongodb/classes/framing.py @@ -1,4 +1,5 @@ """Definition of FramingData data model.""" + from __future__ import annotations from .data_model import DataModel diff --git a/shared/dto/information_value.py b/shared/mongodb/classes/information_value.py similarity index 99% rename from shared/dto/information_value.py rename to shared/mongodb/classes/information_value.py index 45d10f3..fd4bf48 100644 --- a/shared/dto/information_value.py +++ b/shared/mongodb/classes/information_value.py @@ -1,4 +1,5 @@ """Definition of InformationValueData data model.""" + from __future__ import annotations from .data_model import DataModel diff --git a/shared/dto/modality_color.py b/shared/mongodb/classes/modality_color.py similarity index 99% rename from shared/dto/modality_color.py rename to shared/mongodb/classes/modality_color.py index a34d6be..458e568 100644 --- a/shared/dto/modality_color.py +++ b/shared/mongodb/classes/modality_color.py @@ -1,4 +1,5 @@ """Definition of ModalityColorData data model.""" + from __future__ import annotations from .data_model import DataModel diff --git a/shared/dto/modality_depth.py b/shared/mongodb/classes/modality_depth.py similarity index 99% rename from shared/dto/modality_depth.py rename to shared/mongodb/classes/modality_depth.py index 3f8fbe6..e0446f4 100644 --- a/shared/dto/modality_depth.py +++ b/shared/mongodb/classes/modality_depth.py @@ -1,4 +1,5 @@ """Definition of ModalityDepthData data model.""" + from __future__ import annotations from .data_model import DataModel diff --git a/shared/dto/modality_lighting.py b/shared/mongodb/classes/modality_lighting.py similarity index 99% rename from shared/dto/modality_lighting.py rename to shared/mongodb/classes/modality_lighting.py index fb04f4c..508ad57 100644 --- a/shared/dto/modality_lighting.py +++ b/shared/mongodb/classes/modality_lighting.py @@ -1,4 +1,5 @@ """Definition of ModalityLightingData data model.""" + from __future__ import annotations from .data_model import DataModel diff --git a/shared/dto/model_data.py b/shared/mongodb/classes/model_data.py similarity index 100% rename from shared/dto/model_data.py rename to shared/mongodb/classes/model_data.py diff --git a/shared/dto/point_of_view.py b/shared/mongodb/classes/point_of_view.py similarity index 99% rename from shared/dto/point_of_view.py rename to shared/mongodb/classes/point_of_view.py index 98d67c5..b1e4543 100644 --- a/shared/dto/point_of_view.py +++ b/shared/mongodb/classes/point_of_view.py @@ -1,4 +1,5 @@ """Definition of PointOfViewData data model.""" + from __future__ import annotations from .data_model import DataModel diff --git a/shared/dto/salience.py b/shared/mongodb/classes/salience.py similarity index 99% rename from shared/dto/salience.py rename to shared/mongodb/classes/salience.py index 580100d..186132c 100644 --- a/shared/dto/salience.py +++ b/shared/mongodb/classes/salience.py @@ -1,4 +1,5 @@ """Definition of SalienceData data model.""" + from __future__ import annotations from .data_model import DataModel diff --git a/shared/dto/visual_syntax.py b/shared/mongodb/classes/visual_syntax.py similarity index 99% rename from shared/dto/visual_syntax.py rename to shared/mongodb/classes/visual_syntax.py index fd38639..4483e7f 100644 --- a/shared/dto/visual_syntax.py +++ b/shared/mongodb/classes/visual_syntax.py @@ -1,4 +1,5 @@ """Definition of VisualSyntaxData data model.""" + from __future__ import annotations from .data_model import DataModel From 7a8eb93c48fa0c399ed623e2dc5d0ec0a139f79c Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:15:19 +0000 Subject: [PATCH 06/10] updated imports --- misc/model_outputs_from_annotation.py | 4 +- model/src/main.py | 2 +- model/src/models/visual_communication.py | 2 +- shared/mongodb/__init__.py | 6 -- .../mongodb/classes/visual_communication.py | 2 +- .../mongodb/utils/get_visual_communication.py | 2 +- shared/mongodb/utils/upsert_annotation.py | 2 +- shared/mongodb/utils/upsert_prediction.py | 2 +- .../utils/upsert_visual_communication.py | 2 +- web_ui/src/app/init_app.py | 11 +-- web_ui/src/app/layout/labels.py | 84 +++++++++++-------- 11 files changed, 58 insertions(+), 61 deletions(-) diff --git a/misc/model_outputs_from_annotation.py b/misc/model_outputs_from_annotation.py index 1e1ee35..1406b48 100644 --- a/misc/model_outputs_from_annotation.py +++ b/misc/model_outputs_from_annotation.py @@ -1,6 +1,4 @@ -from __future__ import annotations - -from shared.dto import ModelData +from shared.mongodb.classes import ModelData if __name__ == '__main__': # instantiate data object diff --git a/model/src/main.py b/model/src/main.py index b122345..40c6639 100644 --- a/model/src/main.py +++ b/model/src/main.py @@ -10,7 +10,7 @@ from tqdm import tqdm from utils import DEVICE, VCDADataset, load_model from shared.data_store import connect_minio -from shared.dto import ModelData +from shared.mongodb.classes import ModelData from shared.utils import setup_logging if __name__ == '__main__': diff --git a/model/src/models/visual_communication.py b/model/src/models/visual_communication.py index c7403bb..0a841d2 100644 --- a/model/src/models/visual_communication.py +++ b/model/src/models/visual_communication.py @@ -4,7 +4,7 @@ from __future__ import annotations from torch import nn -from shared.dto import ModelData +from shared.mongodb.classes import ModelData from .angle import AngleTail from .contact import ContactTail diff --git a/shared/mongodb/__init__.py b/shared/mongodb/__init__.py index 6ffadda..9550ad5 100644 --- a/shared/mongodb/__init__.py +++ b/shared/mongodb/__init__.py @@ -1,9 +1,3 @@ -"""Database module content.""" - -from __future__ import annotations - -from .classes.dataset import Dataset -from .classes.visual_communication import VisualCommunication from .utils.connect_mongodb import connect_mongodb from .utils.count_documents import count_documents from .utils.get_visual_communication import get_visual_communication diff --git a/shared/mongodb/classes/visual_communication.py b/shared/mongodb/classes/visual_communication.py index 4049a4a..c7ed7ef 100755 --- a/shared/mongodb/classes/visual_communication.py +++ b/shared/mongodb/classes/visual_communication.py @@ -13,7 +13,7 @@ from pydantic import BaseModel, ConfigDict from pymongo.collection import Collection from shared.data_store import get, put -from shared.dto import ModelData +from shared.mongodb.classes import ModelData class VisualCommunication(BaseModel): diff --git a/shared/mongodb/utils/get_visual_communication.py b/shared/mongodb/utils/get_visual_communication.py index edf9d5c..b9c0dbb 100755 --- a/shared/mongodb/utils/get_visual_communication.py +++ b/shared/mongodb/utils/get_visual_communication.py @@ -6,7 +6,7 @@ import logging from pymongo.collection import Collection -from shared.mongodb import VisualCommunication +from shared.mongodb.classes import VisualCommunication from shared.mongodb.exceptions import NoDocumentFoundException diff --git a/shared/mongodb/utils/upsert_annotation.py b/shared/mongodb/utils/upsert_annotation.py index ea7c8ae..9e61bcc 100755 --- a/shared/mongodb/utils/upsert_annotation.py +++ b/shared/mongodb/utils/upsert_annotation.py @@ -4,7 +4,7 @@ import logging from pymongo.collection import Collection -from shared.dto import ModelData +from shared.mongodb.classes import ModelData def upsert_annotation( diff --git a/shared/mongodb/utils/upsert_prediction.py b/shared/mongodb/utils/upsert_prediction.py index 88388d3..0f4d628 100755 --- a/shared/mongodb/utils/upsert_prediction.py +++ b/shared/mongodb/utils/upsert_prediction.py @@ -4,7 +4,7 @@ import logging from pymongo.collection import Collection -from shared.dto import ModelData +from shared.mongodb.classes import ModelData def upsert_prediction( diff --git a/shared/mongodb/utils/upsert_visual_communication.py b/shared/mongodb/utils/upsert_visual_communication.py index b7c9462..a69b9fa 100755 --- a/shared/mongodb/utils/upsert_visual_communication.py +++ b/shared/mongodb/utils/upsert_visual_communication.py @@ -2,7 +2,7 @@ from __future__ import annotations from pymongo.collection import Collection -from shared.mongodb import VisualCommunication +from shared.mongodb.classes import VisualCommunication def upsert_visual_communication( diff --git a/web_ui/src/app/init_app.py b/web_ui/src/app/init_app.py index 9a6c3e5..632d3d4 100644 --- a/web_ui/src/app/init_app.py +++ b/web_ui/src/app/init_app.py @@ -13,14 +13,9 @@ from pydantic import ValidationError from pymongo.collection import Collection from shared.data_store import delete as delete_from_minio -from shared.dto import ModelData -from shared.mongodb import ( - NoDocumentFoundException, - VisualCommunication, - count_documents, - get_visual_communication, - upsert_annotation, -) +from shared.mongodb import count_documents, get_visual_communication, upsert_annotation +from shared.mongodb.classes import ModelData, VisualCommunication +from shared.mongodb.exceptions import NoDocumentFoundException from .layout import app_layout diff --git a/web_ui/src/app/layout/labels.py b/web_ui/src/app/layout/labels.py index c87f4a0..a4ec822 100644 --- a/web_ui/src/app/layout/labels.py +++ b/web_ui/src/app/layout/labels.py @@ -1,20 +1,21 @@ from __future__ import annotations import dash_mantine_components as dmc -from dash import dcc -from dash import html +from dash import dcc, html -from shared.dto import AngleData -from shared.dto import ContactData -from shared.dto import DistanceData -from shared.dto import FramingData -from shared.dto import InformationValueData -from shared.dto import ModalityColorData -from shared.dto import ModalityDepthData -from shared.dto import ModalityLightingData -from shared.dto import PointOfViewData -from shared.dto import SalienceData -from shared.dto import VisualSyntaxData +from shared.mongodb.classes import ( + AngleData, + ContactData, + DistanceData, + FramingData, + InformationValueData, + ModalityColorData, + ModalityDepthData, + ModalityLightingData, + PointOfViewData, + SalienceData, + VisualSyntaxData, +) def generate_visual_syntax_options_map(): @@ -54,54 +55,63 @@ experiential_map = generate_visual_syntax_options_map() experiential_container = dmc.Col( children=[ html.H4('experiential'.title()), - ], span=5, + ], + span=5, ) for title, options in experiential_map.items(): id_dict = {'type': 'annotation', 'index': title.replace('_', '-')} experiential_container.children.append( - dmc.Container([ - html.B(title.title()), - dcc.RadioItems( - options=[text.replace('_', ' ') for text in options], - id=id_dict, - ), - ]), + dmc.Container( + [ + html.B(title.title()), + dcc.RadioItems( + options=[text.replace('_', ' ') for text in options], + id=id_dict, + ), + ], + ), ) # prepare interpersonal container interpersonal_map = generate_interpersonal_options_map() interpersonal_container = dmc.Col( children=[ html.H4('interpersonal'.title()), - ], span=3, + ], + span=3, ) for title, options in interpersonal_map.items(): id_dict = {'type': 'annotation', 'index': title.replace('_', '-')} interpersonal_container.children.append( - dmc.Container([ - html.B(title.title()), - dcc.RadioItems( - options=[text.replace('_', ' ') for text in options], - id=id_dict, - ), - ]), + dmc.Container( + [ + html.B(title.title()), + dcc.RadioItems( + options=[text.replace('_', ' ') for text in options], + id=id_dict, + ), + ], + ), ) # prepare textual container textual_map = generate_textual_options_map() textual_container = dmc.Col( children=[ html.H4('textual'.title()), - ], span=4, + ], + span=4, ) for title, options in textual_map.items(): id_dict = {'type': 'annotation', 'index': title.replace('_', '-')} textual_container.children.append( - dmc.Container([ - html.B(title.title()), - dcc.RadioItems( - options=[text.replace('_', ' ') for text in options], - id=id_dict, - ), - ]), + dmc.Container( + [ + html.B(title.title()), + dcc.RadioItems( + options=[text.replace('_', ' ') for text in options], + id=id_dict, + ), + ], + ), ) labels_element = dmc.Grid( From c7fd78dcf8ebcaf9b8026b22bbba249f07b454b1 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:18:14 +0000 Subject: [PATCH 07/10] removed unused code --- shared/mongodb/utils/save_dataset.py | 38 ---------------------------- 1 file changed, 38 deletions(-) delete mode 100755 shared/mongodb/utils/save_dataset.py diff --git a/shared/mongodb/utils/save_dataset.py b/shared/mongodb/utils/save_dataset.py deleted file mode 100755 index c9bb39e..0000000 --- a/shared/mongodb/utils/save_dataset.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import annotations - -import logging - -from pymongo.collection import Collection - -from shared.mongodb import Dataset - - -def save_dataset( - collection: Collection, - dataset: Dataset, -) -> None: - """Save dataset to database.""" - res = collection.insert_one( - document=dataset.model_dump(), - ) - logging.debug('inserted document: %s', res) - - -if __name__ == '__main__': - from dotenv import load_dotenv - - load_dotenv('local.env') - from shared.mongodb import connect_mongodb, list_names - - # connect to database - collection, db, client = connect_mongodb() - print(client.server_info()) - - name_list = list_names(collection=collection, only_with_annotation=True) - ds = Dataset.new_from_name_list(name_list=name_list) - - print(ds) - # save_dataset( - # collection=collection, - # dataset=ds - # ) From 6f064857768fe5ad26433bbea16c9909041a6f08 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:18:20 +0000 Subject: [PATCH 08/10] updated imports --- misc/image_upload_to_server.py | 3 ++- other/transfer_images_mongo_minio.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/misc/image_upload_to_server.py b/misc/image_upload_to_server.py index 6b0d57f..bef91af 100644 --- a/misc/image_upload_to_server.py +++ b/misc/image_upload_to_server.py @@ -6,7 +6,8 @@ from dotenv import load_dotenv from pymongo.errors import DuplicateKeyError from shared.data_store import connect_minio -from shared.mongodb import VisualCommunication, connect_mongodb +from shared.mongodb import connect_mongodb +from shared.mongodb.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/other/transfer_images_mongo_minio.py b/other/transfer_images_mongo_minio.py index 4d4d819..d308eed 100644 --- a/other/transfer_images_mongo_minio.py +++ b/other/transfer_images_mongo_minio.py @@ -11,7 +11,8 @@ from dotenv import load_dotenv from pymongo.collection import Collection from shared.data_store import connect_minio, put -from shared.mongodb import VisualCommunication, connect_mongodb +from shared.mongodb import connect_mongodb +from shared.mongodb.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST From 1a6cf693466ea43911ea15596e2c26c25b9b3b31 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:21:16 +0000 Subject: [PATCH 09/10] moved code --- shared/mongodb/__init__.py | 15 ++++++++------- shared/mongodb/{utils => src}/__init__.py | 0 shared/mongodb/{ => src}/classes/__init__.py | 0 shared/mongodb/{ => src}/classes/angle.py | 0 shared/mongodb/{ => src}/classes/contact.py | 0 shared/mongodb/{ => src}/classes/data_model.py | 0 shared/mongodb/{ => src}/classes/distance.py | 0 shared/mongodb/{ => src}/classes/framing.py | 0 .../{ => src}/classes/information_value.py | 0 .../mongodb/{ => src}/classes/modality_color.py | 0 .../mongodb/{ => src}/classes/modality_depth.py | 0 .../{ => src}/classes/modality_lighting.py | 0 shared/mongodb/{ => src}/classes/model_data.py | 0 shared/mongodb/{ => src}/classes/point_of_view.py | 0 shared/mongodb/{ => src}/classes/salience.py | 0 .../{ => src}/classes/visual_communication.py | 0 shared/mongodb/{ => src}/classes/visual_syntax.py | 0 shared/mongodb/{utils => src}/connect_mongodb.py | 0 shared/mongodb/{utils => src}/count_documents.py | 0 shared/mongodb/{ => src}/exceptions/__init__.py | 0 .../{ => src}/exceptions/no_document_found.py | 0 shared/mongodb/{utils => src}/get_dataset.py | 0 .../{utils => src}/get_visual_communication.py | 0 shared/mongodb/{utils => src}/list_names.py | 0 .../mongodb/{utils => src}/upsert_annotation.py | 0 .../mongodb/{utils => src}/upsert_prediction.py | 0 .../{utils => src}/upsert_visual_communication.py | 0 27 files changed, 8 insertions(+), 7 deletions(-) rename shared/mongodb/{utils => src}/__init__.py (100%) rename shared/mongodb/{ => src}/classes/__init__.py (100%) rename shared/mongodb/{ => src}/classes/angle.py (100%) rename shared/mongodb/{ => src}/classes/contact.py (100%) rename shared/mongodb/{ => src}/classes/data_model.py (100%) rename shared/mongodb/{ => src}/classes/distance.py (100%) rename shared/mongodb/{ => src}/classes/framing.py (100%) rename shared/mongodb/{ => src}/classes/information_value.py (100%) rename shared/mongodb/{ => src}/classes/modality_color.py (100%) rename shared/mongodb/{ => src}/classes/modality_depth.py (100%) rename shared/mongodb/{ => src}/classes/modality_lighting.py (100%) rename shared/mongodb/{ => src}/classes/model_data.py (100%) rename shared/mongodb/{ => src}/classes/point_of_view.py (100%) rename shared/mongodb/{ => src}/classes/salience.py (100%) rename shared/mongodb/{ => src}/classes/visual_communication.py (100%) rename shared/mongodb/{ => src}/classes/visual_syntax.py (100%) rename shared/mongodb/{utils => src}/connect_mongodb.py (100%) rename shared/mongodb/{utils => src}/count_documents.py (100%) rename shared/mongodb/{ => src}/exceptions/__init__.py (100%) rename shared/mongodb/{ => src}/exceptions/no_document_found.py (100%) rename shared/mongodb/{utils => src}/get_dataset.py (100%) rename shared/mongodb/{utils => src}/get_visual_communication.py (100%) rename shared/mongodb/{utils => src}/list_names.py (100%) rename shared/mongodb/{utils => src}/upsert_annotation.py (100%) rename shared/mongodb/{utils => src}/upsert_prediction.py (100%) rename shared/mongodb/{utils => src}/upsert_visual_communication.py (100%) diff --git a/shared/mongodb/__init__.py b/shared/mongodb/__init__.py index 9550ad5..6ca8e30 100644 --- a/shared/mongodb/__init__.py +++ b/shared/mongodb/__init__.py @@ -1,7 +1,8 @@ -from .utils.connect_mongodb import connect_mongodb -from .utils.count_documents import count_documents -from .utils.get_visual_communication import get_visual_communication -from .utils.list_names import list_names -from .utils.upsert_annotation import upsert_annotation -from .utils.upsert_prediction import upsert_prediction -from .utils.upsert_visual_communication import upsert_visual_communication +from .src import classes, exceptions +from .src.connect_mongodb import connect_mongodb +from .src.count_documents import count_documents +from .src.get_visual_communication import get_visual_communication +from .src.list_names import list_names +from .src.upsert_annotation import upsert_annotation +from .src.upsert_prediction import upsert_prediction +from .src.upsert_visual_communication import upsert_visual_communication diff --git a/shared/mongodb/utils/__init__.py b/shared/mongodb/src/__init__.py similarity index 100% rename from shared/mongodb/utils/__init__.py rename to shared/mongodb/src/__init__.py diff --git a/shared/mongodb/classes/__init__.py b/shared/mongodb/src/classes/__init__.py similarity index 100% rename from shared/mongodb/classes/__init__.py rename to shared/mongodb/src/classes/__init__.py diff --git a/shared/mongodb/classes/angle.py b/shared/mongodb/src/classes/angle.py similarity index 100% rename from shared/mongodb/classes/angle.py rename to shared/mongodb/src/classes/angle.py diff --git a/shared/mongodb/classes/contact.py b/shared/mongodb/src/classes/contact.py similarity index 100% rename from shared/mongodb/classes/contact.py rename to shared/mongodb/src/classes/contact.py diff --git a/shared/mongodb/classes/data_model.py b/shared/mongodb/src/classes/data_model.py similarity index 100% rename from shared/mongodb/classes/data_model.py rename to shared/mongodb/src/classes/data_model.py diff --git a/shared/mongodb/classes/distance.py b/shared/mongodb/src/classes/distance.py similarity index 100% rename from shared/mongodb/classes/distance.py rename to shared/mongodb/src/classes/distance.py diff --git a/shared/mongodb/classes/framing.py b/shared/mongodb/src/classes/framing.py similarity index 100% rename from shared/mongodb/classes/framing.py rename to shared/mongodb/src/classes/framing.py diff --git a/shared/mongodb/classes/information_value.py b/shared/mongodb/src/classes/information_value.py similarity index 100% rename from shared/mongodb/classes/information_value.py rename to shared/mongodb/src/classes/information_value.py diff --git a/shared/mongodb/classes/modality_color.py b/shared/mongodb/src/classes/modality_color.py similarity index 100% rename from shared/mongodb/classes/modality_color.py rename to shared/mongodb/src/classes/modality_color.py diff --git a/shared/mongodb/classes/modality_depth.py b/shared/mongodb/src/classes/modality_depth.py similarity index 100% rename from shared/mongodb/classes/modality_depth.py rename to shared/mongodb/src/classes/modality_depth.py diff --git a/shared/mongodb/classes/modality_lighting.py b/shared/mongodb/src/classes/modality_lighting.py similarity index 100% rename from shared/mongodb/classes/modality_lighting.py rename to shared/mongodb/src/classes/modality_lighting.py diff --git a/shared/mongodb/classes/model_data.py b/shared/mongodb/src/classes/model_data.py similarity index 100% rename from shared/mongodb/classes/model_data.py rename to shared/mongodb/src/classes/model_data.py diff --git a/shared/mongodb/classes/point_of_view.py b/shared/mongodb/src/classes/point_of_view.py similarity index 100% rename from shared/mongodb/classes/point_of_view.py rename to shared/mongodb/src/classes/point_of_view.py diff --git a/shared/mongodb/classes/salience.py b/shared/mongodb/src/classes/salience.py similarity index 100% rename from shared/mongodb/classes/salience.py rename to shared/mongodb/src/classes/salience.py diff --git a/shared/mongodb/classes/visual_communication.py b/shared/mongodb/src/classes/visual_communication.py similarity index 100% rename from shared/mongodb/classes/visual_communication.py rename to shared/mongodb/src/classes/visual_communication.py diff --git a/shared/mongodb/classes/visual_syntax.py b/shared/mongodb/src/classes/visual_syntax.py similarity index 100% rename from shared/mongodb/classes/visual_syntax.py rename to shared/mongodb/src/classes/visual_syntax.py diff --git a/shared/mongodb/utils/connect_mongodb.py b/shared/mongodb/src/connect_mongodb.py similarity index 100% rename from shared/mongodb/utils/connect_mongodb.py rename to shared/mongodb/src/connect_mongodb.py diff --git a/shared/mongodb/utils/count_documents.py b/shared/mongodb/src/count_documents.py similarity index 100% rename from shared/mongodb/utils/count_documents.py rename to shared/mongodb/src/count_documents.py diff --git a/shared/mongodb/exceptions/__init__.py b/shared/mongodb/src/exceptions/__init__.py similarity index 100% rename from shared/mongodb/exceptions/__init__.py rename to shared/mongodb/src/exceptions/__init__.py diff --git a/shared/mongodb/exceptions/no_document_found.py b/shared/mongodb/src/exceptions/no_document_found.py similarity index 100% rename from shared/mongodb/exceptions/no_document_found.py rename to shared/mongodb/src/exceptions/no_document_found.py diff --git a/shared/mongodb/utils/get_dataset.py b/shared/mongodb/src/get_dataset.py similarity index 100% rename from shared/mongodb/utils/get_dataset.py rename to shared/mongodb/src/get_dataset.py diff --git a/shared/mongodb/utils/get_visual_communication.py b/shared/mongodb/src/get_visual_communication.py similarity index 100% rename from shared/mongodb/utils/get_visual_communication.py rename to shared/mongodb/src/get_visual_communication.py diff --git a/shared/mongodb/utils/list_names.py b/shared/mongodb/src/list_names.py similarity index 100% rename from shared/mongodb/utils/list_names.py rename to shared/mongodb/src/list_names.py diff --git a/shared/mongodb/utils/upsert_annotation.py b/shared/mongodb/src/upsert_annotation.py similarity index 100% rename from shared/mongodb/utils/upsert_annotation.py rename to shared/mongodb/src/upsert_annotation.py diff --git a/shared/mongodb/utils/upsert_prediction.py b/shared/mongodb/src/upsert_prediction.py similarity index 100% rename from shared/mongodb/utils/upsert_prediction.py rename to shared/mongodb/src/upsert_prediction.py diff --git a/shared/mongodb/utils/upsert_visual_communication.py b/shared/mongodb/src/upsert_visual_communication.py similarity index 100% rename from shared/mongodb/utils/upsert_visual_communication.py rename to shared/mongodb/src/upsert_visual_communication.py From 66c3f38dba1778f2426cd22638d030b682fb52f9 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:32:14 +0000 Subject: [PATCH 10/10] added ignore for internal module --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b96fd85..9783a98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,3 +114,7 @@ ignore_missing_imports = true [[tool.mypy.overrides]] module = "models.*" ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "shared.mongodb.*" +ignore_missing_imports = true