From e2338ad7101f20941a2d3a5202b9ca5d9292e819 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 20 Oct 2024 20:00:13 +0000 Subject: [PATCH] renamed module --- misc/generate_random_prediction.py | 4 +- misc/get_visual_communication.py | 2 +- misc/image_download.py | 4 +- misc/image_upload.py | 4 +- misc/image_upload_to_server.py | 2 +- misc/prediction_upload.py | 4 +- misc/total_annotated.py | 2 +- misc/total_documents.py | 2 +- other/transfer_images_mongo_minio.py | 2 +- shared/{database => mongodb}/__init__.py | 0 .../{database => mongodb}/classes/__init__.py | 1 + .../{database => mongodb}/classes/dataset.py | 16 ++--- .../classes/exceptions.py | 1 + .../classes/visual_communication.py | 0 .../{database => mongodb}/utils/__init__.py | 0 .../utils/connect_mongodb.py | 0 .../utils/count_documents.py | 7 +-- .../utils/get_dataset.py | 0 .../utils/get_visual_communication.py | 10 ++-- .../{database => mongodb}/utils/list_names.py | 7 +-- .../utils/save_dataset.py | 4 +- .../utils/upsert_annotation.py | 0 .../utils/upsert_prediction.py | 0 .../utils/upsert_visual_communication.py | 12 ++-- web_ui/src/app/init_app.py | 59 ++++++++----------- web_ui/src/main.py | 2 +- 26 files changed, 63 insertions(+), 82 deletions(-) rename shared/{database => mongodb}/__init__.py (100%) rename shared/{database => mongodb}/classes/__init__.py (99%) rename shared/{database => mongodb}/classes/dataset.py (84%) rename shared/{database => mongodb}/classes/exceptions.py (99%) rename shared/{database => mongodb}/classes/visual_communication.py (100%) rename shared/{database => mongodb}/utils/__init__.py (100%) rename shared/{database => mongodb}/utils/connect_mongodb.py (100%) rename shared/{database => mongodb}/utils/count_documents.py (83%) rename shared/{database => mongodb}/utils/get_dataset.py (100%) rename shared/{database => mongodb}/utils/get_visual_communication.py (75%) rename shared/{database => mongodb}/utils/list_names.py (88%) rename shared/{database => mongodb}/utils/save_dataset.py (88%) rename shared/{database => mongodb}/utils/upsert_annotation.py (100%) rename shared/{database => mongodb}/utils/upsert_prediction.py (100%) rename shared/{database => mongodb}/utils/upsert_visual_communication.py (59%) diff --git a/misc/generate_random_prediction.py b/misc/generate_random_prediction.py index edb87f4..61376fd 100644 --- a/misc/generate_random_prediction.py +++ b/misc/generate_random_prediction.py @@ -5,8 +5,8 @@ from pathlib import Path from dotenv import load_dotenv from shared.data_store import connect_minio -from shared.database import connect_mongodb -from shared.database.classes import VisualCommunication +from shared.mongodb import connect_mongodb +from shared.mongodb.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/misc/get_visual_communication.py b/misc/get_visual_communication.py index 725390e..7b62f8d 100644 --- a/misc/get_visual_communication.py +++ b/misc/get_visual_communication.py @@ -5,7 +5,7 @@ from pathlib import Path from dotenv import load_dotenv from shared.data_store import connect_minio -from shared.database import connect_mongodb, get_visual_communication +from shared.mongodb import connect_mongodb, get_visual_communication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/misc/image_download.py b/misc/image_download.py index f84cd25..cc8f28f 100644 --- a/misc/image_download.py +++ b/misc/image_download.py @@ -5,8 +5,8 @@ from pathlib import Path from dotenv import load_dotenv -from shared.database import connect_mongodb -from shared.database.classes import VisualCommunication +from shared.mongodb import connect_mongodb +from shared.mongodb.classes import VisualCommunication if __name__ == '__main__': # prepare env vars diff --git a/misc/image_upload.py b/misc/image_upload.py index 11fa16a..7dcf67b 100644 --- a/misc/image_upload.py +++ b/misc/image_upload.py @@ -6,8 +6,8 @@ from dotenv import load_dotenv from pymongo.errors import DuplicateKeyError from shared.data_store import connect_minio -from shared.database import connect_mongodb -from shared.database.classes import VisualCommunication +from shared.mongodb import connect_mongodb +from shared.mongodb.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/misc/image_upload_to_server.py b/misc/image_upload_to_server.py index 797e6cb..6b0d57f 100644 --- a/misc/image_upload_to_server.py +++ b/misc/image_upload_to_server.py @@ -6,7 +6,7 @@ from dotenv import load_dotenv from pymongo.errors import DuplicateKeyError from shared.data_store import connect_minio -from shared.database import VisualCommunication, connect_mongodb +from shared.mongodb import VisualCommunication, connect_mongodb from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/misc/prediction_upload.py b/misc/prediction_upload.py index a6793ae..4590ede 100644 --- a/misc/prediction_upload.py +++ b/misc/prediction_upload.py @@ -5,8 +5,8 @@ from pathlib import Path from dotenv import load_dotenv from shared.data_store import connect_minio -from shared.database import connect_mongodb, upsert_prediction -from shared.database.classes import VisualCommunication +from shared.mongodb import connect_mongodb, upsert_prediction +from shared.mongodb.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/misc/total_annotated.py b/misc/total_annotated.py index 12c41ad..66f6b04 100644 --- a/misc/total_annotated.py +++ b/misc/total_annotated.py @@ -5,7 +5,7 @@ from pathlib import Path from dotenv import load_dotenv -from shared.database import connect_mongodb, count_documents +from shared.mongodb import connect_mongodb, count_documents if __name__ == '__main__': # prepare env vars diff --git a/misc/total_documents.py b/misc/total_documents.py index fffe1f4..fa8b1c8 100644 --- a/misc/total_documents.py +++ b/misc/total_documents.py @@ -5,7 +5,7 @@ from pathlib import Path from dotenv import load_dotenv -from shared.database import connect_mongodb, count_documents +from shared.mongodb import connect_mongodb, count_documents if __name__ == '__main__': # prepare env vars diff --git a/other/transfer_images_mongo_minio.py b/other/transfer_images_mongo_minio.py index aae76a1..4d4d819 100644 --- a/other/transfer_images_mongo_minio.py +++ b/other/transfer_images_mongo_minio.py @@ -11,7 +11,7 @@ from dotenv import load_dotenv from pymongo.collection import Collection from shared.data_store import connect_minio, put -from shared.database import VisualCommunication, connect_mongodb +from shared.mongodb import VisualCommunication, connect_mongodb from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST diff --git a/shared/database/__init__.py b/shared/mongodb/__init__.py similarity index 100% rename from shared/database/__init__.py rename to shared/mongodb/__init__.py diff --git a/shared/database/classes/__init__.py b/shared/mongodb/classes/__init__.py similarity index 99% rename from shared/database/classes/__init__.py rename to shared/mongodb/classes/__init__.py index 9e260cd..e58720b 100755 --- a/shared/database/classes/__init__.py +++ b/shared/mongodb/classes/__init__.py @@ -1,4 +1,5 @@ """Database classes module content.""" + from __future__ import annotations from .dataset import Dataset diff --git a/shared/database/classes/dataset.py b/shared/mongodb/classes/dataset.py similarity index 84% rename from shared/database/classes/dataset.py rename to shared/mongodb/classes/dataset.py index 0c3fa1a..633c533 100755 --- a/shared/database/classes/dataset.py +++ b/shared/mongodb/classes/dataset.py @@ -1,18 +1,18 @@ """Definition of database Dataset class.""" + from __future__ import annotations import logging import random -from datetime import datetime -from datetime import UTC +from datetime import UTC, datetime -from pydantic import BaseModel -from pydantic import Field +from pydantic import BaseModel, Field from pymongo.collection import Collection class Dataset(BaseModel): """Database Dataset model.""" + create_time: datetime = Field(default_factory=lambda: datetime.now(UTC)) train_names: list[str] test_names: list[str] @@ -44,13 +44,9 @@ class Dataset(BaseModel): num_test = round(num_total * fraction_map['test']) # split data validation_name_list = random.choices(name_list, k=num_validation) - name_list = [ - name for name in name_list if name not in validation_name_list - ] + name_list = [name for name in name_list if name not in validation_name_list] test_name_list = random.choices(name_list, k=num_test) - train_name_list = [ - name for name in name_list if name not in test_name_list - ] + train_name_list = [name for name in name_list if name not in test_name_list] # instantiate object dataset = Dataset( train_names=train_name_list, diff --git a/shared/database/classes/exceptions.py b/shared/mongodb/classes/exceptions.py similarity index 99% rename from shared/database/classes/exceptions.py rename to shared/mongodb/classes/exceptions.py index 5372031..1df8a78 100755 --- a/shared/database/classes/exceptions.py +++ b/shared/mongodb/classes/exceptions.py @@ -1,4 +1,5 @@ """Definition of database exception.""" + from __future__ import annotations diff --git a/shared/database/classes/visual_communication.py b/shared/mongodb/classes/visual_communication.py similarity index 100% rename from shared/database/classes/visual_communication.py rename to shared/mongodb/classes/visual_communication.py diff --git a/shared/database/utils/__init__.py b/shared/mongodb/utils/__init__.py similarity index 100% rename from shared/database/utils/__init__.py rename to shared/mongodb/utils/__init__.py diff --git a/shared/database/utils/connect_mongodb.py b/shared/mongodb/utils/connect_mongodb.py similarity index 100% rename from shared/database/utils/connect_mongodb.py rename to shared/mongodb/utils/connect_mongodb.py diff --git a/shared/database/utils/count_documents.py b/shared/mongodb/utils/count_documents.py similarity index 83% rename from shared/database/utils/count_documents.py rename to shared/mongodb/utils/count_documents.py index 53a4a1c..eb9af6b 100755 --- a/shared/database/utils/count_documents.py +++ b/shared/mongodb/utils/count_documents.py @@ -1,4 +1,5 @@ """Definition of function to count documents in database.""" + from __future__ import annotations from pymongo.collection import Collection @@ -8,10 +9,8 @@ def count_documents( collection: Collection, only_with_annotation: bool = False, ) -> int: - """ - Get the total number of documents - in database that matches the filters. - """ + """Get the total number of documents in database that matches the + filters.""" assert isinstance(collection, Collection) assert isinstance(only_with_annotation, bool) # build query diff --git a/shared/database/utils/get_dataset.py b/shared/mongodb/utils/get_dataset.py similarity index 100% rename from shared/database/utils/get_dataset.py rename to shared/mongodb/utils/get_dataset.py diff --git a/shared/database/utils/get_visual_communication.py b/shared/mongodb/utils/get_visual_communication.py similarity index 75% rename from shared/database/utils/get_visual_communication.py rename to shared/mongodb/utils/get_visual_communication.py index 739d1df..0392215 100755 --- a/shared/database/utils/get_visual_communication.py +++ b/shared/mongodb/utils/get_visual_communication.py @@ -1,12 +1,12 @@ """Definition of function to get visual communication from database.""" + from __future__ import annotations import logging from pymongo.collection import Collection -from shared.database import NoDocumentFoundException -from shared.database import VisualCommunication +from shared.mongodb import NoDocumentFoundException, VisualCommunication def get_visual_communication( @@ -22,16 +22,16 @@ def get_visual_communication( data = collection.aggregate( pipeline=[ { - '$match': query, # find using filters + '$match': query, # find using filters }, { '$sample': { - 'size': 1, # get one random + 'size': 1, # get one random }, }, ], ) - data_list = list(data) # read data from cursor object + data_list = list(data) # read data from cursor object if len(data_list) == 0: raise NoDocumentFoundException() vis_com = VisualCommunication.model_validate(data_list[0]) diff --git a/shared/database/utils/list_names.py b/shared/mongodb/utils/list_names.py similarity index 88% rename from shared/database/utils/list_names.py rename to shared/mongodb/utils/list_names.py index 6184166..86e09dc 100755 --- a/shared/database/utils/list_names.py +++ b/shared/mongodb/utils/list_names.py @@ -1,7 +1,6 @@ -""" -Definition of function to list names -of all visual communication documents in database. -""" +"""Definition of function to list names of all visual communication documents +in database.""" + from __future__ import annotations from pymongo.collection import Collection diff --git a/shared/database/utils/save_dataset.py b/shared/mongodb/utils/save_dataset.py similarity index 88% rename from shared/database/utils/save_dataset.py rename to shared/mongodb/utils/save_dataset.py index 0759152..c9bb39e 100755 --- a/shared/database/utils/save_dataset.py +++ b/shared/mongodb/utils/save_dataset.py @@ -4,7 +4,7 @@ import logging from pymongo.collection import Collection -from shared.database import Dataset +from shared.mongodb import Dataset def save_dataset( @@ -22,7 +22,7 @@ if __name__ == '__main__': from dotenv import load_dotenv load_dotenv('local.env') - from shared.database import connect_mongodb, list_names + from shared.mongodb import connect_mongodb, list_names # connect to database collection, db, client = connect_mongodb() diff --git a/shared/database/utils/upsert_annotation.py b/shared/mongodb/utils/upsert_annotation.py similarity index 100% rename from shared/database/utils/upsert_annotation.py rename to shared/mongodb/utils/upsert_annotation.py diff --git a/shared/database/utils/upsert_prediction.py b/shared/mongodb/utils/upsert_prediction.py similarity index 100% rename from shared/database/utils/upsert_prediction.py rename to shared/mongodb/utils/upsert_prediction.py diff --git a/shared/database/utils/upsert_visual_communication.py b/shared/mongodb/utils/upsert_visual_communication.py similarity index 59% rename from shared/database/utils/upsert_visual_communication.py rename to shared/mongodb/utils/upsert_visual_communication.py index 30c2e7e..b7c9462 100755 --- a/shared/database/utils/upsert_visual_communication.py +++ b/shared/mongodb/utils/upsert_visual_communication.py @@ -2,22 +2,18 @@ from __future__ import annotations from pymongo.collection import Collection -from shared.database import VisualCommunication +from shared.mongodb import VisualCommunication def upsert_visual_communication( collection: Collection, visual_communication_list: list[VisualCommunication], ) -> bool: - """ - Upsert VisualCommunication object in the database. + """Upsert VisualCommunication object in the database. + Returns bool stating success. """ response = collection.insert_many( - [ - vis_com.model_dump() - for vis_com - in visual_communication_list - ], + [vis_com.model_dump() for vis_com in visual_communication_list], ) return response.acknowledged diff --git a/web_ui/src/app/init_app.py b/web_ui/src/app/init_app.py index 481b9b7..9a6c3e5 100644 --- a/web_ui/src/app/init_app.py +++ b/web_ui/src/app/init_app.py @@ -1,28 +1,28 @@ """Definition of init_app function.""" + from __future__ import annotations import logging import os import dash_bootstrap_components as dbc -from dash import ALL -from dash import Dash -from dash import Input -from dash import Output -from dash import State +from dash import ALL, Dash, Input, Output, State from dash_auth import BasicAuth from minio import Minio from pydantic import ValidationError from pymongo.collection import Collection -from .layout import app_layout from shared.data_store import delete as delete_from_minio -from shared.database import count_documents -from shared.database import get_visual_communication -from shared.database import NoDocumentFoundException -from shared.database import upsert_annotation -from shared.database import VisualCommunication from shared.dto import ModelData +from shared.mongodb import ( + NoDocumentFoundException, + VisualCommunication, + count_documents, + get_visual_communication, + upsert_annotation, +) + +from .layout import app_layout def init_app( @@ -101,10 +101,8 @@ def init_app( collection=mongo_collection, only_with_annotation=True, ) - limit = int(num_total/20) - label_str = f"{ - num_handled - }/{num_total}" if num_handled >= limit else '' + limit = int(num_total / 20) + label_str = f'{num_handled}/{num_total}' if num_handled >= limit else '' return num_handled, num_total, label_str # define callback: upload image @@ -122,9 +120,7 @@ def init_app( """Upload image to database through web ui.""" try: # stop if no input - assert ( - content_list is not None - ) and ( + assert (content_list is not None) and ( filename_list is not None ), 'nothing to upload' # handle input @@ -159,7 +155,9 @@ def init_app( client=minio_client, object_name=vis_com.object_name, ) - assert len(failed_filename_list) == 0, f"failed uploading:{ + assert ( + len(failed_filename_list) == 0 + ), f"failed uploading:{ '\n'.join(failed_filename_list) }" except Exception as exc: @@ -208,30 +206,19 @@ def init_app( logging.info('saving annotations to database: %s', vis_com_name) try: # extract option keys - annotation_keys = [ - elem['index'] - for elem in annotation_keys - ] + annotation_keys = [elem['index'] for elem in annotation_keys] # ensure all options are set logging.info(annotation_keys) for option, value in zip(annotation_keys, annotation_values): if value is None: raise ValueError(f"{option} is not set") # prepare data to save - annotation_keys = [ - elem.replace(' ', '_') - for elem - in annotation_keys - ] + annotation_keys = [elem.replace(' ', '_') for elem in annotation_keys] annotation_values = [ - elem.replace(' ', '_').lower() - for elem - in annotation_values + elem.replace(' ', '_').lower() for elem in annotation_values ] annotation_map = { - key: value - for key, value - in zip(annotation_keys, annotation_values) + key: value for key, value in zip(annotation_keys, annotation_values) } # instantiate ModelOutputs object annotations = ModelData.from_annotations(**annotation_map) @@ -273,8 +260,10 @@ def init_app( response[2] = image_src response[3] = annotation_values logging.info( - 'finished getting visual communication: %s', vis_com_name, + 'finished getting visual communication: %s', + vis_com_name, ) return tuple(response) + logging.info('initialised app') return app diff --git a/web_ui/src/main.py b/web_ui/src/main.py index ea48325..6d6fbd5 100644 --- a/web_ui/src/main.py +++ b/web_ui/src/main.py @@ -5,7 +5,7 @@ from __future__ import annotations import os from shared.data_store import connect_minio -from shared.database import connect_mongodb +from shared.mongodb import connect_mongodb from shared.utils import check_env, setup_logging from .app import init_app