From 0ebf7e6ce133c1cde2e07089c1c3f6100982b65d Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 13:00:28 +0200 Subject: [PATCH 01/15] updated logging formatting --- shared/utils/setup_logging.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shared/utils/setup_logging.py b/shared/utils/setup_logging.py index aa4ca14..4c70810 100644 --- a/shared/utils/setup_logging.py +++ b/shared/utils/setup_logging.py @@ -12,10 +12,12 @@ def setup_logging() -> None: fmt = ( '%(asctime)s | ' '%(levelname)s | ' - '%(filename)s | ' + '%(name)s | ' '%(funcName)s | ' '%(message)s' ) datefmt = '%Y-%m-%d %H:%M:%S' logging.basicConfig(format=fmt, datefmt=datefmt, level=level) + # change levels for modules that spam the log + logging.getLogger('pymongo').setLevel(logging.WARNING) logging.debug('finished') -- 2.54.0 From f3aab68d5f14afd516633868f1a488da245809f5 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 16:14:43 +0200 Subject: [PATCH 02/15] added minio interface --- shared/data_store/__init__.py | 5 +++++ shared/data_store/connect.py | 30 +++++++++++++++++++++++++++ shared/data_store/get.py | 27 +++++++++++++++++++++++++ shared/data_store/put.py | 38 +++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 shared/data_store/__init__.py create mode 100644 shared/data_store/connect.py create mode 100644 shared/data_store/get.py create mode 100644 shared/data_store/put.py diff --git a/shared/data_store/__init__.py b/shared/data_store/__init__.py new file mode 100644 index 0000000..f8fcb99 --- /dev/null +++ b/shared/data_store/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from .connect import connect +from .get import get +from .put import put diff --git a/shared/data_store/connect.py b/shared/data_store/connect.py new file mode 100644 index 0000000..0bbf3af --- /dev/null +++ b/shared/data_store/connect.py @@ -0,0 +1,30 @@ +"""Definition of connect function.""" +from __future__ import annotations + +import os + +from minio import Minio + + +def connect( +) -> Minio: + """Connect to MinIO server""" + minio_endpoint = os.getenv('MINIO_ENDPOINT', default=None) + assert isinstance(minio_endpoint, str) + minio_access_key = os.getenv('MINIO_ACCESS_KEY', default=None) + assert isinstance(minio_access_key, str) + minio_secret_key = os.getenv('MINIO_SECRET_KEY', default=None) + assert isinstance(minio_secret_key, str) + minio_bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None) + assert isinstance(minio_bucket_name, str) + # connect client + client = Minio( + endpoint=minio_endpoint, + access_key=minio_access_key, + secret_key=minio_secret_key, + secure=False, + ) + # ensure bucket exists + if not client.bucket_exists(bucket_name=minio_bucket_name): + client.make_bucket(bucket_name=minio_bucket_name) + return client diff --git a/shared/data_store/get.py b/shared/data_store/get.py new file mode 100644 index 0000000..de5660f --- /dev/null +++ b/shared/data_store/get.py @@ -0,0 +1,27 @@ +"""Definition of get function.""" +from __future__ import annotations + +import os +from io import BytesIO + +from minio import Minio + + +def get( + client: Minio, + object_name: str, +) -> BytesIO: + """Get buffer from bucket in MinIO.""" + assert isinstance(client, Minio) + assert isinstance(object_name, str) + bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None) + assert isinstance(bucket_name, str) + # get buffer + try: + response = client.get_object(bucket_name, object_name) + buffer = BytesIO(response.data) + finally: + response.close() + response.release_conn() + buffer.seek(0) + return buffer diff --git a/shared/data_store/put.py b/shared/data_store/put.py new file mode 100644 index 0000000..a9dac0c --- /dev/null +++ b/shared/data_store/put.py @@ -0,0 +1,38 @@ +"""Definition of put function.""" +from __future__ import annotations + +import logging +import os +from hashlib import md5 +from io import BytesIO + +from minio import Minio + + +def put( + client: Minio, + buffer: BytesIO, +) -> str: + """Put buffer in bucket in MinIO and return MD5 checksum as object name.""" + assert isinstance(client, Minio) + assert isinstance(buffer, BytesIO) + bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None) + assert isinstance(bucket_name, str) + # get md5 of image + checksum = md5(buffer.getbuffer()).hexdigest() + # prepare for saving + num_bytes = buffer.tell() + buffer.seek(0) + # send data to bucket + try: + client.put_object( + bucket_name=bucket_name, + object_name=checksum, + length=num_bytes, + data=buffer, + ) + except Exception as exc: + logging.error('failed saving data to MinIO') + raise exc + logging.debug('saved data to %s', checksum) + return checksum -- 2.54.0 From b093154193d0fba0b1bb602e80898603283ef127 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 16:15:45 +0200 Subject: [PATCH 03/15] silenced noisy modules --- shared/utils/setup_logging.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shared/utils/setup_logging.py b/shared/utils/setup_logging.py index 4c70810..4241832 100644 --- a/shared/utils/setup_logging.py +++ b/shared/utils/setup_logging.py @@ -20,4 +20,6 @@ def setup_logging() -> None: logging.basicConfig(format=fmt, datefmt=datefmt, level=level) # change levels for modules that spam the log logging.getLogger('pymongo').setLevel(logging.WARNING) + logging.getLogger('urllib3').setLevel(logging.INFO) + logging.getLogger('PIL').setLevel(logging.INFO) logging.debug('finished') -- 2.54.0 From 9d0648f3c10e236695152363066da72bc8235831 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 16:19:04 +0200 Subject: [PATCH 04/15] transferred images from mongo to minio --- other/transfer_images_mongo_minio.py | 122 +++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 other/transfer_images_mongo_minio.py diff --git a/other/transfer_images_mongo_minio.py b/other/transfer_images_mongo_minio.py new file mode 100644 index 0000000..553ab19 --- /dev/null +++ b/other/transfer_images_mongo_minio.py @@ -0,0 +1,122 @@ +"""Script to move all images from MongoDB to MinIO.""" +from __future__ import annotations + +import logging +from io import BytesIO +from pathlib import Path + +from bson import ObjectId +from dotenv import load_dotenv +from pymongo.collection import Collection + +from shared.data_store import connect as connect_minio +from shared.data_store import put +from shared.database import connect +from shared.database import VisualCommunication +from shared.utils import check_env +from shared.utils import setup_logging + + +def list_mongo_document_ids( + collection: Collection, +) -> list[ObjectId]: + """Get list of all VisualCommunication documents in MongoDB.""" + # prepare query + query = collection.find( + filter={}, + projection={'_id': True}, + ) + # execute query + res_list = list(query) + logging.debug('got %s document(s)', len(res_list)) + # convert result + id_list = [elem['_id'] for elem in res_list] + return id_list + + +def get_visual_communication( + collection: Collection, + doc_id: ObjectId, +) -> VisualCommunication: + """Get image from MongoDB.""" + # prepare query + query = collection.find( + filter={'_id': doc_id}, + projection={'_id': False}, + ) + # execute query + res_list = list(query) + logging.debug('got %s document(s)', len(res_list)) + # convert result + vis_com = VisualCommunication.model_validate(res_list[0]) + return vis_com + + +def update_visual_communication( + collection: Collection, + vis_com: VisualCommunication, +) -> None: + """Update VisualCommunication document in MongoDB.""" + try: + collection.update_one( + filter={ + 'name': vis_com.name, + }, + update={ + '$set': vis_com.model_dump(), + }, + ) + except Exception as exc: + logging.error('failed saving %s', vis_com.name) + logging.debug(exc) + raise exc + + +if __name__ == '__main__': + # load in env file + env_path = Path(__file__).parent.parent / 'server.env' + assert env_path.exists() + load_dotenv(env_path) + # ensure env vars set + check_env() + # setup logging + setup_logging() + # connect to minIO + minio_client = connect_minio() + # connect to MongoDB + collection, db, client = connect() + # list documents in mongoDB + id_list = list_mongo_document_ids(collection) + for doc_id in id_list: + try: + # get image from MongoDB + vis_com = get_visual_communication(collection, doc_id) + except Exception as exc: + logging.debug(exc) + logging.error('failed getting image from document: %s', doc_id) + continue + try: + # save image to buffer + buffer = BytesIO() + vis_com.image.save(buffer, 'png') + # put buffer in minio + object_name = put( + client=minio_client, + buffer=buffer, + ) + except Exception as exc: + logging.debug(exc) + logging.error('failed saving image to minio') + continue + # update visual communication in mongodb + try: + vis_com.image = None # type: ignore + vis_com.object_name = object_name + update_visual_communication(collection, vis_com) + except Exception as exc: + logging.debug(exc) + logging.error( + 'failed updating visual communication %s', vis_com.name, + ) + continue + logging.debug('updated visual communication %s', vis_com.name) -- 2.54.0 From 2e060f07763791b30c53f7c1c9a391d7c5181f71 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 16:19:51 +0200 Subject: [PATCH 05/15] updated to match images being stored in minio --- .../database/classes/visual_communication.py | 62 ++++++++++--------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/shared/database/classes/visual_communication.py b/shared/database/classes/visual_communication.py index f20a0ab..9a9898b 100755 --- a/shared/database/classes/visual_communication.py +++ b/shared/database/classes/visual_communication.py @@ -7,11 +7,12 @@ from base64 import b64encode from io import BytesIO from pathlib import Path +from minio import Minio from PIL import Image from pydantic import BaseModel -from pydantic import field_serializer -from pydantic import field_validator +from shared.data_store import get +from shared.data_store import put from shared.dto import ModelData @@ -19,7 +20,7 @@ class VisualCommunication(BaseModel): """Visual communication model.""" name: str - image: Image.Image + object_name: str annotation: ModelData | None = None prediction: ModelData | None = None @@ -33,12 +34,22 @@ class VisualCommunication(BaseModel): return cls.__name__ @classmethod - def from_file(cls, path: Path) -> VisualCommunication: + def from_file(cls, path: Path, minio_client: Minio) -> VisualCommunication: """Instantiate from file.""" + assert isinstance(path, Path) + assert isinstance(minio_client, Minio) + # determine name name = path.stem + # open and upload image to minio image = Image.open(path) image.load() - return VisualCommunication(name=name, image=image) + buffer = BytesIO() + image.save(buffer, 'png') + object_name = put( + client=minio_client, + buffer=buffer, + ) + return VisualCommunication(name=name, object_name=object_name) @classmethod def decode_image(cls, content: str) -> Image.Image: @@ -46,35 +57,26 @@ class VisualCommunication(BaseModel): _, content_data = content.split(',') return Image.open(BytesIO(b64decode(content_data))) - @field_serializer('image') - @classmethod - def serialize_image(cls, image: Image.Image) -> bytes: # type: ignore - """Convert image to bytes for storage in database.""" - buffer = BytesIO() - image.save(buffer, format='JPEG') - return buffer.getvalue() + def get_image(self, minio_client: Minio) -> Image.Image: + """Load image data from minio.""" + assert isinstance(minio_client, Minio) + # get buffer from minio + buffer = get( + client=minio_client, + object_name=self.object_name, + ) + # convert data to image + im = Image.open(buffer) + return im - @field_validator('image', mode='before') - @classmethod - def convert_to_image( - cls, - image: Image.Image | BytesIO | bytes, - ) -> Image.Image: - """Convert bytes input from database into image.""" - if isinstance(image, bytes): - image = BytesIO(image) - if isinstance(image, BytesIO): - image = Image.open(image) - return image - - def __repr__(self) -> str: - return f"{self.classname()}(name='{self.name}')" - - def webencoded_image(self) -> str: + def webencoded_image(self, minio_client: Minio) -> str: """Convert image to be displayed on webpage.""" + assert isinstance(minio_client, Minio) + # get image from minio + image = self.get_image(minio_client) # convert images to bytes string buffer = BytesIO() - self.image.save(buffer, format='png') + image.save(buffer, format='png') img_enc = b64encode(buffer.getvalue()).decode('utf-8') return f"data:image/png;base64, {img_enc}" -- 2.54.0 From 119282fee954d1f1b80198ad90b71d00d0411f1d Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 16:20:16 +0200 Subject: [PATCH 06/15] tested loading images from minio --- tests/get_visual_communication_test.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/get_visual_communication_test.py b/tests/get_visual_communication_test.py index ed1ad5e..1b406ce 100644 --- a/tests/get_visual_communication_test.py +++ b/tests/get_visual_communication_test.py @@ -1,22 +1,32 @@ from __future__ import annotations -import os from pathlib import Path from dotenv import load_dotenv +from shared.data_store import ( + connect as connect_minio, +) from shared.database import connect from shared.database import get_visual_communication +from shared.utils import check_env +from shared.utils import setup_logging if __name__ == '__main__': - # prepare env vars - env_path = Path(__file__).parent.parent / 'local.env' + # load in env file + env_path = Path(__file__).parent.parent / 'server.env' assert env_path.exists() load_dotenv(env_path) - os.environ['MONGO_HOST'] = 'localhost' - # connect to database + # ensure env vars set + check_env() + # setup logging + setup_logging() + # connect to minIO + minio_client = connect_minio() + # connect to MongoDB collection, db, client = connect() - print(client.server_info()) # get visual communication vis_com = get_visual_communication(collection) - print(vis_com) + print(repr(vis_com)) + image = vis_com.get_image(minio_client=minio_client) + image.show() -- 2.54.0 From b8061466ce6d6b47dd795c904742aa2a485d2ae7 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 16:31:19 +0200 Subject: [PATCH 07/15] updated to use new vis com class --- web_ui/src/app/init_app.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/web_ui/src/app/init_app.py b/web_ui/src/app/init_app.py index 6d17e14..e228995 100644 --- a/web_ui/src/app/init_app.py +++ b/web_ui/src/app/init_app.py @@ -11,6 +11,7 @@ from dash import Input from dash import Output from dash import State from dash_auth import BasicAuth +from minio import Minio from pydantic import ValidationError from pymongo.collection import Collection @@ -25,7 +26,8 @@ from shared.dto import ModelData def init_app( - collection: Collection, + mongo_collection: Collection, + minio_client: Minio, ) -> Dash: """Initialise web UI application.""" # setup app @@ -92,11 +94,11 @@ def init_app( assert isinstance(filename_list, list) # get values from database num_total = count_documents( - collection=collection, + collection=mongo_collection, only_with_annotation=False, ) num_handled = count_documents( - collection=collection, + collection=mongo_collection, only_with_annotation=True, ) limit = int(num_total/20) @@ -138,7 +140,7 @@ def init_app( vis_com_list.append(vis_com) # upsert documents success = upsert_visual_communication( - collection=collection, + collection=mongo_collection, visual_communication_list=vis_com_list, ) if success: @@ -217,7 +219,7 @@ def init_app( annotations = ModelData.from_annotations(**annotation_map) # save data to database upsert_annotation( - collection=collection, + collection=mongo_collection, vis_com_name=vis_com_name, annotations=annotations, ) @@ -231,12 +233,12 @@ def init_app( try: # get data vis_com = get_visual_communication( - collection=collection, + collection=mongo_collection, with_annotation=False, ) # set variables vis_com_name = vis_com.name - image_src = vis_com.webencoded_image() + image_src = vis_com.webencoded_image(minio_client=minio_client) if vis_com.prediction is not None: # TODO: update to use optional predictions pass -- 2.54.0 From d537cdf11df4ccd6d83d55dcf87acacbd4be21bd Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 16:31:35 +0200 Subject: [PATCH 08/15] updated to use new vis com class --- web_ui/src/main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/web_ui/src/main.py b/web_ui/src/main.py index c22b45d..72facfd 100644 --- a/web_ui/src/main.py +++ b/web_ui/src/main.py @@ -4,6 +4,7 @@ from __future__ import annotations import os from .app import init_app +from shared.data_store import connect as connect_minio from shared.database import connect from shared.utils import check_env from shared.utils import setup_logging @@ -17,7 +18,13 @@ setup_logging() # connect to database collection, db, client = connect() +# connect to minio +minio_client = connect_minio() + # initialise application -app = init_app(collection=collection) +app = init_app( + mongo_collection=collection, + minio_client=minio_client, +) server = app.server server.config.update(SECRET_KEY=os.urandom(24)) -- 2.54.0 From e1075b35510caedb9e4a534b01038be26e1fede9 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 23:14:55 +0200 Subject: [PATCH 09/15] ignore outdated class reference --- other/transfer_images_mongo_minio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/other/transfer_images_mongo_minio.py b/other/transfer_images_mongo_minio.py index 553ab19..9a9834f 100644 --- a/other/transfer_images_mongo_minio.py +++ b/other/transfer_images_mongo_minio.py @@ -98,7 +98,7 @@ if __name__ == '__main__': try: # save image to buffer buffer = BytesIO() - vis_com.image.save(buffer, 'png') + vis_com.image.save(buffer, 'png') # type: ignore # put buffer in minio object_name = put( client=minio_client, -- 2.54.0 From d09cb9dd20827f5374bc1c6b8c0cb0c046d13f10 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 23:15:31 +0200 Subject: [PATCH 10/15] added delete function --- shared/data_store/__init__.py | 1 + shared/data_store/delete.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 shared/data_store/delete.py diff --git a/shared/data_store/__init__.py b/shared/data_store/__init__.py index f8fcb99..853af00 100644 --- a/shared/data_store/__init__.py +++ b/shared/data_store/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations from .connect import connect +from .delete import delete from .get import get from .put import put diff --git a/shared/data_store/delete.py b/shared/data_store/delete.py new file mode 100644 index 0000000..b5f21d9 --- /dev/null +++ b/shared/data_store/delete.py @@ -0,0 +1,29 @@ +"""Definition of delete function.""" +from __future__ import annotations + +import logging +import os + +from minio import Minio + + +def delete( + client: Minio, + object_name: str, +) -> None: + """Delete object from MinIO.""" + assert isinstance(client, Minio) + assert isinstance(object_name, str) + bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None) + assert isinstance(bucket_name, str) + # remove object + try: + client.remove_object( + bucket_name=bucket_name, + object_name=object_name, + ) + except Exception as exc: + logging.debug(exc) + logging.error('failed deleting %s', object_name) + else: + logging.debug('deleted %s', object_name) -- 2.54.0 From 123cab2500d20336b94bcf0e169f840c11ab3e9b Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 23:16:42 +0200 Subject: [PATCH 11/15] updated to use new functions --- .../database/classes/visual_communication.py | 61 ++++++++++++++++--- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/shared/database/classes/visual_communication.py b/shared/database/classes/visual_communication.py index 9a9898b..6d63a67 100755 --- a/shared/database/classes/visual_communication.py +++ b/shared/database/classes/visual_communication.py @@ -10,6 +10,7 @@ from pathlib import Path from minio import Minio from PIL import Image from pydantic import BaseModel +from pymongo.collection import Collection from shared.data_store import get from shared.data_store import put @@ -34,23 +35,60 @@ class VisualCommunication(BaseModel): return cls.__name__ @classmethod - def from_file(cls, path: Path, minio_client: Minio) -> VisualCommunication: - """Instantiate from file.""" - assert isinstance(path, Path) + def upload_image_to_minio( + cls, + image: Image.Image, + minio_client: Minio, + ) -> str: + """Upload image to MinIO and return MD5 checksum of hashed image.""" + assert isinstance(image, Image.Image) assert isinstance(minio_client, Minio) - # determine name - name = path.stem - # open and upload image to minio - image = Image.open(path) - image.load() buffer = BytesIO() image.save(buffer, 'png') object_name = put( client=minio_client, buffer=buffer, ) + return object_name + + @classmethod + def from_name_and_image( + cls, + name: str, + image: Image.Image, + minio_client: Minio, + ) -> VisualCommunication: + """ + Instantiate from filename and image + that is automatically uploaded to MinIO. + """ + assert isinstance(name, str) + assert isinstance(image, Image.Image) + assert isinstance(minio_client, Minio) + # upload file to minio + object_name = VisualCommunication.upload_image_to_minio( + image=image, + minio_client=minio_client, + ) return VisualCommunication(name=name, object_name=object_name) + @classmethod + def from_file(cls, path: Path, minio_client: Minio) -> VisualCommunication: + """Instantiate from file.""" + assert isinstance(path, Path) + assert isinstance(minio_client, Minio) + # determine name + name = path.stem + # open image + image = Image.open(path) + image.load() + # instantiate object + return VisualCommunication.from_name_and_image( + name=name, + image=image, + minio_client=minio_client, + ) + @classmethod def decode_image(cls, content: str) -> Image.Image: """Extract image from webencoded content.""" @@ -69,6 +107,13 @@ class VisualCommunication(BaseModel): im = Image.open(buffer) return im + def save_to_mongo(self, collection: Collection) -> None: + """Save self as document in MongoDB.""" + res = collection.insert_one( + document=self.model_dump(), + ) + assert res.acknowledged + def webencoded_image(self, minio_client: Minio) -> str: """Convert image to be displayed on webpage.""" assert isinstance(minio_client, Minio) -- 2.54.0 From 0bd4008e726f6326ad61a44ccac3d04660289381 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 23:17:28 +0200 Subject: [PATCH 12/15] updated to use new class --- tests/generate_random_prediction_test.py | 21 ++++++++++++-- tests/image_upload_test.py | 28 ++++++++++-------- tests/image_upload_to_server_test.py | 27 +++++++++++------- tests/prediction_upload_test.py | 36 +++++++++++------------- 4 files changed, 69 insertions(+), 43 deletions(-) diff --git a/tests/generate_random_prediction_test.py b/tests/generate_random_prediction_test.py index 2fccdd5..acceaf9 100644 --- a/tests/generate_random_prediction_test.py +++ b/tests/generate_random_prediction_test.py @@ -2,10 +2,27 @@ from __future__ import annotations from pathlib import Path -from shared.database.classes import VisualCommunication +from dotenv import load_dotenv +from shared.data_store import connect as connect_minio +from shared.database import connect as connect_mongo +from shared.database.classes import VisualCommunication +from shared.utils import check_env +from shared.utils import setup_logging if __name__ == '__main__': + # load in env file + env_path = Path(__file__).parent.parent / 'server.env' + assert env_path.exists() + load_dotenv(env_path) + # ensure env vars set + check_env() + # setup logging + setup_logging() + # connect to minIO + minio_client = connect_minio() + # connect to MongoDB + collection, db, client = connect_mongo() # get list of image paths test_dir = Path(__file__).parent img_dir = test_dir / 'imgs' @@ -13,7 +30,7 @@ if __name__ == '__main__': print(img_path_list) # instantiate data object vis_com_list = [ - VisualCommunication.from_file(path) + VisualCommunication.from_file(path, minio_client=minio_client) for path in img_path_list ] diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 916db40..46f0dd6 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -1,15 +1,29 @@ from __future__ import annotations -import os from pathlib import Path from dotenv import load_dotenv from pymongo.errors import DuplicateKeyError -from shared.database import connect +from shared.data_store import connect as connect_minio +from shared.database import connect as connect_mongo from shared.database.classes import VisualCommunication +from shared.utils import check_env +from shared.utils import setup_logging if __name__ == '__main__': + # load in env file + env_path = Path(__file__).parent.parent / 'server.env' + assert env_path.exists() + load_dotenv(env_path) + # ensure env vars set + check_env() + # setup logging + setup_logging() + # connect to minIO + minio_client = connect_minio() + # connect to MongoDB + collection, db, client = connect_mongo() # get list of image paths test_dir = Path(__file__).parent img_dir = test_dir / 'imgs' @@ -17,20 +31,12 @@ if __name__ == '__main__': print(img_path_list) # instantiate data object vis_com_list = [ - VisualCommunication.from_file(path) + VisualCommunication.from_file(path, minio_client=minio_client) for path in img_path_list ] for vis_com in vis_com_list: print(repr(vis_com)) - # prepare env vars - env_path = test_dir.parent / 'local.env' - assert env_path.exists() - load_dotenv(env_path) - os.environ['MONGO_HOST'] = 'localhost' - # connect to database - collection, db, client = connect() - print(client.server_info()) # upload images for vis_com in vis_com_list: try: diff --git a/tests/image_upload_to_server_test.py b/tests/image_upload_to_server_test.py index 9ecc79b..ca376a0 100644 --- a/tests/image_upload_to_server_test.py +++ b/tests/image_upload_to_server_test.py @@ -5,10 +5,25 @@ from pathlib import Path from dotenv import load_dotenv from pymongo.errors import DuplicateKeyError -from shared.database import connect +from shared.data_store import connect as connect_minio +from shared.database import connect as connect_mongo from shared.database import VisualCommunication +from shared.utils import check_env +from shared.utils import setup_logging if __name__ == '__main__': + # load in env file + env_path = Path(__file__).parent.parent / 'server.env' + assert env_path.exists() + load_dotenv(env_path) + # ensure env vars set + check_env() + # setup logging + setup_logging() + # connect to minIO + minio_client = connect_minio() + # connect to MongoDB + collection, db, client = connect_mongo() # get list of image paths ext_img_dir = Path('/Volumes/BW-PSSD/Mixed Methods/') assert ext_img_dir.exists() @@ -20,19 +35,11 @@ if __name__ == '__main__': print(f"found {len(img_path_list)} images") # create visual communication objects vis_com_list = [ - VisualCommunication.from_file(path) + VisualCommunication.from_file(path, minio_client=minio_client) for path in img_path_list ] print(f"created {len(vis_com_list)} visual communication objects") - # prepare env vars - env_path = Path(__file__).parent.parent / 'server.env' - assert env_path.exists() - load_dotenv(env_path) - # connect to database - collection, db, client = connect() - assert client.server_info() is not None - print('connected to database') # upload images for vis_com in vis_com_list: try: diff --git a/tests/prediction_upload_test.py b/tests/prediction_upload_test.py index bf667f3..1521eca 100644 --- a/tests/prediction_upload_test.py +++ b/tests/prediction_upload_test.py @@ -1,46 +1,42 @@ from __future__ import annotations -import logging -import os from pathlib import Path from dotenv import load_dotenv -from shared.database import connect +from shared.data_store import connect as connect_minio +from shared.database import connect as connect_mongo from shared.database import upsert_prediction from shared.database.classes import VisualCommunication +from shared.utils import check_env +from shared.utils import setup_logging if __name__ == '__main__': + # load in env file + env_path = Path(__file__).parent.parent / 'server.env' + assert env_path.exists() + load_dotenv(env_path) + # ensure env vars set + check_env() # setup logging - fmt = ( - '%(asctime)s | ' - '%(levelname)s | ' - '%(filename)s | ' - '%(funcName)s | ' - '%(message)s' - ) - datefmt = '%Y-%m-%d %H:%M:%S' - logging.basicConfig(format=fmt, datefmt=datefmt, level=logging.INFO) + setup_logging() + # connect to minIO + minio_client = connect_minio() + # connect to MongoDB + collection, db, client = connect_mongo() # get list of image paths test_dir = Path(__file__).parent img_dir = test_dir / 'imgs' img_path_list = [path for path in img_dir.glob('*.jpeg') if path.is_file()] # instantiate data object vis_com_list = [ - VisualCommunication.from_file(path) + VisualCommunication.from_file(path, minio_client=minio_client) for path in img_path_list ] # generate random predictions for vis_com in vis_com_list: vis_com.generate_random_prediction() - # prepare env vars - env_path = test_dir.parent / 'local.env' - assert env_path.exists() - load_dotenv(env_path) - os.environ['MONGO_HOST'] = 'localhost' - # connect to database - collection, db, client = connect() # upload visual communication for vis_com in vis_com_list: if vis_com.prediction is None: -- 2.54.0 From eaca23f27a744271176b48b18e6631803e9a1990 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 23:18:27 +0200 Subject: [PATCH 13/15] updated to use new class --- web_ui/src/app/init_app.py | 76 +++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/web_ui/src/app/init_app.py b/web_ui/src/app/init_app.py index e228995..481b9b7 100644 --- a/web_ui/src/app/init_app.py +++ b/web_ui/src/app/init_app.py @@ -16,11 +16,11 @@ from pydantic import ValidationError from pymongo.collection import Collection from .layout import app_layout +from shared.data_store import delete as delete_from_minio from shared.database import count_documents from shared.database import get_visual_communication from shared.database import NoDocumentFoundException from shared.database import upsert_annotation -from shared.database import upsert_visual_communication from shared.database import VisualCommunication from shared.dto import ModelData @@ -120,34 +120,52 @@ def init_app( filename_list: list[str] | None, ) -> tuple[str | None, str | None]: """Upload image to database through web ui.""" - # stop early if possible - if content_list is None or filename_list is None: - logging.info('nothing to upload.') - return None, 'nothing to upload'.title() - # build list of visual communication - vis_com_list = [] - for content, filename in zip(content_list, filename_list): - try: - image = VisualCommunication.decode_image(content=content) - vis_com = VisualCommunication( - name=filename, - image=image, - ) - except Exception as exc: - logging.error('failed handling data from file: %s', filename) - logging.debug(exc) - continue - vis_com_list.append(vis_com) - # upsert documents - success = upsert_visual_communication( - collection=mongo_collection, - visual_communication_list=vis_com_list, - ) - if success: - logging.info('uploaded %s files to database', len(vis_com_list)) - return 'successfully uploaded images'.title(), None - logging.error('failed uploading images to database') - return None, 'failed uploading images'.title() + try: + # stop if no input + assert ( + content_list is not None + ) and ( + filename_list is not None + ), 'nothing to upload' + # handle input + failed_filename_list = [] + for content, filename in zip(content_list, filename_list): + try: + # decode image content + image = VisualCommunication.decode_image(content=content) + except Exception: + logging.debug('failed decoding %s', filename) + failed_filename_list.append(filename) + continue + try: + # instantiate to upload image to minio + vis_com = VisualCommunication.from_name_and_image( + name=filename, + image=image, + minio_client=minio_client, + ) + except Exception as exc: + logging.debug(exc) + failed_filename_list.append(filename) + continue + try: + # save to mongodb + vis_com.save_to_mongo(collection=mongo_collection) + except Exception as exc: + logging.debug(exc) + failed_filename_list.append(filename) + # remove document from minio + delete_from_minio( + client=minio_client, + object_name=vis_com.object_name, + ) + assert len(failed_filename_list) == 0, f"failed uploading:{ + '\n'.join(failed_filename_list) + }" + except Exception as exc: + logging.debug(exc) + return None, str(exc).title() + return 'successfully uploaded images'.title(), None # define callback: cycle visual communication data @app.callback( -- 2.54.0 From 962d028c0709ab5a69ba5630eb1cfd6390823c33 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 15 Jun 2024 23:59:39 +0200 Subject: [PATCH 14/15] added minio env vars --- shared/utils/check_env.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/shared/utils/check_env.py b/shared/utils/check_env.py index 0fb3a2f..c6cf480 100644 --- a/shared/utils/check_env.py +++ b/shared/utils/check_env.py @@ -14,6 +14,10 @@ def check_env() -> None: 'MONGO_PASSWORD', 'DASH_AUTH_USERNAME', 'DASH_AUTH_PASSWORD', + 'MINIO_ENDPOINT', + 'MINIO_ACCESS_KEY', + 'MINIO_SECRET_KEY', + 'MINIO_BUCKET_NAME', } for env_var in necesasary_var_list: # ensure env var set -- 2.54.0 From 2ee5ab2b9c10d71a48834edacc47e71bd61d4b76 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sun, 16 Jun 2024 00:06:28 +0200 Subject: [PATCH 15/15] added env var check --- shared/data_store/connect.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/shared/data_store/connect.py b/shared/data_store/connect.py index 0bbf3af..6530e78 100644 --- a/shared/data_store/connect.py +++ b/shared/data_store/connect.py @@ -9,6 +9,21 @@ from minio import Minio def connect( ) -> Minio: """Connect to MinIO server""" + # ensure necessary env vars available + necesasary_var_list = [ + 'MINIO_ENDPOINT', + 'MINIO_ACCESS_KEY', + 'MINIO_SECRET_KEY', + 'MINIO_BUCKET_NAME', + ] + for env_var in necesasary_var_list: + # ensure env var set + assert ( + env_var in os.environ + ), ( + f"environment variable not set: {env_var}" + ) + # prepare arguments minio_endpoint = os.getenv('MINIO_ENDPOINT', default=None) assert isinstance(minio_endpoint, str) minio_access_key = os.getenv('MINIO_ACCESS_KEY', default=None) -- 2.54.0