From a379270add7aa7c71d032525c83781a40e409cd8 Mon Sep 17 00:00:00 2001 From: brian Date: Tue, 18 Mar 2025 19:21:17 +0000 Subject: [PATCH] removed deprecated modules --- shared/datastore/__init__.py | 1 - shared/datastore/src/__init__.py | 1 - shared/datastore/src/datastore_interface.py | 69 ----- shared/datastore/src/datastore_minio.py | 241 ------------------ .../tests/integration/base_crud_test.py | 112 -------- .../datastore/tests/integration/conftest.py | 196 -------------- .../tests/integration/connect_test.py | 10 - .../tests/integration/image_crud_test.py | 82 ------ .../tests/integration/model_crud_test.py | 90 ------- .../tests/unit/datastore_minio_test.py | 230 ----------------- shared/docstore/__init__.py | 8 - shared/docstore/src/__init__.py | 3 - shared/docstore/src/classes/__init__.py | 13 - shared/docstore/src/classes/angle.py | 13 - shared/docstore/src/classes/contact.py | 12 - shared/docstore/src/classes/data_model.py | 67 ----- shared/docstore/src/classes/distance.py | 13 - shared/docstore/src/classes/framing.py | 14 - .../docstore/src/classes/information_value.py | 13 - shared/docstore/src/classes/modality_color.py | 13 - shared/docstore/src/classes/modality_depth.py | 13 - .../docstore/src/classes/modality_lighting.py | 13 - shared/docstore/src/classes/model_data.py | 115 --------- shared/docstore/src/classes/point_of_view.py | 12 - shared/docstore/src/classes/salience.py | 15 -- .../src/classes/visual_communication.py | 122 --------- shared/docstore/src/classes/visual_syntax.py | 28 -- shared/docstore/src/connect_mongodb.py | 30 --- shared/docstore/src/count_documents.py | 20 -- shared/docstore/src/docstore_interface.py | 83 ------ shared/docstore/src/docstore_mongo.py | 241 ------------------ shared/docstore/src/exceptions/__init__.py | 1 - .../src/exceptions/no_document_found.py | 5 - shared/docstore/src/get_dataset.py | 16 -- .../docstore/src/get_visual_communication.py | 40 --- shared/docstore/src/list_names.py | 30 --- shared/docstore/src/upsert_annotation.py | 30 --- shared/docstore/src/upsert_prediction.py | 30 --- .../src/upsert_visual_communication.py | 19 -- .../tests/unit/docstore_mongo_test.py | 45 ---- .../tests/integration/conftest.py | 14 +- 41 files changed, 10 insertions(+), 2113 deletions(-) delete mode 100644 shared/datastore/__init__.py delete mode 100644 shared/datastore/src/__init__.py delete mode 100644 shared/datastore/src/datastore_interface.py delete mode 100644 shared/datastore/src/datastore_minio.py delete mode 100644 shared/datastore/tests/integration/base_crud_test.py delete mode 100644 shared/datastore/tests/integration/conftest.py delete mode 100644 shared/datastore/tests/integration/connect_test.py delete mode 100644 shared/datastore/tests/integration/image_crud_test.py delete mode 100644 shared/datastore/tests/integration/model_crud_test.py delete mode 100644 shared/datastore/tests/unit/datastore_minio_test.py delete mode 100644 shared/docstore/__init__.py delete mode 100755 shared/docstore/src/__init__.py delete mode 100755 shared/docstore/src/classes/__init__.py delete mode 100644 shared/docstore/src/classes/angle.py delete mode 100644 shared/docstore/src/classes/contact.py delete mode 100644 shared/docstore/src/classes/data_model.py delete mode 100644 shared/docstore/src/classes/distance.py delete mode 100644 shared/docstore/src/classes/framing.py delete mode 100644 shared/docstore/src/classes/information_value.py delete mode 100644 shared/docstore/src/classes/modality_color.py delete mode 100644 shared/docstore/src/classes/modality_depth.py delete mode 100644 shared/docstore/src/classes/modality_lighting.py delete mode 100644 shared/docstore/src/classes/model_data.py delete mode 100644 shared/docstore/src/classes/point_of_view.py delete mode 100644 shared/docstore/src/classes/salience.py delete mode 100755 shared/docstore/src/classes/visual_communication.py delete mode 100644 shared/docstore/src/classes/visual_syntax.py delete mode 100644 shared/docstore/src/connect_mongodb.py delete mode 100755 shared/docstore/src/count_documents.py delete mode 100644 shared/docstore/src/docstore_interface.py delete mode 100644 shared/docstore/src/docstore_mongo.py delete mode 100644 shared/docstore/src/exceptions/__init__.py delete mode 100755 shared/docstore/src/exceptions/no_document_found.py delete mode 100755 shared/docstore/src/get_dataset.py delete mode 100755 shared/docstore/src/get_visual_communication.py delete mode 100755 shared/docstore/src/list_names.py delete mode 100755 shared/docstore/src/upsert_annotation.py delete mode 100755 shared/docstore/src/upsert_prediction.py delete mode 100755 shared/docstore/src/upsert_visual_communication.py delete mode 100644 shared/docstore/tests/unit/docstore_mongo_test.py diff --git a/shared/datastore/__init__.py b/shared/datastore/__init__.py deleted file mode 100644 index aec85ae..0000000 --- a/shared/datastore/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .src import Datastore diff --git a/shared/datastore/src/__init__.py b/shared/datastore/src/__init__.py deleted file mode 100644 index 2d860b3..0000000 --- a/shared/datastore/src/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .datastore_minio import DatastoreMinio as Datastore diff --git a/shared/datastore/src/datastore_interface.py b/shared/datastore/src/datastore_interface.py deleted file mode 100644 index 7cd5445..0000000 --- a/shared/datastore/src/datastore_interface.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Definition of datastore interface.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from collections import OrderedDict - -from PIL import Image -from torch.nn import Module - - -class DatastoreInterface(ABC): - """Datastore interface class.""" - - @abstractmethod - def connect( - self, - ) -> None: - pass - - @abstractmethod - def close( - self, - ) -> None: - pass - - @abstractmethod - def __enter__( - self, - ) -> DatastoreInterface: - self.connect() - return self - - @abstractmethod - def __exit__( - self, - exc_type, - exc_val, - exc_tb, - ) -> None: - self.close() - - @abstractmethod - def put_image( - self, - image: Image.Image, - ) -> str: - pass - - @abstractmethod - def get_image( - self, - object_name: str, - ) -> Image.Image: - pass - - @abstractmethod - def put_model( - self, - model: Module, - ) -> str: - pass - - @abstractmethod - def get_model( - self, - object_name: str, - ) -> OrderedDict: - pass diff --git a/shared/datastore/src/datastore_minio.py b/shared/datastore/src/datastore_minio.py deleted file mode 100644 index d12edcb..0000000 --- a/shared/datastore/src/datastore_minio.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Definition of datastore minio implementation.""" - -from __future__ import annotations - -import logging -import os -from collections import OrderedDict -from hashlib import md5 -from io import BytesIO -from traceback import format_exc - -import torch -from minio import Minio -from PIL import Image - -from shared.utils import check_env - -from .datastore_interface import DatastoreInterface - - -class DatastoreMinio(DatastoreInterface): - """Datastore interface.""" - - def __init__(self): - # ensure necessary env vars available - var_list = { - 'MINIO_ENDPOINT', - 'MINIO_ACCESS_KEY', - 'MINIO_SECRET_KEY', - 'MINIO_BUCKET_NAME', - } - check_env(var_list) - # prepare internal variables - self._client: Minio | None = None - self._bucket_name: str | None = None - - def connect(self) -> None: - """Connect to Minio server.""" - # prepare arguments - minio_endpoint = str(os.getenv('MINIO_ENDPOINT')) - minio_access_key = str(os.getenv('MINIO_ACCESS_KEY')) - minio_secret_key = str(os.getenv('MINIO_SECRET_KEY')) - minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) - # connect client - client = Minio( - endpoint=minio_endpoint, - access_key=minio_access_key, - secret_key=minio_secret_key, - secure=False, - ) - # ensure bucket exists - if not client.bucket_exists(bucket_name=minio_bucket_name): - logging.debug('creating bucket: %s', minio_bucket_name) - client.make_bucket(bucket_name=minio_bucket_name) - logging.debug('finished') - # persist state - self._client = client - self._bucket_name = minio_bucket_name - - def close(self) -> None: - """Close connection to Minio server. - - N.B. Minio connection cannot be closed manually. - """ - self._client = None - self._bucket_name = None - - def __enter__(self) -> DatastoreMinio: - self.connect() - return self - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - if any( - ( - exc_type is not None, - exc_val is not None, - exc_tb is not None, - ), - ): - logging.error('error while exiting context') - self.close() - - def _put( - self, - object_name: str, - buffer: BytesIO, - ) -> None: - """Save in-memory buffer as object in Minio.""" - assert isinstance(object_name, str) - assert len(object_name) > 0 - assert isinstance(buffer, BytesIO) - assert isinstance(self._client, Minio) - assert isinstance(self._bucket_name, str) - # prepare for saving - num_bytes = len(buffer.getvalue()) - buffer.seek(0) - # send data to bucket - try: - self._client.put_object( - bucket_name=self._bucket_name, - object_name=object_name, - length=num_bytes, - data=buffer, - ) - logging.debug('saved data to %s', object_name) - except Exception as exc: - logging.error('failed saving data to MinIO') - raise exc - - def _get( - self, - object_name: str, - ) -> BytesIO: - """Get object from Minio as in-memory buffer.""" - assert isinstance(object_name, str) - assert len(object_name) > 0 - assert isinstance(self._client, Minio) - assert isinstance(self._bucket_name, str) - try: - # make request - response = self._client.get_object( - bucket_name=self._bucket_name, - object_name=object_name, - ) - assert response.status == 200 - # get buffer - buffer = BytesIO() - chunk_size = 2**14 - while chunk := response.read(chunk_size): - buffer.write(chunk) - buffer.seek(0) - logging.debug('got %s', object_name) - return buffer - except Exception as exc: - logging.error('failed getting data from MinIO') - logging.debug(format_exc()) - raise exc - finally: - # close connection if established - if 'response' in locals(): - response.close() - response.release_conn() - - def _delete( - self, - object_name: str, - ) -> None: - """Delete object from Minio.""" - assert isinstance(object_name, str) - assert len(object_name) > 0 - assert isinstance(self._client, Minio) - assert isinstance(self._bucket_name, str) - # remove object - try: - self._client.remove_object( - bucket_name=self._bucket_name, - object_name=object_name, - ) - logging.debug('deleted %s', object_name) - except Exception as exc: - logging.error('failed deleting %s', object_name) - logging.debug(format_exc()) - raise exc - - def put_image( - self, - image: Image.Image, - ) -> str: - """Put image in Minio.""" - assert isinstance(image, Image.Image) - # save data to buffer - buffer = BytesIO() - image.save(buffer, 'png') - # get md5 of buffer - checksum = md5(buffer.getbuffer()).hexdigest() - # build object path - object_path = f'images/{checksum}' - # send data to bucket - self._put( - object_name=object_path, - buffer=buffer, - ) - logging.debug('saved data to %s', object_path) - return checksum - - def get_image( - self, - object_name: str, - ) -> Image.Image: - """Get image from Minio.""" - assert isinstance(object_name, str) - assert len(object_name) > 0 - # build object path - object_path = f'images/{object_name}' - # get object from bucket - buffer = self._get( - object_name=object_path, - ) - # convert data to image - image = Image.open(buffer) - logging.debug('got data from %s', object_path) - return image - - def put_model( - self, - model: torch.nn.Module, - ) -> str: - """Put model in Minio.""" - assert isinstance(model, torch.nn.Module) - # save data to buffer - buffer = BytesIO() - torch.save(model.state_dict(), buffer) - # get md5 of image - checksum = md5(buffer.getbuffer()).hexdigest() - # build object path - object_path = f'models/{checksum}' - # send data to bucket - self._put( - object_name=object_path, - buffer=buffer, - ) - logging.debug('saved data to %s', object_path) - return checksum - - def get_model( - self, - object_name: str, - ) -> OrderedDict: - """Get model data from Minio.""" - assert isinstance(object_name, str) - assert len(object_name) > 0 - # build object path - object_path = f'models/{object_name}' - # get object from bucket - buffer = self._get( - object_name=object_path, - ) - # convert data to model checkpoint - model_content = torch.load(buffer) - logging.debug('got data from %s', object_path) - return model_content diff --git a/shared/datastore/tests/integration/base_crud_test.py b/shared/datastore/tests/integration/base_crud_test.py deleted file mode 100644 index 723ffc7..0000000 --- a/shared/datastore/tests/integration/base_crud_test.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Integration tests related to base CRUD functions.""" - -import logging -import os -from io import BytesIO - -import minio -import pytest - -from shared.datastore import Datastore - - -def same_data( - data_a: BytesIO, - data_b: BytesIO, -) -> bool: - """Check if two BytesIO-objects contain the same data.""" - assert isinstance(data_a, BytesIO) - assert isinstance(data_b, BytesIO) - # prepare for being read - data_a.seek(0) - data_b.seek(0) - # convert to bytes - data_a_bytes = data_a.read() - data_b_bytes = data_b.read() - # compare size - if len(data_a_bytes) != len(data_b_bytes): - logging.error( - 'data has different length: %s and %s', - len(data_a_bytes), - len(data_b_bytes), - ) - return False - # compare content - if data_a_bytes != data_b_bytes: - logging.error('data has different bytes') - return False - return True - - -def test_should_get_data( - datastore: Datastore, - data_in_minio, -): - # ARRANGE - data, _, object_name = data_in_minio - # ACT - received_data = datastore._get( - object_name=object_name, - ) - # ASSERT - assert isinstance(data, BytesIO) - assert same_data(data, received_data) - - -def test_should_delete_data( - datastore: Datastore, - data_in_minio, -): - # ARRANGE - _, _, object_name = data_in_minio - # ACT - datastore._delete( - object_name=object_name, - ) - # ASSERT - with pytest.raises(minio.error.S3Error): - _ = datastore._get( - object_name=object_name, - ) - - -def test_should_put_data( - datastore: Datastore, - data, -): - # ARRANGE - minio_object_name = str(os.getenv('MINIO_OBJECT_NAME')) - buffer = BytesIO(data) - # ACT - datastore._put( - object_name=minio_object_name, - buffer=buffer, - ) - received_data = datastore._get( - object_name=minio_object_name, - ) - # ASSERT - assert isinstance(received_data, BytesIO) - assert same_data(received_data, buffer) - - -def test_should_update_data( - datastore: Datastore, - data_in_minio, -): - # ARRANGE - buffer, _, object_name = data_in_minio - # ACT - datastore._put( - object_name=object_name, - buffer=buffer, - ) - received_data = datastore._get( - object_name=object_name, - ) - # ASSERT - assert same_data(received_data, buffer) - - -if __name__ == '__main__': - pytest.main() diff --git a/shared/datastore/tests/integration/conftest.py b/shared/datastore/tests/integration/conftest.py deleted file mode 100644 index 741fb25..0000000 --- a/shared/datastore/tests/integration/conftest.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Integration test configurations.""" - -import os -import random -from collections.abc import Iterator -from hashlib import md5 -from io import BytesIO -from pathlib import Path - -import pytest -import torch -from dotenv import load_dotenv -from PIL import Image - -from model.src.models import VisualCommunicationModel -from shared.datastore import Datastore - -env_var_map = { - 'MINIO_BUCKET_NAME': 'test-bucket', - 'MINIO_OBJECT_NAME': '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX', -} - - -@pytest.fixture(scope='session', autouse=True) -def populate_env( - request: pytest.FixtureRequest, -) -> None: - """Populate environment with variables used for testing.""" - # read env-file for local testing - load_dotenv( - dotenv_path=Path(__file__).parent.parent.parent.parent.parent / 'server.env', - ) - # update env - for key, val in env_var_map.items(): - os.environ[key] = val - - # ensure cleanup - def cleanup_env(): - for key in env_var_map: - _ = os.environ.pop(key, default=None) - - request.addfinalizer(cleanup_env) - - -@pytest.fixture(scope='session') -def datastore( - populate_env, -) -> Iterator[Datastore]: - # prepare arguments - minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) - # connect to minio - datastore_client = Datastore() - datastore_client.connect() - assert datastore_client._client is not None - # ensure bucket exists - if not datastore_client._client.bucket_exists(bucket_name=minio_bucket_name): - datastore_client._client.make_bucket(bucket_name=minio_bucket_name) - # expose client - yield datastore_client - # remove objects left behind by tests - for obj in datastore_client._client.list_objects( - bucket_name=minio_bucket_name, - recursive=True, - ): - datastore_client._client.remove_object( - bucket_name=obj.bucket_name, - object_name=obj.object_name, - ) - # remove bucket - datastore_client._client.remove_bucket(bucket_name=minio_bucket_name) - assert not datastore_client._client.bucket_exists(bucket_name=minio_bucket_name) - # disconnect from minio - datastore_client.close() - - -@pytest.fixture -def data() -> Iterator[bytes]: - # generate random data - num_bytes = 2**21 # 2 MB - data = random.randbytes(n=num_bytes) - # expose data - yield data - - -@pytest.fixture -def data_in_minio( - datastore: Datastore, - data: bytes, -) -> Iterator[tuple[BytesIO, str, str]]: - assert datastore._client is not None - # prepare arguments - minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) - minio_object_name = str(os.getenv('MINIO_OBJECT_NAME')) - # convert data - buffer = BytesIO(data) - # prepare for saving - num_bytes = len(buffer.getvalue()) - buffer.seek(0) - # send data to bucket - datastore._client.put_object( - bucket_name=minio_bucket_name, - object_name=minio_object_name, - length=num_bytes, - data=buffer, - ) - # expose data - yield buffer, minio_bucket_name, minio_object_name - # clean up - datastore._client.remove_object( - bucket_name=minio_bucket_name, - object_name=minio_object_name, - ) - - -@pytest.fixture -def image() -> Iterator[Image.Image]: - # generate image - image = Image.new(mode='RGB', size=(480, 480)) - # expose image - yield image - - -@pytest.fixture -def image_in_minio( - datastore: Datastore, - image: Image.Image, -) -> Iterator[tuple[Image.Image, str]]: - assert datastore._client is not None - # prepare arguments - minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) - # save data to buffer - buffer = BytesIO() - image.save(buffer, 'png') - # get md5 of buffer - checksum = md5(buffer.getbuffer()).hexdigest() - # build object path - object_path = f'images/{checksum}' - # prepare for saving - num_bytes = len(buffer.getvalue()) - buffer.seek(0) - # send data to bucket - datastore._client.put_object( - bucket_name=minio_bucket_name, - object_name=object_path, - length=num_bytes, - data=buffer, - ) - # expose image and object name - yield image, checksum - # cleanup - datastore._client.remove_object( - bucket_name=minio_bucket_name, - object_name=object_path, - ) - - -@pytest.fixture -def model() -> Iterator[torch.nn.Module]: - # generate model - model = VisualCommunicationModel().to('cpu') - # expose model - yield model - - -@pytest.fixture -def model_in_minio( - datastore: Datastore, - model: torch.nn.Module, -) -> Iterator[tuple[torch.nn.Module, str]]: - assert datastore._client is not None - # prepare arguments - minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) - # save data to buffer - buffer = BytesIO() - torch.save(model.state_dict(), buffer) - # get md5 of image - checksum = md5(buffer.getbuffer()).hexdigest() - # build object path - object_path = f'models/{checksum}' - # prepare for saving - num_bytes = len(buffer.getvalue()) - buffer.seek(0) - # send data to bucket - datastore._client.put_object( - bucket_name=minio_bucket_name, - object_name=object_path, - length=num_bytes, - data=buffer, - ) - # expose model and object name - yield model, checksum - # cleanup - datastore._client.remove_object( - bucket_name=minio_bucket_name, - object_name=object_path, - ) diff --git a/shared/datastore/tests/integration/connect_test.py b/shared/datastore/tests/integration/connect_test.py deleted file mode 100644 index ea5868e..0000000 --- a/shared/datastore/tests/integration/connect_test.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Integration test for connect_minio function.""" - -from minio import Minio - -from shared.datastore import Datastore - - -def test_should_return_correct_type(): - with Datastore() as ds: - assert isinstance(ds._client, Minio) diff --git a/shared/datastore/tests/integration/image_crud_test.py b/shared/datastore/tests/integration/image_crud_test.py deleted file mode 100644 index c3b0dbb..0000000 --- a/shared/datastore/tests/integration/image_crud_test.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Integration tests related to image CRUD.""" - -import numpy as np -import pytest -from PIL import Image - -from shared.datastore import Datastore - - -def same_image( - img_a: Image.Image, - img_b: Image.Image, -) -> bool: - """Check if two images contain the same data.""" - assert isinstance(img_a, Image.Image) - assert isinstance(img_b, Image.Image) - # check if images have a comparable number of channels - if img_a.getbands() != img_b.getbands(): - return False - # calculate pixel difference between images - img_a_arr = np.asarray(img_a) - img_b_arr = np.asarray(img_b) - diff = np.subtract(img_a_arr, img_b_arr) - if np.sum(diff) != 0: - return False - return True - - -def test_should_get_image( - datastore: Datastore, - image_in_minio: tuple[Image.Image, str], -): - # ARRANGE - image, object_name = image_in_minio - # ACT - received_image = datastore.get_image( - object_name=object_name, - ) - # ASSERT - assert isinstance(image, Image.Image) - assert same_image(image, received_image) - - -def test_should_put_image( - datastore: Datastore, - image: Image.Image, -): - # ARRANGE - object_name = datastore.put_image( - image=image, - ) - assert isinstance(object_name, str) - assert len(object_name) > 0 - # ACT - received_image = datastore.get_image( - object_name=object_name, - ) - # ASSERT - assert same_image(image, received_image) - - -def test_should_update_image( - datastore: Datastore, - image_in_minio: tuple[Image.Image, str], -): - # ARRANGE - image, object_name = image_in_minio - object_name = datastore.put_image( - image=image, - ) - assert isinstance(object_name, str) - assert len(object_name) > 0 - # ACT - received_image = datastore.get_image( - object_name=object_name, - ) - # ASSERT - assert same_image(image, received_image) - - -if __name__ == '__main__': - pytest.main() diff --git a/shared/datastore/tests/integration/model_crud_test.py b/shared/datastore/tests/integration/model_crud_test.py deleted file mode 100644 index 2f41472..0000000 --- a/shared/datastore/tests/integration/model_crud_test.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Integration tests related to model CRUD.""" - -from collections import OrderedDict - -from torch.nn import Module - -from model.src.models import VisualCommunicationModel -from shared.datastore import Datastore - - -def same_model( - model_a: Module, - model_b: Module, -) -> bool: - """Check if two models are the same class, have the same number of - parameters and contain the same weights.""" - assert isinstance(model_a, Module) - assert isinstance(model_b, Module) - # compare model classes - assert type(model_a) is type(model_b) - # compare number of parameters - params_a = list(model_a.parameters()) - params_b = list(model_b.parameters()) - if len(params_a) != len(params_b): - return False - # compare model weights - for p_a, p_b in zip(params_a, params_b): - if p_a.data.ne(p_b.data).sum() > 0: - return False - return True - - -def test_should_get_model( - datastore: Datastore, - model_in_minio: tuple[Module, str], -): - # ARRANGE - model, object_name = model_in_minio - # ACT - model_data = datastore.get_model( - object_name=object_name, - ) - assert isinstance(model_data, OrderedDict) - received_model = VisualCommunicationModel().to('cpu') - received_model.load_state_dict(model_data) - # ASSERT - assert same_model(model, received_model) - - -def test_should_put_model( - datastore: Datastore, - model: Module, -): - # ARRANGE - object_name = datastore.put_model( - model=model, - ) - assert isinstance(object_name, str) - assert len(object_name) > 0 - # ACT - model_data = datastore.get_model( - object_name=object_name, - ) - assert isinstance(model_data, OrderedDict) - received_model = VisualCommunicationModel().to('cpu') - received_model.load_state_dict(model_data) - # ASSERT - assert same_model(model, received_model) - - -def test_should_update_model( - datastore: Datastore, - model_in_minio: tuple[Module, str], -): - # ARRANGE - model, object_name = model_in_minio - object_name = datastore.put_model( - model=model, - ) - assert isinstance(object_name, str) - assert len(object_name) > 0 - # ACT - model_data = datastore.get_model( - object_name=object_name, - ) - assert isinstance(model_data, OrderedDict) - received_model = VisualCommunicationModel().to('cpu') - received_model.load_state_dict(model_data) - # ASSERT - assert same_model(model, received_model) diff --git a/shared/datastore/tests/unit/datastore_minio_test.py b/shared/datastore/tests/unit/datastore_minio_test.py deleted file mode 100644 index e23ee67..0000000 --- a/shared/datastore/tests/unit/datastore_minio_test.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Definition of unittests for Datastore Minio implementation.""" - -import os -from hashlib import md5 -from io import BytesIO -from unittest import TestCase -from unittest.mock import ANY, MagicMock, patch - -from minio import Minio -from PIL import Image -from urllib3 import BaseHTTPResponse - -from shared.datastore.src.datastore_minio import DatastoreMinio - - -class TestDatastoreMinioImplementation(TestCase): - - def setUp(self): - # define relevant env vars - self.env_var_map = { - 'MINIO_ENDPOINT': '192.168.1.2', - 'MINIO_ACCESS_KEY': 'randomAccess_key', - 'MINIO_SECRET_KEY': 'randomSecret_key', - 'MINIO_BUCKET_NAME': 'test-bucket-name', - } - # set env vars - for key, val in self.env_var_map.items(): - os.environ[key] = val - # set other variables - self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX' - self.image = Image.new(mode='RGB', size=(480, 480)) - self.buffer = BytesIO() - self.image.save(self.buffer, 'png') - self.num_bytes = len(self.buffer.getvalue()) - self.checksum = md5(self.buffer.getbuffer()).hexdigest() - - def tearDown(self): - # clear env vars - for key in self.env_var_map: - _ = os.environ.pop(key, default=None) - - def test_instantiation_should_fail_when_env_not_set(self): - # ensure env not set - self.tearDown() - # run test - with self.assertRaises(OSError): - _ = DatastoreMinio() - - @patch('shared.datastore.src.datastore_minio.Minio') - def test_connect_should_call_Minio_with_env_vars(self, minio_mock): - # ARRANGE - datastore = DatastoreMinio() - minio_mock().bucket_exists.return_value = False - # ACT - datastore.connect() - # ASSERT - minio_mock.assert_called_with( - endpoint=self.env_var_map['MINIO_ENDPOINT'], - access_key=self.env_var_map['MINIO_ACCESS_KEY'], - secret_key=self.env_var_map['MINIO_SECRET_KEY'], - secure=False, - ) - minio_mock().bucket_exists.assert_called_with( - bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], - ) - minio_mock().make_bucket.assert_called_with( - bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], - ) - - def test_close_should_overwrite_private_variables(self): - # ARRANGE - datastore = DatastoreMinio() - datastore._client = MagicMock() - datastore._bucket_name = MagicMock() - assert isinstance(datastore._client, MagicMock) - assert isinstance(datastore._bucket_name, MagicMock) - # ACT - datastore.close() - # ASSERT - assert datastore._client is None - assert datastore._bucket_name is None - - @patch('shared.datastore.src.datastore_minio.DatastoreMinio.connect') - @patch('shared.datastore.src.datastore_minio.DatastoreMinio.close') - def test_context_management_implemented( - self, - mocked_close_method, - mocked_connect_method, - ): - # ARRANGE, ACT and ASSERT - with DatastoreMinio() as ds: - mocked_connect_method.assert_called_once() - assert isinstance(ds, DatastoreMinio) - mocked_close_method.assert_called_once() - - def test_should_call_put_object_with_arguments(self): - # ARRANGE - datastore = DatastoreMinio() - datastore._client = MagicMock(Minio) - datastore._bucket_name = self.env_var_map['MINIO_BUCKET_NAME'] - self.buffer.seek(0) - # ACT - datastore._put( - object_name=self.object_name, - buffer=self.buffer, - ) - # ASSERT - datastore._client.put_object.assert_called_with( - bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], - object_name=self.object_name, - length=self.num_bytes, - data=self.buffer, - ) - - def test_should_call_get_object_with_arguments(self): - # ARRANGE - datastore = DatastoreMinio() - datastore._client = MagicMock(Minio) - datastore._client.get_object.return_value = MagicMock( - BaseHTTPResponse, - status=200, - ) - # datastore._client.get_object.read.side_effect = [b'random ', b'test', b'text'] - datastore._bucket_name = self.env_var_map['MINIO_BUCKET_NAME'] - # ACT - with self.assertRaises(TypeError): # dont care to mock even more... - datastore._get( - object_name=self.object_name, - ) - # ASSERT - datastore._client.get_object.assert_called_with( - bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], - object_name=self.object_name, - ) - - def test_should_call_remove_object_with_arguments(self): - # ARRANGE - datastore = DatastoreMinio() - datastore._client = MagicMock(Minio) - datastore._bucket_name = self.env_var_map['MINIO_BUCKET_NAME'] - # ACT - datastore._delete( - object_name=self.object_name, - ) - # ASSERT - datastore._client.remove_object.assert_called_with( - bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], - object_name=self.object_name, - ) - - def test_put_image_should_call_put_object_with_arguments(self): - # ARRANGE - datastore = DatastoreMinio() - datastore._client = MagicMock(Minio) - datastore._bucket_name = self.env_var_map['MINIO_BUCKET_NAME'] - object_path = f'images/{self.checksum}' - # ACT - datastore.put_image( - image=self.image, - ) - # ASSERT - datastore._client.put_object.assert_called_with( - bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], - object_name=object_path, - length=self.num_bytes, - data=ANY, # saved to different buffer when converting image - ) - - def test_get_image_should_call_get_object_with_arguments(self): - # ARRANGE - datastore = DatastoreMinio() - datastore._client = MagicMock(Minio) - datastore._client.get_object.return_value = MagicMock( - BaseHTTPResponse, - status=200, - ) - datastore._bucket_name = self.env_var_map['MINIO_BUCKET_NAME'] - object_path = f'images/{self.checksum}' - # ACT - with self.assertRaises(TypeError): # dont care to mock even more... - datastore.get_image( - object_name=self.checksum, - ) - # ASSERT - datastore._client.get_object.assert_called_with( - bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], - object_name=object_path, - ) - - # @patch('shared.datastore.src.datastore_minio.torch.serialization.save') - # def test_put_model_should_call_put_object_with_arguments(self, mocked_torch_fn): - # # ARRANGE - # datastore = DatastoreMinio() - # datastore._client = MagicMock(Minio) - # datastore._bucket_name = self.env_var_map['MINIO_BUCKET_NAME'] - # object_path = f'images/{self.checksum}' - # mocked_torch_fn.return_value = None - # model = MagicMock(torch.nn.Module) - # # ACT - # datastore.put_model( - # model=model, - # ) - # # ASSERT - # datastore._client.put_object.assert_called_with( - # bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], - # object_name=object_path, - # length=self.num_bytes, - # data=ANY, # saved to different buffer when converting data - # ) - - def test_get_model_should_call_get_object_with_arguments(self): - # ARRANGE - datastore = DatastoreMinio() - datastore._client = MagicMock(Minio) - datastore._client.get_object.return_value = MagicMock( - BaseHTTPResponse, - status=200, - ) - datastore._bucket_name = self.env_var_map['MINIO_BUCKET_NAME'] - object_path = f'models/{self.checksum}' - # ACT - with self.assertRaises(TypeError): # dont care to mock even more... - datastore.get_model( - object_name=self.checksum, - ) - # ASSERT - datastore._client.get_object.assert_called_with( - bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], - object_name=object_path, - ) diff --git a/shared/docstore/__init__.py b/shared/docstore/__init__.py deleted file mode 100644 index 6ca8e30..0000000 --- a/shared/docstore/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .src import classes, exceptions -from .src.connect_mongodb import connect_mongodb -from .src.count_documents import count_documents -from .src.get_visual_communication import get_visual_communication -from .src.list_names import list_names -from .src.upsert_annotation import upsert_annotation -from .src.upsert_prediction import upsert_prediction -from .src.upsert_visual_communication import upsert_visual_communication diff --git a/shared/docstore/src/__init__.py b/shared/docstore/src/__init__.py deleted file mode 100755 index c6267c3..0000000 --- a/shared/docstore/src/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Database utils module content.""" - -from .connect_mongodb import connect_mongodb diff --git a/shared/docstore/src/classes/__init__.py b/shared/docstore/src/classes/__init__.py deleted file mode 100755 index 84090da..0000000 --- a/shared/docstore/src/classes/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from .angle import AngleData -from .contact import ContactData -from .distance import DistanceData -from .framing import FramingData -from .information_value import InformationValueData -from .modality_color import ModalityColorData -from .modality_depth import ModalityDepthData -from .modality_lighting import ModalityLightingData -from .model_data import ModelData -from .point_of_view import PointOfViewData -from .salience import SalienceData -from .visual_communication import VisualCommunication -from .visual_syntax import VisualSyntaxData diff --git a/shared/docstore/src/classes/angle.py b/shared/docstore/src/classes/angle.py deleted file mode 100644 index 12491da..0000000 --- a/shared/docstore/src/classes/angle.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Definition of Angle data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class AngleData(DataModel): - """Angle data model.""" - - high: float - eye_level: float - low: float diff --git a/shared/docstore/src/classes/contact.py b/shared/docstore/src/classes/contact.py deleted file mode 100644 index fc08a0a..0000000 --- a/shared/docstore/src/classes/contact.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Definition of ContactData data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class ContactData(DataModel): - """ContactData data model.""" - - offer: float - demand: float diff --git a/shared/docstore/src/classes/data_model.py b/shared/docstore/src/classes/data_model.py deleted file mode 100644 index f19b0c8..0000000 --- a/shared/docstore/src/classes/data_model.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Definition of DataModel base class.""" - -import random - -from pydantic import BaseModel, ValidationError -from torch import Tensor - - -class DataModel(BaseModel): - """DataModel base class.""" - - @classmethod - def classname(cls) -> str: - """Return classname.""" - return cls.__name__ - - @classmethod - def list_fields(cls) -> list[str]: - """List options that are stored as attributes.""" - return list(cls.model_fields.keys()) - - @classmethod - def from_random(cls): - """Instantiate with random numbers.""" - kwargs = {field: random.random() for field in cls.list_fields()} - return cls(**kwargs) - - @classmethod - def from_choice(cls, option: str): - """Instantiate from choice.""" - if option is None: - raise ValidationError() - assert isinstance(option, str), 'option is not a string' - allowed_options_list = cls.list_fields() - assert ( - option in allowed_options_list - ), f"{option} is not among allowed fields {allowed_options_list}" - kwargs = {field: 0 for field in cls.list_fields()} - kwargs[option] = 1 - return cls(**kwargs) - - @classmethod - def from_tensor(cls, tensor: Tensor): - """Instantiate from list of values.""" - assert tensor.size(dim=0) == 1, f'tensor batch larger than 1: {tensor}' - data_list = [float(t.item()) for t in tensor[0]] - kwargs = dict(zip(cls.list_fields(), data_list)) - return cls(**kwargs) - - def __repr__(self) -> str: - model_dict = self.model_dump() - model_repr_str = f'{self.classname()}(' - model_repr_str += ', '.join( - [f'{field}={value:.3f}' for field, value in model_dict.items()], - ) - model_repr_str += ')' - return model_repr_str - - def highest_score_field(self) -> str: - """Return name of field with highest score.""" - model_dict = self.model_dump() - return max(model_dict, key=lambda k: model_dict[k]) - - def highest_score_value(self) -> float: - """Return value of field with highest score.""" - model_dict = self.model_dump() - return max(model_dict.values()) diff --git a/shared/docstore/src/classes/distance.py b/shared/docstore/src/classes/distance.py deleted file mode 100644 index 0aebbe6..0000000 --- a/shared/docstore/src/classes/distance.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Definition of DistanceData data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class DistanceData(DataModel): - """DistanceData data model.""" - - long: float - medium: float - close: float diff --git a/shared/docstore/src/classes/framing.py b/shared/docstore/src/classes/framing.py deleted file mode 100644 index eca22e3..0000000 --- a/shared/docstore/src/classes/framing.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Definition of FramingData data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class FramingData(DataModel): - """FramingData data model.""" - - frame_lines: float - empty_space: float - colour_contrast: float - form_contrast: float diff --git a/shared/docstore/src/classes/information_value.py b/shared/docstore/src/classes/information_value.py deleted file mode 100644 index fd4bf48..0000000 --- a/shared/docstore/src/classes/information_value.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Definition of InformationValueData data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class InformationValueData(DataModel): - """InformationValueData data model.""" - - given_new: float - ideal_real: float - central_marginal: float diff --git a/shared/docstore/src/classes/modality_color.py b/shared/docstore/src/classes/modality_color.py deleted file mode 100644 index 458e568..0000000 --- a/shared/docstore/src/classes/modality_color.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Definition of ModalityColorData data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class ModalityColorData(DataModel): - """ModalityColorData data model.""" - - high: float - medium: float - low: float diff --git a/shared/docstore/src/classes/modality_depth.py b/shared/docstore/src/classes/modality_depth.py deleted file mode 100644 index e0446f4..0000000 --- a/shared/docstore/src/classes/modality_depth.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Definition of ModalityDepthData data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class ModalityDepthData(DataModel): - """ModalityDepthData data model.""" - - high: float - medium: float - low: float diff --git a/shared/docstore/src/classes/modality_lighting.py b/shared/docstore/src/classes/modality_lighting.py deleted file mode 100644 index 508ad57..0000000 --- a/shared/docstore/src/classes/modality_lighting.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Definition of ModalityLightingData data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class ModalityLightingData(DataModel): - """ModalityLightingData data model.""" - - high: float - medium: float - low: float diff --git a/shared/docstore/src/classes/model_data.py b/shared/docstore/src/classes/model_data.py deleted file mode 100644 index 5bbbf07..0000000 --- a/shared/docstore/src/classes/model_data.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Definition of ModelData data model.""" - -from __future__ import annotations - -from .angle import AngleData -from .contact import ContactData -from .data_model import DataModel -from .distance import DistanceData -from .framing import FramingData -from .information_value import InformationValueData -from .modality_color import ModalityColorData -from .modality_depth import ModalityDepthData -from .modality_lighting import ModalityLightingData -from .point_of_view import PointOfViewData -from .salience import SalienceData -from .visual_syntax import VisualSyntaxData - - -class ModelData(DataModel): - """ModelData model for data IO with combined ML model.""" - - visual_syntax: VisualSyntaxData - contact: ContactData - angle: AngleData - point_of_view: PointOfViewData - distance: DistanceData - modality_lighting: ModalityLightingData - modality_color: ModalityColorData - modality_depth: ModalityDepthData - information_value: InformationValueData - framing: FramingData - salience: SalienceData - - @classmethod - def from_random(cls) -> ModelData: - """Instantiate with random numbers.""" - kwargs = { - field: field_info.annotation.from_random() # type: ignore - for field, field_info in cls.model_fields.items() - } - return cls(**kwargs) - - @classmethod - def from_prediction_dict( - cls, - prediction_dict: dict, - ) -> ModelData: - """Instantiate from prediction dictionary.""" - kwargs = { - 'visual_syntax': VisualSyntaxData.from_tensor( - prediction_dict['visual_syntax'], - ), - 'contact': ContactData.from_tensor( - prediction_dict['contact'], - ), - 'angle': AngleData.from_tensor( - prediction_dict['angle'], - ), - 'point_of_view': PointOfViewData.from_tensor( - prediction_dict['point_of_view'], - ), - 'distance': DistanceData.from_tensor( - prediction_dict['distance'], - ), - 'modality_lighting': ModalityLightingData.from_tensor( - prediction_dict['modality_lighting'], - ), - 'modality_color': ModalityColorData.from_tensor( - prediction_dict['modality_color'], - ), - 'modality_depth': ModalityDepthData.from_tensor( - prediction_dict['modality_depth'], - ), - 'information_value': InformationValueData.from_tensor( - prediction_dict['information_value'], - ), - 'framing': FramingData.from_tensor( - prediction_dict['framing'], - ), - 'salience': SalienceData.from_tensor( - prediction_dict['salience'], - ), - } - return cls(**kwargs) - - @classmethod - def from_annotations( - cls, - visual_syntax: str, - contact: str, - angle: str, - point_of_view: str, - distance: str, - modality_lighting: str, - modality_color: str, - modality_depth: str, - information_value: str, - framing: str, - salience: str, - ) -> ModelData: - """Instantiate from annotation.""" - kwargs = { - 'visual_syntax': VisualSyntaxData.from_choice(visual_syntax), - 'contact': ContactData.from_choice(contact), - 'angle': AngleData.from_choice(angle), - 'point_of_view': PointOfViewData.from_choice(point_of_view), - 'distance': DistanceData.from_choice(distance), - 'modality_lighting': ModalityLightingData.from_choice(modality_lighting), - 'modality_color': ModalityColorData.from_choice(modality_color), - 'modality_depth': ModalityDepthData.from_choice(modality_depth), - 'information_value': InformationValueData.from_choice(information_value), - 'framing': FramingData.from_choice(framing), - 'salience': SalienceData.from_choice(salience), - } - return cls(**kwargs) diff --git a/shared/docstore/src/classes/point_of_view.py b/shared/docstore/src/classes/point_of_view.py deleted file mode 100644 index b1e4543..0000000 --- a/shared/docstore/src/classes/point_of_view.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Definition of PointOfViewData data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class PointOfViewData(DataModel): - """PointOfViewData data model.""" - - frontal: float - oblique: float diff --git a/shared/docstore/src/classes/salience.py b/shared/docstore/src/classes/salience.py deleted file mode 100644 index 186132c..0000000 --- a/shared/docstore/src/classes/salience.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Definition of SalienceData data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class SalienceData(DataModel): - """SalienceData data model.""" - - size: float - colour: float - tone: float - form: float - positioning: float diff --git a/shared/docstore/src/classes/visual_communication.py b/shared/docstore/src/classes/visual_communication.py deleted file mode 100755 index e39afc2..0000000 --- a/shared/docstore/src/classes/visual_communication.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Definition of VisualCommunication model.""" - -from __future__ import annotations - -import logging -from base64 import b64decode, b64encode -from io import BytesIO -from pathlib import Path - -from minio import Minio -from PIL import Image -from pydantic import BaseModel, ConfigDict -from pymongo.collection import Collection - -from shared.datastore import Datastore -from shared.docstore.src.classes import ModelData - - -class VisualCommunication(BaseModel): - """Visual communication model.""" - - name: str - object_name: str - annotation: ModelData | None = None - prediction: ModelData | None = None - model_config = ConfigDict(arbitrary_types_allowed=True) - - @classmethod - def classname(cls) -> str: - """Return classname.""" - return cls.__name__ - - @classmethod - def upload_image_to_minio( - cls, - image: Image.Image, - minio_client: Minio, - ) -> str: - """Upload image to MinIO and return MD5 checksum of hashed image.""" - assert isinstance(image, Image.Image) - assert isinstance(minio_client, Minio) - with Datastore() as ds: - object_name = ds.put_image( - image=image, - ) - return object_name - - @classmethod - def from_name_and_image( - cls, - name: str, - image: Image.Image, - minio_client: Minio, - ) -> VisualCommunication: - """Instantiate from filename and image that is automatically uploaded - to MinIO.""" - assert isinstance(name, str) - assert isinstance(image, Image.Image) - assert isinstance(minio_client, Minio) - # upload file to minio - object_name = VisualCommunication.upload_image_to_minio( - image=image, - minio_client=minio_client, - ) - return VisualCommunication(name=name, object_name=object_name) - - @classmethod - def from_file(cls, path: Path, minio_client: Minio) -> VisualCommunication: - """Instantiate from file.""" - assert isinstance(path, Path) - assert isinstance(minio_client, Minio) - # determine name - name = path.stem - # open image - image = Image.open(path) - image.load() - # instantiate object - return VisualCommunication.from_name_and_image( - name=name, - image=image, - minio_client=minio_client, - ) - - @classmethod - def decode_image(cls, content: str) -> Image.Image: - """Extract image from webencoded content.""" - _, content_data = content.split(',') - return Image.open(BytesIO(b64decode(content_data))) - - def get_image(self, minio_client: Minio) -> Image.Image: - """Load image data from minio.""" - assert isinstance(minio_client, Minio) - # get image from minio - with Datastore() as ds: - image = ds.get_image( - object_name=self.object_name, - ) - return image - - def save_to_mongo(self, collection: Collection) -> None: - """Save self as document in MongoDB.""" - res = collection.insert_one( - document=self.model_dump(), - ) - assert res.acknowledged - - def webencoded_image(self, minio_client: Minio) -> str: - """Convert image to be displayed on webpage.""" - assert isinstance(minio_client, Minio) - # get image from minio - image = self.get_image(minio_client) - # convert images to bytes string - buffer = BytesIO() - image.save(buffer, format='png') - img_enc = b64encode(buffer.getvalue()).decode('utf-8') - return f"data:image/png;base64, {img_enc}" - - def generate_random_prediction(self, force: bool = False) -> None: - """Generate random prediction values.""" - if not force and self.prediction is not None: - logging.warning('set force=True to overwrite existing values.') - self.prediction = ModelData.from_random() diff --git a/shared/docstore/src/classes/visual_syntax.py b/shared/docstore/src/classes/visual_syntax.py deleted file mode 100644 index 4483e7f..0000000 --- a/shared/docstore/src/classes/visual_syntax.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Definition of VisualSyntaxData data model.""" - -from __future__ import annotations - -from .data_model import DataModel - - -class VisualSyntaxData(DataModel): - """VisualSyntaxData data model.""" - - non_transactional_action: float - non_transactional_reaction: float - unidirectional_transactional_action: float - unidirectional_transactional_reaction: float - bidirectional_transactional_action: float - bidirectional_transactional_reaction: float - conversion: float - speech_process: float - classification_overt_taxonomy: float - analytical_exhaustive: float - analytical_disarranged: float - analytical_temporal: float - analytical_distributed: float - analytical_topological: float - analytical_exploded: float - analytical_inclusive: float - symbolic_suggestive: float - symbolic_attributive: float diff --git a/shared/docstore/src/connect_mongodb.py b/shared/docstore/src/connect_mongodb.py deleted file mode 100644 index c9743bc..0000000 --- a/shared/docstore/src/connect_mongodb.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Definition of function to connect to database using environment -variables.""" - -import logging -import os - -from dotenv import load_dotenv -from pymongo import MongoClient - - -def connect_mongodb(): - """Connect to MongoDB using env vars.""" - # load env vars - load_dotenv() - necessary_env_vars = [ - 'MONGO_HOST', - 'MONGO_DB', - 'MONGO_COLLECTION', - ] - for env_var in necessary_env_vars: - assert env_var in os.environ, f"{env_var} not found" - # connect to database - client = MongoClient(os.getenv('MONGO_HOST')) - db = client[os.getenv('MONGO_DB')] - # extract collection - collection = db[os.getenv('MONGO_COLLECTION')] - # set unique index on "name" - collection.create_index('name', unique=True) - logging.debug('finished') - return collection, db, client diff --git a/shared/docstore/src/count_documents.py b/shared/docstore/src/count_documents.py deleted file mode 100755 index eb9af6b..0000000 --- a/shared/docstore/src/count_documents.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Definition of function to count documents in database.""" - -from __future__ import annotations - -from pymongo.collection import Collection - - -def count_documents( - collection: Collection, - only_with_annotation: bool = False, -) -> int: - """Get the total number of documents in database that matches the - filters.""" - assert isinstance(collection, Collection) - assert isinstance(only_with_annotation, bool) - # build query - query = {} - if only_with_annotation: - query['annotation'] = {'$ne': None} - return collection.count_documents(filter=query) diff --git a/shared/docstore/src/docstore_interface.py b/shared/docstore/src/docstore_interface.py deleted file mode 100644 index eba250c..0000000 --- a/shared/docstore/src/docstore_interface.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Definition of docstore interface.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod - -from shared.docstore.src.classes import ModelData, VisualCommunication - - -class DocstoreInterface(ABC): - """Docstore interface class.""" - - @abstractmethod - def connect( - self, - ) -> None: - pass - - @abstractmethod - def close( - self, - ) -> None: - pass - - @abstractmethod - def __enter__( - self, - ) -> DocstoreInterface: - pass - - @abstractmethod - def __exit__( - self, - exc_type, - exc_val, - exc_tb, - ) -> None: - pass - - @abstractmethod - def count_documents( - self, - only_with_annotation: bool, - ) -> int: - pass - - @abstractmethod - def list_names( - self, - only_with_annotation: bool, - ) -> list[str]: - pass - - @abstractmethod - def upsert_visual_comminucations( - self, - visual_communication_list: list[VisualCommunication], - ) -> None: - pass - - @abstractmethod - def upsert_annotations( - self, - visual_communication_name: str, - annotations: ModelData, - ) -> None: - pass - - @abstractmethod - def upsert_prediction( - self, - visual_communication_name: str, - predictions: ModelData, - ) -> None: - pass - - @abstractmethod - def get_visual_communication( - self, - with_annotation: bool, - name: str | None = None, - ) -> VisualCommunication: - pass diff --git a/shared/docstore/src/docstore_mongo.py b/shared/docstore/src/docstore_mongo.py deleted file mode 100644 index cd665b8..0000000 --- a/shared/docstore/src/docstore_mongo.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Definition of docstore mongodb implementation.""" - -from __future__ import annotations - -import logging -import os -from typing import Any - -from pymongo import MongoClient -from pymongo.collection import Collection -from pymongo.database import Database - -from shared.utils import check_env - -from .classes import ModelData, VisualCommunication -from .docstore_interface import DocstoreInterface -from .exceptions import NoDocumentFoundException - - -class DocstoreMongo(DocstoreInterface): - """Docstore interface.""" - - def __init__(self): - # ensure necessary env vars available - var_list = { - 'MONGO_ENDPOINT', - 'MONGO_DB', - 'MONGO_COLLECTION', - } - check_env(var_list) - # prepare internal variables - self.client: MongoClient | None = None - self.db: Database | None = None - self.collection: Collection | None = None - - def connect(self) -> None: - """Connect to Mongo server.""" - # prepare arguments - mongo_endpoint = str(os.getenv('MONGO_ENDPOINT')) - mongo_database = str(os.getenv('MONGO_DB')) - mongo_collection = str(os.getenv('MONGO_COLLECTION')) - # connect client - client: MongoClient = MongoClient(mongo_endpoint) - database = client[mongo_database] - collection = database[mongo_collection] - # set unique index on 'name' - collection.create_index(keys='name', unique=True) - # persist state - self._client = client - self._database = database - self._collection = collection - - def close(self) -> None: - """Close connection to Mongo server.""" - self._client.close() - self._client = None # type: ignore - self._database = None # type: ignore - self._collection = None # type: ignore - - def __enter__(self) -> DocstoreMongo: - self.connect() - return self - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - if any( - ( - exc_type is not None, - exc_val is not None, - exc_tb is not None, - ), - ): - logging.error('error while exiting context') - self.close() - - def count_documents( - self, - only_with_annotation: bool = False, - ): - """Get the total number of Visual Communication documents matching the - filter.""" - assert isinstance(only_with_annotation, bool) - # build query - query = {} - if only_with_annotation: - query['annotation'] = {'$ne': None} - # execute query - num_docs = self._collection.count_documents(filter=query) - return num_docs - - def list_names( - self, - only_with_annotation: bool, - ) -> list[str]: - """List names of Visual Communication documents that match the - filters.""" - assert isinstance(only_with_annotation, bool) - assert isinstance(self._collection, Collection) - # build query - query = {} - if only_with_annotation: - query['annotation'] = {'$ne': None} - # execute query - res_list = self._collection.find( - filter=query, - projection={ - '_id': False, # don't get id - 'name': True, # include document name - }, - ) - # extract info - name_list = [elem['name'] for elem in res_list] - return name_list - - def upsert_visual_comminucations( - self, - visual_communication_list: list[VisualCommunication], - ) -> None: - """Upsert Visual Communication document in the database.""" - assert isinstance(visual_communication_list, list) - assert all( - isinstance(vis_com, VisualCommunication) - for vis_com in visual_communication_list - ) - assert isinstance(self._collection, Collection) - # convert to dict - doc_list = [vis_com.model_dump() for vis_com in visual_communication_list] - # insert documents - response = self._collection.insert_many(documents=doc_list) - # check response - if not response.acknowledged: - raise OSError('failed inserting documents') - - def upsert_annotations( - self, - visual_communication_name: str, - annotations: ModelData, - ) -> None: - """Upsert annotation for the document with matching name.""" - assert isinstance(visual_communication_name, str) - assert len(visual_communication_name) > 0 - assert isinstance(annotations, ModelData) - assert isinstance(self._collection, Collection) - # convert to dict - doc = annotations.model_dump() - # build query - query = { - 'name': visual_communication_name, - } - update = { - '$set': { - 'annotation': doc, - }, - } - # execute query - res = self._collection.update_one( - filter=query, - update=update, - upsert=True, - ) - # check response - if not res.acknowledged: - raise OSError( - f'failed upserting annotations for {visual_communication_name}', - ) - - def upsert_prediction( - self, - visual_communication_name: str, - predictions: ModelData, - ) -> None: - """Upsert prediction for the document with matching name.""" - assert isinstance(visual_communication_name, str) - assert len(visual_communication_name) > 0 - assert isinstance(annotations, ModelData) - assert isinstance(self._collection, Collection) - # convert to dict - doc = predictions.model_dump() - # build query - query = { - 'name': visual_communication_name, - } - update = { - '$set': { - 'prediction': doc, - }, - } - # execute query - res = self._collection.update_one( - filter=query, - update=update, - upsert=True, - ) - # check response - if not res.acknowledged: - raise OSError( - f'failed upserting predictions for {visual_communication_name}', - ) - - def get_visual_communication( - self, - with_annotation: bool = False, - name: str | None = None, - ) -> VisualCommunication: - """Get a random Visual Communication document that matches annotation - filter. - - If name is not specified, a random document matching filter is - returned. - """ - assert isinstance(with_annotation, bool) - if name is not None: - assert isinstance(name, str) - assert len(name) > 0 - # build query - query: dict[str, Any] = {} - if with_annotation: - query['annotation'] = {'$ne': None} - else: - query['annotation'] = {'$eq': None} - if name is not None: - query['name'] = {'$eq': name} - # execute query - res_list = self._collection.aggregate( - pipeline=[ - { - '$match': query, # find using filters - }, - { - '$sample': { - 'size': 1, # get one random - }, - }, - ], - ) - doc_list = list(res_list) - # check result - if len(doc_list) == 0: - raise NoDocumentFoundException() - # convert - vis_com = VisualCommunication.model_validate(doc_list[0]) - return vis_com diff --git a/shared/docstore/src/exceptions/__init__.py b/shared/docstore/src/exceptions/__init__.py deleted file mode 100644 index 6daba63..0000000 --- a/shared/docstore/src/exceptions/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .no_document_found import NoDocumentFoundException diff --git a/shared/docstore/src/exceptions/no_document_found.py b/shared/docstore/src/exceptions/no_document_found.py deleted file mode 100755 index dfe1c99..0000000 --- a/shared/docstore/src/exceptions/no_document_found.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Definition of database exception.""" - - -class NoDocumentFoundException(Exception): - """Database exception for when no documents are found.""" diff --git a/shared/docstore/src/get_dataset.py b/shared/docstore/src/get_dataset.py deleted file mode 100755 index 0c823cc..0000000 --- a/shared/docstore/src/get_dataset.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Definition of get_dataset function.""" - -from typing import Literal - -from pymongo.collection import Collection - - -def get_dataset( - collection: Collection, - type: Literal['train', 'test', 'validation'], -) -> list[str]: - """Get list of data names for the corresponding type.""" - assert isinstance(collection, Collection) - assert isinstance(type, str) - assert type in ['train', 'test', 'validation'] - return [] diff --git a/shared/docstore/src/get_visual_communication.py b/shared/docstore/src/get_visual_communication.py deleted file mode 100755 index 5c85a88..0000000 --- a/shared/docstore/src/get_visual_communication.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Definition of function to get visual communication from database.""" - -from __future__ import annotations - -import logging - -from pymongo.collection import Collection - -from shared.docstore.src.classes import VisualCommunication -from shared.docstore.src.exceptions import NoDocumentFoundException - - -def get_visual_communication( - collection: Collection, - with_annotation: bool = False, -) -> VisualCommunication: - """Get a random visual communication from the database.""" - query = {} - if with_annotation: - query['annotation'] = {'$ne': None} - else: - query['annotation'] = {'$eq': None} - data = collection.aggregate( - pipeline=[ - { - '$match': query, # find using filters - }, - { - '$sample': { - 'size': 1, # get one random - }, - }, - ], - ) - data_list = list(data) # read data from cursor object - if len(data_list) == 0: - raise NoDocumentFoundException() - vis_com = VisualCommunication.model_validate(data_list[0]) - logging.debug('finished') - return vis_com diff --git a/shared/docstore/src/list_names.py b/shared/docstore/src/list_names.py deleted file mode 100755 index 86e09dc..0000000 --- a/shared/docstore/src/list_names.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Definition of function to list names of all visual communication documents -in database.""" - -from __future__ import annotations - -from pymongo.collection import Collection - - -def list_names( - collection: Collection, - only_with_annotation: bool = True, -) -> list[str]: - """List the names of entries that match the filters.""" - assert isinstance(collection, Collection) - assert isinstance(only_with_annotation, bool) - # build query - query = {} - if only_with_annotation: - query['annotation'] = {'$ne': None} - # execute query - res_list = collection.find( - filter=query, - projection={ - '_id': False, - 'name': True, - }, - ) - # extract information - name_list = [elem['name'] for elem in res_list] - return name_list diff --git a/shared/docstore/src/upsert_annotation.py b/shared/docstore/src/upsert_annotation.py deleted file mode 100755 index 4eec4c0..0000000 --- a/shared/docstore/src/upsert_annotation.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -import logging - -from pymongo.collection import Collection - -from shared.docstore.src.classes import ModelData - - -def upsert_annotation( - collection: Collection, - vis_com_name: str, - annotations: ModelData, -) -> None: - """Upserts annotation data in the database.""" - query = { - 'name': vis_com_name, - } - update = { - '$set': { - 'annotation': annotations.model_dump(), - }, - } - res = collection.update_one( - filter=query, - update=update, - upsert=True, - ) - logging.info('upserted document: %s', res) - logging.info('finished') diff --git a/shared/docstore/src/upsert_prediction.py b/shared/docstore/src/upsert_prediction.py deleted file mode 100755 index 81378a8..0000000 --- a/shared/docstore/src/upsert_prediction.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -import logging - -from pymongo.collection import Collection - -from shared.docstore.src.classes import ModelData - - -def upsert_prediction( - collection: Collection, - vis_com_name: str, - predictions: ModelData, -) -> None: - """Upsert prediction data in the database.""" - query = { - 'name': vis_com_name, - } - update = { - '$set': { - 'prediction': predictions.model_dump(), - }, - } - res = collection.update_one( - filter=query, - update=update, - upsert=True, - ) - logging.debug('upserted document: %s', res) - logging.info('finished') diff --git a/shared/docstore/src/upsert_visual_communication.py b/shared/docstore/src/upsert_visual_communication.py deleted file mode 100755 index b5a863c..0000000 --- a/shared/docstore/src/upsert_visual_communication.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -from pymongo.collection import Collection - -from shared.docstore.src.classes import VisualCommunication - - -def upsert_visual_communication( - collection: Collection, - visual_communication_list: list[VisualCommunication], -) -> bool: - """Upsert VisualCommunication object in the database. - - Returns bool stating success. - """ - response = collection.insert_many( - [vis_com.model_dump() for vis_com in visual_communication_list], - ) - return response.acknowledged diff --git a/shared/docstore/tests/unit/docstore_mongo_test.py b/shared/docstore/tests/unit/docstore_mongo_test.py deleted file mode 100644 index 6e79b39..0000000 --- a/shared/docstore/tests/unit/docstore_mongo_test.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Definition of unittests for Docstore MongoDB implementation.""" - -import os -from unittest import TestCase -from unittest.mock import MagicMock - -from pymongo import MongoClient -from pymongo.collection import Collection -from pymongo.database import Database - -from shared.docstore.src.docstore_mongo import DocstoreMongo - - -class TestDocstoreMongoImplementation(TestCase): - - def setUp(self): - # define relevant env vars - self.env_var_map = { - 'MONGO_ENDPOINT': '192.168.1.2:27017', - 'MONGO_DB': 'visual_critical_discourse_analysis', - 'MONGO_COLLECTION': 'test-collection', - } - # set env vars - for key, val in self.env_var_map.items(): - os.environ[key] = val - # set other variables - self.mongo_client_mock = MagicMock(MongoClient) - self.mongo_database_mock = MagicMock(Database) - self.mongo_collection_mock = MagicMock(Collection) - - def tearDown(self): - # clear env vars - for key in self.env_var_map: - _ = os.environ.pop(key, default=None) - # reset reuseable mocks - self.mongo_client_mock.reset_mock() - self.mongo_database_mock.reset_mock() - self.mongo_collection_mock.reset_mock() - - def test_instantiation_should_fail_when_env_not_set(self): - # ensure env not set - self.tearDown() - # run test - with self.assertRaises(OSError): - _ = DocstoreMongo() diff --git a/shared/repositories/tests/integration/conftest.py b/shared/repositories/tests/integration/conftest.py index ecc1cf0..a44910b 100644 --- a/shared/repositories/tests/integration/conftest.py +++ b/shared/repositories/tests/integration/conftest.py @@ -32,14 +32,17 @@ from shared.repositories.src.implementations import ( random.seed(13) # define test environment variables +necessary_env_vars = { + 'MINIO_ENDPOINT', + 'MINIO_ACCESS_KEY', + 'MINIO_SECRET_KEY', + 'MONGO_ENDPOINT', +} env_var_map = { - 'MINIO_ENDPOINT': '192.168.1.2:9000', - 'MINIO_ACCESS_KEY': '1rMQTezbALeJyOgAbYYE', - 'MINIO_SECRET_KEY': 'O76GM756ybKYLFokZFJneaCoYwpaIbAlOdB7mcc0', + 'MINIO_ENDPOINT': 'localhost:9000', 'MINIO_BUCKET_NAME': 'test-bucket', 'MINIO_OBJECT_NAME': 'test-object', 'MINIO_IMAGE_NAME': 'test-image', - 'MONGO_ENDPOINT': 'mongodb://192.168.1.2:27017/', 'MONGO_DB': 'test-db', 'MONGO_COLLECTION': 'test-collection', } @@ -53,6 +56,9 @@ def setup_env( # load in optional local test environment variables test_env_path = Path(__file__).parent.parent.parent.parent.parent / 'test.env' load_dotenv(test_env_path) + # check if necessary env vars are set + for key in necessary_env_vars: + assert key in os.environ, f'{key} not set' # set testing env vars for key, val in env_var_map.items(): # set env var