added integration tests and repository pattern for image, model and visual communication DTOs
Code Quality Pipeline / Check Code (pull_request) Failing after 3m21s
Code Quality Pipeline / Check Code (pull_request) Failing after 3m21s
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
"""Definition of docstore interface."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from shared.docstore.src.classes import ModelData, VisualCommunication
|
||||
|
||||
|
||||
class DocstoreInterface(ABC):
|
||||
"""Docstore interface class."""
|
||||
|
||||
@abstractmethod
|
||||
def connect(
|
||||
self,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def close(
|
||||
self,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def __enter__(
|
||||
self,
|
||||
) -> DocstoreInterface:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type,
|
||||
exc_val,
|
||||
exc_tb,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def count_documents(
|
||||
self,
|
||||
only_with_annotation: bool,
|
||||
) -> int:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def list_names(
|
||||
self,
|
||||
only_with_annotation: bool,
|
||||
) -> list[str]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def upsert_visual_comminucations(
|
||||
self,
|
||||
visual_communication_list: list[VisualCommunication],
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def upsert_annotations(
|
||||
self,
|
||||
visual_communication_name: str,
|
||||
annotations: ModelData,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def upsert_prediction(
|
||||
self,
|
||||
visual_communication_name: str,
|
||||
predictions: ModelData,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_visual_communication(
|
||||
self,
|
||||
with_annotation: bool,
|
||||
name: str | None = None,
|
||||
) -> VisualCommunication:
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Definition of docstore mongodb implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from pymongo import MongoClient
|
||||
from pymongo.collection import Collection
|
||||
from pymongo.database import Database
|
||||
|
||||
from shared.utils import check_env
|
||||
|
||||
from .classes import ModelData, VisualCommunication
|
||||
from .docstore_interface import DocstoreInterface
|
||||
from .exceptions import NoDocumentFoundException
|
||||
|
||||
|
||||
class DocstoreMongo(DocstoreInterface):
|
||||
"""Docstore interface."""
|
||||
|
||||
def __init__(self):
|
||||
# ensure necessary env vars available
|
||||
var_list = {
|
||||
'MONGO_ENDPOINT',
|
||||
'MONGO_DB',
|
||||
'MONGO_COLLECTION',
|
||||
}
|
||||
check_env(var_list)
|
||||
# prepare internal variables
|
||||
self.client: MongoClient | None = None
|
||||
self.db: Database | None = None
|
||||
self.collection: Collection | None = None
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Connect to Mongo server."""
|
||||
# prepare arguments
|
||||
mongo_endpoint = str(os.getenv('MONGO_ENDPOINT'))
|
||||
mongo_database = str(os.getenv('MONGO_DB'))
|
||||
mongo_collection = str(os.getenv('MONGO_COLLECTION'))
|
||||
# connect client
|
||||
client = MongoClient(mongo_endpoint)
|
||||
database = client[mongo_database]
|
||||
collection = database[mongo_collection]
|
||||
# set unique index on 'name'
|
||||
collection.create_index(keys='name', unique=True)
|
||||
# persist state
|
||||
self._client = client
|
||||
self._database = database
|
||||
self._collection = collection
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close connection to Mongo server."""
|
||||
self._client.close()
|
||||
self._client = None
|
||||
self._database = None
|
||||
self._collection = None
|
||||
|
||||
def __enter__(self) -> DocstoreMongo:
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
||||
if any(
|
||||
(
|
||||
exc_type is not None,
|
||||
exc_val is not None,
|
||||
exc_tb is not None,
|
||||
),
|
||||
):
|
||||
logging.error('error while exiting context')
|
||||
self.close()
|
||||
|
||||
def count_documents(
|
||||
self,
|
||||
only_with_annotation: bool = False,
|
||||
):
|
||||
"""Get the total number of Visual Communication documents matching the
|
||||
filter."""
|
||||
assert isinstance(only_with_annotation, bool)
|
||||
# build query
|
||||
query = {}
|
||||
if only_with_annotation:
|
||||
query['annotation'] = {'$ne': None}
|
||||
# execute query
|
||||
num_docs = self._collection.count_documents(filter=query)
|
||||
return num_docs
|
||||
|
||||
def list_names(
|
||||
self,
|
||||
only_with_annotation: bool,
|
||||
) -> list[str]:
|
||||
"""List names of Visual Communication documents that match the
|
||||
filters."""
|
||||
assert isinstance(only_with_annotation, bool)
|
||||
assert isinstance(self._collection, Collection)
|
||||
# build query
|
||||
query = {}
|
||||
if only_with_annotation:
|
||||
query['annotation'] = {'$ne': None}
|
||||
# execute query
|
||||
res_list = self._collection.find(
|
||||
filter=query,
|
||||
projection={
|
||||
'_id': False, # don't get id
|
||||
'name': True, # include document name
|
||||
},
|
||||
)
|
||||
# extract info
|
||||
name_list = [elem['name'] for elem in res_list]
|
||||
return name_list
|
||||
|
||||
def upsert_visual_comminucations(
|
||||
self,
|
||||
visual_communication_list: list[VisualCommunication],
|
||||
) -> None:
|
||||
"""Upsert Visual Communication document in the database."""
|
||||
assert isinstance(visual_communication_list, list)
|
||||
assert all(
|
||||
isinstance(vis_com, VisualCommunication)
|
||||
for vis_com in visual_communication_list
|
||||
)
|
||||
assert isinstance(self._collection, Collection)
|
||||
# convert to dict
|
||||
doc_list = [vis_com.model_dump() for vis_com in visual_communication_list]
|
||||
# insert documents
|
||||
response = self._collection.insert_many(documents=doc_list)
|
||||
# check response
|
||||
if not response.acknowledged:
|
||||
raise OSError('failed inserting documents')
|
||||
|
||||
def upsert_annotations(
|
||||
self,
|
||||
visual_communication_name: str,
|
||||
annotations: ModelData,
|
||||
) -> None:
|
||||
"""Upsert annotation for the document with matching name."""
|
||||
assert isinstance(visual_communication_name, str)
|
||||
assert len(visual_communication_name) > 0
|
||||
assert isinstance(annotations, ModelData)
|
||||
assert isinstance(self._collection, Collection)
|
||||
# convert to dict
|
||||
doc = annotations.model_dump()
|
||||
# build query
|
||||
query = {
|
||||
'name': visual_communication_name,
|
||||
}
|
||||
update = {
|
||||
'$set': {
|
||||
'annotation': doc,
|
||||
},
|
||||
}
|
||||
# execute query
|
||||
res = self._collection.update_one(
|
||||
filter=query,
|
||||
update=update,
|
||||
upsert=True,
|
||||
)
|
||||
# check response
|
||||
if not res.acknowledged:
|
||||
raise OSError(
|
||||
f'failed upserting annotations for {visual_communication_name}',
|
||||
)
|
||||
|
||||
def upsert_prediction(
|
||||
self,
|
||||
visual_communication_name: str,
|
||||
predictions: ModelData,
|
||||
) -> None:
|
||||
"""Upsert prediction for the document with matching name."""
|
||||
assert isinstance(visual_communication_name, str)
|
||||
assert len(visual_communication_name) > 0
|
||||
assert isinstance(annotations, ModelData)
|
||||
assert isinstance(self._collection, Collection)
|
||||
# convert to dict
|
||||
doc = predictions.model_dump()
|
||||
# build query
|
||||
query = {
|
||||
'name': visual_communication_name,
|
||||
}
|
||||
update = {
|
||||
'$set': {
|
||||
'prediction': doc,
|
||||
},
|
||||
}
|
||||
# execute query
|
||||
res = self._collection.update_one(
|
||||
filter=query,
|
||||
update=update,
|
||||
upsert=True,
|
||||
)
|
||||
# check response
|
||||
if not res.acknowledged:
|
||||
raise OSError(
|
||||
f'failed upserting predictions for {visual_communication_name}',
|
||||
)
|
||||
|
||||
def get_visual_communication(
|
||||
self,
|
||||
with_annotation: bool = False,
|
||||
name: str | None = None,
|
||||
) -> VisualCommunication:
|
||||
"""Get a random Visual Communication document that matches annotation
|
||||
filter.
|
||||
|
||||
If name is not specified, a random document matching filter is
|
||||
returned.
|
||||
"""
|
||||
assert isinstance(with_annotation, bool)
|
||||
if name is not None:
|
||||
assert isinstance(name, str)
|
||||
assert len(name) > 0
|
||||
# build query
|
||||
query: dict[str, Any] = {}
|
||||
if with_annotation:
|
||||
query['annotation'] = {'$ne': None}
|
||||
else:
|
||||
query['annotation'] = {'$eq': None}
|
||||
if name is not None:
|
||||
query['name'] = {'$eq': name}
|
||||
# execute query
|
||||
res_list = self._collection.aggregate(
|
||||
pipeline=[
|
||||
{
|
||||
'$match': query, # find using filters
|
||||
},
|
||||
{
|
||||
'$sample': {
|
||||
'size': 1, # get one random
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
doc_list = list(res_list)
|
||||
# check result
|
||||
if len(doc_list) == 0:
|
||||
raise NoDocumentFoundException()
|
||||
# convert
|
||||
vis_com = VisualCommunication.model_validate(doc_list[0])
|
||||
return vis_com
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Definition of unittests for Docstore MongoDB implementation."""
|
||||
|
||||
import os
|
||||
from unittest import TestCase
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from pymongo import MongoClient
|
||||
from pymongo.collection import Collection
|
||||
from pymongo.database import Database
|
||||
|
||||
from shared.docstore.src.docstore_mongo import DocstoreMongo
|
||||
|
||||
|
||||
class TestDocstoreMongoImplementation(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
# define relevant env vars
|
||||
self.env_var_map = {
|
||||
'MONGO_ENDPOINT': '192.168.1.2:27017',
|
||||
'MONGO_DB': 'visual_critical_discourse_analysis',
|
||||
'MONGO_COLLECTION': 'test-collection',
|
||||
}
|
||||
# set env vars
|
||||
for key, val in self.env_var_map.items():
|
||||
os.environ[key] = val
|
||||
# set other variables
|
||||
self.mongo_client_mock = MagicMock(MongoClient)
|
||||
self.mongo_database_mock = MagicMock(Database)
|
||||
self.mongo_collection_mock = MagicMock(Collection)
|
||||
|
||||
def tearDown(self):
|
||||
# clear env vars
|
||||
for key in self.env_var_map:
|
||||
_ = os.environ.pop(key, default=None)
|
||||
# reset reuseable mocks
|
||||
self.mongo_client_mock.reset_mock()
|
||||
self.mongo_database_mock.reset_mock()
|
||||
self.mongo_collection_mock.reset_mock()
|
||||
|
||||
def test_instantiation_should_fail_when_env_not_set(self):
|
||||
# ensure env not set
|
||||
self.tearDown()
|
||||
# run test
|
||||
with self.assertRaises(OSError):
|
||||
_ = DocstoreMongo()
|
||||
@@ -0,0 +1,8 @@
|
||||
from .src import ImageRepository, ModelRepository, VisualCommunicationRepository
|
||||
from .src.dto import (
|
||||
HexadecimalString,
|
||||
ImageData,
|
||||
ModelData,
|
||||
VisualCommunicationData,
|
||||
VisualCommunicationValues,
|
||||
)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .image_repository import ImageRepository
|
||||
from .model_repository import ModelRepository
|
||||
from .visual_communication_repository import VisualCommunicationRepository
|
||||
@@ -0,0 +1,7 @@
|
||||
from .hexadecimal_string import HexadecimalString
|
||||
from .image_data import ImageData
|
||||
from .model_data import ModelData
|
||||
from .visual_communication_data import (
|
||||
VisualCommunicationData,
|
||||
VisualCommunicationValues,
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Definition of BytesIO Pydantic Annotation."""
|
||||
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
from pydantic.json_schema import JsonSchemaValue
|
||||
from pydantic_core import core_schema
|
||||
|
||||
|
||||
class BytesIOPydanticAnnotation:
|
||||
"""Pydantic annotation that defines input validation, as well as general
|
||||
and json serialization."""
|
||||
|
||||
@classmethod
|
||||
def validate_input(cls, v: Any, handler) -> BytesIO:
|
||||
"""Pydantic-related function to validate input on instantiation."""
|
||||
if isinstance(v, BytesIO):
|
||||
return v
|
||||
s = handler(v)
|
||||
return BytesIO(s)
|
||||
|
||||
@classmethod
|
||||
def __get_pydantic_core_schema__(
|
||||
cls,
|
||||
source_type,
|
||||
_handler,
|
||||
) -> core_schema.CoreSchema:
|
||||
assert source_type is BytesIO
|
||||
return core_schema.no_info_wrap_validator_function(
|
||||
function=cls.validate_input,
|
||||
schema=core_schema.str_schema(),
|
||||
serialization=core_schema.to_string_ser_schema(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __get_pydantic_json_schema__(cls, _core_schema, handler) -> JsonSchemaValue:
|
||||
return handler(core_schema.str_schema())
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Definition of Checksum DTO."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
class HexadecimalString(str):
|
||||
"""Hexadecimal-string class."""
|
||||
|
||||
def __new__(cls, string):
|
||||
# ensure proper input format
|
||||
pattern = r'[0-9-a-fA-F]{32}'
|
||||
match = re.match(pattern, string)
|
||||
if match is None:
|
||||
raise ValueError(f'format does not match a hexadecimal-string: {string}')
|
||||
return super().__new__(cls, string)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
class_name = self.__class__.__name__
|
||||
return f"{class_name}('{self}')"
|
||||
|
||||
def __reduce__(self):
|
||||
return self.__class__, (self,)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Definition of HexadecimalString Pydantic Annotation."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic.json_schema import JsonSchemaValue
|
||||
from pydantic_core import core_schema
|
||||
|
||||
from .hexadecimal_string import HexadecimalString
|
||||
|
||||
|
||||
class HexadecimalStringPydanticAnnotation:
|
||||
"""Pydantic annotation that defines input validation, as well as general
|
||||
and json serialization."""
|
||||
|
||||
@classmethod
|
||||
def validate_input(cls, v: Any, handler) -> HexadecimalString:
|
||||
"""Pydantic-related function to validate input on instantiation."""
|
||||
if isinstance(v, HexadecimalString):
|
||||
return v
|
||||
s = handler(v)
|
||||
return HexadecimalString(s)
|
||||
|
||||
@classmethod
|
||||
def __get_pydantic_core_schema__(
|
||||
cls,
|
||||
source_type,
|
||||
_handler,
|
||||
) -> core_schema.CoreSchema:
|
||||
assert source_type is HexadecimalString
|
||||
return core_schema.no_info_wrap_validator_function(
|
||||
function=cls.validate_input,
|
||||
schema=core_schema.str_schema(),
|
||||
serialization=core_schema.to_string_ser_schema(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __get_pydantic_json_schema__(cls, _core_schema, handler) -> JsonSchemaValue:
|
||||
return handler(core_schema.str_schema())
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Definition of VisualData DTO."""
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from PIL import Image
|
||||
from pydantic import Field
|
||||
|
||||
from .image_pydantic_annotation import ImagePydanticAnnotation
|
||||
from .type_checking_base_model import TypeCheckingBaseModel
|
||||
|
||||
|
||||
class ImageData(TypeCheckingBaseModel):
|
||||
"""Visual data class."""
|
||||
|
||||
image: Annotated[Image.Image, ImagePydanticAnnotation]
|
||||
name: str = Field(min_length=1)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Definition of BytesIO Pydantic Annotation."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image
|
||||
from pydantic.json_schema import JsonSchemaValue
|
||||
from pydantic_core import core_schema
|
||||
|
||||
|
||||
class ImagePydanticAnnotation:
|
||||
"""Pydantic annotation that defines input validation, as well as general
|
||||
and json serialization."""
|
||||
|
||||
@classmethod
|
||||
def validate_input(cls, v: Any, handler) -> Image.Image:
|
||||
"""Pydantic-related function to validate input on instantiation."""
|
||||
if isinstance(v, Image.Image):
|
||||
return v
|
||||
s = handler(v)
|
||||
return Image.open(s)
|
||||
|
||||
@classmethod
|
||||
def __get_pydantic_core_schema__(
|
||||
cls,
|
||||
source_type,
|
||||
_handler,
|
||||
) -> core_schema.CoreSchema:
|
||||
assert source_type is Image.Image
|
||||
return core_schema.no_info_wrap_validator_function(
|
||||
function=cls.validate_input,
|
||||
schema=core_schema.str_schema(),
|
||||
serialization=core_schema.to_string_ser_schema(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __get_pydantic_json_schema__(cls, _core_schema, handler) -> JsonSchemaValue:
|
||||
return handler(core_schema.str_schema())
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Definition of ModelData DTO."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import md5
|
||||
from io import BytesIO
|
||||
from typing import Annotated
|
||||
|
||||
import torch
|
||||
from pydantic import Field
|
||||
|
||||
from .bytes_io_pydantic_annotation import BytesIOPydanticAnnotation
|
||||
from .hexadecimal_string import HexadecimalString
|
||||
from .hexadecimal_string_pydantic_annotation import HexadecimalStringPydanticAnnotation
|
||||
from .type_checking_base_model import TypeCheckingBaseModel
|
||||
|
||||
|
||||
class ModelData(TypeCheckingBaseModel):
|
||||
"""Model Data DTO."""
|
||||
|
||||
buffer: Annotated[BytesIO, BytesIOPydanticAnnotation]
|
||||
buffer_checksum: Annotated[HexadecimalString, HexadecimalStringPydanticAnnotation]
|
||||
class_name: str = Field(
|
||||
min_length=1,
|
||||
description='name of model class to generate data.',
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def calculate_checksum(buffer: BytesIO) -> HexadecimalString:
|
||||
"""Calculate buffer checksum."""
|
||||
checksum = md5(buffer.getbuffer()).hexdigest()
|
||||
return HexadecimalString(checksum)
|
||||
|
||||
@staticmethod
|
||||
def model_to_buffer(model: torch.nn.Module) -> BytesIO:
|
||||
"""Save model to buffer."""
|
||||
assert isinstance(model, torch.nn.Module)
|
||||
buffer = BytesIO()
|
||||
torch.save(model.state_dict(), buffer)
|
||||
return buffer
|
||||
|
||||
@classmethod
|
||||
def from_model(cls, model: torch.nn.Module) -> ModelData:
|
||||
"""Instantiate from torch module."""
|
||||
assert isinstance(model, torch.nn.Module)
|
||||
# get model name
|
||||
class_name = type(model).__name__
|
||||
# save data to buffer
|
||||
buffer = cls.model_to_buffer(model)
|
||||
buffer = BytesIO()
|
||||
torch.save(model.state_dict(), buffer)
|
||||
# calculate checksum
|
||||
buffer_checksum = cls.calculate_checksum(buffer)
|
||||
# instantiate from buffer
|
||||
data = cls(
|
||||
buffer=buffer,
|
||||
buffer_checksum=buffer_checksum,
|
||||
class_name=class_name,
|
||||
)
|
||||
return data
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Definition of TypeCheckingBaseModel class."""
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class TypeCheckingBaseModel(BaseModel):
|
||||
"""BaseModel with added type checking on input types."""
|
||||
|
||||
model_config = ConfigDict(
|
||||
validate_assignment=True, # argument type checking
|
||||
frozen=True, # ensure data immutability
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
from .visual_communication_data import VisualCommunicationData
|
||||
from .visual_communication_values import VisualCommunicationValues
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Definition of AngleValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class AngleValues(ValuesModel):
|
||||
"""Angle values DTO."""
|
||||
|
||||
high: float
|
||||
eye_level: float
|
||||
low: float
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Definition of ContactValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class ContactValues(ValuesModel):
|
||||
"""Contact values DTO."""
|
||||
|
||||
offer: float
|
||||
demand: float
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Definition of DistanceValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class DistanceValues(ValuesModel):
|
||||
"""Distance values DTO."""
|
||||
|
||||
long: float
|
||||
medium: float
|
||||
close: float
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Definition of FramingValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class FramingValues(ValuesModel):
|
||||
"""Framing values DTO."""
|
||||
|
||||
frame_lines: float
|
||||
empty_space: float
|
||||
colour_contrast: float
|
||||
form_contrast: float
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Definition of InformationValueValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class InformationValueValues(ValuesModel):
|
||||
"""Information value values DTO."""
|
||||
|
||||
given_new: float
|
||||
ideal_real: float
|
||||
central_marginal: float
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Definition of ModalityColorValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class ModalityColorValues(ValuesModel):
|
||||
"""Modality color values DTO."""
|
||||
|
||||
high: float
|
||||
medium: float
|
||||
low: float
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Definition of ModalityDepthValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class ModalityDepthValues(ValuesModel):
|
||||
"""Modality depth values DTO."""
|
||||
|
||||
high: float
|
||||
medium: float
|
||||
low: float
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Definition of ModalityLightingValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class ModalityLightingValues(ValuesModel):
|
||||
"""Modality lighting values DTO."""
|
||||
|
||||
high: float
|
||||
medium: float
|
||||
low: float
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Definition of PointOfViewValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class PointOfViewValues(ValuesModel):
|
||||
"""Point-of-view values DTO."""
|
||||
|
||||
frontal: float
|
||||
oblique: float
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Definition of SalienceValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class SalienceValues(ValuesModel):
|
||||
"""Salience values DTO."""
|
||||
|
||||
size: float
|
||||
colour: float
|
||||
tone: float
|
||||
form: float
|
||||
positioning: float
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Definition of ValuesModel base class."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from torch import Tensor
|
||||
|
||||
|
||||
class ValuesModel(BaseModel):
|
||||
"""ValuesModel base class."""
|
||||
|
||||
model_config = ConfigDict(
|
||||
validate_assignment=True, # argument type checking
|
||||
frozen=True, # ensure data immutability
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def list_fields(cls) -> list[str]:
|
||||
"""List options that are stored as attributes."""
|
||||
return list(cls.model_fields.keys())
|
||||
|
||||
@classmethod
|
||||
def from_random(cls):
|
||||
"""Instantiate with random numbers."""
|
||||
kwargs = {field: random.random() for field in cls.list_fields()}
|
||||
return cls(**kwargs)
|
||||
|
||||
@classmethod
|
||||
def from_choice(cls, option: str) -> ValuesModel:
|
||||
"""Instantiate from choice."""
|
||||
assert isinstance(option, str)
|
||||
assert len(option) > 0
|
||||
allowed_options_list = cls.list_fields()
|
||||
if option not in allowed_options_list:
|
||||
raise ValueError(f'option {option} must be in {allowed_options_list}')
|
||||
# generate field values
|
||||
kwargs = {field: 0 for field in allowed_options_list}
|
||||
# set chosen value to max probability
|
||||
kwargs[option] = 1
|
||||
return cls(**kwargs)
|
||||
|
||||
@classmethod
|
||||
def from_tensor(cls, tensor: Tensor):
|
||||
"""Instantiate from list of values."""
|
||||
assert tensor.size(dim=0) == 1, f'tensor batch larger than 1: {tensor}'
|
||||
data_list = [float(t.item()) for t in tensor[0]]
|
||||
kwargs = dict(zip(cls.list_fields(), data_list))
|
||||
return cls(**kwargs)
|
||||
|
||||
def highest_score_field(self) -> str:
|
||||
"""Return name of field with highest score."""
|
||||
model_dict = self.model_dump()
|
||||
return max(model_dict, key=lambda k: model_dict[k])
|
||||
|
||||
def highest_score_value(self) -> float:
|
||||
"""Return value of field with highest score."""
|
||||
model_dict = self.model_dump()
|
||||
return max(model_dict.values())
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Definition of VisualCommunicationData DTO."""
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from ..type_checking_base_model import TypeCheckingBaseModel
|
||||
from .visual_communication_values import VisualCommunicationValues
|
||||
|
||||
|
||||
class VisualCommunicationData(TypeCheckingBaseModel):
|
||||
"""Visual communication data class."""
|
||||
|
||||
name: str = Field(min_length=1)
|
||||
annotation: VisualCommunicationValues | None = None
|
||||
prediction: VisualCommunicationValues | None = None
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Definition of VisualCommunicationValues DTO."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..type_checking_base_model import TypeCheckingBaseModel
|
||||
from .angle_values import AngleValues
|
||||
from .contact_values import ContactValues
|
||||
from .distance_values import DistanceValues
|
||||
from .framing_values import FramingValues
|
||||
from .information_value_values import InformationValueValues
|
||||
from .modality_color_values import ModalityColorValues
|
||||
from .modality_depth_values import ModalityDepthValues
|
||||
from .modality_lighting_values import ModalityLightingValues
|
||||
from .point_of_view_values import PointOfViewValues
|
||||
from .salience_values import SalienceValues
|
||||
from .visual_syntax_values import VisualSyntaxValues
|
||||
|
||||
|
||||
class VisualCommunicationValues(TypeCheckingBaseModel):
|
||||
"""Visual communication values class."""
|
||||
|
||||
visual_syntax: VisualSyntaxValues
|
||||
contact: ContactValues
|
||||
angle: AngleValues
|
||||
point_of_view: PointOfViewValues
|
||||
distance: DistanceValues
|
||||
modality_lighting: ModalityLightingValues
|
||||
modality_color: ModalityColorValues
|
||||
modality_depth: ModalityDepthValues
|
||||
information_value: InformationValueValues
|
||||
framing: FramingValues
|
||||
salience: SalienceValues
|
||||
|
||||
@classmethod
|
||||
def from_random(cls) -> VisualCommunicationValues:
|
||||
"""Create a random instance."""
|
||||
return cls(
|
||||
visual_syntax=VisualSyntaxValues.from_random(),
|
||||
contact=ContactValues.from_random(),
|
||||
angle=AngleValues.from_random(),
|
||||
point_of_view=PointOfViewValues.from_random(),
|
||||
distance=DistanceValues.from_random(),
|
||||
modality_lighting=ModalityLightingValues.from_random(),
|
||||
modality_color=ModalityColorValues.from_random(),
|
||||
modality_depth=ModalityDepthValues.from_random(),
|
||||
information_value=InformationValueValues.from_random(),
|
||||
framing=FramingValues.from_random(),
|
||||
salience=SalienceValues.from_random(),
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Definition of VisualSyntaxValues DTO."""
|
||||
|
||||
from .values_model import ValuesModel
|
||||
|
||||
|
||||
class VisualSyntaxValues(ValuesModel):
|
||||
"""Visual syntax values DTO."""
|
||||
|
||||
non_transactional_action: float
|
||||
non_transactional_reaction: float
|
||||
unidirectional_transactional_action: float
|
||||
unidirectional_transactional_reaction: float
|
||||
bidirectional_transactional_action: float
|
||||
bidirectional_transactional_reaction: float
|
||||
conversion: float
|
||||
speech_process: float
|
||||
classification_overt_taxonomy: float
|
||||
analytical_exhaustive: float
|
||||
analytical_disarranged: float
|
||||
analytical_temporal: float
|
||||
analytical_distributed: float
|
||||
analytical_topological: float
|
||||
analytical_exploded: float
|
||||
analytical_inclusive: float
|
||||
symbolic_suggestive: float
|
||||
symbolic_attributive: float
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Definition of ImageRepository class."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from .dto import ImageData
|
||||
from .implementations import MinioImplementation
|
||||
from .interfaces import ImageInterface
|
||||
|
||||
|
||||
class ImageRepository(ImageInterface, MinioImplementation):
|
||||
"""Image repository class that handles CRUD functionality for
|
||||
VisualData."""
|
||||
|
||||
def __enter__(self) -> ImageRepository:
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _build_path(image_name: str) -> Path:
|
||||
"""Build object path."""
|
||||
assert isinstance(image_name, str)
|
||||
path = Path('images') / image_name
|
||||
return path
|
||||
|
||||
def get_data(self, image_name: str) -> ImageData | None:
|
||||
"""Get Visual data."""
|
||||
assert isinstance(image_name, str)
|
||||
assert len(image_name) > 0
|
||||
# build path
|
||||
path = self._build_path(image_name)
|
||||
# get object from bucket
|
||||
buffer = self._get(path)
|
||||
# handle if no data found
|
||||
if not buffer:
|
||||
return None
|
||||
# convert data
|
||||
image = Image.open(buffer)
|
||||
data = ImageData(image=image, name=image_name)
|
||||
return data
|
||||
|
||||
def put_data(self, data: ImageData) -> None:
|
||||
"""Put visual data."""
|
||||
assert isinstance(data, ImageData)
|
||||
# build path
|
||||
path = self._build_path(data.name)
|
||||
# save image to buffer
|
||||
buffer = BytesIO()
|
||||
data.image.save(buffer, 'png')
|
||||
# put object in bucket
|
||||
self._put(path, buffer)
|
||||
|
||||
def remove_data(self, image_name: str) -> None:
|
||||
"""Remove visual data."""
|
||||
assert isinstance(image_name, str)
|
||||
assert len(image_name) > 0
|
||||
# build path
|
||||
path = self._build_path(image_name)
|
||||
# remove object
|
||||
self._delete(path)
|
||||
|
||||
def list_names(self) -> list[str]:
|
||||
"""List names of all images."""
|
||||
# build path
|
||||
path = self._build_path('')
|
||||
# list object paths
|
||||
obj_path_list = self._list_objects(path)
|
||||
# strip prefix
|
||||
name_list = [obj_path.split('/')[-1] for obj_path in obj_path_list]
|
||||
return name_list
|
||||
@@ -0,0 +1,2 @@
|
||||
from .minio_implementation import MinioImplementation
|
||||
from .mongo_implementation import MongoImplementation
|
||||
@@ -0,0 +1,186 @@
|
||||
"""MinIO implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from traceback import format_exc
|
||||
|
||||
from minio import Minio
|
||||
|
||||
from shared.utils import check_env
|
||||
|
||||
from ..interfaces import DatabaseInterface
|
||||
|
||||
|
||||
class MinioImplementation(DatabaseInterface):
|
||||
"""MinIO basic CRUD implementation."""
|
||||
|
||||
def __init__(self):
|
||||
# ensure necessary env vars available
|
||||
var_list = {
|
||||
'MINIO_ENDPOINT',
|
||||
'MINIO_ACCESS_KEY',
|
||||
'MINIO_SECRET_KEY',
|
||||
'MINIO_BUCKET_NAME',
|
||||
}
|
||||
check_env(var_list)
|
||||
# prepare internal variables
|
||||
self._client: Minio | None = None
|
||||
self._bucket_name: str | None = None
|
||||
|
||||
def connect(self):
|
||||
"""Connect to MinIO server."""
|
||||
# prepare arguments
|
||||
minio_endpoint = str(os.getenv('MINIO_ENDPOINT'))
|
||||
minio_access_key = str(os.getenv('MINIO_ACCESS_KEY'))
|
||||
minio_secret_key = str(os.getenv('MINIO_SECRET_KEY'))
|
||||
minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||
# connect client
|
||||
client = Minio(
|
||||
endpoint=minio_endpoint,
|
||||
access_key=minio_access_key,
|
||||
secret_key=minio_secret_key,
|
||||
secure=False,
|
||||
)
|
||||
# ensure bucket exists
|
||||
if not client.bucket_exists(bucket_name=minio_bucket_name):
|
||||
logging.debug('creating bucket: %s', minio_bucket_name)
|
||||
client.make_bucket(bucket_name=minio_bucket_name)
|
||||
# persist state
|
||||
self._client = client
|
||||
self._bucket_name = minio_bucket_name
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close connection to MinIO server.
|
||||
|
||||
N.B. MinIO connection cannot be closed manually.
|
||||
"""
|
||||
self._client = None
|
||||
self._bucket_name = None
|
||||
|
||||
def connected(self):
|
||||
"""Check connection to Minio."""
|
||||
res = isinstance(self._client, Minio)
|
||||
logging.debug(res)
|
||||
return res
|
||||
|
||||
def __enter__(self) -> MinioImplementation:
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
||||
if any(
|
||||
(
|
||||
exc_type is not None,
|
||||
exc_val is not None,
|
||||
exc_tb is not None,
|
||||
),
|
||||
):
|
||||
logging.error('error while exiting context')
|
||||
self.close()
|
||||
|
||||
def _put(
|
||||
self,
|
||||
path: Path,
|
||||
buffer: BytesIO,
|
||||
) -> None:
|
||||
"""Save in-memory buffer as object in MinIO."""
|
||||
assert isinstance(path, Path)
|
||||
assert isinstance(buffer, BytesIO)
|
||||
assert isinstance(self._client, Minio)
|
||||
assert isinstance(self._bucket_name, str)
|
||||
# prepare for saving
|
||||
num_bytes = len(buffer.getvalue())
|
||||
buffer.seek(0)
|
||||
# send data to bucket
|
||||
try:
|
||||
self._client.put_object(
|
||||
bucket_name=self._bucket_name,
|
||||
object_name=path.as_posix(),
|
||||
length=num_bytes,
|
||||
data=buffer,
|
||||
)
|
||||
logging.debug('saved data to %s', path)
|
||||
except Exception as exc:
|
||||
logging.error('failed saving data to MinIO')
|
||||
raise exc
|
||||
|
||||
def _get(
|
||||
self,
|
||||
path: Path,
|
||||
) -> BytesIO | None:
|
||||
"""Get object from MinIO as in-memory buffer."""
|
||||
assert isinstance(path, Path)
|
||||
assert isinstance(self._client, Minio)
|
||||
assert isinstance(self._bucket_name, str)
|
||||
try:
|
||||
# make request
|
||||
response = self._client.get_object(
|
||||
bucket_name=self._bucket_name,
|
||||
object_name=path.as_posix(),
|
||||
)
|
||||
assert response.status == 200
|
||||
# get buffer
|
||||
buffer = BytesIO()
|
||||
chunk_size = 2**14
|
||||
while chunk := response.read(chunk_size):
|
||||
buffer.write(chunk)
|
||||
buffer.seek(0)
|
||||
logging.debug('got %s', path)
|
||||
return buffer
|
||||
except Exception:
|
||||
logging.error('failed getting data from MinIO')
|
||||
logging.debug(format_exc())
|
||||
return None
|
||||
finally:
|
||||
# close connection if established
|
||||
if 'response' in locals():
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
def _delete(
|
||||
self,
|
||||
path: Path,
|
||||
) -> None:
|
||||
"""Delete object from MinIO."""
|
||||
assert isinstance(path, Path)
|
||||
assert isinstance(self._client, Minio)
|
||||
assert isinstance(self._bucket_name, str)
|
||||
# remove object
|
||||
try:
|
||||
self._client.remove_object(
|
||||
bucket_name=self._bucket_name,
|
||||
object_name=path.as_posix(),
|
||||
)
|
||||
logging.debug('deleted %s', path)
|
||||
except Exception as exc:
|
||||
logging.error('failed deleting %s', path)
|
||||
logging.debug(format_exc())
|
||||
raise exc
|
||||
|
||||
def _list_objects(
|
||||
self,
|
||||
path: Path,
|
||||
) -> list[str]:
|
||||
"""List objects in bucket under path."""
|
||||
assert isinstance(path, Path)
|
||||
assert isinstance(self._client, Minio)
|
||||
assert isinstance(self._bucket_name, str)
|
||||
try:
|
||||
# list objects
|
||||
obj_list = self._client.list_objects(
|
||||
bucket_name=self._bucket_name,
|
||||
prefix=path.as_posix(),
|
||||
recursive=True,
|
||||
)
|
||||
# extract info
|
||||
name_list = [obj.object_name for obj in obj_list]
|
||||
logging.debug('got %s objects matching %s', len(name_list), path)
|
||||
return name_list
|
||||
except Exception as exc:
|
||||
logging.error('failed listing objects under %s', path)
|
||||
logging.debug(format_exc())
|
||||
raise exc
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Mongo implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import traceback
|
||||
|
||||
from pymongo import MongoClient
|
||||
from pymongo.collection import Collection
|
||||
from pymongo.database import Database
|
||||
from pymongo.errors import ServerSelectionTimeoutError
|
||||
|
||||
from shared.utils import check_env
|
||||
|
||||
from ..interfaces import DatabaseInterface
|
||||
|
||||
|
||||
class MongoImplementation(DatabaseInterface):
|
||||
"""MongoDB basic CRUD implementation."""
|
||||
|
||||
def __init__(self):
|
||||
# ensure necessary env vars available
|
||||
var_list = {
|
||||
'MONGO_ENDPOINT',
|
||||
'MONGO_DB',
|
||||
'MONGO_COLLECTION',
|
||||
}
|
||||
check_env(var_list)
|
||||
# prepare internal variables
|
||||
self.client: MongoClient | None = None
|
||||
self.db: Database | None = None
|
||||
self.collection: Collection | None = None
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Connect to Mongo server."""
|
||||
# prepare arguments
|
||||
mongo_endpoint = str(os.getenv('MONGO_ENDPOINT'))
|
||||
mongo_database = str(os.getenv('MONGO_DB'))
|
||||
mongo_collection = str(os.getenv('MONGO_COLLECTION'))
|
||||
# connect client
|
||||
client = MongoClient(mongo_endpoint)
|
||||
database = client[mongo_database]
|
||||
collection = database[mongo_collection]
|
||||
# set unique index on 'name'
|
||||
collection.create_index(keys='name', unique=True)
|
||||
# persist state
|
||||
self._client = client
|
||||
self._database = database
|
||||
self._collection = collection
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close connection to Mongo server."""
|
||||
self._client.close()
|
||||
self._client = None
|
||||
self._database = None
|
||||
self._collection = None
|
||||
|
||||
def connected(self) -> bool:
|
||||
"""Check connection to Mongo."""
|
||||
if self._client is None:
|
||||
return False
|
||||
try:
|
||||
# trigger fetch data
|
||||
_ = self._client.server_info()
|
||||
res = True
|
||||
except ServerSelectionTimeoutError:
|
||||
res = False
|
||||
logging.debug(res)
|
||||
return res
|
||||
|
||||
def __enter__(self) -> MongoImplementation:
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
||||
if any(
|
||||
(
|
||||
exc_type is not None,
|
||||
exc_val is not None,
|
||||
exc_tb is not None,
|
||||
),
|
||||
):
|
||||
logging.error('error while exiting context')
|
||||
traceback.print_exception(exc_type, exc_val, exc_tb)
|
||||
self.close()
|
||||
|
||||
def _save(
|
||||
self,
|
||||
data: dict,
|
||||
query: dict,
|
||||
) -> None:
|
||||
"""Save document in Mongo."""
|
||||
assert isinstance(data, dict)
|
||||
assert isinstance(query, dict)
|
||||
assert 'name' in data
|
||||
assert self.connected()
|
||||
self._collection.update_one(
|
||||
filter=query,
|
||||
update={
|
||||
'$set': data.copy(),
|
||||
},
|
||||
upsert=True,
|
||||
)
|
||||
logging.debug('Save %s', data)
|
||||
|
||||
def _get(
|
||||
self,
|
||||
query: dict,
|
||||
) -> dict | None:
|
||||
"""Get document from Mongo."""
|
||||
assert isinstance(query, dict)
|
||||
assert self.connected()
|
||||
doc = self._collection.find_one(query, projection={'_id': False})
|
||||
logging.debug('Found %s', doc)
|
||||
return doc
|
||||
|
||||
def _delete(
|
||||
self,
|
||||
query: dict,
|
||||
) -> None:
|
||||
"""Remove document from Mongo."""
|
||||
assert isinstance(query, dict)
|
||||
assert self.connected()
|
||||
doc = self._collection.delete_one(query)
|
||||
logging.debug('Deleted %s', doc)
|
||||
|
||||
def _list_documents(self, key='name') -> list[str]:
|
||||
"""List documents in Mongo."""
|
||||
assert isinstance(key, str)
|
||||
assert len(key) > 0
|
||||
# build query
|
||||
doc_list = list(
|
||||
self._collection.find(
|
||||
filter={},
|
||||
projection={
|
||||
'_id': False,
|
||||
key: True,
|
||||
},
|
||||
),
|
||||
)
|
||||
# extract values
|
||||
value_list = [doc[key] for doc in doc_list]
|
||||
logging.debug('Got %s document(s)', len(doc_list))
|
||||
return value_list
|
||||
@@ -0,0 +1,4 @@
|
||||
from .database_interface import DatabaseInterface
|
||||
from .image_interface import ImageInterface
|
||||
from .model_interface import ModelInterface
|
||||
from .visual_communication_interface import VisualCommunicationInterface
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Definition of DatabaseInterface class."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class DatabaseInterface(ABC):
|
||||
"""Interface base class adding 'connect', 'close' and context
|
||||
functionalities."""
|
||||
|
||||
@abstractmethod
|
||||
def connect(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def close(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def connected(self) -> bool:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def __enter__(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
raise NotImplementedError()
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Definition of ImageInterface."""
|
||||
|
||||
from abc import abstractmethod
|
||||
|
||||
from ..dto import ImageData
|
||||
from .database_interface import DatabaseInterface
|
||||
|
||||
|
||||
class ImageInterface(DatabaseInterface):
|
||||
"""Image interface class."""
|
||||
|
||||
@abstractmethod
|
||||
def get_data(self, image_name: str) -> ImageData | None:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def put_data(self, data: ImageData) -> None:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def remove_data(self, image_name: str) -> None:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def list_names(self) -> list[str]:
|
||||
raise NotImplementedError()
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Definition of ModelInterface."""
|
||||
|
||||
from abc import abstractmethod
|
||||
|
||||
from ..dto import ModelData
|
||||
from .database_interface import DatabaseInterface
|
||||
|
||||
|
||||
class ModelInterface(DatabaseInterface):
|
||||
"""Model interface class."""
|
||||
|
||||
@abstractmethod
|
||||
def get_data(self, object_name: str) -> ModelData | None:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def put_data(self, data: ModelData) -> None:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def remove_data(self, object_name: str) -> None:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def list_names(self) -> list[str]:
|
||||
raise NotImplementedError()
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Definition of VisualCommunicationInterface."""
|
||||
|
||||
from abc import abstractmethod
|
||||
|
||||
from ..dto import VisualCommunicationData
|
||||
from .database_interface import DatabaseInterface
|
||||
|
||||
|
||||
class VisualCommunicationInterface(DatabaseInterface):
|
||||
"""Visual communication interface class."""
|
||||
|
||||
@abstractmethod
|
||||
def get_data(self, name: str) -> VisualCommunicationData | None:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def put_data(self, data: VisualCommunicationData) -> None:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def remove_data(self, name: str) -> None:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def list_names(self) -> list[str]:
|
||||
raise NotImplementedError()
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Definition of ModelRepository class."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .dto import HexadecimalString, ModelData
|
||||
from .implementations import MinioImplementation
|
||||
from .interfaces import ModelInterface
|
||||
|
||||
|
||||
class ModelRepository(ModelInterface, MinioImplementation):
|
||||
"""Model repository class that handles CRUD functionality for ModelData."""
|
||||
|
||||
def __enter__(self) -> ModelRepository:
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _prefix() -> Path:
|
||||
"""Object name prefix."""
|
||||
return Path('models')
|
||||
|
||||
@classmethod
|
||||
def _build_object_name(cls, data: ModelData) -> str:
|
||||
"""Build object name from data."""
|
||||
return f'{data.class_name}-{data.buffer_checksum}'
|
||||
|
||||
def get_data(self, object_name: str) -> ModelData | None:
|
||||
"""Get model data."""
|
||||
assert isinstance(object_name, str)
|
||||
# build path
|
||||
path = self._prefix() / object_name
|
||||
# get object from bucket
|
||||
buffer = self._get(path)
|
||||
# handle if no data found
|
||||
if not buffer:
|
||||
return None
|
||||
# extract info
|
||||
class_name, buffer_checksum_str = object_name.split('-')
|
||||
# convert data
|
||||
buffer_checksum = HexadecimalString(buffer_checksum_str)
|
||||
# instantiate data
|
||||
data = ModelData(
|
||||
buffer=buffer,
|
||||
buffer_checksum=buffer_checksum,
|
||||
class_name=class_name,
|
||||
)
|
||||
return data
|
||||
|
||||
def put_data(self, data: ModelData) -> None:
|
||||
"""Put model data."""
|
||||
assert isinstance(data, ModelData)
|
||||
# build object name
|
||||
object_name = self._build_object_name(data)
|
||||
# build path
|
||||
path = self._prefix() / object_name
|
||||
# put object in bucket
|
||||
self._put(path, data.buffer)
|
||||
|
||||
def remove_data(self, object_name: str) -> None:
|
||||
"""Remove model data."""
|
||||
assert isinstance(object_name, str)
|
||||
# build path
|
||||
path = self._prefix() / object_name
|
||||
# remove object
|
||||
self._delete(path)
|
||||
|
||||
def list_names(self) -> list[str]:
|
||||
"""List names of all models."""
|
||||
# build path
|
||||
path = self._prefix()
|
||||
# list object paths
|
||||
obj_path_list = self._list_objects(path)
|
||||
# strip prefix
|
||||
name_list = [obj_path.split('/')[-1] for obj_path in obj_path_list]
|
||||
return name_list
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Definition of VisualCommunicationRepository class."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .dto import VisualCommunicationData
|
||||
from .implementations import MongoImplementation
|
||||
from .interfaces import VisualCommunicationInterface
|
||||
|
||||
|
||||
class VisualCommunicationRepository(VisualCommunicationInterface, MongoImplementation):
|
||||
"""Visual communication repository class that handles CRUD functionality
|
||||
for VisualCommunicationData."""
|
||||
|
||||
def __enter__(self) -> VisualCommunicationRepository:
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
def get_data(self, name: str) -> VisualCommunicationData | None:
|
||||
"""Get visual communication data."""
|
||||
assert isinstance(name, str)
|
||||
assert len(name) > 0
|
||||
# build query
|
||||
query = {'name': name}
|
||||
# get document from mongo
|
||||
doc = self._get(query)
|
||||
# handle if no data found
|
||||
if doc is None:
|
||||
return None
|
||||
# instantiate object
|
||||
data = VisualCommunicationData(**doc)
|
||||
return data
|
||||
|
||||
def put_data(self, data: VisualCommunicationData) -> None:
|
||||
"""Put visual communication data."""
|
||||
assert isinstance(data, VisualCommunicationData)
|
||||
# convert to dict
|
||||
data = data.model_dump(mode='json')
|
||||
# build query
|
||||
query = {'name': data['name']}
|
||||
# save to mongo
|
||||
self._save(data, query)
|
||||
|
||||
def remove_data(self, name: str) -> None:
|
||||
"""Remove visual communication data."""
|
||||
assert isinstance(name, str)
|
||||
assert len(name) > 0
|
||||
# build query
|
||||
query = {'name': name}
|
||||
# remove document from mongo
|
||||
self._delete(query)
|
||||
|
||||
def list_names(self) -> list[str]:
|
||||
"""List names of all documents."""
|
||||
# list documents
|
||||
name_list = self._list_documents(key='name')
|
||||
return name_list
|
||||
@@ -0,0 +1,444 @@
|
||||
"""Integration tests configuration."""
|
||||
|
||||
import os
|
||||
import random
|
||||
from collections.abc import Iterator
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import minio
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from pymongo import MongoClient
|
||||
from testcontainers.minio import MinioContainer
|
||||
from testcontainers.mongodb import MongoDbContainer
|
||||
|
||||
from model.src.models import VisualCommunicationModel
|
||||
from shared.repositories import (
|
||||
HexadecimalString,
|
||||
ImageData,
|
||||
ImageRepository,
|
||||
ModelData,
|
||||
ModelRepository,
|
||||
VisualCommunicationData,
|
||||
VisualCommunicationRepository,
|
||||
VisualCommunicationValues,
|
||||
)
|
||||
from shared.repositories.src.implementations import (
|
||||
MinioImplementation,
|
||||
MongoImplementation,
|
||||
)
|
||||
|
||||
# set random seed for reproducibility
|
||||
random.seed(13)
|
||||
|
||||
# define test static variables
|
||||
MINIO_PORT = 9000
|
||||
MINIO_ACCESS_KEY = 'minioadmin'
|
||||
MINIO_SECRET_KEY = 'minioadmin'
|
||||
MONGO_PORT = 27017
|
||||
MONGO_USERNAME = 'admin'
|
||||
MONGO_PASSWORD = 'password'
|
||||
|
||||
env_var_map = {
|
||||
'MINIO_ENDPOINT': f'localhost:{MINIO_PORT}', # updated when container is running
|
||||
'MINIO_ACCESS_KEY': MINIO_ACCESS_KEY,
|
||||
'MINIO_SECRET_KEY': MINIO_SECRET_KEY,
|
||||
'MINIO_BUCKET_NAME': 'test-bucket',
|
||||
'MINIO_OBJECT_NAME': 'test-object',
|
||||
'MINIO_IMAGE_NAME': 'test-image',
|
||||
'MONGO_ENDPOINT': (
|
||||
f'mongodb://{MONGO_USERNAME}:{MONGO_PASSWORD}' f'@localhost:{MONGO_PORT}/'
|
||||
), # updated when container is running
|
||||
'MONGO_DB': 'test-db',
|
||||
'MONGO_COLLECTION': 'test-collection',
|
||||
}
|
||||
container_map = {
|
||||
'minio': MinioContainer(
|
||||
port=MINIO_PORT,
|
||||
access_key=MINIO_ACCESS_KEY,
|
||||
secret_key=MINIO_SECRET_KEY,
|
||||
),
|
||||
'mongo': MongoDbContainer(
|
||||
port=MONGO_PORT,
|
||||
username=MONGO_USERNAME,
|
||||
password=MONGO_PASSWORD,
|
||||
dbname=env_var_map['MONGO_DB'],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope='session', autouse=True)
|
||||
def setup_infrastructure(request: pytest.FixtureRequest) -> None:
|
||||
"""Prepare infrastructure for integration test."""
|
||||
# prepare infrastructure
|
||||
for container in container_map.values():
|
||||
container.start()
|
||||
# update env var map
|
||||
minio_host = container_map['minio'].get_container_host_ip()
|
||||
minio_port = container_map['minio'].get_exposed_port(MINIO_PORT)
|
||||
env_var_map['MINIO_ENDPOINT'] = f'{minio_host}:{minio_port}'
|
||||
mongo_host = container_map['mongo'].get_container_host_ip()
|
||||
mongo_port = container_map['mongo'].get_exposed_port(MONGO_PORT)
|
||||
env_var_map['MONGO_ENDPOINT'] = (
|
||||
f'mongodb://{MONGO_USERNAME}:{MONGO_PASSWORD}@{mongo_host}:{mongo_port}/'
|
||||
)
|
||||
|
||||
# ensure cleanup
|
||||
def cleanup_infrastructure():
|
||||
for container in container_map.values():
|
||||
container.stop()
|
||||
|
||||
request.addfinalizer(cleanup_infrastructure)
|
||||
|
||||
|
||||
@pytest.fixture(scope='session', autouse=True)
|
||||
def populate_env(
|
||||
request: pytest.FixtureRequest,
|
||||
setup_infrastructure,
|
||||
) -> None:
|
||||
"""Populate environment with variables used for testing."""
|
||||
# update env
|
||||
for key, val in env_var_map.items():
|
||||
os.environ[key] = val
|
||||
|
||||
# ensure cleanup
|
||||
def cleanup_env():
|
||||
for key in env_var_map:
|
||||
_ = os.environ.pop(key, default=None)
|
||||
|
||||
request.addfinalizer(cleanup_env)
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def raw_minio_client(
|
||||
setup_infrastructure,
|
||||
populate_env,
|
||||
) -> Iterator[minio.Minio]:
|
||||
"""Raw Minio client fixture."""
|
||||
# prepare arguments
|
||||
minio_endpoint = str(os.getenv('MINIO_ENDPOINT'))
|
||||
minio_access_key = str(os.getenv('MINIO_ACCESS_KEY'))
|
||||
minio_secret_key = str(os.getenv('MINIO_SECRET_KEY'))
|
||||
minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||
# connect client
|
||||
client = minio.Minio(
|
||||
endpoint=minio_endpoint,
|
||||
access_key=minio_access_key,
|
||||
secret_key=minio_secret_key,
|
||||
secure=False,
|
||||
)
|
||||
# ensure bucket exists
|
||||
if not client.bucket_exists(bucket_name=minio_bucket_name):
|
||||
client.make_bucket(bucket_name=minio_bucket_name)
|
||||
# expose client
|
||||
yield client
|
||||
# cleanup
|
||||
object_list = client.list_objects(minio_bucket_name, recursive=True)
|
||||
for obj in object_list:
|
||||
client.remove_object(
|
||||
bucket_name=obj.bucket_name,
|
||||
object_name=obj.object_name,
|
||||
)
|
||||
client.remove_bucket(minio_bucket_name)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def minio_client(
|
||||
setup_infrastructure,
|
||||
populate_env,
|
||||
) -> Iterator[MinioImplementation]:
|
||||
"""MinioImplementation fixture."""
|
||||
# instantiate and connect client
|
||||
minio_client = MinioImplementation()
|
||||
minio_client.connect()
|
||||
# expose client
|
||||
yield minio_client
|
||||
# cleanup
|
||||
object_name_list = minio_client._list_objects(Path('*'))
|
||||
for name in object_name_list:
|
||||
minio_client._delete(Path(name))
|
||||
minio_client.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def buffer() -> Iterator[BytesIO]:
|
||||
"""Bytes buffer fixture."""
|
||||
# generate reproducible random data
|
||||
data = random.randbytes(n=2**21) # 2 MB
|
||||
# convert data
|
||||
buffer = BytesIO(data)
|
||||
# expose buffer
|
||||
yield buffer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def buffer_in_minio(
|
||||
raw_minio_client: minio.Minio,
|
||||
buffer: BytesIO,
|
||||
) -> Iterator[tuple[Path, BytesIO]]:
|
||||
"""Buffer in Minio fixture."""
|
||||
# prepare arguments
|
||||
minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||
minio_object_name = str(os.getenv('MINIO_OBJECT_NAME'))
|
||||
# prepare for saving
|
||||
num_bytes = len(buffer.getvalue())
|
||||
buffer.seek(0)
|
||||
# put data in bucket
|
||||
raw_minio_client.put_object(
|
||||
bucket_name=minio_bucket_name,
|
||||
object_name=minio_object_name,
|
||||
length=num_bytes,
|
||||
data=buffer,
|
||||
)
|
||||
# expose data
|
||||
yield Path(minio_object_name), buffer
|
||||
# cleanup
|
||||
raw_minio_client.remove_object(
|
||||
bucket_name=minio_bucket_name,
|
||||
object_name=minio_object_name,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def image_data() -> Iterator[ImageData]:
|
||||
"""Image data fixture."""
|
||||
# prepare arguments
|
||||
name = str(os.getenv('MINIO_IMAGE_NAME'))
|
||||
image = Image.new(mode='RGB', size=(480, 480))
|
||||
# instantiate data
|
||||
image_data = ImageData(image=image, name=name)
|
||||
# expose data
|
||||
yield image_data
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def image_data_in_minio(
|
||||
raw_minio_client: minio.Minio,
|
||||
image_data: ImageData,
|
||||
) -> Iterator[ImageData]:
|
||||
"""Image data in Minio fixture."""
|
||||
# prepare arguments
|
||||
minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||
image_name = image_data.name
|
||||
# build object path
|
||||
object_path = ImageRepository._build_path(image_name)
|
||||
# save image to buffer
|
||||
buffer = BytesIO()
|
||||
image_data.image.save(buffer, 'png')
|
||||
# prepare for saving
|
||||
num_bytes = len(buffer.getvalue())
|
||||
buffer.seek(0)
|
||||
# put data in bucket
|
||||
raw_minio_client.put_object(
|
||||
bucket_name=minio_bucket_name,
|
||||
object_name=object_path.as_posix(),
|
||||
length=num_bytes,
|
||||
data=buffer,
|
||||
)
|
||||
# expose data
|
||||
yield image_data
|
||||
# cleanup
|
||||
raw_minio_client.remove_object(
|
||||
bucket_name=minio_bucket_name,
|
||||
object_name=object_path.as_posix(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def image_repo(
|
||||
setup_infrastructure,
|
||||
populate_env,
|
||||
) -> Iterator[ImageRepository]:
|
||||
"""Image repository fixture."""
|
||||
repo = ImageRepository()
|
||||
repo.connect()
|
||||
yield repo
|
||||
repo.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def model_data() -> Iterator[ModelData]:
|
||||
"""Model data fixture."""
|
||||
# prepare arguments
|
||||
vis_com_model = VisualCommunicationModel().to('cpu')
|
||||
class_name = type(vis_com_model).__name__
|
||||
buffer = ModelData.model_to_buffer(vis_com_model)
|
||||
buffer_checksum = HexadecimalString('77dcab1769563654a6e24f92d40f29bd')
|
||||
# instantiate data
|
||||
model_data = ModelData(
|
||||
buffer=buffer,
|
||||
buffer_checksum=buffer_checksum,
|
||||
class_name=class_name,
|
||||
)
|
||||
# expose model
|
||||
yield model_data
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def model_data_in_minio(
|
||||
raw_minio_client: minio.Minio,
|
||||
model_data: ModelData,
|
||||
) -> Iterator[ModelData]:
|
||||
"""Model data in Minio fixture."""
|
||||
# prepare arguments
|
||||
minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||
object_name = ModelRepository._build_object_name(model_data)
|
||||
buffer = model_data.buffer
|
||||
# build object path
|
||||
object_path = ModelRepository._prefix() / object_name
|
||||
# prepare for saving
|
||||
num_bytes = len(buffer.getvalue())
|
||||
buffer.seek(0)
|
||||
# put data in bucket
|
||||
raw_minio_client.put_object(
|
||||
bucket_name=minio_bucket_name,
|
||||
object_name=object_path.as_posix(),
|
||||
length=num_bytes,
|
||||
data=buffer,
|
||||
)
|
||||
# expose data
|
||||
yield model_data
|
||||
# cleanup
|
||||
raw_minio_client.remove_object(
|
||||
bucket_name=minio_bucket_name,
|
||||
object_name=object_path.as_posix(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def model_repo(
|
||||
setup_infrastructure,
|
||||
populate_env,
|
||||
) -> Iterator[ModelRepository]:
|
||||
"""Model repository fixture."""
|
||||
repo = ModelRepository()
|
||||
repo.connect()
|
||||
yield repo
|
||||
repo.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def raw_mongo_client(
|
||||
setup_infrastructure,
|
||||
populate_env,
|
||||
) -> Iterator[MongoClient]:
|
||||
"""Raw mongo client fixture."""
|
||||
# prepare arguments
|
||||
mongo_endpoint = str(os.getenv('MONGO_ENDPOINT'))
|
||||
mongo_database = str(os.getenv('MONGO_DB'))
|
||||
# connect client
|
||||
client = MongoClient(mongo_endpoint)
|
||||
_ = client[mongo_database]
|
||||
# expose client
|
||||
yield client
|
||||
# cleanup
|
||||
client.drop_database(mongo_database)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mongo_client(
|
||||
setup_infrastructure,
|
||||
populate_env,
|
||||
) -> Iterator[MongoImplementation]:
|
||||
"""MongoImplementation fixture."""
|
||||
# instantiate and connect client
|
||||
mongo_client = MongoImplementation()
|
||||
mongo_client.connect()
|
||||
# expose client
|
||||
yield mongo_client
|
||||
# cleanup
|
||||
mongo_client._collection.drop()
|
||||
mongo_client.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dictionary() -> Iterator[dict]:
|
||||
"""Dictionary fixture."""
|
||||
# prepare data
|
||||
data = {
|
||||
'name': 'test-dictionary',
|
||||
'str_key': 'value',
|
||||
'int_key': 100,
|
||||
'float_key': 3.14,
|
||||
'list_key': [1, 2, 3],
|
||||
'dict_key': {
|
||||
'nested_key': 'nested_value',
|
||||
},
|
||||
}
|
||||
# expose data
|
||||
yield data
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dictionary_in_mongo(
|
||||
raw_mongo_client: MongoClient,
|
||||
dictionary: dict,
|
||||
) -> Iterator[dict]:
|
||||
"""Dictionary in Mongo fixture."""
|
||||
# prepare arguments
|
||||
database = str(os.getenv('MONGO_DB'))
|
||||
collection = str(os.getenv('MONGO_COLLECTION'))
|
||||
# save data
|
||||
_ = raw_mongo_client[database][collection].insert_one(dictionary.copy())
|
||||
# expose data
|
||||
yield dictionary
|
||||
# cleanup
|
||||
raw_mongo_client[database][collection].delete_one(dictionary)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def visual_communication_values() -> Iterator[VisualCommunicationValues]:
|
||||
"""Visual communication values fixture."""
|
||||
# instantiate with random values
|
||||
visual_communication_values = VisualCommunicationValues.from_random()
|
||||
# expose values
|
||||
yield visual_communication_values
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def visual_communication_data(
|
||||
visual_communication_values: VisualCommunicationValues,
|
||||
) -> Iterator[VisualCommunicationData]:
|
||||
"""Visual communication data fixture."""
|
||||
# prepare arguments
|
||||
name = 'test-visual-communication'
|
||||
annotation = visual_communication_values
|
||||
# instantiate data
|
||||
visual_communication_data = VisualCommunicationData(
|
||||
name=name,
|
||||
annotation=annotation,
|
||||
)
|
||||
# expose data
|
||||
yield visual_communication_data
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def visual_communication_data_in_mongo(
|
||||
raw_mongo_client: MongoClient,
|
||||
visual_communication_data: VisualCommunicationData,
|
||||
) -> Iterator[VisualCommunicationData]:
|
||||
"""Visual communication data in Mongo fixture."""
|
||||
# prepare arguments
|
||||
database = str(os.getenv('MONGO_DB'))
|
||||
collection = str(os.getenv('MONGO_COLLECTION'))
|
||||
# convert data
|
||||
dictionary = visual_communication_data.model_dump(mode='dict')
|
||||
# save data
|
||||
_ = raw_mongo_client[database][collection].insert_one(dictionary.copy())
|
||||
# expose data
|
||||
yield visual_communication_data
|
||||
# cleanup
|
||||
raw_mongo_client[database][collection].delete_one(dictionary)
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def visual_communication_repo(
|
||||
setup_infrastructure,
|
||||
populate_env,
|
||||
) -> Iterator[VisualCommunicationRepository]:
|
||||
"""Visual communication repository fixture."""
|
||||
repo = VisualCommunicationRepository()
|
||||
repo.connect()
|
||||
yield repo
|
||||
repo.close()
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Integration tests for ImageRepository class."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from shared.repositories import ImageData, ImageRepository
|
||||
|
||||
|
||||
def same_image(
|
||||
img_a: Image.Image,
|
||||
img_b: Image.Image,
|
||||
) -> bool:
|
||||
"""Check if two images contain the same data."""
|
||||
assert isinstance(img_a, Image.Image)
|
||||
assert isinstance(img_b, Image.Image)
|
||||
# check if images have a comparable number of channels
|
||||
if img_a.getbands() != img_b.getbands():
|
||||
return False
|
||||
# calculate pixel difference between images
|
||||
img_a_arr = np.asarray(img_a)
|
||||
img_b_arr = np.asarray(img_b)
|
||||
diff = np.subtract(img_a_arr, img_b_arr)
|
||||
if np.sum(diff) != 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def same_image_data(
|
||||
data_a: ImageData,
|
||||
data_b: ImageData,
|
||||
) -> bool:
|
||||
"""Check if two ImageData-objects contain the same data."""
|
||||
assert isinstance(data_a, ImageData)
|
||||
assert isinstance(data_b, ImageData)
|
||||
# compare names
|
||||
if data_a.name != data_b.name:
|
||||
return False
|
||||
# compare images
|
||||
if not same_image(data_a.image, data_b.image):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def test_should_have_context_handler():
|
||||
"""Test that class has a working context handler implemented."""
|
||||
# ACT
|
||||
with ImageRepository() as repo:
|
||||
# ASSERT
|
||||
assert repo.connected()
|
||||
|
||||
|
||||
def test_should_get_image_data(
|
||||
image_repo: ImageRepository,
|
||||
image_data_in_minio: ImageData,
|
||||
):
|
||||
"""Test getting image data."""
|
||||
# ARRANGE
|
||||
image_name = image_data_in_minio.name
|
||||
# ACT
|
||||
received_image_data = image_repo.get_data(image_name)
|
||||
# ASSERT
|
||||
assert received_image_data is not None
|
||||
assert isinstance(received_image_data, ImageData)
|
||||
assert same_image_data(image_data_in_minio, received_image_data)
|
||||
|
||||
|
||||
def test_should_get_none_when_no_image_data(
|
||||
image_repo: ImageRepository,
|
||||
image_data: ImageData,
|
||||
):
|
||||
"""Test getting None when no data is available."""
|
||||
# ARRANGE
|
||||
image_name = image_data.name
|
||||
# ACT
|
||||
received_image_data = image_repo.get_data(image_name)
|
||||
# ASSERT
|
||||
assert received_image_data is None
|
||||
|
||||
|
||||
def test_should_delete_image_data(
|
||||
image_repo: ImageRepository,
|
||||
image_data_in_minio: ImageData,
|
||||
):
|
||||
"""Test deleting image data."""
|
||||
# ARRANGE
|
||||
image_name = image_data_in_minio.name
|
||||
# ACT
|
||||
image_repo.remove_data(image_name)
|
||||
received_image_data = image_repo.get_data(image_name)
|
||||
# ASSERT
|
||||
assert received_image_data is None
|
||||
|
||||
|
||||
def test_should_put_image_data(
|
||||
image_repo: ImageRepository,
|
||||
image_data: ImageData,
|
||||
):
|
||||
"""Test putting image data."""
|
||||
# ARRANGE
|
||||
image_name = image_data.name
|
||||
# ACT
|
||||
image_repo.put_data(image_data)
|
||||
received_image_data = image_repo.get_data(image_name)
|
||||
# ASSERT
|
||||
assert received_image_data is not None
|
||||
assert same_image_data(image_data, received_image_data)
|
||||
|
||||
|
||||
def test_should_update_image_data(
|
||||
image_repo: ImageRepository,
|
||||
image_data_in_minio: ImageData,
|
||||
):
|
||||
"""Test updating image data."""
|
||||
# ARRANGE
|
||||
updated_image_data = image_data_in_minio.model_copy(
|
||||
update={
|
||||
'image': Image.new(mode='RGB', size=(480, 480), color='white'),
|
||||
},
|
||||
)
|
||||
image_name = updated_image_data.name
|
||||
# ACT
|
||||
image_repo.put_data(updated_image_data)
|
||||
received_image_data = image_repo.get_data(image_name)
|
||||
# ASSERT
|
||||
assert not same_image_data(image_data_in_minio, updated_image_data)
|
||||
assert received_image_data is not None
|
||||
assert same_image_data(updated_image_data, received_image_data)
|
||||
|
||||
|
||||
def test_should_list_names(
|
||||
image_repo: ImageRepository,
|
||||
image_data_in_minio: ImageData,
|
||||
):
|
||||
"""Test get all image names."""
|
||||
# ARRANGE
|
||||
updated_image_data = image_data_in_minio.model_copy(
|
||||
update={
|
||||
'name': 'updated-test-image',
|
||||
},
|
||||
)
|
||||
image_repo.put_data(updated_image_data)
|
||||
expected_name_list = [
|
||||
image_data_in_minio.name,
|
||||
updated_image_data.name,
|
||||
]
|
||||
# ACT
|
||||
name_list = image_repo.list_names()
|
||||
# ASSERT
|
||||
assert len(name_list) == 2
|
||||
for name in name_list:
|
||||
assert name in expected_name_list
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main(['-s', '-v', __file__])
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Integration tests related to Minio implementation."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from shared.repositories.src.implementations import MinioImplementation
|
||||
|
||||
|
||||
def same_data(
|
||||
data_a: BytesIO,
|
||||
data_b: BytesIO,
|
||||
) -> bool:
|
||||
"""Check if two BytesIO-objects contain the same data."""
|
||||
assert isinstance(data_a, BytesIO)
|
||||
assert isinstance(data_b, BytesIO)
|
||||
# prepare for being read
|
||||
data_a.seek(0)
|
||||
data_b.seek(0)
|
||||
# convert to bytes
|
||||
data_a_bytes = data_a.read()
|
||||
data_b_bytes = data_b.read()
|
||||
# compare size
|
||||
if len(data_a_bytes) != len(data_b_bytes):
|
||||
logging.error(
|
||||
'data has different length: %s and %s',
|
||||
len(data_a_bytes),
|
||||
len(data_b_bytes),
|
||||
)
|
||||
return False
|
||||
# compare content
|
||||
if data_a_bytes != data_b_bytes:
|
||||
logging.error('data has different bytes')
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def test_should_connect_to_minio():
|
||||
"""Test connection to Minio."""
|
||||
# ARRANGE
|
||||
client = MinioImplementation()
|
||||
# ACT
|
||||
client.connect()
|
||||
# ASSERT
|
||||
assert client.connected()
|
||||
client.close()
|
||||
|
||||
|
||||
def test_should_have_context_handler():
|
||||
"""Test that class has a working context handler implemented."""
|
||||
# ACT
|
||||
with MinioImplementation() as client:
|
||||
# ASSERT
|
||||
assert client.connected()
|
||||
|
||||
|
||||
def test_should_get_data(
|
||||
minio_client: MinioImplementation,
|
||||
buffer_in_minio: tuple[Path, BytesIO],
|
||||
):
|
||||
"""Test getting data from Minio."""
|
||||
# ARRANGE
|
||||
path, buffer = buffer_in_minio
|
||||
# ACT
|
||||
received_buffer = minio_client._get(path)
|
||||
# ASSERT
|
||||
assert received_buffer is not None
|
||||
assert same_data(received_buffer, buffer)
|
||||
|
||||
|
||||
def test_should_get_none_when_no_data(
|
||||
minio_client: MinioImplementation,
|
||||
):
|
||||
"""Test getting None when no data is available in Minio."""
|
||||
# ARRANGE
|
||||
nonexistent_path = Path('nonexistent-object-name')
|
||||
# ACT
|
||||
received_buffer = minio_client._get(nonexistent_path)
|
||||
# ASSERT
|
||||
assert received_buffer is None
|
||||
|
||||
|
||||
def test_should_delete_data(
|
||||
minio_client: MinioImplementation,
|
||||
buffer_in_minio: tuple[Path, BytesIO],
|
||||
):
|
||||
"""Test deleting data from Minio."""
|
||||
# ARRANGE
|
||||
path, _ = buffer_in_minio
|
||||
# ACT
|
||||
minio_client._delete(path)
|
||||
# ASSERT
|
||||
received_buffer = minio_client._get(path)
|
||||
assert received_buffer is None
|
||||
|
||||
|
||||
def test_should_put_data(
|
||||
minio_client: MinioImplementation,
|
||||
buffer: BytesIO,
|
||||
):
|
||||
"""Test putting data in Minio."""
|
||||
# ARRANGE
|
||||
path = Path(os.getenv('MINIO_OBJECT_NAME', default=''))
|
||||
# ACT
|
||||
minio_client._put(path, buffer)
|
||||
received_buffer = minio_client._get(path)
|
||||
# ASSERT
|
||||
assert received_buffer is not None
|
||||
assert same_data(received_buffer, buffer)
|
||||
|
||||
|
||||
def test_should_update_data(
|
||||
minio_client: MinioImplementation,
|
||||
buffer_in_minio: tuple[Path, BytesIO],
|
||||
):
|
||||
"""Test updating data in Minio."""
|
||||
# ARRANGE
|
||||
path, buffer = buffer_in_minio
|
||||
updated_buffer = BytesIO(buffer.getvalue() + b'extra data')
|
||||
# ACT
|
||||
minio_client._put(path, updated_buffer)
|
||||
received_buffer = minio_client._get(path)
|
||||
# ASSERT
|
||||
assert not same_data(updated_buffer, buffer)
|
||||
assert received_buffer is not None
|
||||
assert same_data(received_buffer, updated_buffer)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main(['-s', '-v', __file__])
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Integration tests for ModelRepository class."""
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
|
||||
from shared.repositories import ModelData, ModelRepository
|
||||
|
||||
|
||||
def same_buffer(
|
||||
buffer_a: BytesIO,
|
||||
buffer_b: BytesIO,
|
||||
) -> bool:
|
||||
"""Check if 2 buffers contain the same data."""
|
||||
assert isinstance(buffer_a, BytesIO)
|
||||
assert isinstance(buffer_b, BytesIO)
|
||||
# read buffers
|
||||
a_values = buffer_a.getvalue()
|
||||
b_values = buffer_b.getvalue()
|
||||
# compare length of buffers
|
||||
if len(a_values) != len(b_values):
|
||||
return False
|
||||
# compare content of buffers
|
||||
if a_values != b_values:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def same_model_data(
|
||||
data_a: ModelData,
|
||||
data_b: ModelData,
|
||||
) -> bool:
|
||||
"""Check if to ModelData-objects contain the same data."""
|
||||
assert isinstance(data_a, ModelData)
|
||||
assert isinstance(data_b, ModelData)
|
||||
# compare names
|
||||
if data_a.buffer_checksum != data_b.buffer_checksum:
|
||||
return False
|
||||
# compare buffer
|
||||
if not same_buffer(data_a.buffer, data_b.buffer):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def test_should_have_context_handler():
|
||||
"""Test that class has a working context handler implemented."""
|
||||
# ACT
|
||||
with ModelRepository() as repo:
|
||||
# ASSERT
|
||||
assert repo.connected()
|
||||
|
||||
|
||||
def test_should_get_model_data(
|
||||
model_repo: ModelRepository,
|
||||
model_data_in_minio: ModelData,
|
||||
):
|
||||
"""Test getting model data."""
|
||||
# ARRANGE
|
||||
object_name = ModelRepository._build_object_name(model_data_in_minio)
|
||||
# ACT
|
||||
received_model_data = model_repo.get_data(object_name)
|
||||
# ASSERT
|
||||
assert received_model_data is not None
|
||||
assert isinstance(received_model_data, ModelData)
|
||||
assert same_model_data(model_data_in_minio, received_model_data)
|
||||
|
||||
|
||||
def test_should_get_none_when_no_model_data(
|
||||
model_repo: ModelRepository,
|
||||
model_data: ModelData,
|
||||
):
|
||||
"""Test getting None whne no data is available."""
|
||||
# ARRANGE
|
||||
object_name = ModelRepository._build_object_name(model_data)
|
||||
# ACT
|
||||
received_model_data = model_repo.get_data(object_name)
|
||||
# ASSERT
|
||||
assert received_model_data is None
|
||||
|
||||
|
||||
def test_should_delete_model_data(
|
||||
model_repo: ModelRepository,
|
||||
model_data_in_minio: ModelData,
|
||||
):
|
||||
"""Test deleting model data."""
|
||||
# ARRANGE
|
||||
object_name = ModelRepository._build_object_name(model_data_in_minio)
|
||||
# ACT
|
||||
model_repo.remove_data(object_name)
|
||||
received_model_data = model_repo.get_data(object_name)
|
||||
# ASSERT
|
||||
assert received_model_data is None
|
||||
|
||||
|
||||
def test_should_put_model_data(
|
||||
model_repo: ModelRepository,
|
||||
model_data: ModelData,
|
||||
):
|
||||
"""Test putting model data."""
|
||||
# ARRANGE
|
||||
object_name = ModelRepository._build_object_name(model_data)
|
||||
# ACT
|
||||
model_repo.put_data(model_data)
|
||||
received_model_data = model_repo.get_data(object_name)
|
||||
# ASSERT
|
||||
assert received_model_data is not None
|
||||
assert same_model_data(model_data, received_model_data)
|
||||
|
||||
|
||||
def test_should_list_names(
|
||||
model_repo: ModelRepository,
|
||||
model_data_in_minio: ModelData,
|
||||
):
|
||||
"""Test get all model names."""
|
||||
# ARRANGE
|
||||
new_buffer_checksum = ModelData.calculate_checksum(model_data_in_minio.buffer)
|
||||
updated_model_data = model_data_in_minio.model_copy(
|
||||
update={
|
||||
'buffer_checksum': new_buffer_checksum,
|
||||
},
|
||||
)
|
||||
model_repo.put_data(updated_model_data)
|
||||
expected_name_list = [
|
||||
ModelRepository._build_object_name(model_data_in_minio),
|
||||
ModelRepository._build_object_name(updated_model_data),
|
||||
]
|
||||
# ACT
|
||||
name_list = model_repo.list_names()
|
||||
# ASSERT
|
||||
assert len(name_list) == 2
|
||||
for name in name_list:
|
||||
assert name in expected_name_list
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main(['-s', '-v', __file__])
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Integration tests related to Mongo implementation."""
|
||||
|
||||
import pytest
|
||||
|
||||
from shared.repositories.src.implementations import MongoImplementation
|
||||
|
||||
|
||||
def test_should_connect_mongo():
|
||||
"""Test connecting to Mongo."""
|
||||
# ARRANGE
|
||||
client = MongoImplementation()
|
||||
# ACT
|
||||
client.connect()
|
||||
# ASSERT
|
||||
assert client.connected()
|
||||
client.close()
|
||||
|
||||
|
||||
def test_should_have_context_handler():
|
||||
"""Test that class has a working context handler implemented."""
|
||||
# ACT
|
||||
with MongoImplementation() as client:
|
||||
# ASSERT
|
||||
assert client.connected()
|
||||
|
||||
|
||||
def test_should_get_data(
|
||||
mongo_client: MongoImplementation,
|
||||
dictionary_in_mongo: dict,
|
||||
):
|
||||
"""Test getting data from mongo."""
|
||||
# ARRANGE
|
||||
name = dictionary_in_mongo['name']
|
||||
query = {'name': name}
|
||||
# ACT
|
||||
received_dictionary = mongo_client._get(query)
|
||||
# ASSERT
|
||||
assert received_dictionary is not None
|
||||
assert isinstance(received_dictionary, dict)
|
||||
assert received_dictionary == dictionary_in_mongo
|
||||
|
||||
|
||||
def test_should_get_none_when_no_data(
|
||||
mongo_client: MongoImplementation,
|
||||
dictionary: dict,
|
||||
):
|
||||
"""Test getting None when no data is available in mongo."""
|
||||
# ARRANGE
|
||||
name = dictionary['name']
|
||||
query = {'name': name}
|
||||
# ACT
|
||||
received_dictionary = mongo_client._get(query)
|
||||
# ASSERT
|
||||
assert received_dictionary is None
|
||||
|
||||
|
||||
def test_should_delete_data(
|
||||
mongo_client: MongoImplementation,
|
||||
dictionary_in_mongo: dict,
|
||||
):
|
||||
"""Test deleting data from mongo."""
|
||||
# ARRANGE
|
||||
name = dictionary_in_mongo['name']
|
||||
query = {'name': name}
|
||||
# ACT
|
||||
mongo_client._delete(query)
|
||||
received_dictionary = mongo_client._get(query)
|
||||
# ASSERT
|
||||
assert received_dictionary is None
|
||||
|
||||
|
||||
def test_should_save_data(
|
||||
mongo_client: MongoImplementation,
|
||||
dictionary: dict,
|
||||
):
|
||||
"""Test saving data to mongo."""
|
||||
# ARRANGE
|
||||
name = dictionary['name']
|
||||
query = {'name': name}
|
||||
# ACT
|
||||
mongo_client._save(
|
||||
data=dictionary,
|
||||
query=query,
|
||||
)
|
||||
# ASSERT
|
||||
received_dictionary = mongo_client._get(query)
|
||||
assert received_dictionary is not None
|
||||
assert isinstance(received_dictionary, dict)
|
||||
assert received_dictionary == dictionary
|
||||
|
||||
|
||||
def test_should_update_data(
|
||||
mongo_client: MongoImplementation,
|
||||
dictionary_in_mongo: dict,
|
||||
):
|
||||
"""Test updating data in mongo."""
|
||||
# ARRANGE
|
||||
name = dictionary_in_mongo['name']
|
||||
query = {'name': name}
|
||||
# ACT
|
||||
dictionary_in_mongo['str-key'] = 'updated-value'
|
||||
mongo_client._save(
|
||||
data=dictionary_in_mongo,
|
||||
query=query,
|
||||
)
|
||||
# ASSERT
|
||||
received_dictionary = mongo_client._get(query)
|
||||
assert received_dictionary is not None
|
||||
assert received_dictionary == dictionary_in_mongo
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main(['-s', '-v', __file__])
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Integration tests for VisualCommunicationRepository class."""
|
||||
|
||||
import pytest
|
||||
|
||||
from shared.repositories import (
|
||||
VisualCommunicationData,
|
||||
VisualCommunicationRepository,
|
||||
VisualCommunicationValues,
|
||||
)
|
||||
|
||||
|
||||
def test_should_have_context_handler():
|
||||
"""Test that class has a working context handler implemented."""
|
||||
# ACT
|
||||
with VisualCommunicationRepository() as repo:
|
||||
# ASSERT
|
||||
assert repo.connected()
|
||||
|
||||
|
||||
def test_should_get_visual_communication_data(
|
||||
visual_communication_repo: VisualCommunicationRepository,
|
||||
visual_communication_data_in_mongo: VisualCommunicationData,
|
||||
):
|
||||
"""Test getting visual communication data."""
|
||||
# ARRANGE
|
||||
name = visual_communication_data_in_mongo.name
|
||||
# ACT
|
||||
received_visual_communication_data = visual_communication_repo.get_data(name)
|
||||
# ASSERT
|
||||
assert received_visual_communication_data is not None
|
||||
assert isinstance(received_visual_communication_data, VisualCommunicationData)
|
||||
assert received_visual_communication_data == visual_communication_data_in_mongo
|
||||
|
||||
|
||||
def test_should_get_none_when_no_visual_communication_data(
|
||||
visual_communication_repo: VisualCommunicationRepository,
|
||||
visual_communication_data: VisualCommunicationData,
|
||||
):
|
||||
"""Test getting None when no data is available."""
|
||||
# ARRANGE
|
||||
name = visual_communication_data.name
|
||||
# ACT
|
||||
received_visual_communication_data = visual_communication_repo.get_data(name)
|
||||
# ASSERT
|
||||
assert received_visual_communication_data is None
|
||||
|
||||
|
||||
def test_should_delete_image_data(
|
||||
visual_communication_repo: VisualCommunicationRepository,
|
||||
visual_communication_data_in_mongo: VisualCommunicationData,
|
||||
):
|
||||
"""Test deleting visual communication data."""
|
||||
# ARRANGE
|
||||
name = visual_communication_data_in_mongo.name
|
||||
# ACT
|
||||
visual_communication_repo.remove_data(name)
|
||||
received_visual_communication_data = visual_communication_repo.get_data(name)
|
||||
# ASSERT
|
||||
assert received_visual_communication_data is None
|
||||
|
||||
|
||||
def test_should_put_visual_communication_data(
|
||||
visual_communication_repo: VisualCommunicationRepository,
|
||||
visual_communication_data: VisualCommunicationData,
|
||||
):
|
||||
"""Test putting visual communication data."""
|
||||
# ARRANGE
|
||||
name = visual_communication_data.name
|
||||
# ACT
|
||||
visual_communication_repo.put_data(visual_communication_data)
|
||||
received_visual_communication_data = visual_communication_repo.get_data(name)
|
||||
# ASSERT
|
||||
assert received_visual_communication_data is not None
|
||||
assert isinstance(received_visual_communication_data, VisualCommunicationData)
|
||||
assert received_visual_communication_data == visual_communication_data
|
||||
|
||||
|
||||
def test_should_update_visual_communication_data(
|
||||
visual_communication_repo: VisualCommunicationRepository,
|
||||
visual_communication_data_in_mongo: VisualCommunicationData,
|
||||
):
|
||||
"""Test updating visual communication data."""
|
||||
# ARRANGE
|
||||
name = visual_communication_data_in_mongo.name
|
||||
updated_annotation = VisualCommunicationValues.from_random()
|
||||
updated_visual_communication = VisualCommunicationData(
|
||||
name=name,
|
||||
annotation=updated_annotation,
|
||||
)
|
||||
# ACT
|
||||
visual_communication_repo.put_data(updated_visual_communication)
|
||||
received_visual_communication_data = visual_communication_repo.get_data(name)
|
||||
# ASSERT
|
||||
assert received_visual_communication_data is not None
|
||||
assert isinstance(received_visual_communication_data, VisualCommunicationData)
|
||||
assert received_visual_communication_data == updated_visual_communication
|
||||
|
||||
|
||||
def test_should_list_names(
|
||||
visual_communication_repo: VisualCommunicationRepository,
|
||||
visual_communication_data_in_mongo: VisualCommunicationData,
|
||||
):
|
||||
"""Test listing names of all documents."""
|
||||
# ARRANGE
|
||||
name = 'test-visual-communication-2'
|
||||
annotation = VisualCommunicationValues.from_random()
|
||||
second_visual_communication = VisualCommunicationData(
|
||||
name=name,
|
||||
annotation=annotation,
|
||||
)
|
||||
visual_communication_repo.put_data(second_visual_communication)
|
||||
expected_name_list = [
|
||||
visual_communication_data_in_mongo.name,
|
||||
second_visual_communication.name,
|
||||
]
|
||||
# ACT
|
||||
name_list = visual_communication_repo.list_names()
|
||||
# ASSERT
|
||||
assert len(name_list) == 2
|
||||
for name in name_list:
|
||||
assert name in expected_name_list
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main(['-s', '-v', __file__])
|
||||
Reference in New Issue
Block a user