From cbbb0441778a81e0e70d0e20ce61fd2e5cc8c62f Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 16 Nov 2024 19:46:37 +0000 Subject: [PATCH] updated integration tests --- .../tests/integration/base_crud_test.py | 77 ++++++++++++ .../datastore/tests/integration/conftest.py | 78 +++++++++++++ .../tests/integration/connect_test.py | 10 ++ .../tests/integration/image_crud_test.py | 53 +++++++++ .../tests/integration/model_crud_test.py | 53 +++++++++ shared/datastore/tests/integration_test.py | 110 ------------------ 6 files changed, 271 insertions(+), 110 deletions(-) create mode 100644 shared/datastore/tests/integration/base_crud_test.py create mode 100644 shared/datastore/tests/integration/conftest.py create mode 100644 shared/datastore/tests/integration/connect_test.py create mode 100644 shared/datastore/tests/integration/image_crud_test.py create mode 100644 shared/datastore/tests/integration/model_crud_test.py delete mode 100644 shared/datastore/tests/integration_test.py diff --git a/shared/datastore/tests/integration/base_crud_test.py b/shared/datastore/tests/integration/base_crud_test.py new file mode 100644 index 0000000..93b4c61 --- /dev/null +++ b/shared/datastore/tests/integration/base_crud_test.py @@ -0,0 +1,77 @@ +"""Integration tests related to base CRUD functions.""" + +from io import BytesIO + +import pytest + +from shared.datastore import connect_minio, delete, get, put + + +def test_should_get_data( + data_in_minio, +): + data, bucket_name, object_name = data_in_minio + client = connect_minio() + received_data = get( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) + assert isinstance(data, BytesIO) + assert received_data == data + + +def test_should_delete_data( + data_in_minio, +): + _, bucket_name, object_name = data_in_minio + client = connect_minio() + delete( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) + with pytest.raises(ValueError): + _ = get( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) + + +def test_should_put_data( + data, +): + buffer, bucket_name, object_name = data + client = connect_minio() + put( + client=client, + buffer=buffer, + bucket_name=bucket_name, + object_name=object_name, + ) + received_data = get( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) + assert received_data == data + + +def test_should_update_data( + data_in_minio, +): + buffer, bucket_name, object_name = data_in_minio + client = connect_minio() + put( + client=client, + buffer=buffer, + bucket_name=bucket_name, + object_name=object_name, + ) + received_data = get( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) + assert received_data == buffer diff --git a/shared/datastore/tests/integration/conftest.py b/shared/datastore/tests/integration/conftest.py new file mode 100644 index 0000000..98a22ac --- /dev/null +++ b/shared/datastore/tests/integration/conftest.py @@ -0,0 +1,78 @@ +"""Integration test configurations.""" + +import os + +import pytest +from PIL import Image +from testcontainers.minio import MinioContainer + +env_var_map = { + 'MINIO_ENDPOINT': 'localhost:9000', + 'MINIO_ACCESS_KEY': 'test-access-key', + 'MINIO_SECRET_KEY': 'test-secret-key', + 'MINIO_BUCKET_NAME': 'test-bucket', +} +container_map = { + 'minio': MinioContainer( + port=env_var_map['MINIO_ENDPOINT'].rsplit(':', maxsplit=1)[-1], + access_key=env_var_map['MINIO_ACCESS_KEY'], + secret_key=env_var_map['MINIO_SECRET_KEY'], + ), +} + + +@pytest.fixture(scope='session', autouse=True) +def setup_infrastructure(request: pytest.FixtureRequest) -> None: + """Prepare infrastructure for integration test.""" + # prepare infrastructure + for container in container_map.values(): + container.start() + # update env var map + minio_host = container_map['minio'].get_container_host_ip() + minio_port = container_map['minio'].get_exposed_port(9000) + env_var_map['MINIO_ENDPOINT'] = f'{minio_host}:{minio_port}' + + # ensure cleanup + def cleanup_infrastructure(): + for container in container_map.values(): + container.stop() + + request.addfinalizer(cleanup_infrastructure) + + +@pytest.fixture(scope='session', autouse=True) +def populate_env( + request: pytest.FixtureRequest, + setup_infrastructure, +) -> None: + """Populate environment with variables used for testing.""" + # update env + for key, val in env_var_map.items(): + os.environ[key] = val + + # ensure cleanup + def cleanup_env(): + for key in env_var_map: + _ = os.environ.pop(key, default=None) + + request.addfinalizer(cleanup_env) + + +@pytest.fixture +def image() -> Image.Image: + # generate image + image = Image.new(mode='RGB', size=(480, 480)) + # expose image + yield image + + +# @pytest.fixture +# def image_in_minio( +# image: Image.Image, +# ) -> tuple(Image.Image, str): +# # + + +# # expose image and object name +# yield image, object_name +# # cleanup diff --git a/shared/datastore/tests/integration/connect_test.py b/shared/datastore/tests/integration/connect_test.py new file mode 100644 index 0000000..25b5aaf --- /dev/null +++ b/shared/datastore/tests/integration/connect_test.py @@ -0,0 +1,10 @@ +"""Integration test for connect_minio function.""" + +from minio import Minio + +from shared.datastore import connect_minio + + +def test_should_return_correct_type(): + client = connect_minio() + assert isinstance(client, Minio) diff --git a/shared/datastore/tests/integration/image_crud_test.py b/shared/datastore/tests/integration/image_crud_test.py new file mode 100644 index 0000000..bb1589e --- /dev/null +++ b/shared/datastore/tests/integration/image_crud_test.py @@ -0,0 +1,53 @@ +"""Integration tests related to image CRUD.""" + +from PIL import Image + +from shared.datastore import connect_minio, get_image, put_image + + +def test_should_get_image( + image_in_minio, +): + image, object_name = image_in_minio + client = connect_minio() + received_image = get_image( + client=client, + object_name=object_name, + ) + assert isinstance(image, Image.Image) + assert received_image == image + + +def test_should_put_image( + image, +): + client = connect_minio() + object_name = put_image( + client=client, + image=image, + ) + assert isinstance(object_name, str) + assert len(object_name) > 0 + received_image = get_image( + client=client, + object_name=object_name, + ) + assert received_image == image + + +def test_should_update_image( + image_in_minio, +): + image, object_name = image_in_minio + client = connect_minio() + object_name = put_image( + client=client, + image=image, + ) + assert isinstance(object_name, str) + assert len(object_name) > 0 + received_image = get_image( + client=client, + object_name=object_name, + ) + assert received_image == image diff --git a/shared/datastore/tests/integration/model_crud_test.py b/shared/datastore/tests/integration/model_crud_test.py new file mode 100644 index 0000000..dd9d859 --- /dev/null +++ b/shared/datastore/tests/integration/model_crud_test.py @@ -0,0 +1,53 @@ +"""Integration tests related to model CRUD.""" + +from torch.nn import Module + +from shared.datastore import connect_minio, get_model, put_model + + +def test_should_get_model( + model_in_minio, +): + model, object_name = model_in_minio + client = connect_minio() + received_model = get_model( + client=client, + object_name=object_name, + ) + assert isinstance(model, Module) + assert received_model == model + + +def test_should_put_model( + model, +): + client = connect_minio() + object_name = put_model( + client=client, + model=model, + ) + assert isinstance(object_name, str) + assert len(object_name) > 0 + received_model = get_model( + client=client, + object_name=object_name, + ) + assert received_model == model + + +def test_should_update_model( + model_in_minio, +): + model, object_name = model_in_minio + client = connect_minio() + object_name = put_model( + client=client, + model=model, + ) + assert isinstance(object_name, str) + assert len(object_name) > 0 + received_model = get_model( + client=client, + object_name=object_name, + ) + assert received_model == model diff --git a/shared/datastore/tests/integration_test.py b/shared/datastore/tests/integration_test.py deleted file mode 100644 index 959d8b1..0000000 --- a/shared/datastore/tests/integration_test.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Integration tests for functions exposed by the datastore module.""" - -import os -from io import BytesIO -from random import randbytes - -import pytest -from minio import Minio -from testcontainers.minio import MinioContainer - -from shared.datastore import connect_minio, get, put -from shared.utils import setup_logging - -MINIO_BUCKET_NAME = 'test-bucket' -minio = MinioContainer() - - -@pytest.fixture(scope='module', autouse=True) -def setup(request: pytest.FixtureRequest): - """Setup function for datastore module integration tests.""" - # start minio - minio.start() - - # ensure minio is stopped after testing - def remove_container(): - minio.stop() - - request.addfinalizer(remove_container) - # extract information - minio_ip = minio.get_container_host_ip() - minio_port = minio.get_exposed_port(port=9000) - os.environ['MINIO_ENDPOINT'] = f'{minio_ip}:{minio_port}' - os.environ['MINIO_ACCESS_KEY'] = minio.access_key - os.environ['MINIO_SECRET_KEY'] = minio.secret_key - os.environ['MINIO_BUCKET_NAME'] = MINIO_BUCKET_NAME - - -def test_connect_minio(): - """Test function connect_minio.""" - minio_client = connect_minio() - assert isinstance(minio_client, Minio) - - -def test_put(): - """Test function put.""" - # connect to minio - minio_client = connect_minio() - # generate random bytes - buffer = BytesIO(randbytes(2**20)) - # put data in Minio - put( - client=minio_client, - buffer=buffer, - bucket_name=MINIO_BUCKET_NAME, - object_name='test_put_data', - ) - - -def test_get(): - """Test function get.""" - # connect to minio - minio_client = connect_minio() - # generate random bytes - buffer = BytesIO(randbytes(2**20)) - # put data in minio - put( - client=minio_client, - buffer=buffer, - bucket_name=MINIO_BUCKET_NAME, - object_name='test_get_data', - ) - # get data back from minio - buffer_rec = get( - client=minio_client, - bucket_name=MINIO_BUCKET_NAME, - object_name='test_get_data', - ) - # check that data are the same - assert buffer.read() == buffer_rec.read() - - -def test_delete(): - """Test function delete.""" - raise NotImplementedError() - - -def test_get_image(): - """Test function get_image.""" - raise NotImplementedError() - - -def test_put_image(): - """Test function put_image.""" - raise NotImplementedError() - - -def test_get_model(): - """Test function get_model.""" - raise NotImplementedError() - - -def test_put_model(): - """Test function put_model.""" - raise NotImplementedError() - - -if __name__ == '__main__': - os.environ['LOG_LEVEL'] = 'debug' - setup_logging() - pytest.main()