Brian Bjarke JensenandCursor
5149299c1b
Convert is_connected to a method that verifies backend liveness via TTL-cached ping (Redis) or bucket_exists (MinIO), with cache invalidation on connect/disconnect. Co-authored-by: Cursor <[email protected]>
502 lines
15 KiB
Python
502 lines
15 KiB
Python
"""Integration tests for the MinioAdapter."""
|
|
|
|
from collections.abc import Generator
|
|
import pytest
|
|
from unittest.mock import MagicMock
|
|
from minio import Minio
|
|
from io import BytesIO
|
|
import random
|
|
import os
|
|
import logging
|
|
from minio import S3Error
|
|
from urllib3.response import BaseHTTPResponse
|
|
from python_repositories.adapters.minio_adapter import MinioAdapter
|
|
from python_repositories.interfaces import ObjectRepositoryInterface
|
|
|
|
|
|
def same_data(
|
|
data_a: BytesIO,
|
|
data_b: BytesIO,
|
|
) -> bool:
|
|
"""Check if two BytesIO-objects contain the same data."""
|
|
assert isinstance(data_a, BytesIO)
|
|
assert isinstance(data_b, BytesIO)
|
|
# prepare for being read
|
|
data_a.seek(0)
|
|
data_b.seek(0)
|
|
# convert to bytes
|
|
data_a_bytes = data_a.read()
|
|
data_b_bytes = data_b.read()
|
|
# compare size
|
|
if len(data_a_bytes) != len(data_b_bytes):
|
|
logging.error(
|
|
"data has different length: %s and %s",
|
|
len(data_a_bytes),
|
|
len(data_b_bytes),
|
|
)
|
|
return False
|
|
# compare content
|
|
if data_a_bytes != data_b_bytes:
|
|
logging.error("data has different bytes")
|
|
return False
|
|
return True
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def data() -> Generator[BytesIO, None, None]:
|
|
"""Provide a sample data bytes for tests."""
|
|
# Generate random bytes
|
|
random_bytes = random.randbytes(2**21) # 2 MiB
|
|
yield BytesIO(random_bytes)
|
|
|
|
|
|
@pytest.fixture(scope="function")
|
|
def data_in_minio(
|
|
raw_minio_client: Minio,
|
|
data: BytesIO,
|
|
) -> Generator[tuple[str, BytesIO], None, None]:
|
|
"""Fixture to set up a known value in Minio before each test."""
|
|
object_name = "test_object"
|
|
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
|
# Upload object
|
|
num_bytes = data.getbuffer().nbytes
|
|
data.seek(0)
|
|
raw_minio_client.put_object(
|
|
bucket_name,
|
|
object_name,
|
|
data,
|
|
length=num_bytes,
|
|
part_size=MinioAdapter.chunk_size,
|
|
)
|
|
# Reset data for reading in tests
|
|
data.seek(0)
|
|
|
|
yield object_name, data
|
|
|
|
# Cleanup
|
|
raw_minio_client.remove_object(bucket_name, object_name)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def minio_adapter() -> Generator[MinioAdapter, None, None]:
|
|
"""Fixture to provide a connected MinioAdapter instance."""
|
|
adapter = MinioAdapter()
|
|
adapter.connect()
|
|
yield adapter
|
|
adapter.disconnect()
|
|
|
|
|
|
@pytest.fixture(scope="function", autouse=True)
|
|
def clear_minio(
|
|
raw_minio_client: Minio,
|
|
) -> None:
|
|
"""Fixture to clear all Minio objects before each test."""
|
|
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
|
# Clear all objects before each test
|
|
objects = raw_minio_client.list_objects(bucket_name, recursive=True)
|
|
for obj in objects:
|
|
if not obj.object_name:
|
|
continue
|
|
raw_minio_client.remove_object(bucket_name, obj.object_name)
|
|
|
|
|
|
def test_should_adhere_to_interface() -> None:
|
|
"""Test that the MinioAdapter adheres to the expected interface."""
|
|
assert issubclass(MinioAdapter, ObjectRepositoryInterface)
|
|
_ = MinioAdapter()
|
|
|
|
|
|
def test_should_have_logger_when_instantiated() -> None:
|
|
"""Test that the MinioAdapter has a logger when instantiated."""
|
|
adapter = MinioAdapter()
|
|
assert hasattr(adapter, "logger")
|
|
|
|
|
|
def test_should_not_be_connected_when_instantiated() -> None:
|
|
"""Test that the MinioAdapter is not connected when instantiated."""
|
|
adapter = MinioAdapter()
|
|
assert not adapter.is_connected()
|
|
|
|
|
|
def test_should_log_info_when_already_connected(
|
|
minio_adapter: MinioAdapter,
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
"""Test that the MinioAdapter logs info when connect is called while already connected."""
|
|
with caplog.at_level(logging.INFO):
|
|
minio_adapter.connect()
|
|
assert "Already connected to Minio" in caplog.text
|
|
|
|
|
|
def test_should_raise_connection_error_when_unable_to_connect(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises a ConnectionError when unable to connect."""
|
|
# Arrange
|
|
monkeypatch.setenv("MINIO_ENDPOINT", "invalid_uri")
|
|
adapter = MinioAdapter()
|
|
with pytest.raises(ConnectionError):
|
|
adapter.connect()
|
|
assert not adapter.is_connected()
|
|
|
|
|
|
def test_should_log_info_when_creating_expected_bucket(
|
|
raw_minio_client: Minio,
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
"""Test that the MinioAdapter logs info when creating the expected bucket."""
|
|
# Arrange
|
|
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
|
raw_minio_client.remove_bucket(bucket_name)
|
|
adapter = MinioAdapter()
|
|
# Act
|
|
with caplog.at_level(logging.INFO):
|
|
adapter.connect()
|
|
# Assert
|
|
assert f"Creating bucket '{bucket_name}'" in caplog.text
|
|
|
|
|
|
def test_should_log_error_on_exception_during_exit(
|
|
minio_container: dict[str, str],
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
"""Test that the MinioAdapter logs an error if an exception occurs during __exit__."""
|
|
try:
|
|
with MinioAdapter() as adapter:
|
|
assert adapter.is_connected()
|
|
raise ValueError("Simulated error")
|
|
except ValueError:
|
|
pass # Expected
|
|
# Assert error was logged
|
|
assert "Error while exiting context" in caplog.text
|
|
|
|
|
|
def test_should_have_context_manager() -> None:
|
|
"""Test that the MinioAdapter can be used as a context manager."""
|
|
with MinioAdapter() as adapter:
|
|
assert adapter._client is not None
|
|
assert adapter._client is None
|
|
|
|
|
|
def test_should_get_data(
|
|
data_in_minio: tuple[str, BytesIO],
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter can get data from a bucket."""
|
|
# Arrange
|
|
object_name, expected_data = data_in_minio
|
|
# Act
|
|
received_data = minio_adapter.get(object_name)
|
|
# Assert
|
|
assert received_data is not None
|
|
assert same_data(expected_data, received_data)
|
|
|
|
|
|
def test_should_get_none_for_nonexistent_object(
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter returns None for a nonexistent object."""
|
|
# Arrange
|
|
object_name = "nonexistent_object"
|
|
# Act
|
|
received_data = minio_adapter.get(object_name)
|
|
# Assert
|
|
assert received_data is None
|
|
|
|
|
|
def test_should_raise_value_error_on_invalid_get_object_name(
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises ValueError when getting with an invalid object name."""
|
|
# Arrange
|
|
invalid_object_names = ["", 123, None]
|
|
# Act & Assert
|
|
for object_name in invalid_object_names:
|
|
with pytest.raises(ValueError):
|
|
minio_adapter.get(object_name) # type: ignore
|
|
|
|
|
|
def test_should_raise_connection_error_on_get_when_not_connected(
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises ConnectionError when getting while not connected."""
|
|
# Arrange
|
|
adapter = MinioAdapter() # not connected
|
|
# Act & Assert
|
|
with pytest.raises(ConnectionError):
|
|
adapter.get("some_object")
|
|
|
|
|
|
def test_should_log_warning_when_getting_nonexistent_object(
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
"""Test that the MinioAdapter logs a warning when getting a nonexistent object."""
|
|
# Arrange
|
|
adapter = MinioAdapter()
|
|
adapter._client = MagicMock(spec=Minio)
|
|
adapter._client.get_object.side_effect = S3Error(
|
|
MagicMock(spec=BaseHTTPResponse),
|
|
"NoSuchKey",
|
|
"",
|
|
"",
|
|
"",
|
|
"",
|
|
bucket_name="test-bucket",
|
|
object_name="missing-object",
|
|
)
|
|
adapter._bucket_name = "test-bucket"
|
|
object_name = "missing-object"
|
|
# Act
|
|
with caplog.at_level("WARNING"):
|
|
result = adapter.get(object_name)
|
|
# Assert
|
|
assert result is None
|
|
assert (
|
|
f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'"
|
|
in caplog.text
|
|
)
|
|
|
|
|
|
def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
"""Test that the MinioAdapter logs an error when getting a nonexistent object."""
|
|
# Arrange
|
|
adapter = MinioAdapter()
|
|
adapter._client = MagicMock(spec=Minio)
|
|
other_s3error = S3Error(
|
|
MagicMock(spec=BaseHTTPResponse),
|
|
"UnhandledError",
|
|
"",
|
|
"",
|
|
"",
|
|
"",
|
|
bucket_name="test-bucket",
|
|
object_name="missing-object",
|
|
)
|
|
adapter._client.get_object.side_effect = other_s3error
|
|
adapter._bucket_name = "test-bucket"
|
|
object_name = "missing-object"
|
|
# Act
|
|
with caplog.at_level("ERROR"):
|
|
result = adapter.get(object_name)
|
|
# Assert
|
|
assert result is None
|
|
assert repr(other_s3error) in caplog.text
|
|
|
|
|
|
def test_should_log_error_when_getting_with_general_exception(
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
"""Test that the MinioAdapter logs an error when getting a nonexistent object."""
|
|
# Arrange
|
|
adapter = MinioAdapter()
|
|
adapter._client = MagicMock(spec=Minio)
|
|
general_exception = Exception("General failure")
|
|
adapter._client.get_object.side_effect = general_exception
|
|
adapter._bucket_name = "test-bucket"
|
|
object_name = "missing-object"
|
|
# Act
|
|
with caplog.at_level("ERROR"):
|
|
result = adapter.get(object_name)
|
|
# Assert
|
|
assert result is None
|
|
assert repr(general_exception) in caplog.text
|
|
|
|
|
|
def test_should_put_data(
|
|
data: BytesIO,
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter can put data into a bucket."""
|
|
# Arrange
|
|
object_name = "new_test_object"
|
|
received_data = minio_adapter.get(object_name)
|
|
assert received_data is None # ensure object does not exist yet
|
|
# Act
|
|
minio_adapter.put(object_name, data)
|
|
# Assert
|
|
received_data = minio_adapter.get(object_name)
|
|
assert received_data is not None
|
|
assert same_data(data, received_data)
|
|
# Cleanup
|
|
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
|
minio_adapter._client.remove_object(bucket_name, object_name) # type: ignore
|
|
|
|
|
|
def test_should_update_data(
|
|
data_in_minio: tuple[str, BytesIO],
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter can update data in a bucket."""
|
|
# Arrange
|
|
object_name, _ = data_in_minio
|
|
new_data = BytesIO(random.randbytes(2**21)) # 2 MiB
|
|
received_data = minio_adapter.get(object_name)
|
|
assert received_data is not None
|
|
assert not same_data(received_data, new_data)
|
|
# Act
|
|
minio_adapter.put(object_name, new_data)
|
|
# Assert
|
|
received_data = minio_adapter.get(object_name)
|
|
assert received_data is not None
|
|
assert same_data(new_data, received_data)
|
|
|
|
|
|
def test_should_raise_value_error_on_invalid_put_object_name(
|
|
data: BytesIO,
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises ValueError when putting with an invalid object name."""
|
|
# Arrange
|
|
invalid_object_names = ["", 123, None]
|
|
# Act & Assert
|
|
for object_name in invalid_object_names:
|
|
with pytest.raises(ValueError):
|
|
minio_adapter.put(object_name, data) # type: ignore
|
|
|
|
|
|
def test_should_raise_value_error_on_invalid_put_data(
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises ValueError when putting with invalid data."""
|
|
# Arrange
|
|
object_name = "valid_object_name"
|
|
invalid_data = ["not_bytesio", 123, None]
|
|
# Act & Assert
|
|
for data in invalid_data:
|
|
with pytest.raises(ValueError):
|
|
minio_adapter.put(object_name, data) # type: ignore
|
|
|
|
|
|
def test_should_raise_value_error_on_invalid_put_content_type(
|
|
data: BytesIO,
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises ValueError when putting with an invalid content type."""
|
|
# Arrange
|
|
object_name = "valid_object_name"
|
|
invalid_content_types = ["", 123, None]
|
|
# Act & Assert
|
|
for content_type in invalid_content_types:
|
|
with pytest.raises(ValueError):
|
|
minio_adapter.put(object_name, data, content_type) # type: ignore
|
|
|
|
|
|
def test_should_raise_connection_error_on_put_when_not_connected(
|
|
data: BytesIO,
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises ConnectionError when putting while not connected."""
|
|
# Arrange
|
|
adapter = MinioAdapter() # not connected
|
|
# Act & Assert
|
|
with pytest.raises(ConnectionError):
|
|
adapter.put("some_object", data)
|
|
|
|
|
|
def test_should_delete_object(
|
|
data_in_minio: tuple[str, BytesIO],
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter can delete an object from a bucket."""
|
|
# Arrange
|
|
object_name, _ = data_in_minio
|
|
received_data = minio_adapter.get(object_name)
|
|
assert received_data is not None # ensure object exists
|
|
# Act
|
|
minio_adapter.delete(object_name)
|
|
# Assert
|
|
received_data = minio_adapter.get(object_name)
|
|
assert received_data is None
|
|
|
|
|
|
def test_should_raise_value_error_on_invalid_delete_object_name(
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises ValueError when deleting with an invalid object name."""
|
|
# Arrange
|
|
invalid_object_names = ["", 123, None]
|
|
# Act & Assert
|
|
for object_name in invalid_object_names:
|
|
with pytest.raises(ValueError):
|
|
minio_adapter.delete(object_name) # type: ignore
|
|
|
|
|
|
def test_should_raise_connection_error_on_delete_when_not_connected(
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises ConnectionError when deleting while not connected."""
|
|
# Arrange
|
|
adapter = MinioAdapter() # not connected
|
|
# Act & Assert
|
|
with pytest.raises(ConnectionError):
|
|
adapter.delete("some_object")
|
|
|
|
|
|
def test_should_list_objects(
|
|
data_in_minio: tuple[str, BytesIO],
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter can list objects in a bucket."""
|
|
# Arrange
|
|
object_name, _ = data_in_minio
|
|
new_data = BytesIO(random.randbytes(2**21)) # 2 MiB
|
|
new_data_name = "another_test_object"
|
|
minio_adapter.put(new_data_name, new_data)
|
|
# Act
|
|
objects = minio_adapter.list_objects()
|
|
# Assert
|
|
assert isinstance(objects, list)
|
|
assert len(objects) == 2
|
|
assert object_name in objects
|
|
assert new_data_name in objects
|
|
|
|
|
|
def test_should_list_objects_with_prefix(
|
|
data_in_minio: tuple[str, BytesIO],
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter can list objects in a bucket with a prefix."""
|
|
# Arrange
|
|
object_name, _ = data_in_minio
|
|
new_data = BytesIO(random.randbytes(2**21)) # 2 MiB
|
|
new_data_name = "prefix_test_object"
|
|
minio_adapter.put(new_data_name, new_data)
|
|
prefix = "prefix_"
|
|
# Act
|
|
objects = minio_adapter.list_objects(prefix)
|
|
# Assert
|
|
assert isinstance(objects, list)
|
|
assert len(objects) == 1
|
|
assert new_data_name in objects
|
|
assert object_name not in objects
|
|
|
|
|
|
def test_should_raise_value_error_on_invalid_list_objects_prefix(
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises ValueError when listing with an invalid prefix."""
|
|
# Arrange
|
|
invalid_prefixes = [123, None]
|
|
# Act & Assert
|
|
for prefix in invalid_prefixes:
|
|
with pytest.raises(ValueError):
|
|
minio_adapter.list_objects(prefix) # type: ignore
|
|
|
|
|
|
def test_should_raise_connection_error_on_list_objects_when_not_connected(
|
|
minio_adapter: MinioAdapter,
|
|
) -> None:
|
|
"""Test that the MinioAdapter raises ConnectionError when listing while not connected."""
|
|
# Arrange
|
|
adapter = MinioAdapter() # not connected
|
|
# Act & Assert
|
|
with pytest.raises(ConnectionError):
|
|
adapter.list_objects()
|
|
|
|
|
|
# allows local debugging by running file as script
|
|
if __name__ == "__main__":
|
|
pytest.main(["-s", "-v", __file__])
|