Merge pull request 'added support for content type when putting object in minio' (#9) from add-support-for-content-type-in-minio into main

Reviewed-on: brian/python-repositories#9

.gitea/workflows/code-quality.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Install dependencies
         env:
           UV_LINK_MODE: copy
-        run: uv sync --extra redis
+        run: uv sync --all-extras
 
       - name: Type check with mypy
         run: uv run mypy .

.gitea/workflows/test.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Install dependencies
         env:
           UV_LINK_MODE: copy
-        run: uv sync --extra redis
+        run: uv sync --all-extras
 
       - name: Run pytest
         env:

pyproject.toml

@@ -60,6 +60,10 @@ namespace_packages = true  # enable namespace packages
 module = "testcontainers.*"
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+module = "minio.*"
+ignore_missing_imports = true
+
 [dependency-groups]
 dev = [
     "mypy>=1.17.1",

python_repositories/adapters/__init__.py

@@ -1,5 +1,13 @@
-from .redis_adapter import RedisAdapter as RedisAdapter
+"""
+Adapters for various backend repositories (e.g., Redis, Minio).
+
+This module exposes concrete implementations for repository interfaces.
+"""
+
+from .redis_adapter import RedisAdapter
+from .minio_adapter import MinioAdapter
 
 __all__ = [
     "RedisAdapter",
+    "MinioAdapter",
 ]

python_repositories/adapters/minio_adapter.py

@@ -3,10 +3,8 @@
 from __future__ import annotations
 import os
 from io import BytesIO
-
+from importlib.util import find_spec
 import structlog
-from minio import Minio, S3Error
-
 from python_utils import check_env
 
 from python_repositories.interfaces import (
@@ -14,6 +12,10 @@ from python_repositories.interfaces import (
     ConnectionAwareInterface,
 )
 
+# Handle optional dependencies
+if find_spec("minio") is not None:
+    import minio
+
 
 class MinioAdapter(
     ContextAwareInterface,
@@ -25,7 +27,7 @@ class MinioAdapter(
     access_key_env_var_name: str = "MINIO_ACCESS_KEY"
     secret_key_env_var_name: str = "MINIO_SECRET_KEY"
     bucket_env_var_name: str = "MINIO_BUCKET"
-    chunk_size: int = 5*2**20  # 5 MiB
+    chunk_size: int = 5 * 2**20  # 5 MiB
 
     def __init__(self) -> None:
         # Setup logger
@@ -42,7 +44,7 @@ class MinioAdapter(
             },
         )
         # Prepare internal variables
-        self._client: Minio | None = None
+        self._client: minio.Minio | None = None
         self._bucket_name: str | None = None
 
     def __enter__(self) -> MinioAdapter:
@@ -77,7 +79,7 @@ class MinioAdapter(
         secret_key = str(os.getenv(self.secret_key_env_var_name))
         bucket = str(os.getenv(self.bucket_env_var_name))
         # Connect client
-        client = Minio(
+        client = minio.Minio(
             endpoint=endpoint,
             access_key=access_key,
             secret_key=secret_key,
@@ -107,17 +109,24 @@ class MinioAdapter(
     @property
     def is_connected(self) -> bool:
         """Check if connected to Minio server."""
-        res = bool(isinstance(self._client, Minio))
+        res = bool(isinstance(self._client, minio.Minio))
         self.logger.debug(res)
         return res
 
-    def _put(self, object_name: str, data: BytesIO) -> None:
+    def _put(
+        self,
+        object_name: str,
+        data: BytesIO,
+        content_type: str = "application/octet-stream",
+    ) -> None:
         """Put an object into the Minio bucket."""
         # Check input
         if not isinstance(object_name, str) or len(object_name) == 0:
             raise ValueError("object_name must be a non-empty string")
         if not isinstance(data, BytesIO) or data.getbuffer().nbytes == 0:
             raise ValueError("data must be a non-empty BytesIO object")
+        if not isinstance(content_type, str) or len(content_type) == 0:
+            raise ValueError("content_type must be a non-empty string")
         # Check connection
         if self._client is None or not self.is_connected:
             raise ConnectionError("Not connected to Minio")
@@ -127,13 +136,16 @@ class MinioAdapter(
         # Send data to bucket
         # N.B. bucket name is set when connecting
         self._client.put_object(
-            bucket_name=self._bucket_name,  # type: ignore
+            bucket_name=self._bucket_name,
             object_name=object_name,
             data=data,
             length=num_bytes,
             part_size=self.chunk_size,
+            content_type=content_type,
+        )
+        self.logger.debug(
+            f"Put object '{object_name}' into bucket '{self._bucket_name}'"
         )
-        self.logger.debug(f"Put object '{object_name}' into bucket '{self._bucket_name}'")
 
     def _get(self, object_name: str) -> BytesIO | None:
         """Get an object from the Minio bucket."""
@@ -147,7 +159,7 @@ class MinioAdapter(
         # N.B. bucket name is set when connecting
         try:
             response = self._client.get_object(
-                bucket_name=self._bucket_name,  # type: ignore
+                bucket_name=self._bucket_name,
                 object_name=object_name,
             )
             # Get buffered data
@@ -155,11 +167,15 @@ class MinioAdapter(
             while chunk := response.read(self.chunk_size):
                 buffer.write(chunk)
             buffer.seek(0)
-            self.logger.debug(f"Got object '{object_name}' from bucket '{self._bucket_name}'")
+            self.logger.debug(
+                f"Got object '{object_name}' from bucket '{self._bucket_name}'"
+            )
             return buffer
-        except S3Error as exc:
+        except minio.S3Error as exc:
             if exc.code == "NoSuchKey":
-                self.logger.warning(f"Object '{object_name}' not found in bucket '{self._bucket_name}'")
+                self.logger.warning(
+                    f"Object '{object_name}' not found in bucket '{self._bucket_name}'"
+                )
             else:
                 self.logger.error(repr(exc))
         except Exception as exc:  # pylint: disable=broad-except
@@ -177,10 +193,12 @@ class MinioAdapter(
         # Delete object from bucket
         # N.B. bucket name is set when connecting
         self._client.remove_object(
-            bucket_name=self._bucket_name,  # type: ignore
+            bucket_name=self._bucket_name,
             object_name=object_name,
         )
-        self.logger.debug(f"Deleted object '{object_name}' from bucket '{self._bucket_name}'")
+        self.logger.debug(
+            f"Deleted object '{object_name}' from bucket '{self._bucket_name}'"
+        )
 
     def _list_objects(self, prefix: str = "") -> list[str]:
         """List objects in the Minio bucket with an optional prefix."""
@@ -193,10 +211,14 @@ class MinioAdapter(
             raise ConnectionError("Not connected to Minio")
         # List objects in bucket
         objects = self._client.list_objects(
-            bucket_name=self._bucket_name,  # type: ignore
+            bucket_name=self._bucket_name,
             prefix=prefix,
             recursive=True,
         )
-        object_names = [obj.object_name for obj in objects if obj.object_name is not None]
-        self.logger.debug(f"Listed {len(object_names)} object(s) in bucket '{self._bucket_name}' with prefix '{prefix}'")
+        object_names = [
+            obj.object_name for obj in objects if obj.object_name is not None
+        ]
+        self.logger.debug(
+            f"Listed {len(object_names)} object(s) in bucket '{self._bucket_name}' with prefix '{prefix}'"
+        )
         return object_names

python_repositories/adapters/redis_adapter.py

@@ -2,13 +2,10 @@
 
 from __future__ import annotations
 from typing import cast
-
+from importlib.util import find_spec
 import os
 import structlog
 
-import redis
-from redis.commands.json.path import Path as RedisPath
-
 from python_utils import check_env
 
 from python_repositories.interfaces import (
@@ -16,6 +13,11 @@ from python_repositories.interfaces import (
     ConnectionAwareInterface,
 )
 
+# Handle optional dependencies
+if find_spec("redis") is not None:
+    import redis
+    from redis.commands.json.path import Path as RedisPath
+
 
 class RedisAdapter(
     ContextAwareInterface,
@@ -24,7 +26,7 @@ class RedisAdapter(
     """Redis adapter exposing basic CRUD functionality."""
 
     uri_env_var_name: str = "REDIS_URI"
-    path: str = RedisPath.root_path()
+    path: str = "."  # JSON root path, updated in __init__
     encoding: str = "UTF-8"
 
     def __init__(self) -> None:
@@ -36,6 +38,7 @@ class RedisAdapter(
         check_env(self.uri_env_var_name)
         # Prepare internal variables
         self._client: redis.Redis | None = None
+        self.path: str = RedisPath.root_path()
 
     def __enter__(self) -> RedisAdapter:
         """Enter the context."""

tests/integration/conftest.py

@@ -37,7 +37,7 @@ def configure_logging() -> None:
 
 
 @pytest.fixture(scope="session")
-def redis_container() -> Generator[str, None, None]:
+def redis_container() -> Generator[str]:
     """Set up a Redis container for testing and yield the Redis URI."""
     # Start container
     container = RedisContainer(
@@ -56,7 +56,7 @@ def redis_container() -> Generator[str, None, None]:
 
 
 @pytest.fixture(scope="session")
-def minio_container() -> Generator[dict[str, str], None, None]:
+def minio_container() -> Generator[dict[str, str]]:
     """Set up a Minio container for testing and yield the Minio URI."""
     # Start container
     container = MinioContainer(
@@ -138,4 +138,4 @@ def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio]:
     # Cleanup
     objects = client.list_objects(bucket_name, recursive=True)
     for obj in objects:
-        client.remove_object(bucket_name, obj.object_name)
+        client.remove_object(bucket_name, obj.object_name)

tests/integration/minio_adapter_test.py

@@ -4,14 +4,14 @@
 """Integration tests for the MinioAdapter."""
 
 from collections.abc import Generator
-from minio import S3Error
 import pytest
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock
 from minio import Minio
 from io import BytesIO
 import random
 import os
 import logging
+from minio import S3Error
 from python_repositories.adapters.minio_adapter import MinioAdapter
 
 
@@ -31,14 +31,14 @@ def same_data(
     # compare size
     if len(data_a_bytes) != len(data_b_bytes):
         logging.error(
-            'data has different length: %s and %s',
+            "data has different length: %s and %s",
             len(data_a_bytes),
             len(data_b_bytes),
         )
         return False
     # compare content
     if data_a_bytes != data_b_bytes:
-        logging.error('data has different bytes')
+        logging.error("data has different bytes")
         return False
     return True
 
@@ -187,7 +187,7 @@ def test_should_get_data(
     # Arrange
     object_name, expected_data = data_in_minio
     # Act
-    received_data = minio_adapter._get(object_name)  # type: ignore
+    received_data = minio_adapter._get(object_name)
     # Assert
     assert received_data is not None
     assert same_data(expected_data, received_data)
@@ -200,7 +200,7 @@ def test_should_get_none_for_nonexistent_object(
     # Arrange
     object_name = "nonexistent_object"
     # Act
-    received_data = minio_adapter._get(object_name)  # type: ignore
+    received_data = minio_adapter._get(object_name)
     # Assert
     assert received_data is None
 
@@ -225,7 +225,7 @@ def test_should_raise_connection_error_on_get_when_not_connected(
     adapter = MinioAdapter()  # not connected
     # Act & Assert
     with pytest.raises(ConnectionError):
-        adapter._get("some_object")  # type: ignore
+        adapter._get("some_object")
 
 
 def test_should_log_warning_when_getting_nonexistent_object(
@@ -235,7 +235,16 @@ def test_should_log_warning_when_getting_nonexistent_object(
     # Arrange
     adapter = MinioAdapter()
     adapter._client = MagicMock(spec=Minio)
-    adapter._client.get_object.side_effect = S3Error(code="NoSuchKey", message="", resource="", request_id="", host_id="", response="", bucket_name="test-bucket", object_name="missing-object")
+    adapter._client.get_object.side_effect = S3Error(
+        code="NoSuchKey",
+        message="",
+        resource="",
+        request_id="",
+        host_id="",
+        response="",
+        bucket_name="test-bucket",
+        object_name="missing-object",
+    )
     adapter._bucket_name = "test-bucket"
     object_name = "missing-object"
     # Act
@@ -243,7 +252,10 @@ def test_should_log_warning_when_getting_nonexistent_object(
         result = adapter._get(object_name)
     # Assert
     assert result is None
-    assert f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'" in caplog.text
+    assert (
+        f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'"
+        in caplog.text
+    )
 
 
 def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
@@ -253,7 +265,16 @@ def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
     # Arrange
     adapter = MinioAdapter()
     adapter._client = MagicMock(spec=Minio)
-    other_s3error = S3Error(code="UnhandledError", message="", resource="", request_id="", host_id="", response="", bucket_name="test-bucket", object_name="missing-object")
+    other_s3error = S3Error(
+        code="UnhandledError",
+        message="",
+        resource="",
+        request_id="",
+        host_id="",
+        response="",
+        bucket_name="test-bucket",
+        object_name="missing-object",
+    )
     adapter._client.get_object.side_effect = other_s3error
     adapter._bucket_name = "test-bucket"
     object_name = "missing-object"
@@ -291,17 +312,17 @@ def test_should_put_data(
     """Test that the MinioAdapter can put data into a bucket."""
     # Arrange
     object_name = "new_test_object"
-    received_data = minio_adapter._get(object_name)  # type: ignore
+    received_data = minio_adapter._get(object_name)
     assert received_data is None  # ensure object does not exist yet
     # Act
-    minio_adapter._put(object_name, data)  # type: ignore
+    minio_adapter._put(object_name, data)
     # Assert
-    received_data = minio_adapter._get(object_name)  # type: ignore
+    received_data = minio_adapter._get(object_name)
     assert received_data is not None
     assert same_data(data, received_data)
     # Cleanup
     bucket_name = str(os.getenv("MINIO_BUCKET"))
-    minio_adapter._client.remove_object(bucket_name, object_name)
+    minio_adapter._client.remove_object(bucket_name, object_name)  # type: ignore
 
 
 def test_should_update_data(
@@ -312,13 +333,13 @@ def test_should_update_data(
     # Arrange
     object_name, _ = data_in_minio
     new_data = BytesIO(random.randbytes(2**21))  # 2 MiB
-    received_data = minio_adapter._get(object_name)  # type: ignore
+    received_data = minio_adapter._get(object_name)
     assert received_data is not None
     assert not same_data(received_data, new_data)
     # Act
-    minio_adapter._put(object_name, new_data)  # type: ignore
+    minio_adapter._put(object_name, new_data)
     # Assert
-    received_data = minio_adapter._get(object_name)  # type: ignore
+    received_data = minio_adapter._get(object_name)
     assert received_data is not None
     assert same_data(new_data, received_data)
 
@@ -349,6 +370,20 @@ def test_should_raise_value_error_on_invalid_put_data(
             minio_adapter._put(object_name, data)  # type: ignore
 
 
+def test_should_raise_value_error_on_invalid_put_content_type(
+    data: BytesIO,
+    minio_adapter: MinioAdapter,
+) -> None:
+    """Test that the MinioAdapter raises ValueError when putting with an invalid content type."""
+    # Arrange
+    object_name = "valid_object_name"
+    invalid_content_types = ["", 123, None]
+    # Act & Assert
+    for content_type in invalid_content_types:
+        with pytest.raises(ValueError):
+            minio_adapter._put(object_name, data, content_type)  # type: ignore
+
+
 def test_should_raise_connection_error_on_put_when_not_connected(
     data: BytesIO,
     minio_adapter: MinioAdapter,
@@ -358,7 +393,7 @@ def test_should_raise_connection_error_on_put_when_not_connected(
     adapter = MinioAdapter()  # not connected
     # Act & Assert
     with pytest.raises(ConnectionError):
-        adapter._put("some_object", data)  # type: ignore
+        adapter._put("some_object", data)
 
 
 def test_should_delete_object(
@@ -368,12 +403,12 @@ def test_should_delete_object(
     """Test that the MinioAdapter can delete an object from a bucket."""
     # Arrange
     object_name, _ = data_in_minio
-    received_data = minio_adapter._get(object_name)  # type: ignore
+    received_data = minio_adapter._get(object_name)
     assert received_data is not None  # ensure object exists
     # Act
-    minio_adapter._delete(object_name)  # type: ignore
+    minio_adapter._delete(object_name)
     # Assert
-    received_data = minio_adapter._get(object_name)  # type: ignore
+    received_data = minio_adapter._get(object_name)
     assert received_data is None
 
 
@@ -397,7 +432,7 @@ def test_should_raise_connection_error_on_delete_when_not_connected(
     adapter = MinioAdapter()  # not connected
     # Act & Assert
     with pytest.raises(ConnectionError):
-        adapter._delete("some_object")  # type: ignore
+        adapter._delete("some_object")
 
 
 def test_should_list_objects(
@@ -409,9 +444,9 @@ def test_should_list_objects(
     object_name, _ = data_in_minio
     new_data = BytesIO(random.randbytes(2**21))  # 2 MiB
     new_data_name = "another_test_object"
-    minio_adapter._put(new_data_name, new_data)  # type: ignore
+    minio_adapter._put(new_data_name, new_data)
     # Act
-    objects = minio_adapter._list_objects()  # type: ignore
+    objects = minio_adapter._list_objects()
     # Assert
     assert isinstance(objects, list)
     assert len(objects) == 2
@@ -428,10 +463,10 @@ def test_should_list_objects_with_prefix(
     object_name, _ = data_in_minio
     new_data = BytesIO(random.randbytes(2**21))  # 2 MiB
     new_data_name = "prefix_test_object"
-    minio_adapter._put(new_data_name, new_data)  # type: ignore
+    minio_adapter._put(new_data_name, new_data)
     prefix = "prefix_"
     # Act
-    objects = minio_adapter._list_objects(prefix)  # type: ignore
+    objects = minio_adapter._list_objects(prefix)
     # Assert
     assert isinstance(objects, list)
     assert len(objects) == 1
@@ -459,7 +494,7 @@ def test_should_raise_connection_error_on_list_objects_when_not_connected(
     adapter = MinioAdapter()  # not connected
     # Act & Assert
     with pytest.raises(ConnectionError):
-        adapter._list_objects()  # type: ignore
+        adapter._list_objects()
 
 
 # allows local debugging by running file as script

tests/integration/redis_adapter_test.py

@@ -134,7 +134,7 @@ def test_should_get_value(
     # Arrange
     key, data = data_in_redis
     # Act
-    value = redis_adapter._get(key)  # type: ignore
+    value = redis_adapter._get(key)
     # Assert
     assert value is not None
     assert value == data
@@ -145,7 +145,7 @@ def test_should_get_none_for_missing_key(
 ) -> None:
     """Test that getting a non-existent key returns None."""
     # Act
-    value = redis_adapter._get("nonexistent_key")  # type: ignore
+    value = redis_adapter._get("nonexistent_key")
     # Assert
     assert value is None
 
@@ -180,12 +180,12 @@ def test_should_set_value(
     """Test that the RedisAdapter can set a value."""
     # Arrange
     key = "test_key"
-    received_data = redis_adapter._get(key)  # type: ignore
+    received_data = redis_adapter._get(key)
     assert received_data is None  # Ensure key does not exist
     # Act
     redis_adapter._set(key, data)
     # Assert
-    received_data = redis_adapter._get(key)  # type: ignore
+    received_data = redis_adapter._get(key)
     assert received_data is not None
     assert received_data == data
 
@@ -198,7 +198,7 @@ def test_should_update_value(
     # Arrange
     key, _ = data_in_redis
     new_data = {"new_key": "new_value"}
-    received_data = redis_adapter._get(key)  # type: ignore
+    received_data = redis_adapter._get(key)
     assert received_data is not None
     assert received_data != new_data
     # Act
@@ -253,7 +253,7 @@ def test_should_delete_key(
     """Test that deleting a key removes it from Redis."""
     # Arrange
     key, _ = data_in_redis
-    received_data = redis_adapter._get(key)  # type: ignore
+    received_data = redis_adapter._get(key)
     assert received_data is not None  # Ensure key exists
     # Act
     redis_adapter._delete(key)