From 93fffe14e2e52617860467a1c9c5604003fd757b Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Mon, 15 Sep 2025 00:43:20 +0200 Subject: [PATCH 1/9] added optional group minio --- pyproject.toml | 3 ++ uv.lock | 98 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b9e7b8d..b67dae3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,9 @@ dependencies = [ redis = [ "redis>=6.4.0", ] +minio = [ + "minio>=7.2.16", +] [tool.uv.sources] python-utils = { index = "gitea" } diff --git a/uv.lock b/uv.lock index b19ead3..32b62f7 100644 --- a/uv.lock +++ b/uv.lock @@ -2,7 +2,8 @@ version = 1 revision = 3 requires-python = ">=3.12" resolution-markers = [ - "python_full_version >= '3.13'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", "python_full_version < '3.13'", ] @@ -29,6 +30,49 @@ wheels = [ { url = "http://10.0.0.2:5001/index/anyio/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1" }, ] +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "http://10.0.0.2:5001/index/" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "http://10.0.0.2:5001/index/argon2-cffi/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1" } +wheels = [ + { url = "http://10.0.0.2:5001/index/argon2-cffi/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "25.1.0" +source = { registry = "http://10.0.0.2:5001/index/" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d" } +wheels = [ + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:3d3f05610594151994ca9ccb3c771115bdb4daef161976a266f0dd8aa9996b8f" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8b8efee945193e667a396cbc7b4fb7d357297d6234d30a489905d96caabde56b" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3c6702abc36bf3ccba3f802b799505def420a1b7039862014a65db3205967f5a" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1c70058c6ab1e352304ac7e3b52554daadacd8d453c1752e547c76e9c99ac44" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2fd3bfbff3c5d74fef31a722f729bf93500910db650c925c2d6ef879a7e51cb" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4f9665de60b1b0e99bcd6be4f17d90339698ce954cfd8d9cf4f91c995165a92" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ba92837e4a9aa6a508c8d2d7883ed5a8f6c308c89a4790e1e447a220deb79a85" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp314-cp314t-win32.whl", hash = "sha256:84a461d4d84ae1295871329b346a97f68eade8c53b6ed9a7ca2d7467f3c8ff6f" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b55aec3565b65f56455eebc9b9f34130440404f27fe21c3b375bf1ea4d8fbae6" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:87c33a52407e4c41f3b70a9c2d3f6056d88b10dad7695be708c5021673f55623" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp39-abi3-win32.whl", hash = "sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp39-abi3-win_amd64.whl", hash = "sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98" }, + { url = "http://10.0.0.2:5001/index/argon2-cffi-bindings/argon2_cffi_bindings-25.1.0-cp39-abi3-win_arm64.whl", hash = "sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94" }, +] + [[package]] name = "authlib" version = "1.6.3" @@ -475,6 +519,22 @@ wheels = [ { url = "http://10.0.0.2:5001/index/mdurl/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8" }, ] +[[package]] +name = "minio" +version = "7.2.16" +source = { registry = "http://10.0.0.2:5001/index/" } +dependencies = [ + { name = "argon2-cffi" }, + { name = "certifi" }, + { name = "pycryptodome" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "http://10.0.0.2:5001/index/minio/minio-7.2.16.tar.gz", hash = "sha256:81e365c8494d591d8204a63ee7596bfdf8a7d06ad1b1507d6b9c1664a95f299a" } +wheels = [ + { url = "http://10.0.0.2:5001/index/minio/minio-7.2.16-py3-none-any.whl", hash = "sha256:9288ab988ca57c181eb59a4c96187b293131418e28c164392186c2b89026b223" }, +] + [[package]] name = "mypy" version = "1.18.1" @@ -616,6 +676,36 @@ wheels = [ { url = "http://10.0.0.2:5001/index/pycparser/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934" }, ] +[[package]] +name = "pycryptodome" +version = "3.23.0" +source = { registry = "http://10.0.0.2:5001/index/" } +sdist = { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef" } +wheels = [ + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2" }, + { url = "http://10.0.0.2:5001/index/pycryptodome/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c" }, +] + [[package]] name = "pydantic" version = "2.9.2" @@ -723,6 +813,9 @@ dependencies = [ ] [package.optional-dependencies] +minio = [ + { name = "minio" }, +] redis = [ { name = "redis" }, ] @@ -742,11 +835,12 @@ dev = [ [package.metadata] requires-dist = [ + { name = "minio", marker = "extra == 'minio'", specifier = ">=7.2.16" }, { name = "python-utils", specifier = ">=0.1.0", index = "https://gitea.gt-proj.com/api/packages/brian/pypi/simple/" }, { name = "redis", marker = "extra == 'redis'", specifier = ">=6.4.0" }, { name = "structlog", specifier = ">=25.4.0" }, ] -provides-extras = ["redis"] +provides-extras = ["redis", "minio"] [package.metadata.requires-dev] dev = [ From cc99fe5a2f8d343333d9b05430cec5c07d23f4ee Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Mon, 15 Sep 2025 00:43:32 +0200 Subject: [PATCH 2/9] minor corrections --- tests/integration/redis_adapter_test.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/integration/redis_adapter_test.py b/tests/integration/redis_adapter_test.py index 46ac8cc..bb57f24 100644 --- a/tests/integration/redis_adapter_test.py +++ b/tests/integration/redis_adapter_test.py @@ -10,7 +10,7 @@ from redis.commands.json.path import Path as RedisPath from python_repositories.adapters.redis_adapter import RedisAdapter -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def data() -> Generator[dict[str, str]]: """Provide a sample data dictionary for tests.""" yield {"foo": "bar"} @@ -115,7 +115,6 @@ def test_should_log_error_on_exception_during_exit( raise ValueError("Simulated error") except ValueError: pass # Expected - # Assert error was logged assert "Error while exiting context" in caplog.text @@ -135,8 +134,9 @@ def test_should_get_value( # Arrange key, data = data_in_redis # Act - value = redis_adapter._get(key) + value = redis_adapter._get(key) # type: ignore # Assert + assert value is not None assert value == data @@ -145,7 +145,7 @@ def test_should_get_none_for_missing_key( ) -> None: """Test that getting a non-existent key returns None.""" # Act - value = redis_adapter._get("nonexistent_key") + value = redis_adapter._get("nonexistent_key") # type: ignore # Assert assert value is None @@ -167,7 +167,7 @@ def test_should_raise_connection_error_on_get_when_not_connected( ) -> None: """Test that the RedisAdapter raises ConnectionError when getting while not connected.""" # Arrange - adapter = RedisAdapter() + adapter = RedisAdapter() # not connected # Act & Assert with pytest.raises(ConnectionError): adapter._get("some_key") @@ -180,10 +180,14 @@ def test_should_set_value( """Test that the RedisAdapter can set a value.""" # Arrange key = "test_key" + received_data = redis_adapter._get(key) # type: ignore + assert received_data is None # Ensure key does not exist # Act redis_adapter._set(key, data) # Assert - assert redis_adapter._get(key) == data + received_data = redis_adapter._get(key) # type: ignore + assert received_data is not None + assert received_data == data def test_should_update_value( @@ -194,6 +198,9 @@ def test_should_update_value( # Arrange key, _ = data_in_redis new_data = {"new_key": "new_value"} + received_data = redis_adapter._get(key) # type: ignore + assert received_data is not None + assert received_data != new_data # Act redis_adapter._set(key, new_data) # Assert @@ -246,6 +253,8 @@ def test_should_delete_key( """Test that deleting a key removes it from Redis.""" # Arrange key, _ = data_in_redis + received_data = redis_adapter._get(key) # type: ignore + assert received_data is not None # Ensure key exists # Act redis_adapter._delete(key) # Assert From a3b5443e0dcf2715893a4ea3b5114a0a0bd19881 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Mon, 15 Sep 2025 00:43:43 +0200 Subject: [PATCH 3/9] added minio adapter and tests --- python_repositories/adapters/minio_adapter.py | 202 ++++++++ tests/integration/conftest.py | 65 ++- tests/integration/minio_adapter_test.py | 467 ++++++++++++++++++ 3 files changed, 732 insertions(+), 2 deletions(-) create mode 100644 python_repositories/adapters/minio_adapter.py create mode 100644 tests/integration/minio_adapter_test.py diff --git a/python_repositories/adapters/minio_adapter.py b/python_repositories/adapters/minio_adapter.py new file mode 100644 index 0000000..51ea42b --- /dev/null +++ b/python_repositories/adapters/minio_adapter.py @@ -0,0 +1,202 @@ +"""Definition of MinioAdapter class.""" + +from __future__ import annotations +import os +from io import BytesIO + +import structlog +from minio import Minio, S3Error + +from python_utils import check_env + +from python_repositories.interfaces import ( + ContextAwareInterface, + ConnectionAwareInterface, +) + + +class MinioAdapter( + ContextAwareInterface, + ConnectionAwareInterface, +): + """Minio adapter exposing basic CRUD functionality.""" + + endpoint_env_var_name: str = "MINIO_ENDPOINT" + access_key_env_var_name: str = "MINIO_ACCESS_KEY" + secret_key_env_var_name: str = "MINIO_SECRET_KEY" + bucket_env_var_name: str = "MINIO_BUCKET" + chunk_size: int = 5*2**20 # 5 MiB + + def __init__(self) -> None: + # Setup logger + self.logger = structlog.get_logger( + self.__class__.__name__, + ) + # Check environment variables + check_env( + { + self.endpoint_env_var_name, + self.access_key_env_var_name, + self.secret_key_env_var_name, + self.bucket_env_var_name, + }, + ) + # Prepare internal variables + self._client: Minio | None = None + self._bucket_name: str | None = None + + def __enter__(self) -> MinioAdapter: + """Enter the context.""" + self.connect() + return self + + def __exit__( + self, exc_type: type | None, exc_val: object | None, exc_tb: object | None + ) -> None: + """Exit the context.""" + ctx_info = {"exc_type": exc_type, "exc_val": exc_val, "exc_tb": exc_tb} + if any( + ( + exc_type is not None, + exc_val is not None, + exc_tb is not None, + ), + ): + self.logger.error("Error while exiting context", **ctx_info) + self.disconnect() + + def connect(self) -> None: + """Connect to the Minio server.""" + # Stop if already connected + if self.is_connected: + self.logger.info("Already connected to Minio") + return + # Prepare arguments + endpoint = str(os.getenv(self.endpoint_env_var_name)) + access_key = str(os.getenv(self.access_key_env_var_name)) + secret_key = str(os.getenv(self.secret_key_env_var_name)) + bucket = str(os.getenv(self.bucket_env_var_name)) + # Connect client + client = Minio( + endpoint=endpoint, + access_key=access_key, + secret_key=secret_key, + secure=False, + ) + # Test the connection by listing buckets (will raise if connection fails) + try: + _ = client.list_buckets() + except Exception as exc: # pylint: disable=broad-except + raise ConnectionError(f"Could not connect to Minio at {endpoint}") from exc + # Ensure bucket exists + if not client.bucket_exists(bucket): + self.logger.info(f"Creating bucket '{bucket}'") + client.make_bucket(bucket) + # Persist information + self._client = client + self._bucket_name = bucket + + def disconnect(self) -> None: + """Disconnect from the Minio server.""" + # Close connection + # N.B. Minio client does not have a close method, but we include this for symmetry with other adapters + # Reset client + self._client = None + self._bucket_name = None + + @property + def is_connected(self) -> bool: + """Check if connected to Minio server.""" + res = bool(isinstance(self._client, Minio)) + self.logger.debug(res) + return res + + def _put(self, object_name: str, data: BytesIO) -> None: + """Put an object into the Minio bucket.""" + # Check input + if not isinstance(object_name, str) or len(object_name) == 0: + raise ValueError("object_name must be a non-empty string") + if not isinstance(data, BytesIO) or data.getbuffer().nbytes == 0: + raise ValueError("data must be a non-empty BytesIO object") + # Check connection + if self._client is None or not self.is_connected: + raise ConnectionError("Not connected to Minio") + # Prepare buffer for reading + num_bytes = data.getbuffer().nbytes + data.seek(0) + # Send data to bucket + # N.B. bucket name is set when connecting + self._client.put_object( + bucket_name=self._bucket_name, # type: ignore + object_name=object_name, + data=data, + length=num_bytes, + part_size=self.chunk_size, + ) + self.logger.debug(f"Put object '{object_name}' into bucket '{self._bucket_name}'") + + def _get(self, object_name: str) -> BytesIO | None: + """Get an object from the Minio bucket.""" + # Check input + if not isinstance(object_name, str) or len(object_name) == 0: + raise ValueError("object_name must be a non-empty string") + # Check connection + if self._client is None or not self.is_connected: + raise ConnectionError("Not connected to Minio") + # Get data from bucket + # N.B. bucket name is set when connecting + try: + response = self._client.get_object( + bucket_name=self._bucket_name, # type: ignore + object_name=object_name, + ) + # Get buffered data + buffer = BytesIO() + while chunk := response.read(self.chunk_size): + buffer.write(chunk) + buffer.seek(0) + self.logger.debug(f"Got object '{object_name}' from bucket '{self._bucket_name}'") + return buffer + except S3Error as exc: + if exc.code == "NoSuchKey": + self.logger.warning(f"Object '{object_name}' not found in bucket '{self._bucket_name}'") + else: + self.logger.error(repr(exc)) + except Exception as exc: # pylint: disable=broad-except + self.logger.error(repr(exc)) + return None + + def _delete(self, object_name: str) -> None: + """Delete an object from the Minio bucket.""" + # Check input + if not isinstance(object_name, str) or len(object_name) == 0: + raise ValueError("object_name must be a non-empty string") + # Check connection + if self._client is None or not self.is_connected: + raise ConnectionError("Not connected to Minio") + # Delete object from bucket + # N.B. bucket name is set when connecting + self._client.remove_object( + bucket_name=self._bucket_name, # type: ignore + object_name=object_name, + ) + self.logger.debug(f"Deleted object '{object_name}' from bucket '{self._bucket_name}'") + + def _list_objects(self, prefix: str = "") -> list[str]: + """List objects in the Minio bucket with an optional prefix.""" + # Check input + if not isinstance(prefix, str): + raise ValueError("prefix must be a string") + # Check connection + # N.B. bucket name is set when connecting + if self._client is None or not self.is_connected: + raise ConnectionError("Not connected to Minio") + # List objects in bucket + objects = self._client.list_objects( + bucket_name=self._bucket_name, # type: ignore + prefix=prefix, + recursive=True, + ) + object_names = [obj.object_name for obj in objects if obj.object_name is not None] + self.logger.debug(f"Listed {len(object_names)} object(s) in bucket '{self._bucket_name}' with prefix '{prefix}'") + return object_names diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e7547e5..208357c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,8 +6,14 @@ import pytest import redis import structlog import logging +from minio import Minio from testcontainers.redis import RedisContainer +from testcontainers.minio import MinioContainer + +MINIO_ACCESS_KEY = "minioadmin" +MINIO_SECRET_KEY = "minioadmin" +MINIO_BUCKET = "test-bucket" @pytest.fixture(scope="session", autouse=True) @@ -16,6 +22,10 @@ def configure_logging() -> None: # Configure structlog structlog.configure( processors=[ + structlog.stdlib.filter_by_level, + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + structlog.processors.TimeStamper(fmt="iso"), structlog.processors.JSONRenderer(), ], logger_factory=structlog.stdlib.LoggerFactory(), @@ -27,12 +37,11 @@ def configure_logging() -> None: @pytest.fixture(scope="session") -def redis_container() -> Generator[str]: +def redis_container() -> Generator[str, None, None]: """Set up a Redis container for testing and yield the Redis URI.""" # Start container container = RedisContainer( image="redis/redis-stack:7.2.0-v0", - port=6379, ) container.start() # Set environment variable for Redis URI @@ -46,13 +55,42 @@ def redis_container() -> Generator[str]: container.stop() +@pytest.fixture(scope="session") +def minio_container() -> Generator[dict[str, str], None, None]: + """Set up a Minio container for testing and yield the Minio URI.""" + # Start container + container = MinioContainer( + image="minio/minio:latest", + access_key=MINIO_ACCESS_KEY, + secret_key=MINIO_SECRET_KEY, + ) + container.start() + # Build environment variables dictionary + minio_host = container.get_container_host_ip() + minio_port = container.get_exposed_port(9000) + minio_endpoint = f"{minio_host}:{minio_port}" + env_vars = { + "MINIO_ENDPOINT": minio_endpoint, + "MINIO_ACCESS_KEY": MINIO_ACCESS_KEY, + "MINIO_SECRET_KEY": MINIO_SECRET_KEY, + "MINIO_BUCKET": MINIO_BUCKET, + } + + yield env_vars + + # Stop container + container.stop() + + @pytest.fixture(scope="session", autouse=True) def set_environment_variables( redis_container: str, + minio_container: dict[str, str], ) -> Generator[dict[str, str]]: """Set environment variables needed for tests.""" # Build environment variables dictionary env_vars = {"REDIS_URI": redis_container} + env_vars.update(minio_container) # Set environment variables for key, value in env_vars.items(): os.environ[key] = value @@ -78,3 +116,26 @@ def raw_redis_client(redis_container: str) -> Generator[redis.Redis]: # Cleanup client.flushall() client.close() + + +@pytest.fixture(scope="session") +def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio]: + """Provide a raw Minio client connected to the test Minio container.""" + # Connect client + client = Minio( + endpoint=minio_container["MINIO_ENDPOINT"], + access_key=minio_container["MINIO_ACCESS_KEY"], + secret_key=minio_container["MINIO_SECRET_KEY"], + secure=False, + ) + # Ensure bucket exists + bucket_name = minio_container["MINIO_BUCKET"] + if not client.bucket_exists(bucket_name): + client.make_bucket(bucket_name) + + yield client + + # Cleanup + objects = client.list_objects(bucket_name, recursive=True) + for obj in objects: + client.remove_object(bucket_name, obj.object_name) \ No newline at end of file diff --git a/tests/integration/minio_adapter_test.py b/tests/integration/minio_adapter_test.py new file mode 100644 index 0000000..47e3ca1 --- /dev/null +++ b/tests/integration/minio_adapter_test.py @@ -0,0 +1,467 @@ +# pylint: disable=protected-access +# The above line disables pylint's protected member access warnings for this file, +# allowing tests to access MinioAdapter's internal methods as needed for integration testing. +"""Integration tests for the MinioAdapter.""" + +from collections.abc import Generator +from minio import S3Error +import pytest +from unittest.mock import MagicMock, patch +from minio import Minio +from io import BytesIO +import random +import os +import logging +from python_repositories.adapters.minio_adapter import MinioAdapter + + +def same_data( + data_a: BytesIO, + data_b: BytesIO, +) -> bool: + """Check if two BytesIO-objects contain the same data.""" + assert isinstance(data_a, BytesIO) + assert isinstance(data_b, BytesIO) + # prepare for being read + data_a.seek(0) + data_b.seek(0) + # convert to bytes + data_a_bytes = data_a.read() + data_b_bytes = data_b.read() + # compare size + if len(data_a_bytes) != len(data_b_bytes): + logging.error( + 'data has different length: %s and %s', + len(data_a_bytes), + len(data_b_bytes), + ) + return False + # compare content + if data_a_bytes != data_b_bytes: + logging.error('data has different bytes') + return False + return True + + +@pytest.fixture(scope="module") +def data() -> Generator[BytesIO]: + """Provide a sample data bytes for tests.""" + # Generate random bytes + random_bytes = random.randbytes(2**21) # 2 MiB + yield BytesIO(random_bytes) + + +@pytest.fixture(scope="function") +def data_in_minio( + raw_minio_client: Minio, + data: BytesIO, +) -> Generator[tuple[str, BytesIO]]: + """Fixture to set up a known value in Minio before each test.""" + object_name = "test_object" + bucket_name = str(os.getenv("MINIO_BUCKET")) + # Upload object + num_bytes = data.getbuffer().nbytes + data.seek(0) + raw_minio_client.put_object( + bucket_name, + object_name, + data, + length=num_bytes, + part_size=MinioAdapter.chunk_size, + ) + # Reset data for reading in tests + data.seek(0) + + yield object_name, data + + # Cleanup + raw_minio_client.remove_object(bucket_name, object_name) + + +@pytest.fixture(scope="module") +def minio_adapter() -> Generator[MinioAdapter]: + """Fixture to provide a connected MinioAdapter instance.""" + adapter = MinioAdapter() + adapter.connect() + yield adapter + adapter.disconnect() + + +@pytest.fixture(scope="function", autouse=True) +def clear_minio( + raw_minio_client: Minio, +) -> None: + """Fixture to clear all Minio objects before each test.""" + bucket_name = str(os.getenv("MINIO_BUCKET")) + # Clear all objects before each test + objects = raw_minio_client.list_objects(bucket_name, recursive=True) + for obj in objects: + if not obj.object_name: + continue + raw_minio_client.remove_object(bucket_name, obj.object_name) + + +def test_should_adhere_to_interface() -> None: + """Test that the MinioAdapter adheres to the expected interface.""" + # Instantiation fails if interface not adhered to + _ = MinioAdapter() + + +def test_should_have_logger_when_instantiated() -> None: + """Test that the MinioAdapter has a logger when instantiated.""" + adapter = MinioAdapter() + assert hasattr(adapter, "logger") + + +def test_should_not_be_connected_when_instantiated() -> None: + """Test that the MinioAdapter is not connected when instantiated.""" + adapter = MinioAdapter() + assert not adapter.is_connected + + +def test_should_log_info_when_already_connected( + minio_adapter: MinioAdapter, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that the MinioAdapter logs info when connect is called while already connected.""" + with caplog.at_level(logging.INFO): + minio_adapter.connect() + assert "Already connected to Minio" in caplog.text + + +def test_should_raise_connection_error_when_unable_to_connect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test that the MinioAdapter raises a ConnectionError when unable to connect.""" + # Arrange + monkeypatch.setenv("MINIO_ENDPOINT", "invalid_uri") + adapter = MinioAdapter() + with pytest.raises(ConnectionError): + adapter.connect() + assert not adapter.is_connected + + +def test_should_log_info_when_creating_expected_bucket( + raw_minio_client: Minio, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that the MinioAdapter logs info when creating the expected bucket.""" + # Arrange + bucket_name = str(os.getenv("MINIO_BUCKET")) + raw_minio_client.remove_bucket(bucket_name) + adapter = MinioAdapter() + # Act + with caplog.at_level(logging.INFO): + adapter.connect() + # Assert + assert f"Creating bucket '{bucket_name}'" in caplog.text + + +def test_should_log_error_on_exception_during_exit( + minio_container: dict[str, str], + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that the MinioAdapter logs an error if an exception occurs during __exit__.""" + try: + with MinioAdapter() as adapter: + assert adapter.is_connected + raise ValueError("Simulated error") + except ValueError: + pass # Expected + # Assert error was logged + assert "Error while exiting context" in caplog.text + + +def test_should_have_context_manager() -> None: + """Test that the MinioAdapter can be used as a context manager.""" + with MinioAdapter() as adapter: + assert adapter._client is not None + assert adapter._client is None + + +def test_should_get_data( + data_in_minio: tuple[str, BytesIO], + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter can get data from a bucket.""" + # Arrange + object_name, expected_data = data_in_minio + # Act + received_data = minio_adapter._get(object_name) # type: ignore + # Assert + assert received_data is not None + assert same_data(expected_data, received_data) + + +def test_should_get_none_for_nonexistent_object( + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter returns None for a nonexistent object.""" + # Arrange + object_name = "nonexistent_object" + # Act + received_data = minio_adapter._get(object_name) # type: ignore + # Assert + assert received_data is None + + +def test_should_raise_value_error_on_invalid_get_object_name( + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter raises ValueError when getting with an invalid object name.""" + # Arrange + invalid_object_names = ["", 123, None] + # Act & Assert + for object_name in invalid_object_names: + with pytest.raises(ValueError): + minio_adapter._get(object_name) # type: ignore + + +def test_should_raise_connection_error_on_get_when_not_connected( + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter raises ConnectionError when getting while not connected.""" + # Arrange + adapter = MinioAdapter() # not connected + # Act & Assert + with pytest.raises(ConnectionError): + adapter._get("some_object") # type: ignore + + +def test_should_log_warning_when_getting_nonexistent_object( + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that the MinioAdapter logs a warning when getting a nonexistent object.""" + # Arrange + adapter = MinioAdapter() + adapter._client = MagicMock(spec=Minio) + adapter._client.get_object.side_effect = S3Error(code="NoSuchKey", message="", resource="", request_id="", host_id="", response="", bucket_name="test-bucket", object_name="missing-object") + adapter._bucket_name = "test-bucket" + object_name = "missing-object" + # Act + with caplog.at_level("WARNING"): + result = adapter._get(object_name) + # Assert + assert result is None + assert f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'" in caplog.text + + +def test_should_log_error_when_getting_with_s3error_other_than_no_such_key( + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that the MinioAdapter logs an error when getting a nonexistent object.""" + # Arrange + adapter = MinioAdapter() + adapter._client = MagicMock(spec=Minio) + other_s3error = S3Error(code="UnhandledError", message="", resource="", request_id="", host_id="", response="", bucket_name="test-bucket", object_name="missing-object") + adapter._client.get_object.side_effect = other_s3error + adapter._bucket_name = "test-bucket" + object_name = "missing-object" + # Act + with caplog.at_level("ERROR"): + result = adapter._get(object_name) + # Assert + assert result is None + assert repr(other_s3error) in caplog.text + + +def test_should_log_error_when_getting_with_general_exception( + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that the MinioAdapter logs an error when getting a nonexistent object.""" + # Arrange + adapter = MinioAdapter() + adapter._client = MagicMock(spec=Minio) + general_exception = Exception("General failure") + adapter._client.get_object.side_effect = general_exception + adapter._bucket_name = "test-bucket" + object_name = "missing-object" + # Act + with caplog.at_level("ERROR"): + result = adapter._get(object_name) + # Assert + assert result is None + assert repr(general_exception) in caplog.text + + +def test_should_put_data( + data: BytesIO, + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter can put data into a bucket.""" + # Arrange + object_name = "new_test_object" + received_data = minio_adapter._get(object_name) # type: ignore + assert received_data is None # ensure object does not exist yet + # Act + minio_adapter._put(object_name, data) # type: ignore + # Assert + received_data = minio_adapter._get(object_name) # type: ignore + assert received_data is not None + assert same_data(data, received_data) + # Cleanup + bucket_name = str(os.getenv("MINIO_BUCKET")) + minio_adapter._client.remove_object(bucket_name, object_name) + + +def test_should_update_data( + data_in_minio: tuple[str, BytesIO], + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter can update data in a bucket.""" + # Arrange + object_name, _ = data_in_minio + new_data = BytesIO(random.randbytes(2**21)) # 2 MiB + received_data = minio_adapter._get(object_name) # type: ignore + assert received_data is not None + assert not same_data(received_data, new_data) + # Act + minio_adapter._put(object_name, new_data) # type: ignore + # Assert + received_data = minio_adapter._get(object_name) # type: ignore + assert received_data is not None + assert same_data(new_data, received_data) + + +def test_should_raise_value_error_on_invalid_put_object_name( + data: BytesIO, + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter raises ValueError when putting with an invalid object name.""" + # Arrange + invalid_object_names = ["", 123, None] + # Act & Assert + for object_name in invalid_object_names: + with pytest.raises(ValueError): + minio_adapter._put(object_name, data) # type: ignore + + +def test_should_raise_value_error_on_invalid_put_data( + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter raises ValueError when putting with invalid data.""" + # Arrange + object_name = "valid_object_name" + invalid_data = ["not_bytesio", 123, None] + # Act & Assert + for data in invalid_data: + with pytest.raises(ValueError): + minio_adapter._put(object_name, data) # type: ignore + + +def test_should_raise_connection_error_on_put_when_not_connected( + data: BytesIO, + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter raises ConnectionError when putting while not connected.""" + # Arrange + adapter = MinioAdapter() # not connected + # Act & Assert + with pytest.raises(ConnectionError): + adapter._put("some_object", data) # type: ignore + + +def test_should_delete_object( + data_in_minio: tuple[str, BytesIO], + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter can delete an object from a bucket.""" + # Arrange + object_name, _ = data_in_minio + received_data = minio_adapter._get(object_name) # type: ignore + assert received_data is not None # ensure object exists + # Act + minio_adapter._delete(object_name) # type: ignore + # Assert + received_data = minio_adapter._get(object_name) # type: ignore + assert received_data is None + + +def test_should_raise_value_error_on_invalid_delete_object_name( + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter raises ValueError when deleting with an invalid object name.""" + # Arrange + invalid_object_names = ["", 123, None] + # Act & Assert + for object_name in invalid_object_names: + with pytest.raises(ValueError): + minio_adapter._delete(object_name) # type: ignore + + +def test_should_raise_connection_error_on_delete_when_not_connected( + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter raises ConnectionError when deleting while not connected.""" + # Arrange + adapter = MinioAdapter() # not connected + # Act & Assert + with pytest.raises(ConnectionError): + adapter._delete("some_object") # type: ignore + + +def test_should_list_objects( + data_in_minio: tuple[str, BytesIO], + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter can list objects in a bucket.""" + # Arrange + object_name, _ = data_in_minio + new_data = BytesIO(random.randbytes(2**21)) # 2 MiB + new_data_name = "another_test_object" + minio_adapter._put(new_data_name, new_data) # type: ignore + # Act + objects = minio_adapter._list_objects() # type: ignore + # Assert + assert isinstance(objects, list) + assert len(objects) == 2 + assert object_name in objects + assert new_data_name in objects + + +def test_should_list_objects_with_prefix( + data_in_minio: tuple[str, BytesIO], + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter can list objects in a bucket with a prefix.""" + # Arrange + object_name, _ = data_in_minio + new_data = BytesIO(random.randbytes(2**21)) # 2 MiB + new_data_name = "prefix_test_object" + minio_adapter._put(new_data_name, new_data) # type: ignore + prefix = "prefix_" + # Act + objects = minio_adapter._list_objects(prefix) # type: ignore + # Assert + assert isinstance(objects, list) + assert len(objects) == 1 + assert new_data_name in objects + assert object_name not in objects + + +def test_should_raise_value_error_on_invalid_list_objects_prefix( + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter raises ValueError when listing with an invalid prefix.""" + # Arrange + invalid_prefixes = [123, None] + # Act & Assert + for prefix in invalid_prefixes: + with pytest.raises(ValueError): + minio_adapter._list_objects(prefix) # type: ignore + + +def test_should_raise_connection_error_on_list_objects_when_not_connected( + minio_adapter: MinioAdapter, +) -> None: + """Test that the MinioAdapter raises ConnectionError when listing while not connected.""" + # Arrange + adapter = MinioAdapter() # not connected + # Act & Assert + with pytest.raises(ConnectionError): + adapter._list_objects() # type: ignore + + +# allows local debugging by running file as script +if __name__ == "__main__": + pytest.main(["-s", "-v", __file__]) From c45ba3c5d54462262a1a1ec62fc04070300571c3 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Mon, 15 Sep 2025 00:53:26 +0200 Subject: [PATCH 4/9] mypy fixes --- pyproject.toml | 4 ++ python_repositories/adapters/minio_adapter.py | 8 ++-- tests/integration/minio_adapter_test.py | 40 +++++++++---------- tests/integration/redis_adapter_test.py | 12 +++--- 4 files changed, 34 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b67dae3..bf86a63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,10 @@ namespace_packages = true # enable namespace packages module = "testcontainers.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "minio.*" +ignore_missing_imports = true + [dependency-groups] dev = [ "mypy>=1.17.1", diff --git a/python_repositories/adapters/minio_adapter.py b/python_repositories/adapters/minio_adapter.py index 51ea42b..f78f535 100644 --- a/python_repositories/adapters/minio_adapter.py +++ b/python_repositories/adapters/minio_adapter.py @@ -127,7 +127,7 @@ class MinioAdapter( # Send data to bucket # N.B. bucket name is set when connecting self._client.put_object( - bucket_name=self._bucket_name, # type: ignore + bucket_name=self._bucket_name, object_name=object_name, data=data, length=num_bytes, @@ -147,7 +147,7 @@ class MinioAdapter( # N.B. bucket name is set when connecting try: response = self._client.get_object( - bucket_name=self._bucket_name, # type: ignore + bucket_name=self._bucket_name, object_name=object_name, ) # Get buffered data @@ -177,7 +177,7 @@ class MinioAdapter( # Delete object from bucket # N.B. bucket name is set when connecting self._client.remove_object( - bucket_name=self._bucket_name, # type: ignore + bucket_name=self._bucket_name, object_name=object_name, ) self.logger.debug(f"Deleted object '{object_name}' from bucket '{self._bucket_name}'") @@ -193,7 +193,7 @@ class MinioAdapter( raise ConnectionError("Not connected to Minio") # List objects in bucket objects = self._client.list_objects( - bucket_name=self._bucket_name, # type: ignore + bucket_name=self._bucket_name, prefix=prefix, recursive=True, ) diff --git a/tests/integration/minio_adapter_test.py b/tests/integration/minio_adapter_test.py index 47e3ca1..a9a4752 100644 --- a/tests/integration/minio_adapter_test.py +++ b/tests/integration/minio_adapter_test.py @@ -187,7 +187,7 @@ def test_should_get_data( # Arrange object_name, expected_data = data_in_minio # Act - received_data = minio_adapter._get(object_name) # type: ignore + received_data = minio_adapter._get(object_name) # Assert assert received_data is not None assert same_data(expected_data, received_data) @@ -200,7 +200,7 @@ def test_should_get_none_for_nonexistent_object( # Arrange object_name = "nonexistent_object" # Act - received_data = minio_adapter._get(object_name) # type: ignore + received_data = minio_adapter._get(object_name) # Assert assert received_data is None @@ -225,7 +225,7 @@ def test_should_raise_connection_error_on_get_when_not_connected( adapter = MinioAdapter() # not connected # Act & Assert with pytest.raises(ConnectionError): - adapter._get("some_object") # type: ignore + adapter._get("some_object") def test_should_log_warning_when_getting_nonexistent_object( @@ -291,17 +291,17 @@ def test_should_put_data( """Test that the MinioAdapter can put data into a bucket.""" # Arrange object_name = "new_test_object" - received_data = minio_adapter._get(object_name) # type: ignore + received_data = minio_adapter._get(object_name) assert received_data is None # ensure object does not exist yet # Act - minio_adapter._put(object_name, data) # type: ignore + minio_adapter._put(object_name, data) # Assert - received_data = minio_adapter._get(object_name) # type: ignore + received_data = minio_adapter._get(object_name) assert received_data is not None assert same_data(data, received_data) # Cleanup bucket_name = str(os.getenv("MINIO_BUCKET")) - minio_adapter._client.remove_object(bucket_name, object_name) + minio_adapter._client.remove_object(bucket_name, object_name) # type: ignore def test_should_update_data( @@ -312,13 +312,13 @@ def test_should_update_data( # Arrange object_name, _ = data_in_minio new_data = BytesIO(random.randbytes(2**21)) # 2 MiB - received_data = minio_adapter._get(object_name) # type: ignore + received_data = minio_adapter._get(object_name) assert received_data is not None assert not same_data(received_data, new_data) # Act - minio_adapter._put(object_name, new_data) # type: ignore + minio_adapter._put(object_name, new_data) # Assert - received_data = minio_adapter._get(object_name) # type: ignore + received_data = minio_adapter._get(object_name) assert received_data is not None assert same_data(new_data, received_data) @@ -358,7 +358,7 @@ def test_should_raise_connection_error_on_put_when_not_connected( adapter = MinioAdapter() # not connected # Act & Assert with pytest.raises(ConnectionError): - adapter._put("some_object", data) # type: ignore + adapter._put("some_object", data) def test_should_delete_object( @@ -368,12 +368,12 @@ def test_should_delete_object( """Test that the MinioAdapter can delete an object from a bucket.""" # Arrange object_name, _ = data_in_minio - received_data = minio_adapter._get(object_name) # type: ignore + received_data = minio_adapter._get(object_name) assert received_data is not None # ensure object exists # Act - minio_adapter._delete(object_name) # type: ignore + minio_adapter._delete(object_name) # Assert - received_data = minio_adapter._get(object_name) # type: ignore + received_data = minio_adapter._get(object_name) assert received_data is None @@ -397,7 +397,7 @@ def test_should_raise_connection_error_on_delete_when_not_connected( adapter = MinioAdapter() # not connected # Act & Assert with pytest.raises(ConnectionError): - adapter._delete("some_object") # type: ignore + adapter._delete("some_object") def test_should_list_objects( @@ -409,9 +409,9 @@ def test_should_list_objects( object_name, _ = data_in_minio new_data = BytesIO(random.randbytes(2**21)) # 2 MiB new_data_name = "another_test_object" - minio_adapter._put(new_data_name, new_data) # type: ignore + minio_adapter._put(new_data_name, new_data) # Act - objects = minio_adapter._list_objects() # type: ignore + objects = minio_adapter._list_objects() # Assert assert isinstance(objects, list) assert len(objects) == 2 @@ -428,10 +428,10 @@ def test_should_list_objects_with_prefix( object_name, _ = data_in_minio new_data = BytesIO(random.randbytes(2**21)) # 2 MiB new_data_name = "prefix_test_object" - minio_adapter._put(new_data_name, new_data) # type: ignore + minio_adapter._put(new_data_name, new_data) prefix = "prefix_" # Act - objects = minio_adapter._list_objects(prefix) # type: ignore + objects = minio_adapter._list_objects(prefix) # Assert assert isinstance(objects, list) assert len(objects) == 1 @@ -459,7 +459,7 @@ def test_should_raise_connection_error_on_list_objects_when_not_connected( adapter = MinioAdapter() # not connected # Act & Assert with pytest.raises(ConnectionError): - adapter._list_objects() # type: ignore + adapter._list_objects() # allows local debugging by running file as script diff --git a/tests/integration/redis_adapter_test.py b/tests/integration/redis_adapter_test.py index bb57f24..293ae9a 100644 --- a/tests/integration/redis_adapter_test.py +++ b/tests/integration/redis_adapter_test.py @@ -134,7 +134,7 @@ def test_should_get_value( # Arrange key, data = data_in_redis # Act - value = redis_adapter._get(key) # type: ignore + value = redis_adapter._get(key) # Assert assert value is not None assert value == data @@ -145,7 +145,7 @@ def test_should_get_none_for_missing_key( ) -> None: """Test that getting a non-existent key returns None.""" # Act - value = redis_adapter._get("nonexistent_key") # type: ignore + value = redis_adapter._get("nonexistent_key") # Assert assert value is None @@ -180,12 +180,12 @@ def test_should_set_value( """Test that the RedisAdapter can set a value.""" # Arrange key = "test_key" - received_data = redis_adapter._get(key) # type: ignore + received_data = redis_adapter._get(key) assert received_data is None # Ensure key does not exist # Act redis_adapter._set(key, data) # Assert - received_data = redis_adapter._get(key) # type: ignore + received_data = redis_adapter._get(key) assert received_data is not None assert received_data == data @@ -198,7 +198,7 @@ def test_should_update_value( # Arrange key, _ = data_in_redis new_data = {"new_key": "new_value"} - received_data = redis_adapter._get(key) # type: ignore + received_data = redis_adapter._get(key) assert received_data is not None assert received_data != new_data # Act @@ -253,7 +253,7 @@ def test_should_delete_key( """Test that deleting a key removes it from Redis.""" # Arrange key, _ = data_in_redis - received_data = redis_adapter._get(key) # type: ignore + received_data = redis_adapter._get(key) assert received_data is not None # Ensure key exists # Act redis_adapter._delete(key) From 5a725077753fad1bf91557546b1397800ac4c2dd Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Mon, 15 Sep 2025 00:53:38 +0200 Subject: [PATCH 5/9] ruff format --- python_repositories/adapters/minio_adapter.py | 26 +++++++++++----- tests/integration/conftest.py | 2 +- tests/integration/minio_adapter_test.py | 31 ++++++++++++++++--- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/python_repositories/adapters/minio_adapter.py b/python_repositories/adapters/minio_adapter.py index f78f535..560ca2c 100644 --- a/python_repositories/adapters/minio_adapter.py +++ b/python_repositories/adapters/minio_adapter.py @@ -25,7 +25,7 @@ class MinioAdapter( access_key_env_var_name: str = "MINIO_ACCESS_KEY" secret_key_env_var_name: str = "MINIO_SECRET_KEY" bucket_env_var_name: str = "MINIO_BUCKET" - chunk_size: int = 5*2**20 # 5 MiB + chunk_size: int = 5 * 2**20 # 5 MiB def __init__(self) -> None: # Setup logger @@ -133,7 +133,9 @@ class MinioAdapter( length=num_bytes, part_size=self.chunk_size, ) - self.logger.debug(f"Put object '{object_name}' into bucket '{self._bucket_name}'") + self.logger.debug( + f"Put object '{object_name}' into bucket '{self._bucket_name}'" + ) def _get(self, object_name: str) -> BytesIO | None: """Get an object from the Minio bucket.""" @@ -155,11 +157,15 @@ class MinioAdapter( while chunk := response.read(self.chunk_size): buffer.write(chunk) buffer.seek(0) - self.logger.debug(f"Got object '{object_name}' from bucket '{self._bucket_name}'") + self.logger.debug( + f"Got object '{object_name}' from bucket '{self._bucket_name}'" + ) return buffer except S3Error as exc: if exc.code == "NoSuchKey": - self.logger.warning(f"Object '{object_name}' not found in bucket '{self._bucket_name}'") + self.logger.warning( + f"Object '{object_name}' not found in bucket '{self._bucket_name}'" + ) else: self.logger.error(repr(exc)) except Exception as exc: # pylint: disable=broad-except @@ -180,7 +186,9 @@ class MinioAdapter( bucket_name=self._bucket_name, object_name=object_name, ) - self.logger.debug(f"Deleted object '{object_name}' from bucket '{self._bucket_name}'") + self.logger.debug( + f"Deleted object '{object_name}' from bucket '{self._bucket_name}'" + ) def _list_objects(self, prefix: str = "") -> list[str]: """List objects in the Minio bucket with an optional prefix.""" @@ -197,6 +205,10 @@ class MinioAdapter( prefix=prefix, recursive=True, ) - object_names = [obj.object_name for obj in objects if obj.object_name is not None] - self.logger.debug(f"Listed {len(object_names)} object(s) in bucket '{self._bucket_name}' with prefix '{prefix}'") + object_names = [ + obj.object_name for obj in objects if obj.object_name is not None + ] + self.logger.debug( + f"Listed {len(object_names)} object(s) in bucket '{self._bucket_name}' with prefix '{prefix}'" + ) return object_names diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 208357c..a1efc3b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -138,4 +138,4 @@ def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio]: # Cleanup objects = client.list_objects(bucket_name, recursive=True) for obj in objects: - client.remove_object(bucket_name, obj.object_name) \ No newline at end of file + client.remove_object(bucket_name, obj.object_name) diff --git a/tests/integration/minio_adapter_test.py b/tests/integration/minio_adapter_test.py index a9a4752..4dc9821 100644 --- a/tests/integration/minio_adapter_test.py +++ b/tests/integration/minio_adapter_test.py @@ -31,14 +31,14 @@ def same_data( # compare size if len(data_a_bytes) != len(data_b_bytes): logging.error( - 'data has different length: %s and %s', + "data has different length: %s and %s", len(data_a_bytes), len(data_b_bytes), ) return False # compare content if data_a_bytes != data_b_bytes: - logging.error('data has different bytes') + logging.error("data has different bytes") return False return True @@ -235,7 +235,16 @@ def test_should_log_warning_when_getting_nonexistent_object( # Arrange adapter = MinioAdapter() adapter._client = MagicMock(spec=Minio) - adapter._client.get_object.side_effect = S3Error(code="NoSuchKey", message="", resource="", request_id="", host_id="", response="", bucket_name="test-bucket", object_name="missing-object") + adapter._client.get_object.side_effect = S3Error( + code="NoSuchKey", + message="", + resource="", + request_id="", + host_id="", + response="", + bucket_name="test-bucket", + object_name="missing-object", + ) adapter._bucket_name = "test-bucket" object_name = "missing-object" # Act @@ -243,7 +252,10 @@ def test_should_log_warning_when_getting_nonexistent_object( result = adapter._get(object_name) # Assert assert result is None - assert f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'" in caplog.text + assert ( + f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'" + in caplog.text + ) def test_should_log_error_when_getting_with_s3error_other_than_no_such_key( @@ -253,7 +265,16 @@ def test_should_log_error_when_getting_with_s3error_other_than_no_such_key( # Arrange adapter = MinioAdapter() adapter._client = MagicMock(spec=Minio) - other_s3error = S3Error(code="UnhandledError", message="", resource="", request_id="", host_id="", response="", bucket_name="test-bucket", object_name="missing-object") + other_s3error = S3Error( + code="UnhandledError", + message="", + resource="", + request_id="", + host_id="", + response="", + bucket_name="test-bucket", + object_name="missing-object", + ) adapter._client.get_object.side_effect = other_s3error adapter._bucket_name = "test-bucket" object_name = "missing-object" From 800d702d7aa68d4701cf3fc686559a50f7f6c600 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Mon, 15 Sep 2025 00:54:01 +0200 Subject: [PATCH 6/9] pyupgrade --- tests/integration/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a1efc3b..9b4bfc6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -37,7 +37,7 @@ def configure_logging() -> None: @pytest.fixture(scope="session") -def redis_container() -> Generator[str, None, None]: +def redis_container() -> Generator[str]: """Set up a Redis container for testing and yield the Redis URI.""" # Start container container = RedisContainer( @@ -56,7 +56,7 @@ def redis_container() -> Generator[str, None, None]: @pytest.fixture(scope="session") -def minio_container() -> Generator[dict[str, str], None, None]: +def minio_container() -> Generator[dict[str, str]]: """Set up a Minio container for testing and yield the Minio URI.""" # Start container container = MinioContainer( From c7bced42541057367642a68b9b8eb05fdb046abe Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Mon, 15 Sep 2025 00:56:21 +0200 Subject: [PATCH 7/9] ruff lint fixes --- tests/integration/minio_adapter_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/minio_adapter_test.py b/tests/integration/minio_adapter_test.py index 4dc9821..cc5da13 100644 --- a/tests/integration/minio_adapter_test.py +++ b/tests/integration/minio_adapter_test.py @@ -4,14 +4,14 @@ """Integration tests for the MinioAdapter.""" from collections.abc import Generator -from minio import S3Error import pytest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock from minio import Minio from io import BytesIO import random import os import logging +from minio import S3Error from python_repositories.adapters.minio_adapter import MinioAdapter From 3260a3ffb1b89f798c2955e1ed958477ab0c1584 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Mon, 15 Sep 2025 00:59:40 +0200 Subject: [PATCH 8/9] added new optional package to ci --- .gitea/workflows/code-quality.yml | 2 +- .gitea/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/code-quality.yml b/.gitea/workflows/code-quality.yml index 1788bbe..b636ec6 100644 --- a/.gitea/workflows/code-quality.yml +++ b/.gitea/workflows/code-quality.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies env: UV_LINK_MODE: copy - run: uv sync --extra redis + run: uv sync --extra redis,minio - name: Type check with mypy run: uv run mypy . diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 4fbdc7f..81d964f 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies env: UV_LINK_MODE: copy - run: uv sync --extra redis + run: uv sync --extra redis,minio - name: Run pytest env: From f5af30328d28b8add4dda931ec9f889fe1ec0226 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Mon, 15 Sep 2025 01:03:34 +0200 Subject: [PATCH 9/9] auto install all extra packages --- .gitea/workflows/code-quality.yml | 2 +- .gitea/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/code-quality.yml b/.gitea/workflows/code-quality.yml index b636ec6..8891941 100644 --- a/.gitea/workflows/code-quality.yml +++ b/.gitea/workflows/code-quality.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies env: UV_LINK_MODE: copy - run: uv sync --extra redis,minio + run: uv sync --all-extras - name: Type check with mypy run: uv run mypy . diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 81d964f..a75c48c 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies env: UV_LINK_MODE: copy - run: uv sync --extra redis,minio + run: uv sync --all-extras - name: Run pytest env: