Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c2e6a96c5d | ||
|
|
547a1b3a90 | ||
|
|
39baf9badc | ||
|
|
4a5b3b631f | ||
|
|
1431f455c1 | ||
|
|
eafc717045 | ||
|
|
9a9b7b2985 | ||
|
|
5b9573d499 | ||
|
|
50444af982 | ||
|
|
2f19fcc972 | ||
|
|
3bd65895ec | ||
|
|
5311d49fa6 |
@@ -7,3 +7,4 @@ MINIO_ACCESS_KEY=minioadmin
|
||||
MINIO_SECRET_KEY=minioadmin
|
||||
MINIO_BUCKET=my-bucket
|
||||
MINIO_SECURE=false
|
||||
MINIO_CREATE_BUCKET_IF_MISSING=true
|
||||
|
||||
@@ -34,18 +34,33 @@ Requires Redis with the RedisJSON module (e.g. redis-stack).
|
||||
| -------------------- | ---------------------------------------------------- |
|
||||
| `REDIS_URI` | Redis connection URL (e.g. `redis://localhost:6379`) |
|
||||
|
||||
For key discovery:
|
||||
|
||||
- `list_keys(pattern)` is simple and returns a `list[str]`, but it uses Redis `KEYS` and may block on large datasets.
|
||||
- `scan_keys(pattern, *, count=None)` is preferred for production use and yields keys incrementally via Redis `SCAN`.
|
||||
|
||||
Example:
|
||||
|
||||
```python
|
||||
for key in repo.scan_keys("user:*"):
|
||||
print(key)
|
||||
```
|
||||
|
||||
### MinIO (`ObjectRepositoryInterface`)
|
||||
|
||||
| Environment variable | Description |
|
||||
| -------------------- | ------------------------------------------- |
|
||||
| -------------------------------- | ------------------------------------------------------------------------ |
|
||||
| `MINIO_ENDPOINT` | MinIO server endpoint |
|
||||
| `MINIO_ACCESS_KEY` | Access key |
|
||||
| `MINIO_SECRET_KEY` | Secret key |
|
||||
| `MINIO_BUCKET` | Bucket name (created on connect if missing) |
|
||||
| `MINIO_BUCKET` | Bucket name |
|
||||
| `MINIO_SECURE` | Use HTTPS (`true`/`false`; default: `true`) |
|
||||
| `MINIO_CREATE_BUCKET_IF_MISSING` | Auto-create `MINIO_BUCKET` on connect (`true`/`false`; default: `false`) |
|
||||
|
||||
Copy [`.env.example`](.env.example) to `.env` for local development. `RedisConfig.from_env()` and `MinioConfig.from_env()` load `.env` automatically when resolving configuration from the environment.
|
||||
|
||||
For production, it is recommended to leave `MINIO_CREATE_BUCKET_IF_MISSING` unset so that `connect()` fails fast if the expected bucket is missing. For local development, you will often want `MINIO_SECURE=false` and `MINIO_CREATE_BUCKET_IF_MISSING=true`.
|
||||
|
||||
## Configuration injection
|
||||
|
||||
Adapters accept optional `config` and `client` keyword arguments for explicit setup and testing:
|
||||
@@ -61,6 +76,7 @@ minio = MinioAdapter(
|
||||
secret_key="minioadmin",
|
||||
bucket="my-bucket",
|
||||
secure=False,
|
||||
# create_bucket_if_missing=True, # convenient for local dev
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "python-repositories"
|
||||
version = "0.5.1"
|
||||
version = "2.0.0"
|
||||
description = "Various python repository interfaces exposed as a python package."
|
||||
authors = [
|
||||
{ name = "Brian Bjarke Jensen", email = "[email protected]" }
|
||||
|
||||
@@ -26,6 +26,7 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
||||
secret_key_env_var_name: str = "MINIO_SECRET_KEY"
|
||||
bucket_env_var_name: str = "MINIO_BUCKET"
|
||||
secure_env_var_name: str = "MINIO_SECURE"
|
||||
create_bucket_if_missing_env_var_name: str = "MINIO_CREATE_BUCKET_IF_MISSING"
|
||||
chunk_size: int = 5 * 2**20 # 5 MiB
|
||||
connection_name: str = "Minio"
|
||||
|
||||
@@ -36,15 +37,16 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
||||
client: minio.Minio | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if config is None:
|
||||
if client is not None:
|
||||
if client is not None and config is None:
|
||||
raise ValueError("config is required when client is provided")
|
||||
if config is None:
|
||||
config = MinioConfig.from_env(
|
||||
self.endpoint_env_var_name,
|
||||
self.access_key_env_var_name,
|
||||
self.secret_key_env_var_name,
|
||||
self.bucket_env_var_name,
|
||||
self.secure_env_var_name,
|
||||
self.create_bucket_if_missing_env_var_name,
|
||||
)
|
||||
self._config = config
|
||||
self._client_injected = client is not None
|
||||
@@ -88,6 +90,10 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
raise ConnectionError(f"Could not connect to Minio at {endpoint}") from exc
|
||||
if not client.bucket_exists(bucket):
|
||||
if not self._config.create_bucket_if_missing:
|
||||
raise ConnectionError(
|
||||
f"Bucket '{bucket}' does not exist on Minio at {endpoint}"
|
||||
)
|
||||
self.logger.info(f"Creating bucket '{bucket}'")
|
||||
client.make_bucket(bucket)
|
||||
self._client = client
|
||||
@@ -172,15 +178,12 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
||||
self.logger.warning(
|
||||
f"Object '{object_name}' not found in bucket '{self._bucket_name}'"
|
||||
)
|
||||
else:
|
||||
self.logger.error(repr(exc))
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
self.logger.error(repr(exc))
|
||||
return None
|
||||
raise
|
||||
finally:
|
||||
if response is not None:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
return None
|
||||
|
||||
def delete(self, object_name: str) -> None:
|
||||
"""Delete an object from the Minio bucket."""
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Definition of RedisAdapter class."""
|
||||
|
||||
from __future__ import annotations
|
||||
from collections.abc import Iterator
|
||||
from typing import cast
|
||||
|
||||
from python_repositories.adapters.connection_aware_adapter import (
|
||||
@@ -34,9 +35,9 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
||||
client: redis.Redis | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if config is None:
|
||||
if client is not None:
|
||||
if client is not None and config is None:
|
||||
raise ValueError("config is required when client is provided")
|
||||
if config is None:
|
||||
config = RedisConfig.from_env(self.uri_env_var_name)
|
||||
self._config = config
|
||||
self._client_injected = client is not None
|
||||
@@ -136,11 +137,13 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
||||
self._client.json().delete(key)
|
||||
self.logger.debug(f"Deleted {key}")
|
||||
|
||||
def list_keys(self, pattern: str) -> list[str]:
|
||||
"""List keys in Redis matching a pattern."""
|
||||
# Check input
|
||||
def _validate_pattern(self, pattern: str) -> None:
|
||||
if not isinstance(pattern, str) or len(pattern) == 0:
|
||||
raise ValueError("Pattern must be a non-empty string")
|
||||
|
||||
def list_keys(self, pattern: str) -> list[str]:
|
||||
"""List keys in Redis using KEYS; may block on large datasets."""
|
||||
self._validate_pattern(pattern)
|
||||
# Check connection
|
||||
self._require_connected()
|
||||
assert self._client is not None
|
||||
@@ -152,3 +155,29 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
||||
keys: list[str] = [key.decode(self.encoding) for key in keys_raw]
|
||||
self.logger.debug(f"Got {keys} matching {pattern}")
|
||||
return keys
|
||||
|
||||
def scan_keys(
|
||||
self,
|
||||
pattern: str,
|
||||
*,
|
||||
count: int | None = None,
|
||||
) -> Iterator[str]:
|
||||
"""Yield keys in Redis using SCAN to avoid blocking large datasets."""
|
||||
self._validate_pattern(pattern)
|
||||
self._require_connected()
|
||||
assert self._client is not None
|
||||
client = self._client
|
||||
|
||||
def _decode(key: bytes | str) -> str:
|
||||
return key if isinstance(key, str) else key.decode(self.encoding)
|
||||
|
||||
def _iter() -> Iterator[str]:
|
||||
scan_iter = (
|
||||
client.scan_iter(match=pattern, count=count)
|
||||
if count is not None
|
||||
else client.scan_iter(match=pattern)
|
||||
)
|
||||
for key_raw in scan_iter:
|
||||
yield _decode(key_raw)
|
||||
|
||||
return _iter()
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Parse boolean values from environment variables."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
_TRUTHY = frozenset({"1", "true", "yes", "on"})
|
||||
_FALSY = frozenset({"0", "false", "no", "off"})
|
||||
|
||||
|
||||
def env_bool(name: str, default: bool) -> bool:
|
||||
"""Parse an environment variable as a boolean value."""
|
||||
raw = os.getenv(name)
|
||||
if raw is None:
|
||||
return default
|
||||
normalized = raw.strip().lower()
|
||||
if normalized in _TRUTHY:
|
||||
return True
|
||||
if normalized in _FALSY:
|
||||
return False
|
||||
raise ValueError(f"Invalid boolean value for {name}: {raw!r}")
|
||||
@@ -8,21 +8,7 @@ from dataclasses import dataclass
|
||||
from python_utils import check_env
|
||||
|
||||
from python_repositories.config.dotenv_loader import load_dotenv
|
||||
|
||||
_TRUTHY = frozenset({"1", "true", "yes", "on"})
|
||||
_FALSY = frozenset({"0", "false", "no", "off"})
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
raw = os.getenv(name)
|
||||
if raw is None:
|
||||
return default
|
||||
normalized = raw.strip().lower()
|
||||
if normalized in _TRUTHY:
|
||||
return True
|
||||
if normalized in _FALSY:
|
||||
return False
|
||||
raise ValueError(f"Invalid boolean value for {name}: {raw!r}")
|
||||
from python_repositories.config.env_bool import env_bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -34,6 +20,7 @@ class MinioConfig:
|
||||
secret_key: str
|
||||
bucket: str
|
||||
secure: bool = True
|
||||
create_bucket_if_missing: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_env(
|
||||
@@ -43,6 +30,7 @@ class MinioConfig:
|
||||
secret_key_env_var_name: str = "MINIO_SECRET_KEY",
|
||||
bucket_env_var_name: str = "MINIO_BUCKET",
|
||||
secure_env_var_name: str = "MINIO_SECURE",
|
||||
create_bucket_if_missing_env_var_name: str = ("MINIO_CREATE_BUCKET_IF_MISSING"),
|
||||
*,
|
||||
use_dotenv: bool = True,
|
||||
) -> MinioConfig:
|
||||
@@ -61,5 +49,9 @@ class MinioConfig:
|
||||
access_key=str(os.getenv(access_key_env_var_name)),
|
||||
secret_key=str(os.getenv(secret_key_env_var_name)),
|
||||
bucket=str(os.getenv(bucket_env_var_name)),
|
||||
secure=_env_bool(secure_env_var_name, default=True),
|
||||
secure=env_bool(secure_env_var_name, default=True),
|
||||
create_bucket_if_missing=env_bool(
|
||||
create_bucket_if_missing_env_var_name,
|
||||
default=False,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Definition of JsonRepositoryInterface abstract base class."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
@@ -25,3 +26,13 @@ class JsonRepositoryInterface(ABC):
|
||||
def list_keys(self, pattern: str) -> list[str]:
|
||||
"""List keys matching a glob pattern."""
|
||||
...
|
||||
|
||||
def scan_keys(
|
||||
self,
|
||||
pattern: str,
|
||||
*,
|
||||
count: int | None = None,
|
||||
) -> Iterator[str]:
|
||||
"""Yield keys matching a glob pattern incrementally."""
|
||||
del count
|
||||
yield from self.list_keys(pattern)
|
||||
|
||||
@@ -9,7 +9,11 @@ class ObjectRepositoryInterface(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def get(self, object_name: str) -> BytesIO | None:
|
||||
"""Get an object by name."""
|
||||
"""Get an object by name.
|
||||
|
||||
Returns None when the object does not exist. Raises ConnectionError when
|
||||
not connected. Other backend errors propagate to the caller.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -2,23 +2,52 @@
|
||||
|
||||
import logging
|
||||
from collections.abc import Generator
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
import redis
|
||||
import structlog
|
||||
from minio import Minio
|
||||
from testcontainers.core.container import DockerContainer
|
||||
from testcontainers.core.waiting_utils import WaitStrategy, WaitStrategyTarget
|
||||
from testcontainers.minio import MinioContainer
|
||||
|
||||
from python_repositories.config import MinioConfig, RedisConfig
|
||||
from tests.integration.redis_container_test import REDIS_PORT, RedisTestContainer
|
||||
|
||||
collect_ignore = ["redis_container_test.py"]
|
||||
REDIS_PORT = 6379
|
||||
|
||||
MINIO_ACCESS_KEY = "minioadmin"
|
||||
MINIO_SECRET_KEY = "minioadmin"
|
||||
MINIO_BUCKET = "test-bucket"
|
||||
|
||||
|
||||
class _RedisPingWaitStrategy(WaitStrategy):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.with_transient_exceptions(redis.exceptions.ConnectionError)
|
||||
|
||||
def wait_until_ready(self, container: WaitStrategyTarget) -> None:
|
||||
redis_container = cast("RedisTestContainer", container)
|
||||
if not self._poll(lambda: redis_container.get_client().ping()):
|
||||
raise redis.exceptions.ConnectionError("Could not connect to Redis")
|
||||
|
||||
|
||||
class RedisTestContainer(DockerContainer):
|
||||
"""Redis container using wait strategies instead of the deprecated decorator."""
|
||||
|
||||
def __init__(self, image: str, port: int = REDIS_PORT) -> None:
|
||||
super().__init__(image, _wait_strategy=_RedisPingWaitStrategy())
|
||||
self.port = port
|
||||
self.with_exposed_ports(self.port)
|
||||
|
||||
def get_client(self, **kwargs: Any) -> redis.Redis:
|
||||
return redis.Redis(
|
||||
host=self.get_container_host_ip(),
|
||||
port=self.get_exposed_port(self.port),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def configure_logging() -> None:
|
||||
"""Configure logging for the test session."""
|
||||
|
||||
@@ -6,6 +6,7 @@ import os
|
||||
import random
|
||||
|
||||
import pytest
|
||||
from minio import Minio
|
||||
|
||||
from python_repositories.examples.artifact_object_repository import (
|
||||
ArtifactObjectRepository,
|
||||
@@ -19,8 +20,10 @@ pytestmark = pytest.mark.integration
|
||||
def set_example_env(
|
||||
redis_container: str,
|
||||
minio_container: dict[str, str],
|
||||
raw_minio_client: Minio,
|
||||
) -> Generator[None, None, None]:
|
||||
"""Set env vars so example repositories can use from_env() defaults."""
|
||||
_ = raw_minio_client
|
||||
env_vars = {"REDIS_URI": redis_container, **minio_container}
|
||||
for key, value in env_vars.items():
|
||||
os.environ[key] = value
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Integration tests for the MinioAdapter."""
|
||||
|
||||
from collections.abc import Generator
|
||||
from dataclasses import replace
|
||||
import logging
|
||||
import random
|
||||
from io import BytesIO
|
||||
@@ -118,17 +119,37 @@ def test_should_raise_connection_error_when_unable_to_connect() -> None:
|
||||
assert not adapter.is_connected()
|
||||
|
||||
|
||||
def test_should_log_info_when_creating_expected_bucket(
|
||||
def test_connect_raises_when_bucket_missing(
|
||||
raw_minio_client: Minio,
|
||||
minio_config: MinioConfig,
|
||||
) -> None:
|
||||
"""Test that connect fails when the configured bucket is missing."""
|
||||
bucket_name = minio_config.bucket
|
||||
raw_minio_client.remove_bucket(bucket_name)
|
||||
adapter = MinioAdapter(config=minio_config)
|
||||
|
||||
try:
|
||||
with pytest.raises(ConnectionError, match="does not exist"):
|
||||
adapter.connect()
|
||||
assert not adapter.is_connected()
|
||||
finally:
|
||||
raw_minio_client.make_bucket(bucket_name)
|
||||
|
||||
|
||||
def test_connect_creates_bucket_when_create_bucket_if_missing_enabled(
|
||||
raw_minio_client: Minio,
|
||||
minio_config: MinioConfig,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter logs info when creating the expected bucket."""
|
||||
"""Test that connect can create the configured bucket when enabled."""
|
||||
bucket_name = minio_config.bucket
|
||||
raw_minio_client.remove_bucket(bucket_name)
|
||||
adapter = MinioAdapter(config=minio_config)
|
||||
config = replace(minio_config, create_bucket_if_missing=True)
|
||||
adapter = MinioAdapter(config=config)
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
adapter.connect()
|
||||
|
||||
assert f"Creating bucket '{bucket_name}'" in caplog.text
|
||||
|
||||
|
||||
@@ -218,10 +239,8 @@ def test_should_log_warning_when_getting_nonexistent_object(
|
||||
)
|
||||
|
||||
|
||||
def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter logs an error for unhandled S3 errors."""
|
||||
def test_should_reraise_s3error_other_than_no_such_key() -> None:
|
||||
"""Test that the MinioAdapter re-raises unhandled S3 errors."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
other_s3error = S3Error(
|
||||
@@ -236,25 +255,20 @@ def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
|
||||
)
|
||||
mock_client.get_object.side_effect = other_s3error
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
with caplog.at_level("ERROR"):
|
||||
result = adapter.get("missing-object")
|
||||
assert result is None
|
||||
assert repr(other_s3error) in caplog.text
|
||||
with pytest.raises(S3Error) as exc_info:
|
||||
adapter.get("missing-object")
|
||||
assert exc_info.value.code == "UnhandledError"
|
||||
|
||||
|
||||
def test_should_log_error_when_getting_with_general_exception(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter logs an error on general exceptions during get."""
|
||||
def test_should_reraise_general_exception() -> None:
|
||||
"""Test that the MinioAdapter re-raises general exceptions during get."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
general_exception = Exception("General failure")
|
||||
mock_client.get_object.side_effect = general_exception
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
with caplog.at_level("ERROR"):
|
||||
result = adapter.get("missing-object")
|
||||
assert result is None
|
||||
assert repr(general_exception) in caplog.text
|
||||
with pytest.raises(Exception, match="General failure"):
|
||||
adapter.get("missing-object")
|
||||
|
||||
|
||||
def test_should_put_data(
|
||||
|
||||
@@ -254,5 +254,34 @@ def test_should_raise_connection_error_on_list_keys_when_not_connected(
|
||||
adapter.list_keys("some_pattern")
|
||||
|
||||
|
||||
def test_should_scan_keys(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test scanning keys matching a pattern returns correct keys."""
|
||||
redis_adapter.set("key1", {"a": 1})
|
||||
redis_adapter.set("key2", {"b": 2})
|
||||
keys = set(redis_adapter.scan_keys("key*"))
|
||||
assert keys == {"key1", "key2"}
|
||||
|
||||
|
||||
def test_should_raise_value_error_on_invalid_scan_keys_pattern(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ValueError when scanning keys with an invalid pattern."""
|
||||
invalid_patterns = ["", 123, None]
|
||||
for pattern in invalid_patterns:
|
||||
with pytest.raises(ValueError):
|
||||
list(redis_adapter.scan_keys(pattern)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_should_raise_connection_error_on_scan_keys_when_not_connected(
|
||||
redis_config: RedisConfig,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ConnectionError when scanning keys while not connected."""
|
||||
adapter = RedisAdapter(config=redis_config)
|
||||
with pytest.raises(ConnectionError):
|
||||
list(adapter.scan_keys("some_pattern"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main(["-s", "-v", __file__])
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
"""Redis test container without testcontainers' deprecated wait decorator."""
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import redis
|
||||
from testcontainers.core.container import DockerContainer
|
||||
from testcontainers.core.waiting_utils import WaitStrategy, WaitStrategyTarget
|
||||
|
||||
REDIS_PORT = 6379
|
||||
|
||||
|
||||
class _RedisPingWaitStrategy(WaitStrategy):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.with_transient_exceptions(redis.exceptions.ConnectionError)
|
||||
|
||||
def wait_until_ready(self, container: WaitStrategyTarget) -> None:
|
||||
redis_container = cast("RedisTestContainer", container)
|
||||
if not self._poll(lambda: redis_container.get_client().ping()):
|
||||
raise redis.exceptions.ConnectionError("Could not connect to Redis")
|
||||
|
||||
|
||||
class RedisTestContainer(DockerContainer):
|
||||
"""Redis container using wait strategies instead of the deprecated decorator."""
|
||||
|
||||
def __init__(self, image: str, port: int = REDIS_PORT) -> None:
|
||||
super().__init__(image, _wait_strategy=_RedisPingWaitStrategy())
|
||||
self.port = port
|
||||
self.with_exposed_ports(self.port)
|
||||
|
||||
def get_client(self, **kwargs: Any) -> redis.Redis:
|
||||
return redis.Redis(
|
||||
host=self.get_container_host_ip(),
|
||||
port=self.get_exposed_port(self.port),
|
||||
**kwargs,
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Unit tests for env_bool."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from python_repositories.config.env_bool import env_bool
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
("true", True),
|
||||
("1", True),
|
||||
("yes", True),
|
||||
("on", True),
|
||||
("false", False),
|
||||
("0", False),
|
||||
("no", False),
|
||||
("off", False),
|
||||
],
|
||||
)
|
||||
def test_env_bool_parses_truthy_and_falsy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
value: str,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
monkeypatch.setenv("TEST_BOOL", value)
|
||||
assert env_bool("TEST_BOOL", default=not expected) is expected
|
||||
|
||||
|
||||
def test_env_bool_returns_default_when_unset(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("TEST_BOOL", raising=False)
|
||||
assert env_bool("TEST_BOOL", default=True) is True
|
||||
assert env_bool("TEST_BOOL", default=False) is False
|
||||
|
||||
|
||||
def test_env_bool_raises_for_invalid_value(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("TEST_BOOL", "not-a-bool")
|
||||
with pytest.raises(ValueError, match="TEST_BOOL"):
|
||||
env_bool("TEST_BOOL", default=False)
|
||||
@@ -80,3 +80,24 @@ def test_instantiation_fails_when_list_keys_not_implemented() -> None:
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
_ = Incomplete() # type: ignore
|
||||
|
||||
|
||||
def test_scan_keys_defaults_to_list_keys() -> None:
|
||||
"""Test that the default scan_keys implementation delegates to list_keys."""
|
||||
|
||||
class Complete(JsonRepositoryInterface):
|
||||
def get(self, key: str) -> dict | None:
|
||||
return None
|
||||
|
||||
def set(self, key: str, data: dict) -> None:
|
||||
pass
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
pass
|
||||
|
||||
def list_keys(self, pattern: str) -> list[str]:
|
||||
return [f"{pattern}-1", f"{pattern}-2"]
|
||||
|
||||
repository = Complete()
|
||||
|
||||
assert list(repository.scan_keys("user")) == ["user-1", "user-2"]
|
||||
|
||||
@@ -2,10 +2,12 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from minio import Minio
|
||||
from minio import Minio, S3Error
|
||||
from urllib3.response import BaseHTTPResponse
|
||||
|
||||
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||
from python_repositories.interfaces import ObjectRepositoryInterface
|
||||
@@ -93,6 +95,44 @@ def test_connect_disconnects_before_reconnect(
|
||||
new_client.list_buckets.assert_called_once()
|
||||
|
||||
|
||||
def test_connect_raises_when_bucket_missing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = False
|
||||
|
||||
monkeypatch.setattr(
|
||||
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||
lambda *args, **kwargs: mock_client,
|
||||
)
|
||||
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||
|
||||
with pytest.raises(ConnectionError, match="does not exist"):
|
||||
adapter.connect()
|
||||
|
||||
mock_client.make_bucket.assert_not_called()
|
||||
|
||||
|
||||
def test_connect_creates_bucket_when_create_bucket_if_missing_enabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = False
|
||||
config = replace(TEST_MINIO_CONFIG, create_bucket_if_missing=True)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||
lambda *args, **kwargs: mock_client,
|
||||
)
|
||||
|
||||
adapter = MinioAdapter(config=config)
|
||||
adapter.connect()
|
||||
|
||||
mock_client.bucket_exists.assert_called_once_with(config.bucket)
|
||||
mock_client.make_bucket.assert_called_once_with(config.bucket)
|
||||
|
||||
|
||||
def test_get_closes_response_on_success() -> None:
|
||||
"""get() must close and release the get_object HTTP response."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
@@ -119,8 +159,63 @@ def test_get_closes_response_when_read_fails() -> None:
|
||||
mock_client.get_object.return_value = mock_response
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
result = adapter.get("some-object")
|
||||
with pytest.raises(OSError, match="connection reset"):
|
||||
adapter.get("some-object")
|
||||
|
||||
assert result is None
|
||||
mock_response.close.assert_called_once()
|
||||
mock_response.release_conn.assert_called_once()
|
||||
|
||||
|
||||
def test_get_returns_none_for_no_such_key() -> None:
|
||||
"""get() returns None when the object does not exist."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
mock_client.get_object.side_effect = S3Error(
|
||||
MagicMock(spec=BaseHTTPResponse),
|
||||
"NoSuchKey",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
bucket_name="test-bucket",
|
||||
object_name="missing-object",
|
||||
)
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
result = adapter.get("missing-object")
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_get_reraises_other_s3_errors() -> None:
|
||||
"""get() re-raises S3 errors other than NoSuchKey."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
other_s3error = S3Error(
|
||||
MagicMock(spec=BaseHTTPResponse),
|
||||
"AccessDenied",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
bucket_name="test-bucket",
|
||||
object_name="some-object",
|
||||
)
|
||||
mock_client.get_object.side_effect = other_s3error
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
with pytest.raises(S3Error) as exc_info:
|
||||
adapter.get("some-object")
|
||||
|
||||
assert exc_info.value.code == "AccessDenied"
|
||||
|
||||
|
||||
def test_get_reraises_general_exception_from_get_object() -> None:
|
||||
"""get() re-raises unexpected exceptions from get_object."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
mock_client.get_object.side_effect = Exception("General failure")
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
with pytest.raises(Exception, match="General failure"):
|
||||
adapter.get("some-object")
|
||||
|
||||
@@ -22,6 +22,7 @@ def test_from_env_loads_all_fields(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
assert config.secret_key == "secret"
|
||||
assert config.bucket == "my-bucket"
|
||||
assert config.secure is True
|
||||
assert config.create_bucket_if_missing is False
|
||||
|
||||
|
||||
def test_from_env_defaults_secure_to_true_when_unset(
|
||||
@@ -33,33 +34,27 @@ def test_from_env_defaults_secure_to_true_when_unset(
|
||||
assert config.secure is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
("true", True),
|
||||
("1", True),
|
||||
("yes", True),
|
||||
("false", False),
|
||||
("0", False),
|
||||
("no", False),
|
||||
],
|
||||
)
|
||||
def test_from_env_parses_secure(
|
||||
def test_from_env_reads_secure_from_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_set_required_minio_env(monkeypatch)
|
||||
monkeypatch.setenv("MINIO_SECURE", "false")
|
||||
assert MinioConfig.from_env(use_dotenv=False).secure is False
|
||||
|
||||
|
||||
def test_from_env_reads_create_bucket_if_missing_from_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
value: str,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
_set_required_minio_env(monkeypatch)
|
||||
monkeypatch.setenv("MINIO_SECURE", value)
|
||||
config = MinioConfig.from_env(use_dotenv=False)
|
||||
assert config.secure is expected
|
||||
monkeypatch.setenv("MINIO_CREATE_BUCKET_IF_MISSING", "true")
|
||||
assert MinioConfig.from_env(use_dotenv=False).create_bucket_if_missing is True
|
||||
|
||||
|
||||
def test_from_env_raises_for_invalid_secure(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_from_env_defaults_create_bucket_if_missing_to_false_when_unset(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_set_required_minio_env(monkeypatch)
|
||||
monkeypatch.setenv("MINIO_SECURE", "not-a-bool")
|
||||
with pytest.raises(ValueError, match="MINIO_SECURE"):
|
||||
MinioConfig.from_env(use_dotenv=False)
|
||||
monkeypatch.delenv("MINIO_CREATE_BUCKET_IF_MISSING", raising=False)
|
||||
config = MinioConfig.from_env(use_dotenv=False)
|
||||
assert config.create_bucket_if_missing is False
|
||||
|
||||
|
||||
def test_from_env_raises_when_endpoint_missing(
|
||||
|
||||
@@ -114,3 +114,53 @@ def test_connect_closes_existing_non_injected_client(
|
||||
|
||||
stale_client.close.assert_called_once()
|
||||
assert adapter._client is new_client
|
||||
|
||||
|
||||
def test_scan_keys_yields_decoded_keys() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
mock_client.scan_iter.return_value = iter([b"key1", b"key2"])
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
keys = list(adapter.scan_keys("key*"))
|
||||
|
||||
assert keys == ["key1", "key2"]
|
||||
mock_client.scan_iter.assert_called_once_with(match="key*")
|
||||
mock_client.keys.assert_not_called()
|
||||
|
||||
|
||||
def test_scan_keys_forwards_count() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
mock_client.scan_iter.return_value = iter([b"key1"])
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
keys = list(adapter.scan_keys("key*", count=50))
|
||||
|
||||
assert keys == ["key1"]
|
||||
mock_client.scan_iter.assert_called_once_with(match="key*", count=50)
|
||||
|
||||
|
||||
def test_list_keys_raises_value_error_on_invalid_pattern() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
invalid_patterns = ["", 123, None]
|
||||
for pattern in invalid_patterns:
|
||||
with pytest.raises(ValueError, match="Pattern must be a non-empty string"):
|
||||
adapter.list_keys(pattern) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_scan_keys_raises_value_error_on_invalid_pattern() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
invalid_patterns = ["", 123, None]
|
||||
for pattern in invalid_patterns:
|
||||
with pytest.raises(ValueError, match="Pattern must be a non-empty string"):
|
||||
list(adapter.scan_keys(pattern)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_scan_keys_raises_connection_error_when_not_connected() -> None:
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||
|
||||
with pytest.raises(ConnectionError):
|
||||
list(adapter.scan_keys("key*"))
|
||||
|
||||
Reference in New Issue
Block a user