diff --git a/README.md b/README.md index 795e81f..a4d734d 100644 --- a/README.md +++ b/README.md @@ -161,10 +161,12 @@ uv sync --all-extras uv run pre-commit install # once per clone — runs hooks on git commit uv run pytest tests/unit/ -v # fast, no Docker uv run pytest -m "not integration" -v # all non-Docker tests +uv run pytest tests/integration/redis/ -v # Redis container only +uv run pytest tests/integration/minio/ -v # MinIO container only uv run pytest -v # full suite (requires Docker) ``` -Integration tests are marked with `@pytest.mark.integration` and require Docker (testcontainers). Run unit tests alone for quick local feedback. +Integration tests are marked with `@pytest.mark.integration` and require Docker (testcontainers). Backend-specific markers (`needs_redis`, `needs_minio`) let you run only the containers a test module needs. Run unit tests alone for quick local feedback. ### CI base image diff --git a/pyproject.toml b/pyproject.toml index 07ef3ce..0be389c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,8 @@ pythonpath = ["."] addopts = "--import-mode=importlib" markers = [ "integration: tests requiring Docker containers (deselect with '-m \"not integration\"')", + "needs_redis: integration test requiring a Redis container", + "needs_minio: integration test requiring a MinIO container", ] [tool.coverage.run] diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4aacb1e..c1802cb 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,51 +1,9 @@ """Integration tests configuration.""" -from collections.abc import Generator import logging -from typing import Any, cast -from minio import Minio import pytest -import redis import structlog -from testcontainers.core.container import DockerContainer -from testcontainers.core.waiting_utils import WaitStrategy, WaitStrategyTarget -from testcontainers.minio import MinioContainer - -from python_repositories.config import MinioConfig, RedisConfig - -REDIS_PORT = 6379 - -MINIO_ACCESS_KEY = "minioadmin" -MINIO_SECRET_KEY = "minioadmin" -MINIO_BUCKET = "test-bucket" - - -class _RedisPingWaitStrategy(WaitStrategy): - def __init__(self) -> None: - super().__init__() - self.with_transient_exceptions(redis.exceptions.ConnectionError) - - def wait_until_ready(self, container: WaitStrategyTarget) -> None: - redis_container = cast("RedisTestContainer", container) - if not self._poll(lambda: redis_container.get_client().ping()): - raise redis.exceptions.ConnectionError("Could not connect to Redis") - - -class RedisTestContainer(DockerContainer): - """Redis container using wait strategies instead of the deprecated decorator.""" - - def __init__(self, image: str, port: int = REDIS_PORT) -> None: - super().__init__(image, _wait_strategy=_RedisPingWaitStrategy()) - self.port = port - self.with_exposed_ports(self.port) - - def get_client(self, **kwargs: Any) -> redis.Redis: - return redis.Redis( - host=self.get_container_host_ip(), - port=self.get_exposed_port(self.port), - **kwargs, - ) @pytest.fixture(scope="session", autouse=True) @@ -64,96 +22,3 @@ def configure_logging() -> None: cache_logger_on_first_use=True, ) logging.basicConfig(level=logging.ERROR) - - -@pytest.fixture(scope="session") -def redis_container() -> Generator[str, None, None]: - """Set up a Redis container for testing and yield the Redis URI.""" - container = RedisTestContainer( - image="redis/redis-stack:7.2.0-v0", - ) - container.start() - redis_host = container.get_container_host_ip() - redis_port = container.get_exposed_port(REDIS_PORT) - redis_uri = f"redis://{redis_host}:{redis_port}" - - yield redis_uri - - container.stop() - - -@pytest.fixture(scope="session") -def minio_container() -> Generator[dict[str, str], None, None]: - """Set up a Minio container for testing and yield connection settings.""" - container = MinioContainer( - image="minio/minio:latest", - access_key=MINIO_ACCESS_KEY, - secret_key=MINIO_SECRET_KEY, - ) - container.start() - minio_host = container.get_container_host_ip() - minio_port = container.get_exposed_port(9000) - minio_endpoint = f"{minio_host}:{minio_port}" - env_vars = { - "MINIO_ENDPOINT": minio_endpoint, - "MINIO_ACCESS_KEY": MINIO_ACCESS_KEY, - "MINIO_SECRET_KEY": MINIO_SECRET_KEY, - "MINIO_BUCKET": MINIO_BUCKET, - "MINIO_SECURE": "false", - } - - yield env_vars - - container.stop() - - -@pytest.fixture(scope="session") -def redis_config(redis_container: str) -> RedisConfig: - """Provide RedisConfig built from the test container.""" - return RedisConfig(uri=redis_container) - - -@pytest.fixture(scope="session") -def minio_config(minio_container: dict[str, str]) -> MinioConfig: - """Provide MinioConfig built from the test container.""" - return MinioConfig( - endpoint=minio_container["MINIO_ENDPOINT"], - access_key=minio_container["MINIO_ACCESS_KEY"], - secret_key=minio_container["MINIO_SECRET_KEY"], - bucket=minio_container["MINIO_BUCKET"], - secure=False, - ) - - -@pytest.fixture(scope="session") -def raw_redis_client(redis_container: str) -> Generator[redis.Redis, None, None]: - """Provide a raw Redis client connected to the test Redis container.""" - client = redis.Redis.from_url( - url=redis_container, - socket_connect_timeout=10, - ) - - yield client - - client.flushall() - client.close() - - -@pytest.fixture(scope="session") -def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio, None, None]: - """Provide a raw Minio client connected to the test Minio container.""" - client = Minio( - endpoint=minio_container["MINIO_ENDPOINT"], - access_key=minio_container["MINIO_ACCESS_KEY"], - secret_key=minio_container["MINIO_SECRET_KEY"], - secure=False, - ) - bucket_name = minio_container["MINIO_BUCKET"] - if not client.bucket_exists(bucket_name): - client.make_bucket(bucket_name) - - yield client - - objects = client.list_objects(bucket_name, recursive=True) - for obj in objects: - client.remove_object(bucket_name, obj.object_name) diff --git a/tests/integration/examples/conftest.py b/tests/integration/examples/conftest.py new file mode 100644 index 0000000..e7f6555 --- /dev/null +++ b/tests/integration/examples/conftest.py @@ -0,0 +1,55 @@ +"""Examples integration test fixtures.""" + +from collections.abc import Generator + +from minio import Minio +import pytest +import redis + +from python_repositories.config import MinioConfig, RedisConfig +from tests.integration.minio._containers import ( + minio_config_from_env, + minio_env, + raw_minio_client_from_env, +) +from tests.integration.redis._containers import ( + raw_redis_client_from_container, + redis_config_from_container, + redis_uri, +) + + +@pytest.fixture(scope="session") +def redis_container() -> Generator[str, None, None]: + """Set up a Redis container for testing and yield the Redis URI.""" + yield from redis_uri() + + +@pytest.fixture(scope="session") +def redis_config(redis_container: str) -> RedisConfig: + """Provide RedisConfig built from the test container.""" + return redis_config_from_container(redis_container) + + +@pytest.fixture(scope="session") +def raw_redis_client(redis_container: str) -> Generator[redis.Redis, None, None]: + """Provide a raw Redis client connected to the test Redis container.""" + yield from raw_redis_client_from_container(redis_container) + + +@pytest.fixture(scope="session") +def minio_container() -> Generator[dict[str, str], None, None]: + """Set up a Minio container for testing and yield connection settings.""" + yield from minio_env() + + +@pytest.fixture(scope="session") +def minio_config(minio_container: dict[str, str]) -> MinioConfig: + """Provide MinioConfig built from the test container.""" + return minio_config_from_env(minio_container) + + +@pytest.fixture(scope="session") +def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio, None, None]: + """Provide a raw Minio client connected to the test Minio container.""" + yield from raw_minio_client_from_env(minio_container) diff --git a/tests/integration/examples_test.py b/tests/integration/examples/examples_test.py similarity index 95% rename from tests/integration/examples_test.py rename to tests/integration/examples/examples_test.py index db08e4a..e7309bb 100644 --- a/tests/integration/examples_test.py +++ b/tests/integration/examples/examples_test.py @@ -13,7 +13,11 @@ from python_repositories.examples.artifact_object_repository import ( ) from python_repositories.examples.user_json_repository import UserJsonRepository -pytestmark = pytest.mark.integration +pytestmark = [ + pytest.mark.integration, + pytest.mark.needs_redis, + pytest.mark.needs_minio, +] @pytest.fixture(scope="module", autouse=True) diff --git a/tests/integration/minio/_containers.py b/tests/integration/minio/_containers.py new file mode 100644 index 0000000..1e390af --- /dev/null +++ b/tests/integration/minio/_containers.py @@ -0,0 +1,86 @@ +"""MinIO container session helpers for integration tests.""" + +from collections.abc import Generator + +from minio import Minio +from testcontainers.minio import MinioContainer + +from python_repositories.config import MinioConfig + +MINIO_ACCESS_KEY = "minioadmin" +MINIO_SECRET_KEY = "minioadmin" +MINIO_BUCKET = "test-bucket" + +_minio_env: dict[str, str] | None = None +_minio_container: MinioContainer | None = None +_raw_minio_client: Minio | None = None +_minio_env_refs = 0 +_raw_minio_client_refs = 0 + + +def minio_env() -> Generator[dict[str, str], None, None]: + """Yield session-scoped MinIO connection settings, starting the container once.""" + global _minio_env, _minio_container, _minio_env_refs + if _minio_env is None: + _minio_container = MinioContainer( + image="minio/minio:latest", + access_key=MINIO_ACCESS_KEY, + secret_key=MINIO_SECRET_KEY, + ) + _minio_container.start() + minio_host = _minio_container.get_container_host_ip() + minio_port = _minio_container.get_exposed_port(9000) + minio_endpoint = f"{minio_host}:{minio_port}" + _minio_env = { + "MINIO_ENDPOINT": minio_endpoint, + "MINIO_ACCESS_KEY": MINIO_ACCESS_KEY, + "MINIO_SECRET_KEY": MINIO_SECRET_KEY, + "MINIO_BUCKET": MINIO_BUCKET, + "MINIO_SECURE": "false", + } + + _minio_env_refs += 1 + yield _minio_env + _minio_env_refs -= 1 + + if _minio_env_refs == 0 and _minio_container is not None: + _minio_container.stop() + _minio_container = None + _minio_env = None + + +def minio_config_from_env(env: dict[str, str]) -> MinioConfig: + """Build MinioConfig from container environment settings.""" + return MinioConfig( + endpoint=env["MINIO_ENDPOINT"], + access_key=env["MINIO_ACCESS_KEY"], + secret_key=env["MINIO_SECRET_KEY"], + bucket=env["MINIO_BUCKET"], + secure=False, + ) + + +def raw_minio_client_from_env(env: dict[str, str]) -> Generator[Minio, None, None]: + """Yield a session-scoped raw MinIO client, reusing one client per session.""" + global _raw_minio_client, _raw_minio_client_refs + if _raw_minio_client is None: + _raw_minio_client = Minio( + endpoint=env["MINIO_ENDPOINT"], + access_key=env["MINIO_ACCESS_KEY"], + secret_key=env["MINIO_SECRET_KEY"], + secure=False, + ) + bucket_name = env["MINIO_BUCKET"] + if not _raw_minio_client.bucket_exists(bucket_name): + _raw_minio_client.make_bucket(bucket_name) + + _raw_minio_client_refs += 1 + yield _raw_minio_client + _raw_minio_client_refs -= 1 + + if _raw_minio_client_refs == 0 and _raw_minio_client is not None: + bucket_name = env["MINIO_BUCKET"] + objects = _raw_minio_client.list_objects(bucket_name, recursive=True) + for obj in objects: + _raw_minio_client.remove_object(bucket_name, obj.object_name) + _raw_minio_client = None diff --git a/tests/integration/minio/conftest.py b/tests/integration/minio/conftest.py new file mode 100644 index 0000000..ef7cff1 --- /dev/null +++ b/tests/integration/minio/conftest.py @@ -0,0 +1,31 @@ +"""MinIO integration test fixtures.""" + +from collections.abc import Generator + +from minio import Minio +import pytest + +from python_repositories.config import MinioConfig +from tests.integration.minio._containers import ( + minio_config_from_env, + minio_env, + raw_minio_client_from_env, +) + + +@pytest.fixture(scope="session") +def minio_container() -> Generator[dict[str, str], None, None]: + """Set up a Minio container for testing and yield connection settings.""" + yield from minio_env() + + +@pytest.fixture(scope="session") +def minio_config(minio_container: dict[str, str]) -> MinioConfig: + """Provide MinioConfig built from the test container.""" + return minio_config_from_env(minio_container) + + +@pytest.fixture(scope="session") +def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio, None, None]: + """Provide a raw Minio client connected to the test Minio container.""" + yield from raw_minio_client_from_env(minio_container) diff --git a/tests/integration/minio_adapter_test.py b/tests/integration/minio/minio_adapter_test.py similarity index 99% rename from tests/integration/minio_adapter_test.py rename to tests/integration/minio/minio_adapter_test.py index f03f695..21d12c7 100644 --- a/tests/integration/minio_adapter_test.py +++ b/tests/integration/minio/minio_adapter_test.py @@ -15,7 +15,7 @@ from python_repositories.adapters.minio_adapter import MinioAdapter from python_repositories.config import MinioConfig from tests.conftest import TEST_MINIO_CONFIG -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.integration, pytest.mark.needs_minio] def same_data( diff --git a/tests/integration/redis/_containers.py b/tests/integration/redis/_containers.py new file mode 100644 index 0000000..83a1901 --- /dev/null +++ b/tests/integration/redis/_containers.py @@ -0,0 +1,90 @@ +"""Redis container session helpers for integration tests.""" + +from collections.abc import Generator +from typing import Any, cast + +import redis +from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import WaitStrategy, WaitStrategyTarget + +from python_repositories.config import RedisConfig + +REDIS_PORT = 6379 + + +class _RedisPingWaitStrategy(WaitStrategy): + def __init__(self) -> None: + super().__init__() + self.with_transient_exceptions(redis.exceptions.ConnectionError) + + def wait_until_ready(self, container: WaitStrategyTarget) -> None: + redis_container = cast("RedisTestContainer", container) + if not self._poll(lambda: redis_container.get_client().ping()): + raise redis.exceptions.ConnectionError("Could not connect to Redis") + + +class RedisTestContainer(DockerContainer): + """Redis container using wait strategies instead of the deprecated decorator.""" + + def __init__(self, image: str, port: int = REDIS_PORT) -> None: + super().__init__(image, _wait_strategy=_RedisPingWaitStrategy()) + self.port = port + self.with_exposed_ports(self.port) + + def get_client(self, **kwargs: Any) -> redis.Redis: + return redis.Redis( + host=self.get_container_host_ip(), + port=self.get_exposed_port(self.port), + **kwargs, + ) + + +_redis_uri: str | None = None +_redis_container: RedisTestContainer | None = None +_raw_redis_client: redis.Redis | None = None +_redis_uri_refs = 0 +_raw_redis_client_refs = 0 + + +def redis_uri() -> Generator[str, None, None]: + """Yield a session-scoped Redis URI, starting the container once.""" + global _redis_uri, _redis_container, _redis_uri_refs + if _redis_uri is None: + _redis_container = RedisTestContainer(image="redis/redis-stack:7.2.0-v0") + _redis_container.start() + redis_host = _redis_container.get_container_host_ip() + redis_port = _redis_container.get_exposed_port(REDIS_PORT) + _redis_uri = f"redis://{redis_host}:{redis_port}" + + _redis_uri_refs += 1 + yield _redis_uri + _redis_uri_refs -= 1 + + if _redis_uri_refs == 0 and _redis_container is not None: + _redis_container.stop() + _redis_container = None + _redis_uri = None + + +def redis_config_from_container(uri: str) -> RedisConfig: + """Build RedisConfig from a container URI.""" + return RedisConfig(uri=uri) + + +def raw_redis_client_from_container(uri: str) -> Generator[redis.Redis, None, None]: + """Yield a session-scoped raw Redis client, reusing one client per session.""" + global _raw_redis_client, _raw_redis_client_refs + if _raw_redis_client is None: + _raw_redis_client = redis.Redis.from_url( + url=uri, + socket_connect_timeout=10, + ) + + _raw_redis_client_refs += 1 + yield _raw_redis_client + _raw_redis_client_refs -= 1 + + if _raw_redis_client_refs == 0 and _raw_redis_client is not None: + _raw_redis_client.flushall() + _raw_redis_client.close() + _raw_redis_client = None diff --git a/tests/integration/redis/conftest.py b/tests/integration/redis/conftest.py new file mode 100644 index 0000000..f77beb0 --- /dev/null +++ b/tests/integration/redis/conftest.py @@ -0,0 +1,31 @@ +"""Redis integration test fixtures.""" + +from collections.abc import Generator + +import pytest +import redis + +from python_repositories.config import RedisConfig +from tests.integration.redis._containers import ( + raw_redis_client_from_container, + redis_config_from_container, + redis_uri, +) + + +@pytest.fixture(scope="session") +def redis_container() -> Generator[str, None, None]: + """Set up a Redis container for testing and yield the Redis URI.""" + yield from redis_uri() + + +@pytest.fixture(scope="session") +def redis_config(redis_container: str) -> RedisConfig: + """Provide RedisConfig built from the test container.""" + return redis_config_from_container(redis_container) + + +@pytest.fixture(scope="session") +def raw_redis_client(redis_container: str) -> Generator[redis.Redis, None, None]: + """Provide a raw Redis client connected to the test Redis container.""" + yield from raw_redis_client_from_container(redis_container) diff --git a/tests/integration/redis_adapter_test.py b/tests/integration/redis/redis_adapter_test.py similarity index 99% rename from tests/integration/redis_adapter_test.py rename to tests/integration/redis/redis_adapter_test.py index cd2ea42..4369d22 100644 --- a/tests/integration/redis_adapter_test.py +++ b/tests/integration/redis/redis_adapter_test.py @@ -10,7 +10,7 @@ from redis.commands.json.path import Path as RedisPath from python_repositories.adapters.redis_adapter import RedisAdapter from python_repositories.config import RedisConfig -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.integration, pytest.mark.needs_redis] @pytest.fixture(scope="module")