Merge pull request '[chore] Scope integration fixtures per backend subdirectory' (#54) from cursor/scoped-integration-fixtures into main

Reviewed-on: https://gitea.lille-vemmelund.dk/LilleVemmelund/python-repositories/pulls/54

README.md

@@ -161,10 +161,12 @@ uv sync --all-extras
 uv run pre-commit install   # once per clone — runs hooks on git commit
 uv run pytest tests/unit/ -v                          # fast, no Docker
 uv run pytest -m "not integration" -v                 # all non-Docker tests
+uv run pytest tests/integration/redis/ -v             # Redis container only
+uv run pytest tests/integration/minio/ -v             # MinIO container only
 uv run pytest -v                                      # full suite (requires Docker)
 ```
 
-Integration tests are marked with `@pytest.mark.integration` and require Docker (testcontainers). Run unit tests alone for quick local feedback.
+Integration tests are marked with `@pytest.mark.integration` and require Docker (testcontainers). Backend-specific markers (`needs_redis`, `needs_minio`) let you run only the containers a test module needs. Run unit tests alone for quick local feedback.
 
 ### CI base image
 

pyproject.toml

@@ -43,6 +43,8 @@ pythonpath = ["."]
 addopts = "--import-mode=importlib"
 markers = [
     "integration: tests requiring Docker containers (deselect with '-m \"not integration\"')",
+    "needs_redis: integration test requiring a Redis container",
+    "needs_minio: integration test requiring a MinIO container",
 ]
 
 [tool.coverage.run]

tests/integration/conftest.py

@@ -1,51 +1,9 @@
 """Integration tests configuration."""
 
-from collections.abc import Generator
 import logging
-from typing import Any, cast
 
-from minio import Minio
 import pytest
-import redis
 import structlog
-from testcontainers.core.container import DockerContainer
-from testcontainers.core.waiting_utils import WaitStrategy, WaitStrategyTarget
-from testcontainers.minio import MinioContainer
-
-from python_repositories.config import MinioConfig, RedisConfig
-
-REDIS_PORT = 6379
-
-MINIO_ACCESS_KEY = "minioadmin"
-MINIO_SECRET_KEY = "minioadmin"
-MINIO_BUCKET = "test-bucket"
-
-
-class _RedisPingWaitStrategy(WaitStrategy):
-    def __init__(self) -> None:
-        super().__init__()
-        self.with_transient_exceptions(redis.exceptions.ConnectionError)
-
-    def wait_until_ready(self, container: WaitStrategyTarget) -> None:
-        redis_container = cast("RedisTestContainer", container)
-        if not self._poll(lambda: redis_container.get_client().ping()):
-            raise redis.exceptions.ConnectionError("Could not connect to Redis")
-
-
-class RedisTestContainer(DockerContainer):
-    """Redis container using wait strategies instead of the deprecated decorator."""
-
-    def __init__(self, image: str, port: int = REDIS_PORT) -> None:
-        super().__init__(image, _wait_strategy=_RedisPingWaitStrategy())
-        self.port = port
-        self.with_exposed_ports(self.port)
-
-    def get_client(self, **kwargs: Any) -> redis.Redis:
-        return redis.Redis(
-            host=self.get_container_host_ip(),
-            port=self.get_exposed_port(self.port),
-            **kwargs,
-        )
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -64,96 +22,3 @@ def configure_logging() -> None:
         cache_logger_on_first_use=True,
     )
     logging.basicConfig(level=logging.ERROR)
-
-
-@pytest.fixture(scope="session")
-def redis_container() -> Generator[str, None, None]:
-    """Set up a Redis container for testing and yield the Redis URI."""
-    container = RedisTestContainer(
-        image="redis/redis-stack:7.2.0-v0",
-    )
-    container.start()
-    redis_host = container.get_container_host_ip()
-    redis_port = container.get_exposed_port(REDIS_PORT)
-    redis_uri = f"redis://{redis_host}:{redis_port}"
-
-    yield redis_uri
-
-    container.stop()
-
-
-@pytest.fixture(scope="session")
-def minio_container() -> Generator[dict[str, str], None, None]:
-    """Set up a Minio container for testing and yield connection settings."""
-    container = MinioContainer(
-        image="minio/minio:latest",
-        access_key=MINIO_ACCESS_KEY,
-        secret_key=MINIO_SECRET_KEY,
-    )
-    container.start()
-    minio_host = container.get_container_host_ip()
-    minio_port = container.get_exposed_port(9000)
-    minio_endpoint = f"{minio_host}:{minio_port}"
-    env_vars = {
-        "MINIO_ENDPOINT": minio_endpoint,
-        "MINIO_ACCESS_KEY": MINIO_ACCESS_KEY,
-        "MINIO_SECRET_KEY": MINIO_SECRET_KEY,
-        "MINIO_BUCKET": MINIO_BUCKET,
-        "MINIO_SECURE": "false",
-    }
-
-    yield env_vars
-
-    container.stop()
-
-
-@pytest.fixture(scope="session")
-def redis_config(redis_container: str) -> RedisConfig:
-    """Provide RedisConfig built from the test container."""
-    return RedisConfig(uri=redis_container)
-
-
-@pytest.fixture(scope="session")
-def minio_config(minio_container: dict[str, str]) -> MinioConfig:
-    """Provide MinioConfig built from the test container."""
-    return MinioConfig(
-        endpoint=minio_container["MINIO_ENDPOINT"],
-        access_key=minio_container["MINIO_ACCESS_KEY"],
-        secret_key=minio_container["MINIO_SECRET_KEY"],
-        bucket=minio_container["MINIO_BUCKET"],
-        secure=False,
-    )
-
-
-@pytest.fixture(scope="session")
-def raw_redis_client(redis_container: str) -> Generator[redis.Redis, None, None]:
-    """Provide a raw Redis client connected to the test Redis container."""
-    client = redis.Redis.from_url(
-        url=redis_container,
-        socket_connect_timeout=10,
-    )
-
-    yield client
-
-    client.flushall()
-    client.close()
-
-
-@pytest.fixture(scope="session")
-def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio, None, None]:
-    """Provide a raw Minio client connected to the test Minio container."""
-    client = Minio(
-        endpoint=minio_container["MINIO_ENDPOINT"],
-        access_key=minio_container["MINIO_ACCESS_KEY"],
-        secret_key=minio_container["MINIO_SECRET_KEY"],
-        secure=False,
-    )
-    bucket_name = minio_container["MINIO_BUCKET"]
-    if not client.bucket_exists(bucket_name):
-        client.make_bucket(bucket_name)
-
-    yield client
-
-    objects = client.list_objects(bucket_name, recursive=True)
-    for obj in objects:
-        client.remove_object(bucket_name, obj.object_name)

tests/integration/examples/conftest.py

@@ -0,0 +1,55 @@
+"""Examples integration test fixtures."""
+
+from collections.abc import Generator
+
+from minio import Minio
+import pytest
+import redis
+
+from python_repositories.config import MinioConfig, RedisConfig
+from tests.integration.minio._containers import (
+    minio_config_from_env,
+    minio_env,
+    raw_minio_client_from_env,
+)
+from tests.integration.redis._containers import (
+    raw_redis_client_from_container,
+    redis_config_from_container,
+    redis_uri,
+)
+
+
+@pytest.fixture(scope="session")
+def redis_container() -> Generator[str, None, None]:
+    """Set up a Redis container for testing and yield the Redis URI."""
+    yield from redis_uri()
+
+
+@pytest.fixture(scope="session")
+def redis_config(redis_container: str) -> RedisConfig:
+    """Provide RedisConfig built from the test container."""
+    return redis_config_from_container(redis_container)
+
+
+@pytest.fixture(scope="session")
+def raw_redis_client(redis_container: str) -> Generator[redis.Redis, None, None]:
+    """Provide a raw Redis client connected to the test Redis container."""
+    yield from raw_redis_client_from_container(redis_container)
+
+
+@pytest.fixture(scope="session")
+def minio_container() -> Generator[dict[str, str], None, None]:
+    """Set up a Minio container for testing and yield connection settings."""
+    yield from minio_env()
+
+
+@pytest.fixture(scope="session")
+def minio_config(minio_container: dict[str, str]) -> MinioConfig:
+    """Provide MinioConfig built from the test container."""
+    return minio_config_from_env(minio_container)
+
+
+@pytest.fixture(scope="session")
+def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio, None, None]:
+    """Provide a raw Minio client connected to the test Minio container."""
+    yield from raw_minio_client_from_env(minio_container)

tests/integration/examples/examples_test.py

@@ -13,7 +13,11 @@ from python_repositories.examples.artifact_object_repository import (
 )
 from python_repositories.examples.user_json_repository import UserJsonRepository
 
-pytestmark = pytest.mark.integration
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.needs_redis,
+    pytest.mark.needs_minio,
+]
 
 
 @pytest.fixture(scope="module", autouse=True)

tests/integration/minio/_containers.py

@@ -0,0 +1,86 @@
+"""MinIO container session helpers for integration tests."""
+
+from collections.abc import Generator
+
+from minio import Minio
+from testcontainers.minio import MinioContainer
+
+from python_repositories.config import MinioConfig
+
+MINIO_ACCESS_KEY = "minioadmin"
+MINIO_SECRET_KEY = "minioadmin"
+MINIO_BUCKET = "test-bucket"
+
+_minio_env: dict[str, str] | None = None
+_minio_container: MinioContainer | None = None
+_raw_minio_client: Minio | None = None
+_minio_env_refs = 0
+_raw_minio_client_refs = 0
+
+
+def minio_env() -> Generator[dict[str, str], None, None]:
+    """Yield session-scoped MinIO connection settings, starting the container once."""
+    global _minio_env, _minio_container, _minio_env_refs
+    if _minio_env is None:
+        _minio_container = MinioContainer(
+            image="minio/minio:latest",
+            access_key=MINIO_ACCESS_KEY,
+            secret_key=MINIO_SECRET_KEY,
+        )
+        _minio_container.start()
+        minio_host = _minio_container.get_container_host_ip()
+        minio_port = _minio_container.get_exposed_port(9000)
+        minio_endpoint = f"{minio_host}:{minio_port}"
+        _minio_env = {
+            "MINIO_ENDPOINT": minio_endpoint,
+            "MINIO_ACCESS_KEY": MINIO_ACCESS_KEY,
+            "MINIO_SECRET_KEY": MINIO_SECRET_KEY,
+            "MINIO_BUCKET": MINIO_BUCKET,
+            "MINIO_SECURE": "false",
+        }
+
+    _minio_env_refs += 1
+    yield _minio_env
+    _minio_env_refs -= 1
+
+    if _minio_env_refs == 0 and _minio_container is not None:
+        _minio_container.stop()
+        _minio_container = None
+        _minio_env = None
+
+
+def minio_config_from_env(env: dict[str, str]) -> MinioConfig:
+    """Build MinioConfig from container environment settings."""
+    return MinioConfig(
+        endpoint=env["MINIO_ENDPOINT"],
+        access_key=env["MINIO_ACCESS_KEY"],
+        secret_key=env["MINIO_SECRET_KEY"],
+        bucket=env["MINIO_BUCKET"],
+        secure=False,
+    )
+
+
+def raw_minio_client_from_env(env: dict[str, str]) -> Generator[Minio, None, None]:
+    """Yield a session-scoped raw MinIO client, reusing one client per session."""
+    global _raw_minio_client, _raw_minio_client_refs
+    if _raw_minio_client is None:
+        _raw_minio_client = Minio(
+            endpoint=env["MINIO_ENDPOINT"],
+            access_key=env["MINIO_ACCESS_KEY"],
+            secret_key=env["MINIO_SECRET_KEY"],
+            secure=False,
+        )
+        bucket_name = env["MINIO_BUCKET"]
+        if not _raw_minio_client.bucket_exists(bucket_name):
+            _raw_minio_client.make_bucket(bucket_name)
+
+    _raw_minio_client_refs += 1
+    yield _raw_minio_client
+    _raw_minio_client_refs -= 1
+
+    if _raw_minio_client_refs == 0 and _raw_minio_client is not None:
+        bucket_name = env["MINIO_BUCKET"]
+        objects = _raw_minio_client.list_objects(bucket_name, recursive=True)
+        for obj in objects:
+            _raw_minio_client.remove_object(bucket_name, obj.object_name)
+        _raw_minio_client = None

tests/integration/minio/conftest.py

@@ -0,0 +1,31 @@
+"""MinIO integration test fixtures."""
+
+from collections.abc import Generator
+
+from minio import Minio
+import pytest
+
+from python_repositories.config import MinioConfig
+from tests.integration.minio._containers import (
+    minio_config_from_env,
+    minio_env,
+    raw_minio_client_from_env,
+)
+
+
+@pytest.fixture(scope="session")
+def minio_container() -> Generator[dict[str, str], None, None]:
+    """Set up a Minio container for testing and yield connection settings."""
+    yield from minio_env()
+
+
+@pytest.fixture(scope="session")
+def minio_config(minio_container: dict[str, str]) -> MinioConfig:
+    """Provide MinioConfig built from the test container."""
+    return minio_config_from_env(minio_container)
+
+
+@pytest.fixture(scope="session")
+def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio, None, None]:
+    """Provide a raw Minio client connected to the test Minio container."""
+    yield from raw_minio_client_from_env(minio_container)

tests/integration/minio/minio_adapter_test.py

@@ -15,7 +15,7 @@ from python_repositories.adapters.minio_adapter import MinioAdapter
 from python_repositories.config import MinioConfig
 from tests.conftest import TEST_MINIO_CONFIG
 
-pytestmark = pytest.mark.integration
+pytestmark = [pytest.mark.integration, pytest.mark.needs_minio]
 
 
 def same_data(

tests/integration/redis/_containers.py

@@ -0,0 +1,90 @@
+"""Redis container session helpers for integration tests."""
+
+from collections.abc import Generator
+from typing import Any, cast
+
+import redis
+from testcontainers.core.container import DockerContainer
+from testcontainers.core.waiting_utils import WaitStrategy, WaitStrategyTarget
+
+from python_repositories.config import RedisConfig
+
+REDIS_PORT = 6379
+
+
+class _RedisPingWaitStrategy(WaitStrategy):
+    def __init__(self) -> None:
+        super().__init__()
+        self.with_transient_exceptions(redis.exceptions.ConnectionError)
+
+    def wait_until_ready(self, container: WaitStrategyTarget) -> None:
+        redis_container = cast("RedisTestContainer", container)
+        if not self._poll(lambda: redis_container.get_client().ping()):
+            raise redis.exceptions.ConnectionError("Could not connect to Redis")
+
+
+class RedisTestContainer(DockerContainer):
+    """Redis container using wait strategies instead of the deprecated decorator."""
+
+    def __init__(self, image: str, port: int = REDIS_PORT) -> None:
+        super().__init__(image, _wait_strategy=_RedisPingWaitStrategy())
+        self.port = port
+        self.with_exposed_ports(self.port)
+
+    def get_client(self, **kwargs: Any) -> redis.Redis:
+        return redis.Redis(
+            host=self.get_container_host_ip(),
+            port=self.get_exposed_port(self.port),
+            **kwargs,
+        )
+
+
+_redis_uri: str | None = None
+_redis_container: RedisTestContainer | None = None
+_raw_redis_client: redis.Redis | None = None
+_redis_uri_refs = 0
+_raw_redis_client_refs = 0
+
+
+def redis_uri() -> Generator[str, None, None]:
+    """Yield a session-scoped Redis URI, starting the container once."""
+    global _redis_uri, _redis_container, _redis_uri_refs
+    if _redis_uri is None:
+        _redis_container = RedisTestContainer(image="redis/redis-stack:7.2.0-v0")
+        _redis_container.start()
+        redis_host = _redis_container.get_container_host_ip()
+        redis_port = _redis_container.get_exposed_port(REDIS_PORT)
+        _redis_uri = f"redis://{redis_host}:{redis_port}"
+
+    _redis_uri_refs += 1
+    yield _redis_uri
+    _redis_uri_refs -= 1
+
+    if _redis_uri_refs == 0 and _redis_container is not None:
+        _redis_container.stop()
+        _redis_container = None
+        _redis_uri = None
+
+
+def redis_config_from_container(uri: str) -> RedisConfig:
+    """Build RedisConfig from a container URI."""
+    return RedisConfig(uri=uri)
+
+
+def raw_redis_client_from_container(uri: str) -> Generator[redis.Redis, None, None]:
+    """Yield a session-scoped raw Redis client, reusing one client per session."""
+    global _raw_redis_client, _raw_redis_client_refs
+    if _raw_redis_client is None:
+        _raw_redis_client = redis.Redis.from_url(
+            url=uri,
+            socket_connect_timeout=10,
+        )
+
+    _raw_redis_client_refs += 1
+    yield _raw_redis_client
+    _raw_redis_client_refs -= 1
+
+    if _raw_redis_client_refs == 0 and _raw_redis_client is not None:
+        _raw_redis_client.flushall()
+        _raw_redis_client.close()
+        _raw_redis_client = None

tests/integration/redis/conftest.py

@@ -0,0 +1,31 @@
+"""Redis integration test fixtures."""
+
+from collections.abc import Generator
+
+import pytest
+import redis
+
+from python_repositories.config import RedisConfig
+from tests.integration.redis._containers import (
+    raw_redis_client_from_container,
+    redis_config_from_container,
+    redis_uri,
+)
+
+
+@pytest.fixture(scope="session")
+def redis_container() -> Generator[str, None, None]:
+    """Set up a Redis container for testing and yield the Redis URI."""
+    yield from redis_uri()
+
+
+@pytest.fixture(scope="session")
+def redis_config(redis_container: str) -> RedisConfig:
+    """Provide RedisConfig built from the test container."""
+    return redis_config_from_container(redis_container)
+
+
+@pytest.fixture(scope="session")
+def raw_redis_client(redis_container: str) -> Generator[redis.Redis, None, None]:
+    """Provide a raw Redis client connected to the test Redis container."""
+    yield from raw_redis_client_from_container(redis_container)

tests/integration/redis/redis_adapter_test.py

@@ -10,7 +10,7 @@ from redis.commands.json.path import Path as RedisPath
 from python_repositories.adapters.redis_adapter import RedisAdapter
 from python_repositories.config import RedisConfig
 
-pytestmark = pytest.mark.integration
+pytestmark = [pytest.mark.integration, pytest.mark.needs_redis]
 
 
 @pytest.fixture(scope="module")