Add QueueRepositoryInterface with memory and file-backed adapters.

Provide a generic disk-backed FIFO queue with configurable path, retention, and dedup keys so consumers can buffer items across restarts without optional extras.

Co-authored-by: Cursor <[email protected]>

tests/unit/adapters_test.py

@@ -46,6 +46,8 @@ assert JsonRepositoryInterface is not None
 assert "python_repositories.adapters.redis_adapter" not in sys.modules
 assert "python_repositories.adapters.minio_adapter" not in sys.modules
 assert "python_repositories.adapters.postgres_adapter" not in sys.modules
+assert "python_repositories.adapters.memory_queue_adapter" not in sys.modules
+assert "python_repositories.adapters.file_backed_queue_adapter" not in sys.modules
 """
     result = subprocess.run(
         [sys.executable, "-c", script],
@@ -59,11 +61,19 @@ assert "python_repositories.adapters.postgres_adapter" not in sys.modules
 
 def test_lazy_adapter_load_succeeds_when_extra_present() -> None:
     """Adapters load when their optional dependencies are installed."""
-    from python_repositories import MinioAdapter, PostgresAdapter, RedisAdapter
+    from python_repositories import (
+        FileBackedQueueAdapter,
+        MemoryQueueAdapter,
+        MinioAdapter,
+        PostgresAdapter,
+        RedisAdapter,
+    )
 
     assert RedisAdapter.__name__ == "RedisAdapter"
     assert MinioAdapter.__name__ == "MinioAdapter"
     assert PostgresAdapter.__name__ == "PostgresAdapter"
+    assert MemoryQueueAdapter.__name__ == "MemoryQueueAdapter"
+    assert FileBackedQueueAdapter.__name__ == "FileBackedQueueAdapter"
 
 
 def test_redis_adapter_import_error_without_extra() -> None:
@@ -149,9 +159,13 @@ def test_adapters_dir_exposes_lazy_exports() -> None:
     """dir(adapters) includes lazy adapter names for tab completion."""
     import python_repositories.adapters as adapters
 
-    assert {"RedisAdapter", "MinioAdapter", "PostgresAdapter"}.issubset(
-        set(dir(adapters))
-    )
+    assert {
+        "RedisAdapter",
+        "MinioAdapter",
+        "PostgresAdapter",
+        "MemoryQueueAdapter",
+        "FileBackedQueueAdapter",
+    }.issubset(set(dir(adapters)))
 
 
 def test_adapters_getattr_raises_for_unknown() -> None:
@@ -167,3 +181,5 @@ def test_top_level_dir_exposes_lazy_exports() -> None:
     assert "RedisAdapter" in dir(python_repositories)
     assert "MinioAdapter" in dir(python_repositories)
     assert "PostgresAdapter" in dir(python_repositories)
+    assert "MemoryQueueAdapter" in dir(python_repositories)
+    assert "FileBackedQueueAdapter" in dir(python_repositories)

tests/unit/file_backed_queue_adapter_test.py

@@ -0,0 +1,241 @@
+"""Unit tests for FileBackedQueueAdapter."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+
+from python_repositories.adapters.file_backed_queue_adapter import (
+    FileBackedQueueAdapter,
+)
+from python_repositories.adapters.memory_queue_adapter import MemoryQueueAdapter
+from python_repositories.config.file_queue_config import FileQueueConfig
+from python_repositories.interfaces.queue_repository_interface import (
+    QueueRepositoryInterface,
+)
+
+
+def _config(path: Path, **kwargs: object) -> FileQueueConfig:
+    return FileQueueConfig(path=path, **kwargs)  # type: ignore[arg-type]
+
+
+def test_implements_interface() -> None:
+    assert issubclass(FileBackedQueueAdapter, QueueRepositoryInterface)
+
+
+def test_ops_require_connect(tmp_path: Path) -> None:
+    queue = FileBackedQueueAdapter(config=_config(tmp_path / "q.jsonl"))
+    with pytest.raises(RuntimeError, match="not connected"):
+        queue.enqueue([{"id": 1}])
+    with pytest.raises(RuntimeError, match="not connected"):
+        queue.dequeue_batch()
+    with pytest.raises(RuntimeError, match="not connected"):
+        queue.size()
+    with pytest.raises(RuntimeError, match="not connected"):
+        queue.evict_older_than(datetime.now(UTC))
+
+
+def test_enqueue_dequeue_persists_and_compacts(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    with FileBackedQueueAdapter(config=_config(path, dedup_keys=("id",))) as queue:
+        queue.enqueue([{"id": 1}, {"id": 2}, {"id": 1}])
+        assert queue.size() == 2
+        assert path.is_file()
+        lines = path.read_text(encoding="utf-8").strip().splitlines()
+        assert len(lines) == 2
+        batch = queue.dequeue_batch(max_items=1)
+        assert batch == [{"id": 1}]
+        remaining = path.read_text(encoding="utf-8").strip().splitlines()
+        assert len(remaining) == 1
+        assert queue.size() == 1
+
+
+def test_replay_on_connect(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    config = _config(path, dedup_keys=("id",), age_key="created_at", max_age_hours=24)
+    now = datetime.now(UTC)
+    with FileBackedQueueAdapter(config=config) as queue:
+        queue.enqueue(
+            [
+                {"id": 1, "created_at": now - timedelta(hours=1)},
+                {"id": 2, "created_at": now - timedelta(minutes=10)},
+            ]
+        )
+    with FileBackedQueueAdapter(config=config) as queue:
+        assert queue.size() == 2
+        assert [item["id"] for item in queue.dequeue_batch(max_items=10)] == [1, 2]
+
+
+def test_replay_filters_by_max_age(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    now = datetime.now(UTC)
+    path.write_text(
+        "\n".join(
+            [
+                f'{{"id": "old", "created_at": "{(now - timedelta(hours=30)).isoformat()}"}}',
+                f'{{"id": "new", "created_at": "{(now - timedelta(hours=1)).isoformat()}"}}',
+            ]
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    config = _config(path, dedup_keys=("id",), age_key="created_at", max_age_hours=24)
+    with FileBackedQueueAdapter(config=config) as queue:
+        assert queue.size() == 1
+        assert queue.dequeue_batch(max_items=10)[0]["id"] == "new"
+
+
+def test_connect_skips_corrupt_and_invalid_lines(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    path.write_text(
+        "\n".join(
+            [
+                "",
+                "not-json",
+                "[1, 2]",
+                '{"id": 1}',
+                '{"name": "missing-id"}',
+            ]
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    config = _config(path, dedup_keys=("id",))
+    with FileBackedQueueAdapter(config=config) as queue:
+        assert queue.size() == 1
+        assert queue.dequeue_batch(max_items=10) == [{"id": 1}]
+
+
+def test_connect_skips_missing_or_invalid_age_on_replay(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    now = datetime.now(UTC)
+    path.write_text(
+        "\n".join(
+            [
+                '{"id": 1}',
+                '{"id": 2, "created_at": "not-a-date"}',
+                f'{{"id": 3, "created_at": "{now.isoformat()}"}}',
+            ]
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    config = _config(path, dedup_keys=("id",), age_key="created_at")
+    with FileBackedQueueAdapter(config=config) as queue:
+        assert queue.size() == 1
+        assert queue.dequeue_batch(max_items=10)[0]["id"] == 3
+
+
+def test_connect_is_idempotent(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    queue = FileBackedQueueAdapter(config=_config(path))
+    queue.connect()
+    queue.enqueue([{"id": 1}])
+    queue.connect()
+    assert queue.size() == 1
+    queue.disconnect()
+    assert not queue.is_connected()
+
+
+def test_evict_compacts_file(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    config = _config(path, dedup_keys=("id",), age_key="created_at", max_age_hours=24)
+    now = datetime.now(UTC)
+    with FileBackedQueueAdapter(config=config) as queue:
+        queue.enqueue(
+            [
+                {"id": 1, "created_at": now - timedelta(hours=2)},
+                {"id": 2, "created_at": now - timedelta(minutes=5)},
+            ]
+        )
+        removed = queue.evict_older_than(now - timedelta(hours=1))
+        assert removed == 1
+        assert queue.size() == 1
+        lines = [line for line in path.read_text(encoding="utf-8").splitlines() if line]
+        assert len(lines) == 1
+
+
+def test_auto_evict_on_enqueue(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    config = _config(path, dedup_keys=("id",), age_key="created_at", max_age_hours=1)
+    now = datetime.now(UTC)
+    with FileBackedQueueAdapter(config=config) as queue:
+        queue.enqueue([{"id": 1, "created_at": now - timedelta(hours=2)}])
+        # Item is enqueued then immediately age-evicted.
+        assert queue.size() == 0
+
+
+def test_from_env_when_config_omitted(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    path = tmp_path / "from-env.jsonl"
+    monkeypatch.setenv("FILE_QUEUE_PATH", str(path))
+    with FileBackedQueueAdapter() as queue:
+        queue.enqueue([{"id": 1}])
+        assert queue.size() == 1
+
+
+def test_injected_memory(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    memory = MemoryQueueAdapter(dedup_keys=("id",))
+    config = _config(path, dedup_keys=("id",))
+    with FileBackedQueueAdapter(config=config, memory=memory) as queue:
+        queue.enqueue([{"id": 1}])
+        assert memory.size() == 1
+
+
+def test_empty_enqueue_and_dequeue_skip_compact(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    with FileBackedQueueAdapter(config=_config(path)) as queue:
+        queue.enqueue([])
+        assert queue.dequeue_batch(max_items=10) == []
+        assert queue.evict_older_than(datetime.now(UTC)) == 0
+        assert not path.exists() or path.read_text(encoding="utf-8") == ""
+
+
+def test_reconnect_replays_after_disconnect(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    queue = FileBackedQueueAdapter(config=_config(path, dedup_keys=("id",)))
+    queue.connect()
+    queue.enqueue([{"id": 1}])
+    queue.disconnect()
+    queue.connect()
+    assert queue.size() == 1
+    queue.disconnect()
+
+
+def test_json_default_rejects_unsupported() -> None:
+    from python_repositories.adapters.file_backed_queue_adapter import _json_default
+
+    with pytest.raises(TypeError):
+        _json_default(object())
+
+
+def test_disconnect_ignores_fsync_oserror(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    path = tmp_path / "items.jsonl"
+    queue = FileBackedQueueAdapter(config=_config(path))
+    queue.connect()
+
+    def boom(_fd: int) -> None:
+        raise OSError("fsync failed")
+
+    monkeypatch.setattr(
+        "python_repositories.adapters.file_backed_queue_adapter.os.fsync",
+        boom,
+    )
+    queue.disconnect()
+    assert not queue.is_connected()
+
+
+def test_append_items_requires_open_handle(tmp_path: Path) -> None:
+    path = tmp_path / "items.jsonl"
+    queue = FileBackedQueueAdapter(config=_config(path))
+    queue.connect()
+    queue._append_handle = None  # noqa: SLF001
+    with pytest.raises(RuntimeError, match="append handle is not open"):
+        queue._append_items([{"id": 1}])  # noqa: SLF001
+    queue._connected = False  # noqa: SLF001

tests/unit/file_queue_config_test.py

@@ -0,0 +1,56 @@
+"""Unit tests for FileQueueConfig."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from python_repositories.config import FileQueueConfig
+
+
+def test_defaults() -> None:
+    config = FileQueueConfig(path=Path("/tmp/queue.jsonl"))
+    assert config.path == Path("/tmp/queue.jsonl")
+    assert config.max_age_hours == 24
+    assert config.dedup_keys == ()
+    assert config.age_key is None
+
+
+def test_from_env_loads_path(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("FILE_QUEUE_PATH", "/data/items.jsonl")
+    monkeypatch.delenv("FILE_QUEUE_MAX_AGE_HOURS", raising=False)
+    config = FileQueueConfig.from_env(use_dotenv=False)
+    assert config.path == Path("/data/items.jsonl")
+    assert config.max_age_hours == 24
+
+
+def test_from_env_loads_max_age_hours(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("FILE_QUEUE_PATH", "/data/items.jsonl")
+    monkeypatch.setenv("FILE_QUEUE_MAX_AGE_HOURS", "48")
+    config = FileQueueConfig.from_env(use_dotenv=False)
+    assert config.max_age_hours == 48
+
+
+def test_from_env_raises_when_path_missing(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("FILE_QUEUE_PATH", raising=False)
+    with pytest.raises(Exception):
+        FileQueueConfig.from_env(use_dotenv=False)
+
+
+def test_from_env_respects_custom_env_var_names(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("CUSTOM_QUEUE_PATH", "/custom/q.jsonl")
+    monkeypatch.setenv("CUSTOM_MAX_AGE", "12")
+    config = FileQueueConfig.from_env(
+        "CUSTOM_QUEUE_PATH",
+        max_age_hours_env_var_name="CUSTOM_MAX_AGE",
+        dedup_keys=("id",),
+        age_key="created_at",
+        use_dotenv=False,
+    )
+    assert config.path == Path("/custom/q.jsonl")
+    assert config.max_age_hours == 12
+    assert config.dedup_keys == ("id",)
+    assert config.age_key == "created_at"

tests/unit/memory_queue_adapter_test.py

@@ -0,0 +1,144 @@
+"""Unit tests for MemoryQueueAdapter."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+import threading
+
+import pytest
+
+from python_repositories.adapters.memory_queue_adapter import (
+    MemoryQueueAdapter,
+    _parse_age,
+)
+from python_repositories.interfaces.queue_repository_interface import (
+    QueueRepositoryInterface,
+)
+
+
+def test_implements_interface() -> None:
+    assert issubclass(MemoryQueueAdapter, QueueRepositoryInterface)
+
+
+def test_fifo_without_dedup() -> None:
+    queue = MemoryQueueAdapter()
+    queue.enqueue([{"id": 1}, {"id": 2}, {"id": 3}])
+    assert queue.size() == 3
+    assert queue.dequeue_batch(max_items=2) == [{"id": 1}, {"id": 2}]
+    assert queue.dequeue_batch(max_items=10) == [{"id": 3}]
+    assert queue.dequeue_batch() == []
+    assert queue.size() == 0
+
+
+def test_empty_enqueue_is_noop() -> None:
+    queue = MemoryQueueAdapter()
+    queue.enqueue([])
+    assert queue.size() == 0
+    assert queue.enqueue_and_return_added([]) == []
+
+
+def test_dedup_keeps_first() -> None:
+    queue = MemoryQueueAdapter(dedup_keys=("id",))
+    queue.enqueue([{"id": 1, "v": "a"}, {"id": 1, "v": "b"}, {"id": 2, "v": "c"}])
+    assert queue.size() == 2
+    assert queue.dequeue_batch(max_items=10) == [
+        {"id": 1, "v": "a"},
+        {"id": 2, "v": "c"},
+    ]
+
+
+def test_missing_dedup_key_raises() -> None:
+    queue = MemoryQueueAdapter(dedup_keys=("id",))
+    with pytest.raises(ValueError, match="missing dedup key"):
+        queue.enqueue([{"name": "x"}])
+
+
+def test_missing_age_key_raises() -> None:
+    queue = MemoryQueueAdapter(age_key="created_at")
+    with pytest.raises(ValueError, match="missing age key"):
+        queue.enqueue([{"id": 1}])
+
+
+def test_evict_older_than() -> None:
+    queue = MemoryQueueAdapter(age_key="created_at", dedup_keys=("id",))
+    now = datetime.now(UTC)
+    queue.enqueue(
+        [
+            {"id": 1, "created_at": now - timedelta(hours=2)},
+            {"id": 2, "created_at": now - timedelta(minutes=30)},
+            {"id": 3, "created_at": (now - timedelta(hours=3)).isoformat()},
+        ]
+    )
+    removed = queue.evict_older_than(now - timedelta(hours=1))
+    assert removed == 2
+    assert queue.size() == 1
+    remaining = queue.dequeue_batch(max_items=10)
+    assert remaining[0]["id"] == 2
+
+
+def test_evict_without_age_key_is_noop() -> None:
+    queue = MemoryQueueAdapter()
+    queue.enqueue([{"id": 1}])
+    assert queue.evict_older_than(datetime.now(UTC)) == 0
+    assert queue.size() == 1
+
+
+def test_parse_age_rejects_unsupported_type() -> None:
+    with pytest.raises(ValueError, match="age value must be"):
+        _parse_age(123)
+
+
+def test_parse_age_aware_datetime() -> None:
+    from datetime import timezone
+
+    eastern = timezone(timedelta(hours=-5))
+    aware = datetime(2024, 1, 1, 12, 0, 0, tzinfo=eastern)
+    assert _parse_age(aware) == datetime(2024, 1, 1, 17, 0, 0, tzinfo=UTC)
+
+
+def test_parse_age_naive_datetime() -> None:
+    naive = datetime(2024, 1, 1, 12, 0, 0)
+    assert _parse_age(naive) == datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC)
+
+
+def test_parse_age_zulu_string() -> None:
+    assert _parse_age("2024-01-01T00:00:00Z") == datetime(2024, 1, 1, 0, 0, 0, tzinfo=UTC)
+
+
+def test_parse_age_naive_string() -> None:
+    assert _parse_age("2024-01-01T00:00:00") == datetime(2024, 1, 1, 0, 0, 0, tzinfo=UTC)
+
+
+def test_dequeue_rejects_negative_max_items() -> None:
+    queue = MemoryQueueAdapter()
+    with pytest.raises(ValueError, match="max_items"):
+        queue.dequeue_batch(max_items=-1)
+
+
+def test_clear_and_snapshot() -> None:
+    queue = MemoryQueueAdapter()
+    queue.enqueue([{"id": 1}, {"id": 2}])
+    assert queue.snapshot() == [{"id": 1}, {"id": 2}]
+    queue.clear()
+    assert queue.size() == 0
+    assert queue.snapshot() == []
+
+
+def test_thread_safety_smoke() -> None:
+    queue = MemoryQueueAdapter(dedup_keys=("id",))
+    errors: list[BaseException] = []
+
+    def worker(start: int) -> None:
+        try:
+            for i in range(start, start + 50):
+                queue.enqueue([{"id": i}])
+        except BaseException as exc:  # noqa: BLE001
+            errors.append(exc)
+
+    threads = [threading.Thread(target=worker, args=(i * 50,)) for i in range(4)]
+    for thread in threads:
+        thread.start()
+    for thread in threads:
+        thread.join()
+    assert errors == []
+    assert queue.size() == 200

tests/unit/queue_repository_interface_test.py

@@ -0,0 +1,100 @@
+"""Unit tests for QueueRepositoryInterface."""
+
+from datetime import UTC, datetime
+from typing import Any
+
+import pytest
+
+from python_repositories.interfaces.queue_repository_interface import (
+    QueueRepositoryInterface,
+)
+
+
+class InMemoryQueueRepo:
+    """Plain class that satisfies QueueRepositoryInterface without inheritance."""
+
+    def enqueue(self, items: list[dict[str, Any]]) -> None:
+        pass
+
+    def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
+        del max_items
+        return []
+
+    def size(self) -> int:
+        return 0
+
+    def evict_older_than(self, cutoff: datetime) -> int:
+        del cutoff
+        return 0
+
+
+def accepts_queue_repo(repo: QueueRepositoryInterface) -> None:
+    """Type-checking hook for QueueRepositoryInterface structural subtyping."""
+    repo.size()
+
+
+def test_instantiation_fails_when_enqueue_not_implemented() -> None:
+    class Incomplete(QueueRepositoryInterface):
+        def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
+            return []
+
+        def size(self) -> int:
+            return 0
+
+        def evict_older_than(self, cutoff: datetime) -> int:
+            return 0
+
+    with pytest.raises(TypeError):
+        _ = Incomplete()  # type: ignore
+
+
+def test_instantiation_fails_when_dequeue_batch_not_implemented() -> None:
+    class Incomplete(QueueRepositoryInterface):
+        def enqueue(self, items: list[dict[str, Any]]) -> None:
+            pass
+
+        def size(self) -> int:
+            return 0
+
+        def evict_older_than(self, cutoff: datetime) -> int:
+            return 0
+
+    with pytest.raises(TypeError):
+        _ = Incomplete()  # type: ignore
+
+
+def test_instantiation_fails_when_size_not_implemented() -> None:
+    class Incomplete(QueueRepositoryInterface):
+        def enqueue(self, items: list[dict[str, Any]]) -> None:
+            pass
+
+        def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
+            return []
+
+        def evict_older_than(self, cutoff: datetime) -> int:
+            return 0
+
+    with pytest.raises(TypeError):
+        _ = Incomplete()  # type: ignore
+
+
+def test_instantiation_fails_when_evict_older_than_not_implemented() -> None:
+    class Incomplete(QueueRepositoryInterface):
+        def enqueue(self, items: list[dict[str, Any]]) -> None:
+            pass
+
+        def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
+            return []
+
+        def size(self) -> int:
+            return 0
+
+    with pytest.raises(TypeError):
+        _ = Incomplete()  # type: ignore
+
+
+def test_structural_subtyping() -> None:
+    repo: QueueRepositoryInterface = InMemoryQueueRepo()
+    accepts_queue_repo(repo)
+    assert isinstance(repo, QueueRepositoryInterface)
+    assert repo.evict_older_than(datetime.now(UTC)) == 0