Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9a9b7b2985 | ||
|
|
5b9573d499 | ||
|
|
50444af982 | ||
|
|
2f19fcc972 | ||
|
|
3bd65895ec | ||
|
|
5311d49fa6 | ||
|
|
703bb9521f | ||
|
|
8e47ebf4c6 | ||
|
|
9a800bf553 | ||
|
|
8c34534187 | ||
|
|
b1210bddf2 | ||
|
|
39b104383c | ||
|
|
beb2b5128e | ||
|
|
ce062be411 | ||
|
|
a381d45650 | ||
|
|
5c8cd841b6 | ||
|
|
34632980ba | ||
|
|
5393efc6cd | ||
|
|
e39fc96ac1 | ||
|
|
c1f4826d78 | ||
|
|
0315a33a3b | ||
|
|
5d33ec6091 | ||
|
|
9538e4d26d | ||
|
|
14b8d2798c | ||
|
|
d3cbb65bbc |
@@ -2,9 +2,11 @@
|
|||||||
|
|
||||||
If this PR changes files under `python_repositories/`, the **title must** start with one of:
|
If this PR changes files under `python_repositories/`, the **title must** start with one of:
|
||||||
|
|
||||||
- `[patch]` or `[fix]` — bug fix (0.3.1 → 0.3.2)
|
- `[patch]` or `[fix]` — bug fix (`1.2.3` → `1.2.4`)
|
||||||
- `[minor]` or `[feat]` — new feature (0.3.1 → 0.4.0)
|
- `[minor]` or `[feat]` — new feature (`1.2.3` → `1.3.0`)
|
||||||
- `[major]` or `[breaking]` — breaking change (0.3.1 → 1.0.0)
|
- `[major]` or `[breaking]` — breaking change (`1.2.3` → `2.0.0`)
|
||||||
|
|
||||||
|
Current version: see [`pyproject.toml`](pyproject.toml) on `main`.
|
||||||
|
|
||||||
Docs-, CI-, or test-only PRs do not need a prefix.
|
Docs-, CI-, or test-only PRs do not need a prefix.
|
||||||
|
|
||||||
|
|||||||
@@ -16,8 +16,14 @@ jobs:
|
|||||||
|
|
||||||
- name: Check PR title when source files change
|
- name: Check PR title when source files change
|
||||||
env:
|
env:
|
||||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
API_URL: ${{ vars.API_URL }}
|
||||||
|
REPO_OWNER: ${{ github.repository_owner }}
|
||||||
|
REPO_NAME: ${{ github.event.repository.name }}
|
||||||
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||||
|
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
git fetch origin "${{ github.base_ref }}"
|
git fetch origin "${{ github.base_ref }}"
|
||||||
|
PR_TITLE=$(scripts/ci/fetch-pr-title.sh)
|
||||||
|
echo "Live PR title: ${PR_TITLE}"
|
||||||
git diff --name-only "origin/${{ github.base_ref }}...HEAD" \
|
git diff --name-only "origin/${{ github.base_ref }}...HEAD" \
|
||||||
| scripts/ci/check-pr-title.sh "$PR_TITLE"
|
| scripts/ci/check-pr-title.sh "$PR_TITLE"
|
||||||
|
|||||||
+108
-4
@@ -7,7 +7,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
unit-tests:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
@@ -26,10 +26,114 @@ jobs:
|
|||||||
UV_LINK_MODE: copy
|
UV_LINK_MODE: copy
|
||||||
run: uv sync --all-extras
|
run: uv sync --all-extras
|
||||||
|
|
||||||
- name: Run pytest
|
- name: Run unit tests
|
||||||
|
run: |
|
||||||
|
# --cov-fail-under=0: partial coverage only; floor is checked in coverage-report.
|
||||||
|
uv run pytest tests/unit/ -m "not integration" \
|
||||||
|
--cov=python_repositories \
|
||||||
|
--cov-report= \
|
||||||
|
--cov-fail-under=0
|
||||||
|
|
||||||
|
- name: Upload unit coverage
|
||||||
|
uses: https://github.com/christopherHX/gitea-upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: coverage-unit
|
||||||
|
path: .coverage
|
||||||
|
retention-days: 1
|
||||||
|
compression-level: 0
|
||||||
|
|
||||||
|
integration-tests:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version-file: .python-version
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
run: pip install uv
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
env:
|
env:
|
||||||
PYTHONPATH: .
|
UV_LINK_MODE: copy
|
||||||
run: uv run pytest --cov=python_repositories --cov-report=term-missing > coverage.txt
|
run: uv sync --all-extras
|
||||||
|
|
||||||
|
- name: Verify Docker
|
||||||
|
run: docker info
|
||||||
|
|
||||||
|
- name: Run integration tests
|
||||||
|
run: |
|
||||||
|
# --cov-fail-under=0: partial coverage only; floor is checked in coverage-report.
|
||||||
|
uv run pytest -m integration \
|
||||||
|
--cov=python_repositories \
|
||||||
|
--cov-report= \
|
||||||
|
--cov-fail-under=0
|
||||||
|
|
||||||
|
- name: Upload integration coverage
|
||||||
|
uses: https://github.com/christopherHX/gitea-upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: coverage-integration
|
||||||
|
path: .coverage
|
||||||
|
retention-days: 1
|
||||||
|
compression-level: 0
|
||||||
|
|
||||||
|
coverage-report:
|
||||||
|
# Merges unit + integration coverage and enforces fail_under from pyproject.toml.
|
||||||
|
needs: [unit-tests, integration-tests]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.event_name == 'pull_request' || github.event_name == 'push'
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version-file: .python-version
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
run: pip install uv
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
env:
|
||||||
|
UV_LINK_MODE: copy
|
||||||
|
run: uv sync --all-extras
|
||||||
|
|
||||||
|
- name: Download unit coverage
|
||||||
|
uses: https://github.com/christopherHX/gitea-download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: coverage-unit
|
||||||
|
path: coverage-unit
|
||||||
|
|
||||||
|
- name: Download integration coverage
|
||||||
|
uses: https://github.com/christopherHX/gitea-download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: coverage-integration
|
||||||
|
path: coverage-integration
|
||||||
|
|
||||||
|
- name: Combine coverage report
|
||||||
|
run: |
|
||||||
|
# --fail-under=0 so the full report is always written before the floor check.
|
||||||
|
uv run coverage combine coverage-unit/.coverage coverage-integration/.coverage
|
||||||
|
uv run coverage report -m --include='python_repositories/*' --fail-under=0 > coverage.txt
|
||||||
|
|
||||||
|
- name: Show coverage report
|
||||||
|
run: cat coverage.txt
|
||||||
|
|
||||||
|
- name: Enforce coverage floor
|
||||||
|
run: |
|
||||||
|
# Reads fail_under from pyproject.toml; only combined coverage is evaluated here.
|
||||||
|
FAIL_UNDER=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['tool']['coverage']['report']['fail_under'])")
|
||||||
|
echo "Checking combined coverage against ${FAIL_UNDER}% floor..."
|
||||||
|
if uv run coverage report --fail-under="$FAIL_UNDER" --include='python_repositories/*'; then
|
||||||
|
echo "Coverage floor met."
|
||||||
|
else
|
||||||
|
echo "::error::Combined coverage is below the ${FAIL_UNDER}% floor"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Post coverage summary to PR
|
- name: Post coverage summary to PR
|
||||||
if: github.event_name == 'pull_request'
|
if: github.event_name == 'pull_request'
|
||||||
|
|||||||
@@ -34,6 +34,18 @@ Requires Redis with the RedisJSON module (e.g. redis-stack).
|
|||||||
| -------------------- | ---------------------------------------------------- |
|
| -------------------- | ---------------------------------------------------- |
|
||||||
| `REDIS_URI` | Redis connection URL (e.g. `redis://localhost:6379`) |
|
| `REDIS_URI` | Redis connection URL (e.g. `redis://localhost:6379`) |
|
||||||
|
|
||||||
|
For key discovery:
|
||||||
|
|
||||||
|
- `list_keys(pattern)` is simple and returns a `list[str]`, but it uses Redis `KEYS` and may block on large datasets.
|
||||||
|
- `scan_keys(pattern, *, count=None)` is preferred for production use and yields keys incrementally via Redis `SCAN`.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```python
|
||||||
|
for key in repo.scan_keys("user:*"):
|
||||||
|
print(key)
|
||||||
|
```
|
||||||
|
|
||||||
### MinIO (`ObjectRepositoryInterface`)
|
### MinIO (`ObjectRepositoryInterface`)
|
||||||
|
|
||||||
| Environment variable | Description |
|
| Environment variable | Description |
|
||||||
@@ -148,6 +160,15 @@ uv run pytest -v # full suite (requires Doc
|
|||||||
|
|
||||||
Integration tests are marked with `@pytest.mark.integration` and require Docker (testcontainers). Run unit tests alone for quick local feedback.
|
Integration tests are marked with `@pytest.mark.integration` and require Docker (testcontainers). Run unit tests alone for quick local feedback.
|
||||||
|
|
||||||
|
CI runs unit and integration tests in parallel with coverage, then merges `.coverage` artifacts in a follow-up job (via [christopherhx/gitea-\*-artifact@v4](https://github.com/christopherHX/gitea-upload-artifact) for Gitea 1.26 compatibility). Combined coverage must be at least **90%**; the floor is set by [`fail_under` in `pyproject.toml`](pyproject.toml#L45-L48) and enforced after merging unit and integration coverage, not on unit-only runs.
|
||||||
|
|
||||||
|
To check coverage locally (requires Docker for the full suite):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run pytest --cov=python_repositories --cov-report=
|
||||||
|
uv run coverage report
|
||||||
|
```
|
||||||
|
|
||||||
`pre-commit` is included in the dev dependency group. `uv sync` installs the CLI, but git does not run hooks until you install them with `pre-commit install` (one time per clone). After that, commits run the checks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml) (ruff, mypy, pyupgrade, prettier, and general file hygiene).
|
`pre-commit` is included in the dev dependency group. `uv sync` installs the CLI, but git does not run hooks until you install them with `pre-commit install` (one time per clone). After that, commits run the checks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml) (ruff, mypy, pyupgrade, prettier, and general file hygiene).
|
||||||
|
|
||||||
To run all hooks manually without committing:
|
To run all hooks manually without committing:
|
||||||
@@ -173,9 +194,9 @@ Docs-, CI-, and test-only PRs do not need a prefix and will not trigger a releas
|
|||||||
|
|
||||||
| Prefix | Bump | Example |
|
| Prefix | Bump | Example |
|
||||||
| ------------------------- | ---------- | --------------------- |
|
| ------------------------- | ---------- | --------------------- |
|
||||||
| `[patch]` or `[fix]` | patch | `0.3.1` → `0.3.2` |
|
| `[patch]` or `[fix]` | patch | `1.2.3` → `1.2.4` |
|
||||||
| `[minor]` or `[feat]` | minor | `0.3.1` → `0.4.0` |
|
| `[minor]` or `[feat]` | minor | `1.2.3` → `1.3.0` |
|
||||||
| `[major]` or `[breaking]` | major | `0.3.1` → `1.0.0` |
|
| `[major]` or `[breaking]` | major | `1.2.3` → `2.0.0` |
|
||||||
| _(none)_ | no release | docs / CI / deps only |
|
| _(none)_ | no release | docs / CI / deps only |
|
||||||
|
|
||||||
Example titles:
|
Example titles:
|
||||||
@@ -189,4 +210,4 @@ Release notes are generated from commits since the previous tag (see [`scripts/c
|
|||||||
|
|
||||||
### Manual release
|
### Manual release
|
||||||
|
|
||||||
You can still push a `v*.*.*` tag manually; `publish.yml` will build and publish. The current baseline version is **0.3.1**.
|
You can still push a `v*.*.*` tag manually; `publish.yml` will build and publish. The current released version is in [`pyproject.toml`](pyproject.toml) (and on the latest `v*.*.*` tag).
|
||||||
|
|||||||
+9
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "python-repositories"
|
name = "python-repositories"
|
||||||
version = "0.5.0"
|
version = "1.1.0"
|
||||||
description = "Various python repository interfaces exposed as a python package."
|
description = "Various python repository interfaces exposed as a python package."
|
||||||
authors = [
|
authors = [
|
||||||
{ name = "Brian Bjarke Jensen", email = "[email protected]" }
|
{ name = "Brian Bjarke Jensen", email = "[email protected]" }
|
||||||
@@ -39,6 +39,14 @@ markers = [
|
|||||||
"integration: tests requiring Docker containers (deselect with '-m \"not integration\"')",
|
"integration: tests requiring Docker containers (deselect with '-m \"not integration\"')",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[tool.coverage.run]
|
||||||
|
source = ["python_repositories"]
|
||||||
|
|
||||||
|
[tool.coverage.report]
|
||||||
|
fail_under = 100
|
||||||
|
show_missing = true
|
||||||
|
precision = 2
|
||||||
|
|
||||||
[tool.uv.sources]
|
[tool.uv.sources]
|
||||||
python-utils = { index = "gitea" }
|
python-utils = { index = "gitea" }
|
||||||
|
|
||||||
|
|||||||
@@ -152,6 +152,7 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
assert self._client is not None and self._bucket_name is not None
|
assert self._client is not None and self._bucket_name is not None
|
||||||
# Get data from bucket
|
# Get data from bucket
|
||||||
# N.B. bucket name is set when connecting
|
# N.B. bucket name is set when connecting
|
||||||
|
response = None
|
||||||
try:
|
try:
|
||||||
response = self._client.get_object(
|
response = self._client.get_object(
|
||||||
bucket_name=self._bucket_name,
|
bucket_name=self._bucket_name,
|
||||||
@@ -171,11 +172,12 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
self.logger.warning(
|
self.logger.warning(
|
||||||
f"Object '{object_name}' not found in bucket '{self._bucket_name}'"
|
f"Object '{object_name}' not found in bucket '{self._bucket_name}'"
|
||||||
)
|
)
|
||||||
else:
|
return None
|
||||||
self.logger.error(repr(exc))
|
raise
|
||||||
except Exception as exc: # pylint: disable=broad-except
|
finally:
|
||||||
self.logger.error(repr(exc))
|
if response is not None:
|
||||||
return None
|
response.close()
|
||||||
|
response.release_conn()
|
||||||
|
|
||||||
def delete(self, object_name: str) -> None:
|
def delete(self, object_name: str) -> None:
|
||||||
"""Delete an object from the Minio bucket."""
|
"""Delete an object from the Minio bucket."""
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""Definition of RedisAdapter class."""
|
"""Definition of RedisAdapter class."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
from collections.abc import Iterator
|
||||||
from typing import cast
|
from typing import cast
|
||||||
|
|
||||||
from python_repositories.adapters.connection_aware_adapter import (
|
from python_repositories.adapters.connection_aware_adapter import (
|
||||||
@@ -136,11 +137,13 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
self._client.json().delete(key)
|
self._client.json().delete(key)
|
||||||
self.logger.debug(f"Deleted {key}")
|
self.logger.debug(f"Deleted {key}")
|
||||||
|
|
||||||
def list_keys(self, pattern: str) -> list[str]:
|
def _validate_pattern(self, pattern: str) -> None:
|
||||||
"""List keys in Redis matching a pattern."""
|
|
||||||
# Check input
|
|
||||||
if not isinstance(pattern, str) or len(pattern) == 0:
|
if not isinstance(pattern, str) or len(pattern) == 0:
|
||||||
raise ValueError("Pattern must be a non-empty string")
|
raise ValueError("Pattern must be a non-empty string")
|
||||||
|
|
||||||
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
|
"""List keys in Redis using KEYS; may block on large datasets."""
|
||||||
|
self._validate_pattern(pattern)
|
||||||
# Check connection
|
# Check connection
|
||||||
self._require_connected()
|
self._require_connected()
|
||||||
assert self._client is not None
|
assert self._client is not None
|
||||||
@@ -152,3 +155,29 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
keys: list[str] = [key.decode(self.encoding) for key in keys_raw]
|
keys: list[str] = [key.decode(self.encoding) for key in keys_raw]
|
||||||
self.logger.debug(f"Got {keys} matching {pattern}")
|
self.logger.debug(f"Got {keys} matching {pattern}")
|
||||||
return keys
|
return keys
|
||||||
|
|
||||||
|
def scan_keys(
|
||||||
|
self,
|
||||||
|
pattern: str,
|
||||||
|
*,
|
||||||
|
count: int | None = None,
|
||||||
|
) -> Iterator[str]:
|
||||||
|
"""Yield keys in Redis using SCAN to avoid blocking large datasets."""
|
||||||
|
self._validate_pattern(pattern)
|
||||||
|
self._require_connected()
|
||||||
|
assert self._client is not None
|
||||||
|
client = self._client
|
||||||
|
|
||||||
|
def _decode(key: bytes | str) -> str:
|
||||||
|
return key if isinstance(key, str) else key.decode(self.encoding)
|
||||||
|
|
||||||
|
def _iter() -> Iterator[str]:
|
||||||
|
scan_iter = (
|
||||||
|
client.scan_iter(match=pattern, count=count)
|
||||||
|
if count is not None
|
||||||
|
else client.scan_iter(match=pattern)
|
||||||
|
)
|
||||||
|
for key_raw in scan_iter:
|
||||||
|
yield _decode(key_raw)
|
||||||
|
|
||||||
|
return _iter()
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
"""Definition of JsonRepositoryInterface abstract base class."""
|
"""Definition of JsonRepositoryInterface abstract base class."""
|
||||||
|
|
||||||
|
from collections.abc import Iterator
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
|
|
||||||
@@ -25,3 +26,13 @@ class JsonRepositoryInterface(ABC):
|
|||||||
def list_keys(self, pattern: str) -> list[str]:
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
"""List keys matching a glob pattern."""
|
"""List keys matching a glob pattern."""
|
||||||
...
|
...
|
||||||
|
|
||||||
|
def scan_keys(
|
||||||
|
self,
|
||||||
|
pattern: str,
|
||||||
|
*,
|
||||||
|
count: int | None = None,
|
||||||
|
) -> Iterator[str]:
|
||||||
|
"""Yield keys matching a glob pattern incrementally."""
|
||||||
|
del count
|
||||||
|
yield from self.list_keys(pattern)
|
||||||
|
|||||||
@@ -9,7 +9,11 @@ class ObjectRepositoryInterface(ABC):
|
|||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get(self, object_name: str) -> BytesIO | None:
|
def get(self, object_name: str) -> BytesIO | None:
|
||||||
"""Get an object by name."""
|
"""Get an object by name.
|
||||||
|
|
||||||
|
Returns None when the object does not exist. Raises ConnectionError when
|
||||||
|
not connected. Other backend errors propagate to the caller.
|
||||||
|
"""
|
||||||
...
|
...
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
|
|||||||
Executable
+16
@@ -0,0 +1,16 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Fetch the current PR title from the Gitea API.
|
||||||
|
# Requires: API_URL, REPO_OWNER, REPO_NAME, PR_NUMBER, CI_RUNNER_TOKEN
|
||||||
|
|
||||||
|
: "${API_URL:?API_URL is required}"
|
||||||
|
: "${REPO_OWNER:?REPO_OWNER is required}"
|
||||||
|
: "${REPO_NAME:?REPO_NAME is required}"
|
||||||
|
: "${PR_NUMBER:?PR_NUMBER is required}"
|
||||||
|
: "${CI_RUNNER_TOKEN:?CI_RUNNER_TOKEN is required}"
|
||||||
|
|
||||||
|
curl -sf \
|
||||||
|
"${API_URL}/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PR_NUMBER}" \
|
||||||
|
-H "Authorization: token ${CI_RUNNER_TOKEN}" \
|
||||||
|
| python3 -c 'import json, sys; print(json.load(sys.stdin)["title"])'
|
||||||
@@ -218,10 +218,8 @@ def test_should_log_warning_when_getting_nonexistent_object(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
|
def test_should_reraise_s3error_other_than_no_such_key() -> None:
|
||||||
caplog: pytest.LogCaptureFixture,
|
"""Test that the MinioAdapter re-raises unhandled S3 errors."""
|
||||||
) -> None:
|
|
||||||
"""Test that the MinioAdapter logs an error for unhandled S3 errors."""
|
|
||||||
mock_client = MagicMock(spec=Minio)
|
mock_client = MagicMock(spec=Minio)
|
||||||
mock_client.bucket_exists.return_value = True
|
mock_client.bucket_exists.return_value = True
|
||||||
other_s3error = S3Error(
|
other_s3error = S3Error(
|
||||||
@@ -236,25 +234,20 @@ def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
|
|||||||
)
|
)
|
||||||
mock_client.get_object.side_effect = other_s3error
|
mock_client.get_object.side_effect = other_s3error
|
||||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
with caplog.at_level("ERROR"):
|
with pytest.raises(S3Error) as exc_info:
|
||||||
result = adapter.get("missing-object")
|
adapter.get("missing-object")
|
||||||
assert result is None
|
assert exc_info.value.code == "UnhandledError"
|
||||||
assert repr(other_s3error) in caplog.text
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_error_when_getting_with_general_exception(
|
def test_should_reraise_general_exception() -> None:
|
||||||
caplog: pytest.LogCaptureFixture,
|
"""Test that the MinioAdapter re-raises general exceptions during get."""
|
||||||
) -> None:
|
|
||||||
"""Test that the MinioAdapter logs an error on general exceptions during get."""
|
|
||||||
mock_client = MagicMock(spec=Minio)
|
mock_client = MagicMock(spec=Minio)
|
||||||
mock_client.bucket_exists.return_value = True
|
mock_client.bucket_exists.return_value = True
|
||||||
general_exception = Exception("General failure")
|
general_exception = Exception("General failure")
|
||||||
mock_client.get_object.side_effect = general_exception
|
mock_client.get_object.side_effect = general_exception
|
||||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
with caplog.at_level("ERROR"):
|
with pytest.raises(Exception, match="General failure"):
|
||||||
result = adapter.get("missing-object")
|
adapter.get("missing-object")
|
||||||
assert result is None
|
|
||||||
assert repr(general_exception) in caplog.text
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_put_data(
|
def test_should_put_data(
|
||||||
|
|||||||
@@ -254,5 +254,34 @@ def test_should_raise_connection_error_on_list_keys_when_not_connected(
|
|||||||
adapter.list_keys("some_pattern")
|
adapter.list_keys("some_pattern")
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_scan_keys(
|
||||||
|
redis_adapter: RedisAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test scanning keys matching a pattern returns correct keys."""
|
||||||
|
redis_adapter.set("key1", {"a": 1})
|
||||||
|
redis_adapter.set("key2", {"b": 2})
|
||||||
|
keys = set(redis_adapter.scan_keys("key*"))
|
||||||
|
assert keys == {"key1", "key2"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_value_error_on_invalid_scan_keys_pattern(
|
||||||
|
redis_adapter: RedisAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that the RedisAdapter raises ValueError when scanning keys with an invalid pattern."""
|
||||||
|
invalid_patterns = ["", 123, None]
|
||||||
|
for pattern in invalid_patterns:
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
list(redis_adapter.scan_keys(pattern)) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_on_scan_keys_when_not_connected(
|
||||||
|
redis_config: RedisConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that the RedisAdapter raises ConnectionError when scanning keys while not connected."""
|
||||||
|
adapter = RedisAdapter(config=redis_config)
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
list(adapter.scan_keys("some_pattern"))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
pytest.main(["-s", "-v", __file__])
|
pytest.main(["-s", "-v", __file__])
|
||||||
|
|||||||
@@ -117,3 +117,24 @@ def test_adapters_subpackage_lazy_import_succeeds() -> None:
|
|||||||
from python_repositories.adapters import RedisAdapter
|
from python_repositories.adapters import RedisAdapter
|
||||||
|
|
||||||
assert RedisAdapter.__name__ == "RedisAdapter"
|
assert RedisAdapter.__name__ == "RedisAdapter"
|
||||||
|
|
||||||
|
|
||||||
|
def test_adapters_dir_exposes_lazy_exports() -> None:
|
||||||
|
"""dir(adapters) includes lazy adapter names for tab completion."""
|
||||||
|
import python_repositories.adapters as adapters
|
||||||
|
|
||||||
|
assert {"RedisAdapter", "MinioAdapter"}.issubset(set(dir(adapters)))
|
||||||
|
|
||||||
|
|
||||||
|
def test_adapters_getattr_raises_for_unknown() -> None:
|
||||||
|
"""Unknown adapter names raise AttributeError."""
|
||||||
|
import python_repositories.adapters as adapters
|
||||||
|
|
||||||
|
with pytest.raises(AttributeError, match="has no attribute 'NoSuchAdapter'"):
|
||||||
|
_ = adapters.NoSuchAdapter
|
||||||
|
|
||||||
|
|
||||||
|
def test_top_level_dir_exposes_lazy_exports() -> None:
|
||||||
|
"""dir(python_repositories) includes lazy adapter names for tab completion."""
|
||||||
|
assert "RedisAdapter" in dir(python_repositories)
|
||||||
|
assert "MinioAdapter" in dir(python_repositories)
|
||||||
|
|||||||
@@ -80,3 +80,24 @@ def test_instantiation_fails_when_list_keys_not_implemented() -> None:
|
|||||||
|
|
||||||
with pytest.raises(TypeError):
|
with pytest.raises(TypeError):
|
||||||
_ = Incomplete() # type: ignore
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_keys_defaults_to_list_keys() -> None:
|
||||||
|
"""Test that the default scan_keys implementation delegates to list_keys."""
|
||||||
|
|
||||||
|
class Complete(JsonRepositoryInterface):
|
||||||
|
def get(self, key: str) -> dict | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set(self, key: str, data: dict) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, key: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
|
return [f"{pattern}-1", f"{pattern}-2"]
|
||||||
|
|
||||||
|
repository = Complete()
|
||||||
|
|
||||||
|
assert list(repository.scan_keys("user")) == ["user-1", "user-2"]
|
||||||
|
|||||||
@@ -5,7 +5,8 @@ from __future__ import annotations
|
|||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from minio import Minio
|
from minio import Minio, S3Error
|
||||||
|
from urllib3.response import BaseHTTPResponse
|
||||||
|
|
||||||
from python_repositories.adapters.minio_adapter import MinioAdapter
|
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||||
from python_repositories.interfaces import ObjectRepositoryInterface
|
from python_repositories.interfaces import ObjectRepositoryInterface
|
||||||
@@ -53,3 +54,129 @@ def test_raises_when_client_provided_without_config() -> None:
|
|||||||
mock_client = MagicMock(spec=Minio)
|
mock_client = MagicMock(spec=Minio)
|
||||||
with pytest.raises(ValueError, match="config is required"):
|
with pytest.raises(ValueError, match="config is required"):
|
||||||
MinioAdapter(client=mock_client)
|
MinioAdapter(client=mock_client)
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_succeeds() -> None:
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.list_buckets.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_raises_on_failure() -> None:
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.list_buckets.side_effect = Exception("connection lost")
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError, match="Could not connect to Minio"):
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_disconnects_before_reconnect(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
stale_client = MagicMock(spec=Minio)
|
||||||
|
new_client = MagicMock(spec=Minio)
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||||
|
adapter._client = stale_client
|
||||||
|
adapter._bucket_name = None
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||||
|
lambda *args, **kwargs: new_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
assert adapter._client is new_client
|
||||||
|
new_client.list_buckets.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_closes_response_on_success() -> None:
|
||||||
|
"""get() must close and release the get_object HTTP response."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.read.side_effect = [b"data", b""]
|
||||||
|
mock_client.get_object.return_value = mock_response
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
result = adapter.get("some-object")
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result.read() == b"data"
|
||||||
|
mock_response.close.assert_called_once()
|
||||||
|
mock_response.release_conn.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_closes_response_when_read_fails() -> None:
|
||||||
|
"""get() must close and release the response even if read() raises."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.read.side_effect = OSError("connection reset")
|
||||||
|
mock_client.get_object.return_value = mock_response
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(OSError, match="connection reset"):
|
||||||
|
adapter.get("some-object")
|
||||||
|
|
||||||
|
mock_response.close.assert_called_once()
|
||||||
|
mock_response.release_conn.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_returns_none_for_no_such_key() -> None:
|
||||||
|
"""get() returns None when the object does not exist."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
mock_client.get_object.side_effect = S3Error(
|
||||||
|
MagicMock(spec=BaseHTTPResponse),
|
||||||
|
"NoSuchKey",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
bucket_name="test-bucket",
|
||||||
|
object_name="missing-object",
|
||||||
|
)
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
result = adapter.get("missing-object")
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_reraises_other_s3_errors() -> None:
|
||||||
|
"""get() re-raises S3 errors other than NoSuchKey."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
other_s3error = S3Error(
|
||||||
|
MagicMock(spec=BaseHTTPResponse),
|
||||||
|
"AccessDenied",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
bucket_name="test-bucket",
|
||||||
|
object_name="some-object",
|
||||||
|
)
|
||||||
|
mock_client.get_object.side_effect = other_s3error
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(S3Error) as exc_info:
|
||||||
|
adapter.get("some-object")
|
||||||
|
|
||||||
|
assert exc_info.value.code == "AccessDenied"
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_reraises_general_exception_from_get_object() -> None:
|
||||||
|
"""get() re-raises unexpected exceptions from get_object."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
mock_client.get_object.side_effect = Exception("General failure")
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(Exception, match="General failure"):
|
||||||
|
adapter.get("some-object")
|
||||||
|
|||||||
@@ -69,3 +69,98 @@ def test_subclass_custom_env_var_name(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||||||
monkeypatch.setenv("CUSTOM_REDIS_URI", "redis://custom:6379")
|
monkeypatch.setenv("CUSTOM_REDIS_URI", "redis://custom:6379")
|
||||||
adapter = CustomEnvRedisAdapter()
|
adapter = CustomEnvRedisAdapter()
|
||||||
assert adapter._config.uri == "redis://custom:6379"
|
assert adapter._config.uri == "redis://custom:6379"
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_succeeds_when_ping_ok() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
mock_client.ping.return_value = True
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.ping.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_raises_when_ping_false() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
mock_client.ping.return_value = False
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError, match="Could not connect to Redis"):
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_raises_on_redis_error() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
mock_client.ping.side_effect = redis.ConnectionError("connection lost")
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError, match="Could not connect to Redis"):
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_closes_existing_non_injected_client(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
stale_client = MagicMock(spec=redis.Redis)
|
||||||
|
new_client = MagicMock(spec=redis.Redis)
|
||||||
|
new_client.ping.return_value = True
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
adapter._client = stale_client
|
||||||
|
|
||||||
|
monkeypatch.setattr("redis.Redis.from_url", lambda *args, **kwargs: new_client)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
stale_client.close.assert_called_once()
|
||||||
|
assert adapter._client is new_client
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_keys_yields_decoded_keys() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
mock_client.scan_iter.return_value = iter([b"key1", b"key2"])
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
keys = list(adapter.scan_keys("key*"))
|
||||||
|
|
||||||
|
assert keys == ["key1", "key2"]
|
||||||
|
mock_client.scan_iter.assert_called_once_with(match="key*")
|
||||||
|
mock_client.keys.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_keys_forwards_count() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
mock_client.scan_iter.return_value = iter([b"key1"])
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
keys = list(adapter.scan_keys("key*", count=50))
|
||||||
|
|
||||||
|
assert keys == ["key1"]
|
||||||
|
mock_client.scan_iter.assert_called_once_with(match="key*", count=50)
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_keys_raises_value_error_on_invalid_pattern() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
invalid_patterns = ["", 123, None]
|
||||||
|
for pattern in invalid_patterns:
|
||||||
|
with pytest.raises(ValueError, match="Pattern must be a non-empty string"):
|
||||||
|
adapter.list_keys(pattern) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_keys_raises_value_error_on_invalid_pattern() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
invalid_patterns = ["", 123, None]
|
||||||
|
for pattern in invalid_patterns:
|
||||||
|
with pytest.raises(ValueError, match="Pattern must be a non-empty string"):
|
||||||
|
list(adapter.scan_keys(pattern)) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_keys_raises_connection_error_when_not_connected() -> None:
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
list(adapter.scan_keys("key*"))
|
||||||
|
|||||||
@@ -1056,7 +1056,7 @@ wheels = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "python-repositories"
|
name = "python-repositories"
|
||||||
version = "0.5.0"
|
version = "1.1.0"
|
||||||
source = { editable = "." }
|
source = { editable = "." }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "python-dotenv" },
|
{ name = "python-dotenv" },
|
||||||
|
|||||||
Reference in New Issue
Block a user