Compare commits
154
Commits
v0.2.2
...
ac82f71347
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac82f71347 | ||
|
|
24d5ea14d6 | ||
|
|
b8d8176877 | ||
|
|
a78320b434 | ||
|
|
1822d886b1 | ||
|
|
4f58c32dd6 | ||
|
|
88ea7f06ab | ||
|
|
2b5718ba41 | ||
|
|
b6538f9e91 | ||
|
|
b37b2e6e50 | ||
|
|
1babb52d09 | ||
|
|
63ee17d544 | ||
|
|
2e38bd2406 | ||
|
|
dc6f8a3e89 | ||
|
|
f28d3c4844 | ||
|
|
1cbcc1c19b | ||
|
|
a0e5c9dcca | ||
|
|
44b15cb21a | ||
|
|
f6ee9a3724 | ||
|
|
0829354cc4 | ||
|
|
a0d509a2a3 | ||
|
|
9b7b15a67e | ||
|
|
cd5b84b88e | ||
|
|
6d02f32ca5 | ||
|
|
57b396bcd3 | ||
|
|
093e538d5b | ||
|
|
b28ac6e803 | ||
|
|
6c66fe4e21 | ||
|
|
089a1287f8 | ||
|
|
0a5bd39801 | ||
|
|
683ee332d3 | ||
|
|
9df5032d44 | ||
|
|
4f6f33f36e | ||
|
|
10d43ce198 | ||
|
|
4f7b67203a | ||
|
|
a27e9e5763 | ||
|
|
fd0045d3e0 | ||
|
|
515f4bbeae | ||
|
|
ef31fb4b4e | ||
|
|
707f11f578 | ||
|
|
b1d34f8f0f | ||
|
|
565879dd52 | ||
|
|
0d0f430f6f | ||
|
|
5ac40b1ba5 | ||
|
|
fa4ff1f03c | ||
|
|
0a39151ba0 | ||
|
|
82a702db0c | ||
|
|
b886a7c147 | ||
|
|
4518a93a3a | ||
|
|
a8b7b59484 | ||
|
|
0e6941dccf | ||
|
|
a6428ccd29 | ||
|
|
655fdc4ee8 | ||
|
|
e031c549e5 | ||
|
|
49b6238b7d | ||
|
|
0549358fe1 | ||
|
|
33efbd1005 | ||
|
|
24126e4044 | ||
|
|
f3b6cdbc9e | ||
|
|
fa2554ef5f | ||
|
|
b21fd247c5 | ||
|
|
441a21204d | ||
|
|
f2a8581de2 | ||
|
|
3b05260323 | ||
|
|
369295b8ae | ||
|
|
95c78596e7 | ||
|
|
42e885edd5 | ||
|
|
bd4794d1c0 | ||
|
|
3dc412cd15 | ||
|
|
b8d88e6703 | ||
|
|
67bb88fbb4 | ||
|
|
c2e6a96c5d | ||
|
|
547a1b3a90 | ||
|
|
39baf9badc | ||
|
|
4a5b3b631f | ||
|
|
1431f455c1 | ||
|
|
eafc717045 | ||
|
|
9a9b7b2985 | ||
|
|
5b9573d499 | ||
|
|
50444af982 | ||
|
|
2f19fcc972 | ||
|
|
3bd65895ec | ||
|
|
5311d49fa6 | ||
|
|
703bb9521f | ||
|
|
8e47ebf4c6 | ||
|
|
9a800bf553 | ||
|
|
8c34534187 | ||
|
|
b1210bddf2 | ||
|
|
39b104383c | ||
|
|
beb2b5128e | ||
|
|
ce062be411 | ||
|
|
a381d45650 | ||
|
|
5c8cd841b6 | ||
|
|
34632980ba | ||
|
|
5393efc6cd | ||
|
|
e39fc96ac1 | ||
|
|
c1f4826d78 | ||
|
|
0315a33a3b | ||
|
|
5d33ec6091 | ||
|
|
9538e4d26d | ||
|
|
14b8d2798c | ||
|
|
d3cbb65bbc | ||
|
|
7dbb1fc8ab | ||
|
|
918e5e6ba4 | ||
|
|
3aa91280ec | ||
|
|
7991dabdbc | ||
|
|
366831ac52 | ||
|
|
a98905daa7 | ||
|
|
7ed1b34233 | ||
|
|
3236f27610 | ||
|
|
ac487d91f8 | ||
|
|
a049ce327c | ||
|
|
7b4cf8421b | ||
|
|
b5a92e6222 | ||
|
|
7b950e0d93 | ||
|
|
fe0c477dff | ||
|
|
e7e0fc8c1d | ||
|
|
34fc046868 | ||
|
|
5149299c1b | ||
|
|
fef9552cbf | ||
|
|
e9b21895f4 | ||
|
|
f5953906c1 | ||
|
|
1f02195c27 | ||
|
|
f092ca2022 | ||
|
|
c6eed43d70 | ||
|
|
9e4b1e2c5e | ||
|
|
0096df78a5 | ||
|
|
3073bfbf50 | ||
|
|
30a1de32e8 | ||
|
|
8465957976 | ||
|
|
3ecb9c3136 | ||
|
|
0f31170209 | ||
|
|
e3cab5e0b6 | ||
|
|
f428a974f6 | ||
|
|
a3b9912107 | ||
|
|
abaa078471 | ||
|
|
9864f0cc9d | ||
|
|
e39b467b6b | ||
|
|
85c1a64e14 | ||
|
|
a8491c005c | ||
|
|
2bb361e3a3 | ||
|
|
c5169df10d | ||
|
|
497533f0e3 | ||
|
|
e98fd3b90d | ||
|
|
f4280f15c1 | ||
|
|
d9b9fa49a2 | ||
|
|
9d25f7c722 | ||
|
|
163581526d | ||
|
|
933c4ee7c5 | ||
|
|
2b0d98f84c | ||
|
|
729fa48505 | ||
|
|
fa011be862 | ||
|
|
2e9330949c | ||
|
|
3ab8da92fb |
@@ -0,0 +1,58 @@
|
|||||||
|
---
|
||||||
|
description: After first branch push, provide copy-ready PR title and body
|
||||||
|
alwaysApply: true
|
||||||
|
---
|
||||||
|
|
||||||
|
# PR Title and Comment After First Push
|
||||||
|
|
||||||
|
The user creates pull requests manually. After the **first push** of a feature branch to remote in a session, always end your response with a copy-ready PR title and PR comment.
|
||||||
|
|
||||||
|
## When to provide it
|
||||||
|
|
||||||
|
- After the first successful `git push` (or `git push -u origin <branch>`) for a branch in the current task
|
||||||
|
- Do **not** create the PR with `gh pr create` unless explicitly asked
|
||||||
|
- On later pushes to the same branch, only repeat if the user asks or the change set meaningfully updates what the PR should say
|
||||||
|
|
||||||
|
## PR title
|
||||||
|
|
||||||
|
- Use a fitting version-bump prefix when changes touch `python_repositories/`:
|
||||||
|
- `[patch]` or `[fix]` — bug fix
|
||||||
|
- `[minor]` or `[feat]` — new feature
|
||||||
|
- `[major]` or `[breaking]` — breaking change
|
||||||
|
- Docs-, CI-, or test-only changes: no prefix required
|
||||||
|
- Keep the title concise and descriptive
|
||||||
|
|
||||||
|
## PR comment format
|
||||||
|
|
||||||
|
Provide two separate fenced code blocks the user can copy directly:
|
||||||
|
|
||||||
|
1. **PR title** — single line in a code block
|
||||||
|
2. **PR comment** — body with `## Summary` and `## Test plan` sections, checklist items reflecting what was actually run
|
||||||
|
|
||||||
|
Example structure:
|
||||||
|
|
||||||
|
````
|
||||||
|
PR title:
|
||||||
|
|
||||||
|
```
|
||||||
|
[minor] Short description of the change
|
||||||
|
```
|
||||||
|
|
||||||
|
PR comment:
|
||||||
|
|
||||||
|
```
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Bullet points of what changed and why
|
||||||
|
|
||||||
|
## Test plan
|
||||||
|
|
||||||
|
- [x] Commands or checks that were run
|
||||||
|
```
|
||||||
|
````
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- Match the repo PR template ([`.gitea/PULL_REQUEST_TEMPLATE.md`](.gitea/PULL_REQUEST_TEMPLATE.md))
|
||||||
|
- Be accurate: only mark test-plan items done if they were run
|
||||||
|
- Keep prose clear; the user copies this verbatim into Gitea
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
---
|
||||||
|
description: Unit test file naming and placement conventions
|
||||||
|
globs: tests/**/*.py
|
||||||
|
alwaysApply: false
|
||||||
|
---
|
||||||
|
|
||||||
|
# Test Organization
|
||||||
|
|
||||||
|
Unit tests use a one-to-one mapping between source modules and test files.
|
||||||
|
|
||||||
|
## Naming
|
||||||
|
|
||||||
|
- `python_repositories/<path>/<module>.py` → `tests/unit/<module>_test.py`
|
||||||
|
- Test files must end with `_test.py` (enforced by pre-commit `name-tests-test`)
|
||||||
|
|
||||||
|
## Placement
|
||||||
|
|
||||||
|
- Add tests to the existing `*_test.py` for the module under test
|
||||||
|
- Do not create cross-cutting test files (e.g. `structural_typing_test.py`); colocate with the relevant interface/adapter/config test file
|
||||||
|
- Shared fixtures → `tests/conftest.py`
|
||||||
|
- Module-specific helper classes → the matching test file
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
```
|
||||||
|
json_repository_interface.py → tests/unit/json_repository_interface_test.py
|
||||||
|
redis_adapter.py → tests/unit/redis_adapter_test.py
|
||||||
|
redis_config.py → tests/unit/redis_config_test.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Integration tests live under `tests/integration/<backend>/` with the same `_test.py` suffix.
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
.git
|
||||||
|
.github
|
||||||
|
.gitea
|
||||||
|
.mypy_cache
|
||||||
|
.pytest_cache
|
||||||
|
.ruff_cache
|
||||||
|
.venv
|
||||||
|
__pycache__
|
||||||
|
*.pyc
|
||||||
|
.coverage
|
||||||
|
coverage-unit
|
||||||
|
coverage-integration
|
||||||
|
coverage.txt
|
||||||
|
dist
|
||||||
|
build
|
||||||
|
*.egg-info
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
# Redis (requires redis extra)
|
||||||
|
REDIS_URI=redis://localhost:6379
|
||||||
|
|
||||||
|
# MinIO (requires minio extra)
|
||||||
|
MINIO_ENDPOINT=localhost:9000
|
||||||
|
MINIO_ACCESS_KEY=minioadmin
|
||||||
|
MINIO_SECRET_KEY=minioadmin
|
||||||
|
MINIO_BUCKET=my-bucket
|
||||||
|
MINIO_SECURE=false
|
||||||
|
MINIO_CREATE_BUCKET_IF_MISSING=true
|
||||||
|
|
||||||
|
# Postgres (requires postgres extra)
|
||||||
|
POSTGRES_URI=postgresql://localhost/mydb
|
||||||
|
POSTGRES_TABLE=users
|
||||||
|
POSTGRES_PRIMARY_KEY=id
|
||||||
|
|
||||||
|
# File-backed queue (no optional extra)
|
||||||
|
FILE_QUEUE_PATH=/tmp/python-repositories-queue.jsonl
|
||||||
|
# FILE_QUEUE_MAX_AGE_HOURS=24
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
## Version bump
|
||||||
|
|
||||||
|
If this PR changes files under `python_repositories/`, the **title must** start with one of:
|
||||||
|
|
||||||
|
- `[patch]` or `[fix]` — bug fix (`1.2.3` → `1.2.4`)
|
||||||
|
- `[minor]` or `[feat]` — new feature (`1.2.3` → `1.3.0`)
|
||||||
|
- `[major]` or `[breaking]` — breaking change (`1.2.3` → `2.0.0`)
|
||||||
|
|
||||||
|
Current version: see [`pyproject.toml`](pyproject.toml) on `main`.
|
||||||
|
|
||||||
|
Docs-, CI-, or test-only PRs do not need a prefix.
|
||||||
|
|
||||||
|
**Example title:** `[minor] Add streaming support to RedisAdapter`
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
<!-- What changed and why -->
|
||||||
|
|
||||||
|
## Test plan
|
||||||
|
|
||||||
|
- [ ] Integration tests pass locally
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# Build and publish the per-repo CI base image to the Gitea container registry.
|
||||||
|
|
||||||
|
name: Build CI Image
|
||||||
|
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: "0 2 * * *"
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- pyproject.toml
|
||||||
|
- uv.lock
|
||||||
|
- docker/ci/**
|
||||||
|
|
||||||
|
env:
|
||||||
|
REGISTRY: gitea.lille-vemmelund.dk
|
||||||
|
REGISTRY_USER: ci-bot
|
||||||
|
IMAGE: gitea.lille-vemmelund.dk/lillevemmelund/python-repositories-ci
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-and-push:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Log in to Gitea container registry
|
||||||
|
env:
|
||||||
|
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: |
|
||||||
|
echo "$CI_RUNNER_TOKEN" | docker login "$REGISTRY" -u "$REGISTRY_USER" --password-stdin
|
||||||
|
|
||||||
|
- name: Build CI image
|
||||||
|
env:
|
||||||
|
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: |
|
||||||
|
echo "$CI_RUNNER_TOKEN" > /tmp/uv_token
|
||||||
|
docker build --network=host -f docker/ci/Dockerfile \
|
||||||
|
--secret id=uv_token,src=/tmp/uv_token \
|
||||||
|
-t "${IMAGE}:latest" \
|
||||||
|
.
|
||||||
|
rm -f /tmp/uv_token
|
||||||
|
|
||||||
|
- name: Tag and push CI image
|
||||||
|
run: |
|
||||||
|
STAMP="$(date -u +%Y%m%d%H%M)"
|
||||||
|
docker tag "${IMAGE}:latest" "${IMAGE}:${STAMP}"
|
||||||
|
docker push "${IMAGE}:latest"
|
||||||
|
docker push "${IMAGE}:${STAMP}"
|
||||||
|
echo "Pushed ${IMAGE}:latest and ${IMAGE}:${STAMP}"
|
||||||
@@ -8,23 +8,17 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
code-quality:
|
code-quality:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Sync dependencies
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ inputs.python-version }}
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
env:
|
||||||
UV_LINK_MODE: copy
|
UV_LINK_MODE: copy
|
||||||
run: uv sync --all-extras
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
|
||||||
- name: Type check with mypy
|
- name: Type check with mypy
|
||||||
run: uv run mypy .
|
run: uv run mypy .
|
||||||
@@ -36,9 +30,7 @@ jobs:
|
|||||||
run: uv run ruff format --check .
|
run: uv run ruff format --check .
|
||||||
|
|
||||||
- name: Pyupgrade check
|
- name: Pyupgrade check
|
||||||
run: uv run pyupgrade --py313-plus $(git ls-files '*.py') && git diff --exit-code
|
run: uv run pyupgrade --py312-plus $(git ls-files '*.py') && git diff --exit-code
|
||||||
|
|
||||||
- name: Prettier format
|
- name: Prettier format
|
||||||
run: |
|
run: uv run pre-commit run prettier --all-files
|
||||||
npm install --save-dev --save-exact prettier
|
|
||||||
npx prettier --check .
|
|
||||||
|
|||||||
@@ -7,30 +7,49 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
update-check:
|
update-check:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
with:
|
||||||
python-version: ${{ inputs.python-version }}
|
token: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
|
||||||
- name: Install uv and pip-review
|
- name: Sync dependencies
|
||||||
run: |
|
env:
|
||||||
pip install uv pip-review
|
UV_LINK_MODE: copy
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras
|
||||||
|
|
||||||
- name: Update dependencies in pyproject.toml
|
- name: Upgrade dependencies
|
||||||
|
env:
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv lock --upgrade
|
||||||
|
|
||||||
|
- name: Update pre-commit hook revs
|
||||||
|
env:
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv run pre-commit autoupdate
|
||||||
|
|
||||||
|
- name: Sync upgraded dependencies
|
||||||
|
env:
|
||||||
|
UV_LINK_MODE: copy
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras
|
||||||
|
|
||||||
|
- name: Smoke check
|
||||||
run: |
|
run: |
|
||||||
uv pip update --all --group dev
|
uv run ruff check .
|
||||||
uv pip compile --all
|
uv run ruff format --check .
|
||||||
|
uv run mypy .
|
||||||
|
|
||||||
- name: Commit and push changes
|
- name: Commit and push changes
|
||||||
uses: peter-evans/create-pull-request@v6
|
env:
|
||||||
with:
|
API_URL: ${{ vars.API_URL }}
|
||||||
commit-message: "chore(deps): update dependencies [automated]"
|
REPO_OWNER: ${{ github.repository_owner }}
|
||||||
branch: "renovate/auto-deps-update"
|
REPO_NAME: ${{ github.event.repository.name }}
|
||||||
title: "chore(deps): update dependencies"
|
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
body: "This PR was created automatically to update dependencies."
|
run: scripts/ci/create-deps-update-pr.sh
|
||||||
token: ${{ secrets.CI_RUNNER_TOKEN }}
|
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
name: PR Title Check
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
check-title:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Check PR title when source files change
|
||||||
|
env:
|
||||||
|
API_URL: ${{ vars.API_URL }}
|
||||||
|
REPO_OWNER: ${{ github.repository_owner }}
|
||||||
|
REPO_NAME: ${{ github.event.repository.name }}
|
||||||
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||||
|
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: |
|
||||||
|
git fetch origin "${{ github.base_ref }}"
|
||||||
|
PR_TITLE=$(scripts/ci/fetch-pr-title.sh)
|
||||||
|
echo "Live PR title: ${PR_TITLE}"
|
||||||
|
git diff --name-only "origin/${{ github.base_ref }}...HEAD" \
|
||||||
|
| scripts/ci/check-pr-title.sh "$PR_TITLE"
|
||||||
@@ -9,29 +9,24 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-and-publish:
|
build-and-publish:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ inputs.python-version }}
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Update version in pyproject.toml to match tag
|
- name: Update version in pyproject.toml to match tag
|
||||||
run: |
|
env:
|
||||||
TAG_VERSION="${GITHUB_REF##*/}"
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
VERSION="${TAG_VERSION#v}"
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
sed -i.bak -E "s/^version = \".*\"/version = \"${VERSION}\"/" pyproject.toml
|
run: scripts/ci/bump-version.sh --from-tag "${GITHUB_REF##*/}"
|
||||||
rm pyproject.toml.bak
|
|
||||||
|
|
||||||
- name: Build package
|
- name: Build package
|
||||||
|
env:
|
||||||
|
UV_LINK_MODE: copy
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
uv sync --no-dev
|
uv sync --no-dev --frozen
|
||||||
uv build
|
uv build
|
||||||
|
|
||||||
- name: Publish to Gitea Package Registry
|
- name: Publish to Gitea Package Registry
|
||||||
|
|||||||
@@ -0,0 +1,105 @@
|
|||||||
|
name: Release on merge to main
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
release:
|
||||||
|
if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
|
||||||
|
runs-on: python-repositories-ci
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
token: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
|
||||||
|
- name: Parse merge commit
|
||||||
|
id: meta
|
||||||
|
env:
|
||||||
|
COMMIT_MSG: ${{ github.event.head_commit.message }}
|
||||||
|
run: scripts/ci/parse-merge-commit.sh "$COMMIT_MSG"
|
||||||
|
|
||||||
|
- name: Bump version
|
||||||
|
if: steps.meta.outputs.bump != 'skip'
|
||||||
|
id: bump
|
||||||
|
env:
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: scripts/ci/bump-version.sh "${{ steps.meta.outputs.bump }}"
|
||||||
|
|
||||||
|
- name: Generate release notes
|
||||||
|
if: steps.meta.outputs.bump != 'skip'
|
||||||
|
id: notes
|
||||||
|
env:
|
||||||
|
PR_TITLE: ${{ steps.meta.outputs.pr_title }}
|
||||||
|
run: |
|
||||||
|
PREV_TAG=$(git describe --tags --abbrev=0)
|
||||||
|
NEW_TAG="v${{ steps.bump.outputs.version }}"
|
||||||
|
scripts/ci/generate-release-notes.sh "$NEW_TAG" "$PR_TITLE" "$PREV_TAG"
|
||||||
|
|
||||||
|
- name: Update CHANGELOG
|
||||||
|
if: steps.meta.outputs.bump != 'skip'
|
||||||
|
env:
|
||||||
|
PR_TITLE: ${{ steps.meta.outputs.pr_title }}
|
||||||
|
run: |
|
||||||
|
PREV_TAG=$(git describe --tags --abbrev=0)
|
||||||
|
VERSION="${{ steps.bump.outputs.version }}"
|
||||||
|
scripts/ci/update-changelog.sh "$VERSION" "$PR_TITLE" "$PREV_TAG"
|
||||||
|
|
||||||
|
- name: Commit, tag, and push
|
||||||
|
if: steps.meta.outputs.bump != 'skip'
|
||||||
|
env:
|
||||||
|
VERSION: ${{ steps.bump.outputs.version }}
|
||||||
|
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: |
|
||||||
|
git config user.name "CI Bot"
|
||||||
|
git config user.email "[email protected]"
|
||||||
|
git add pyproject.toml uv.lock CHANGELOG.md
|
||||||
|
git commit -m "chore: release v${VERSION} [skip ci]"
|
||||||
|
git tag "v${VERSION}"
|
||||||
|
git remote set-url origin "https://x-access-token:${CI_RUNNER_TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
|
||||||
|
git push origin main
|
||||||
|
git push origin "v${VERSION}"
|
||||||
|
|
||||||
|
- name: Create Gitea release
|
||||||
|
if: steps.meta.outputs.bump != 'skip'
|
||||||
|
env:
|
||||||
|
API_URL: ${{ vars.API_URL }}
|
||||||
|
REPO_OWNER: ${{ github.repository_owner }}
|
||||||
|
REPO_NAME: ${{ github.event.repository.name }}
|
||||||
|
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
VERSION: ${{ steps.bump.outputs.version }}
|
||||||
|
PR_TITLE: ${{ steps.meta.outputs.pr_title }}
|
||||||
|
RELEASE_BODY: ${{ steps.notes.outputs.body }}
|
||||||
|
run: |
|
||||||
|
SUMMARY=$(echo "$PR_TITLE" | sed -E 's/^\[(patch|fix|minor|feat|major|breaking)\][[:space:]]*//i')
|
||||||
|
export RELEASE_NAME="v${VERSION} — ${SUMMARY}"
|
||||||
|
PAYLOAD=$(python3 -c 'import json, os; print(json.dumps({"tag_name": "v" + os.environ["VERSION"], "name": os.environ["RELEASE_NAME"], "body": os.environ["RELEASE_BODY"]}))')
|
||||||
|
curl -s -X POST \
|
||||||
|
"${API_URL}/repos/${REPO_OWNER}/${REPO_NAME}/releases" \
|
||||||
|
-H "Authorization: token ${CI_RUNNER_TOKEN}" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "$PAYLOAD"
|
||||||
|
|
||||||
|
# Publish here instead of relying on publish.yml tag trigger: release commits use
|
||||||
|
# [skip ci], which suppresses tag-push workflows, and Gitea does not emit release
|
||||||
|
# events for releases created by the CI bot account.
|
||||||
|
- name: Build package
|
||||||
|
if: steps.meta.outputs.bump != 'skip'
|
||||||
|
env:
|
||||||
|
UV_LINK_MODE: copy
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: |
|
||||||
|
uv sync --no-dev --frozen
|
||||||
|
uv build
|
||||||
|
|
||||||
|
- name: Publish to Gitea Package Registry
|
||||||
|
if: steps.meta.outputs.bump != 'skip'
|
||||||
|
env:
|
||||||
|
UV_PUBLISH_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
UV_PUBLISH_URL: ${{ vars.REPOSITORY_URL }}
|
||||||
|
run: uv publish
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
name: Security Audit
|
|
||||||
|
|
||||||
on:
|
|
||||||
schedule:
|
|
||||||
- cron: "0 0 * * 0" # Weekly
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
safety:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ inputs.python-version }}
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
|
||||||
UV_LINK_MODE: copy
|
|
||||||
run: uv sync
|
|
||||||
|
|
||||||
- name: Run safety check
|
|
||||||
run: uv run safety check
|
|
||||||
@@ -7,23 +7,17 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
safety:
|
safety:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Sync dependencies
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ inputs.python-version }}
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
env:
|
||||||
UV_LINK_MODE: copy
|
UV_LINK_MODE: copy
|
||||||
run: uv sync
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
|
||||||
- name: Run safety check
|
- name: Run safety check
|
||||||
run: uv run safety check
|
run: uv run safety check
|
||||||
|
|||||||
@@ -1,38 +0,0 @@
|
|||||||
name: Sync pyproject.toml Version
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_run:
|
|
||||||
workflows: ["Publish Python Package"]
|
|
||||||
types:
|
|
||||||
- completed
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
commit-version-update:
|
|
||||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ inputs.python-version }}
|
|
||||||
|
|
||||||
- name: Configure git
|
|
||||||
run: |
|
|
||||||
git config user.name "CI Bot"
|
|
||||||
git config user.email "[email protected]"
|
|
||||||
|
|
||||||
- name: Update version in pyproject.toml to match tag
|
|
||||||
run: |
|
|
||||||
TAG_VERSION="${{ github.event.workflow_run.head_branch }}"
|
|
||||||
VERSION="${TAG_VERSION#v}"
|
|
||||||
sed -i.bak -E "s/^version = \".*\"/version = \"${VERSION}\"/" pyproject.toml
|
|
||||||
rm pyproject.toml.bak
|
|
||||||
|
|
||||||
- name: Commit and push changes
|
|
||||||
run: |
|
|
||||||
git add pyproject.toml
|
|
||||||
git commit -m "chore: update version in pyproject.toml to ${VERSION} [skip ci]" || echo "No changes to commit"
|
|
||||||
git push
|
|
||||||
+117
-19
@@ -7,31 +7,118 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
unit-tests:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Sync dependencies
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ inputs.python-version }}
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
env:
|
||||||
UV_LINK_MODE: copy
|
UV_LINK_MODE: copy
|
||||||
run: uv sync --all-extras
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
|
||||||
- name: Run pytest
|
- name: Run unit tests
|
||||||
|
run: |
|
||||||
|
# --cov-fail-under=0: partial coverage only; floor is checked in coverage-report.
|
||||||
|
uv run pytest tests/unit/ -m "not integration" \
|
||||||
|
--cov=python_repositories \
|
||||||
|
--cov-report= \
|
||||||
|
--cov-fail-under=0
|
||||||
|
|
||||||
|
- name: Upload unit coverage
|
||||||
|
uses: https://github.com/christopherHX/gitea-upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: coverage-unit
|
||||||
|
path: .coverage
|
||||||
|
retention-days: 1
|
||||||
|
compression-level: 0
|
||||||
|
|
||||||
|
integration-tests:
|
||||||
|
runs-on: python-repositories-ci
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Sync dependencies
|
||||||
env:
|
env:
|
||||||
PYTHONPATH: .
|
UV_LINK_MODE: copy
|
||||||
run: uv run pytest --cov=python_repositories --cov-report=term-missing > coverage.txt
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
|
||||||
|
- name: Verify Docker
|
||||||
|
run: docker info
|
||||||
|
|
||||||
|
- name: Run integration tests
|
||||||
|
run: |
|
||||||
|
# --cov-fail-under=0: partial coverage only; floor is checked in coverage-report.
|
||||||
|
uv run pytest -m integration \
|
||||||
|
--cov=python_repositories \
|
||||||
|
--cov-report= \
|
||||||
|
--cov-fail-under=0
|
||||||
|
|
||||||
|
- name: Upload integration coverage
|
||||||
|
uses: https://github.com/christopherHX/gitea-upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: coverage-integration
|
||||||
|
path: .coverage
|
||||||
|
retention-days: 1
|
||||||
|
compression-level: 0
|
||||||
|
|
||||||
|
coverage-report:
|
||||||
|
# Merges unit + integration coverage and enforces fail_under from pyproject.toml.
|
||||||
|
needs: [unit-tests, integration-tests]
|
||||||
|
runs-on: python-repositories-ci
|
||||||
|
if: github.event_name == 'pull_request' || github.event_name == 'push'
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Sync dependencies
|
||||||
|
env:
|
||||||
|
UV_LINK_MODE: copy
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
|
||||||
|
- name: Download unit coverage
|
||||||
|
uses: https://github.com/christopherHX/gitea-download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: coverage-unit
|
||||||
|
path: coverage-unit
|
||||||
|
|
||||||
|
- name: Download integration coverage
|
||||||
|
uses: https://github.com/christopherHX/gitea-download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: coverage-integration
|
||||||
|
path: coverage-integration
|
||||||
|
|
||||||
|
- name: Combine coverage report
|
||||||
|
run: |
|
||||||
|
# --fail-under=0 so the full report is always written before the floor check.
|
||||||
|
uv run coverage combine coverage-unit/.coverage coverage-integration/.coverage
|
||||||
|
uv run coverage report -m --include='python_repositories/*' --fail-under=0 > coverage.txt
|
||||||
|
|
||||||
|
- name: Show coverage report
|
||||||
|
run: cat coverage.txt
|
||||||
|
|
||||||
|
- name: Enforce coverage floor
|
||||||
|
run: |
|
||||||
|
# Reads fail_under from pyproject.toml; only combined coverage is evaluated here.
|
||||||
|
FAIL_UNDER=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['tool']['coverage']['report']['fail_under'])")
|
||||||
|
echo "Checking combined coverage against ${FAIL_UNDER}% floor..."
|
||||||
|
if uv run coverage report --fail-under="$FAIL_UNDER" --include='python_repositories/*'; then
|
||||||
|
echo "Coverage floor met."
|
||||||
|
else
|
||||||
|
echo "::error::Combined coverage is below the ${FAIL_UNDER}% floor"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Post coverage summary to PR
|
- name: Post coverage summary to PR
|
||||||
|
if: github.event_name == 'pull_request'
|
||||||
env:
|
env:
|
||||||
API_URL: ${{ vars.API_URL }}
|
API_URL: ${{ vars.API_URL }}
|
||||||
REPO_OWNER: ${{ github.repository_owner }}
|
REPO_OWNER: ${{ github.repository_owner }}
|
||||||
@@ -39,9 +126,20 @@ jobs:
|
|||||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||||
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
COVERAGE=$(cat coverage.txt)
|
PAYLOAD=$(python3 -c '
|
||||||
COMMENT_BODY="**Test Coverage Report:**\n\`\`\`\n$COVERAGE\n\`\`\`"
|
import json
|
||||||
curl -s -X POST "$API_URL/repos/$REPO_OWNER/$REPO_NAME/issues/$PR_NUMBER/comments" \
|
import pathlib
|
||||||
|
|
||||||
|
coverage = pathlib.Path("coverage.txt").read_text()
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"body": f"**Test Coverage Report:**\n```\n{coverage}\n```",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
')
|
||||||
|
curl -sf -X POST "$API_URL/repos/$REPO_OWNER/$REPO_NAME/issues/$PR_NUMBER/comments" \
|
||||||
-H "Authorization: token $CI_RUNNER_TOKEN" \
|
-H "Authorization: token $CI_RUNNER_TOKEN" \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-d "{\"body\": \"$COMMENT_BODY\"}"
|
-d "$PAYLOAD"
|
||||||
|
|||||||
@@ -173,4 +173,3 @@ cython_debug/
|
|||||||
|
|
||||||
# PyPI configuration file
|
# PyPI configuration file
|
||||||
.pypirc
|
.pypirc
|
||||||
|
|
||||||
|
|||||||
+21
-17
@@ -1,7 +1,7 @@
|
|||||||
repos:
|
repos:
|
||||||
# General repository hygiene hooks
|
# General repository hygiene hooks
|
||||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
rev: v4.5.0
|
rev: v6.0.0
|
||||||
hooks:
|
hooks:
|
||||||
- id: trailing-whitespace
|
- id: trailing-whitespace
|
||||||
- id: end-of-file-fixer
|
- id: end-of-file-fixer
|
||||||
@@ -11,30 +11,34 @@ repos:
|
|||||||
- id: name-tests-test
|
- id: name-tests-test
|
||||||
- id: check-merge-conflict
|
- id: check-merge-conflict
|
||||||
|
|
||||||
# Python linting and formatting with Ruff
|
# Python tooling via uv (versions pinned in uv.lock)
|
||||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
- repo: local
|
||||||
rev: v0.12.8
|
|
||||||
hooks:
|
hooks:
|
||||||
- id: ruff-check
|
- id: ruff-check
|
||||||
args: [--fix]
|
name: ruff check
|
||||||
|
entry: uv run ruff check --fix
|
||||||
|
language: system
|
||||||
|
types: [python]
|
||||||
- id: ruff-format
|
- id: ruff-format
|
||||||
|
name: ruff format
|
||||||
# Static type checking with mypy
|
entry: uv run ruff format
|
||||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
language: system
|
||||||
rev: v1.8.0
|
types: [python]
|
||||||
hooks:
|
|
||||||
- id: mypy
|
- id: mypy
|
||||||
|
name: mypy
|
||||||
# Python syntax modernization with pyupgrade
|
entry: uv run mypy .
|
||||||
- repo: https://github.com/asottile/pyupgrade
|
language: system
|
||||||
rev: v3.20.0
|
types: [python]
|
||||||
hooks:
|
pass_filenames: false
|
||||||
- id: pyupgrade
|
- id: pyupgrade
|
||||||
args: ["--py311-plus"]
|
name: pyupgrade
|
||||||
|
entry: uv run pyupgrade --py312-plus
|
||||||
|
language: system
|
||||||
|
types: [python]
|
||||||
|
|
||||||
# Formatting for Markdown, JSON, and YAML with Prettier
|
# Formatting for Markdown, JSON, and YAML with Prettier
|
||||||
- repo: https://github.com/pre-commit/mirrors-prettier
|
- repo: https://github.com/pre-commit/mirrors-prettier
|
||||||
rev: v3.1.0
|
rev: v4.0.0-alpha.8
|
||||||
hooks:
|
hooks:
|
||||||
- id: prettier
|
- id: prettier
|
||||||
files: "\\.(md|json|yaml|yml)$"
|
files: "\\.(md|json|yaml|yml)$"
|
||||||
|
|||||||
+375
@@ -0,0 +1,375 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||||
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [2.2.0] - 2026-07-12
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Add Postgres table adapter with TableRepositoryInterface
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- b37b2e6 Merge pull request '[minor] Add Postgres table adapter with TableRepositoryInterface' (#58) from cursor/postgres-table-adapter into main
|
||||||
|
- 1babb52 Apply ruff formatting to postgres adapter and tests.
|
||||||
|
- 63ee17d Apply prettier formatting to README and CHANGELOG.
|
||||||
|
- 2e38bd2 Fix ruff import ordering in postgres test files.
|
||||||
|
- dc6f8a3 Add Postgres table adapter with TableRepositoryInterface.
|
||||||
|
|
||||||
|
## [2.1.0] - 2026-07-11
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Add runtime-checkable Protocol typing to public interfaces
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 1cbcc1c Merge pull request '[minor] Add runtime-checkable Protocol typing to public interfaces' (#57) from cursor/protocol-abc-interfaces into main
|
||||||
|
- a0e5c9d Document test organization and add Cursor workflow rules.
|
||||||
|
- 44b15cb Move structural typing tests into interface unit test files.
|
||||||
|
- f6ee9a3 Add runtime-checkable Protocol typing to all public interfaces.
|
||||||
|
- 0829354 Merge pull request 'Add async future note and remove redundant pytest pythonpath' (#56) from cursor/readme-async-and-test-path-cleanup into main
|
||||||
|
- a0d509a Format CHANGELOG.md with Prettier.
|
||||||
|
- 9b7b15a Document possible async support and drop redundant pytest pythonpath.
|
||||||
|
|
||||||
|
## [2.0.5] - 2026-07-10
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Allow empty payloads in Redis and MinIO adapters
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 6d02f32 Merge pull request '[patch] Allow empty payloads in Redis and MinIO adapters' (#55) from cursor/allow-empty-payloads into main
|
||||||
|
- 57b396b Allow empty payloads in Redis and MinIO adapters.
|
||||||
|
- 093e538 Merge pull request '[chore] Scope integration fixtures per backend subdirectory' (#54) from cursor/scoped-integration-fixtures into main
|
||||||
|
- b28ac6e Scope integration fixtures per backend subdirectory.
|
||||||
|
- 6c66fe4 Merge pull request 'Add consumer-facing CHANGELOG.md maintained by release CI' (#53) from cursor/consumer-changelog into main
|
||||||
|
- 089a128 Format CHANGELOG.md with Prettier
|
||||||
|
- 0a5bd39 Add consumer-facing CHANGELOG.md maintained by release CI.
|
||||||
|
- 683ee33 Merge pull request '[chore] add repository URL and Python 3.12 packaging metadata' (#52) from cursor/add-packaging-metadata into main
|
||||||
|
- 9df5032 Add repository URL and Python 3.12 packaging metadata.
|
||||||
|
- 4f6f33f Merge pull request 'Drop types-redis in favor of redis inline types' (#51) from cursor/drop-types-redis into main
|
||||||
|
- 10d43ce Drop types-redis in favor of redis inline types.
|
||||||
|
|
||||||
|
## [2.0.4] - 2026-07-10
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Include optional dependencies in security audit
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 4f7b672 chore: release v2.0.4 [skip ci]
|
||||||
|
- a27e9e5 Merge pull request '[fix] Include optional dependencies in security audit' (#50) from cursor/security-audit-all-extras into main
|
||||||
|
- fd0045d Include optional dependencies in security audit.
|
||||||
|
- 515f4bb Merge pull request 'revert 707f11f578f007c7152582ea524b66dacea50111' (#49) from brian-patch-1 into main
|
||||||
|
- ef31fb4 revert 707f11f578f007c7152582ea524b66dacea50111
|
||||||
|
- 707f11f chore: release v2.0.4 [skip ci]
|
||||||
|
- b1d34f8 Merge pull request '[patch] Unify dev tooling via uv so pre-commit, CI, and the update bot stay aligned' (#48) from chore/align-dev-tooling-via-uv into main
|
||||||
|
- 565879d Unify dev tooling via uv so pre-commit, CI, and the update bot stay aligned.
|
||||||
|
- 0d0f430 Merge pull request 'Fix automated package publishing in release workflow' (#47) from fix/release-workflow-publish into main
|
||||||
|
- 5ac40b1 Publish packages from the release workflow so automated releases reach the registry.
|
||||||
|
|
||||||
|
## [2.0.3] - 2026-07-10
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Use structured fields for adapter debug and info logs
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- fa4ff1f chore: release v2.0.3 [skip ci]
|
||||||
|
- 0a39151 Merge pull request '[patch] Use structured fields for adapter debug and info logs' (#46) from cursor/structured-adapter-logging into main
|
||||||
|
- 82a702d Fix mypy error in Minio adapter log assertion.
|
||||||
|
- b886a7c Use structured fields for adapter debug and info logs.
|
||||||
|
- 4518a93 Merge pull request 'Migrate Python workflows to python-repositories-ci runner' (#41) from cursor/migrate-python-repositories-ci into main
|
||||||
|
- a8b7b59 Migrate Python workflows to python-repositories-ci runner label.
|
||||||
|
- 0e6941d Merge pull request 'Use lowercase CI image registry owner for Docker compatibility' (#45) from cursor/fix-ci-registry-owner-lowercase into main
|
||||||
|
- a6428cc Use lowercase CI image registry owner for Docker compatibility.
|
||||||
|
- 655fdc4 Merge pull request 'Fix CI image registry owner to match Gitea org slug' (#44) from cursor/fix-ci-registry-owner-case into main
|
||||||
|
- e031c54 Fix CI image registry owner to match Gitea org slug.
|
||||||
|
- 49b6238 Merge pull request 'Rename CI registry owner from brian to lille-vemmelund' (#43) from cursor/rename-ci-owner-to-lille-vemmelund into main
|
||||||
|
- 0549358 Keep python-utils package index under brian owner.
|
||||||
|
- 33efbd1 Update CI registry owner from brian to lille-vemmelund.
|
||||||
|
- 24126e4 Merge pull request 'Fix CI image build failures on runners with isolated DinD DNS' (#42) from fix/ci-image-build-dns into main
|
||||||
|
- f3b6cdb Fix CI image build failures on runners with isolated DinD DNS.
|
||||||
|
- fa2554e Merge pull request 'Add nightly CI base image and migrate workflows to python-repositories-ci' (#40) from cursor/ci-base-image into main
|
||||||
|
- b21fd24 Revert workflows to ubuntu-latest until CI image is seeded.
|
||||||
|
- 441a212 Add local CI image build script and document bootstrap flow.
|
||||||
|
- f2a8581 Read Python version from .python-version in CI image build.
|
||||||
|
- 3b05260 Add nightly CI base image and migrate workflows to python-repositories-ci.
|
||||||
|
|
||||||
|
## [2.0.2] - 2026-07-09
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Use dict[str, Any] for JSON repository document types
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 369295b chore: release v2.0.2 [skip ci]
|
||||||
|
- 95c7859 Merge pull request '[patch] Use dict[str, Any] for JSON repository document types' (#38) from cursor/json-dict-str-any-typing into main
|
||||||
|
- 42e885e Use dict[str, Any] for JSON repository document types.
|
||||||
|
|
||||||
|
## [2.0.1] - 2026-07-09
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Align Redis and MinIO connect() idempotency
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- bd4794d chore: release v2.0.1 [skip ci]
|
||||||
|
- 3dc412c Merge pull request '[patch] Align Redis and MinIO connect() idempotency' (#37) from cursor/connect-idempotency-alignment into main
|
||||||
|
- b8d88e6 Cover injected-client connect when client was cleared by disconnect.
|
||||||
|
- 67bb88f Align Redis and MinIO connect() idempotency in ConnectionAwareAdapter.
|
||||||
|
|
||||||
|
## [2.0.0] - 2026-07-09
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Make MinIO bucket creation opt-in with production-safe defaults
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- c2e6a96 chore: release v2.0.0 [skip ci]
|
||||||
|
- 547a1b3 Merge pull request '[breaking] Make MinIO bucket creation opt-in with production-safe defaults' (#36) from cursor/minio-bucket-creation-opt-in into main
|
||||||
|
- 39baf9b Fix examples integration test by provisioning MinIO bucket in fixture.
|
||||||
|
- 4a5b3b6 Make MinIO bucket creation opt-in with production-safe defaults.
|
||||||
|
- 1431f45 Merge pull request 'Move Redis test container helpers into integration conftest' (#35) from cursor/move-redis-container-to-conftest into main
|
||||||
|
- eafc717 Move Redis test container helpers into integration conftest.
|
||||||
|
|
||||||
|
## [1.1.0] - 2026-07-08
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Add Redis scan_keys iterator API
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 9a9b7b2 chore: release v1.1.0 [skip ci]
|
||||||
|
- 5b9573d Merge pull request '[minor] Add Redis scan_keys iterator API' (#34) from cursor/redis-scan-keys into main
|
||||||
|
- 50444af Add Redis scan_keys iterator API.
|
||||||
|
|
||||||
|
## [1.0.0] - 2026-07-08
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Clarify MinIO get() error semantics to match Redis behavior
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 2f19fcc chore: release v1.0.0 [skip ci]
|
||||||
|
- 3bd6589 Merge pull request '[breaking] Clarify MinIO get() error semantics to match Redis behavior' (#33) from cursor/minio-error-semantics into main
|
||||||
|
- 5311d49 Clarify MinIO get() error semantics to match Redis behavior.
|
||||||
|
|
||||||
|
## [0.5.1] - 2026-07-07
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Fix MinIO get_object response leak in MinioAdapter.get()
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 703bb95 chore: release v0.5.1 [skip ci]
|
||||||
|
- 8e47ebf Merge pull request '[patch] Fix MinIO get_object response leak in MinioAdapter.get()' (#32) from fix/minio-get-object-response-cleanup into main
|
||||||
|
- 9a800bf Fetch live PR title in pr-title-check CI workflow.
|
||||||
|
- 8c34534 Fix MinIO get_object response leak in MinioAdapter.get().
|
||||||
|
- b1210bd Merge pull request 'Reach 100% combined coverage with targeted unit tests' (#31) from cursor/balanced-100-coverage into main
|
||||||
|
- 39b1043 Fix mypy unreachable-code warning in minio reconnect test.
|
||||||
|
- beb2b51 Reach 100% combined coverage with targeted unit tests.
|
||||||
|
- ce062be Merge pull request 'Enforce 90% combined coverage floor in CI' (#29) from cursor/enforce-coverage-floor into main
|
||||||
|
- a381d45 Lower combined coverage floor from 95% to 90%.
|
||||||
|
- 5c8cd84 Document where combined coverage floor is enforced in CI.
|
||||||
|
- 3463298 Skip coverage floor check in partial CI test jobs.
|
||||||
|
- 5393efc Enforce a 95% combined coverage floor in CI and local runs.
|
||||||
|
- e39fc96 Merge pull request 'Split CI into parallel unit, integration, and coverage report jobs' (#28) from cursor/split-ci-test-jobs into main
|
||||||
|
- c1f4826 Use christopherhx v4 artifact actions for parallel coverage merge.
|
||||||
|
- 0315a33 Avoid cross-job artifacts; combine coverage with --cov-append.
|
||||||
|
- 5d33ec6 Use artifact action v3 for Gitea Actions compatibility.
|
||||||
|
- 9538e4d Split CI into parallel unit, integration, and coverage report jobs.
|
||||||
|
- 14b8d27 Merge pull request 'Replace hardcoded version examples in release docs' (#27) from cursor/fix-version-doc-drift into main
|
||||||
|
- d3cbb65 Use version-agnostic examples in release docs.
|
||||||
|
|
||||||
|
## [0.5.0] - 2026-07-06
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Add config and client injection with test reorganization
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 7dbb1fc chore: release v0.5.0 [skip ci]
|
||||||
|
- 918e5e6 Merge pull request '[minor] Add config and client injection with test reorganization' (#25) from cursor/config-client-injection into main
|
||||||
|
- 3aa9128 Load MINIO_SECURE from env and replace adapter asserts with explicit errors.
|
||||||
|
- 7991dab Add config and client injection with test reorganization.
|
||||||
|
- 366831a Merge pull request 'chore(deps): update dependencies' (#26) from renovate/auto-deps-update into main
|
||||||
|
- a98905d chore(deps): update dependencies [automated]
|
||||||
|
- 7ed1b34 Merge pull request 'Sync uv.lock on release to prevent CI failures' (#24) from cursor/sync-uv-lock-on-release into main
|
||||||
|
- 3236f27 manually updated package version
|
||||||
|
- ac487d9 Sync uv.lock on release to keep CI clean.
|
||||||
|
|
||||||
|
## [0.4.1] - 2026-07-05
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Extract shared connection lifecycle into ConnectionAwareAdapter
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- a049ce3 chore: release v0.4.1 [skip ci]
|
||||||
|
- 7b4cf84 Merge pull request '[patch] Extract shared connection lifecycle into ConnectionAwareAdapter' (#23) from cursor/extract-connection-aware-adapter into main
|
||||||
|
- b5a92e6 Sync uv.lock with pyproject.toml version 0.4.0.
|
||||||
|
- 7b950e0 Extract shared connection lifecycle into ConnectionAwareAdapter.
|
||||||
|
|
||||||
|
## [0.4.0] - 2026-07-05
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Strengthen is_connected with cached health probes
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- fe0c477 chore: release v0.4.0 [skip ci]
|
||||||
|
- e7e0fc8 Merge pull request '[minor] Strengthen is_connected with cached health probes' (#22) from cursor/strengthen-is-connected-health-checks into main
|
||||||
|
- 34fc046 Sync uv.lock with pyproject.toml version 0.3.2.
|
||||||
|
- 5149299 Strengthen is_connected with cached health probes.
|
||||||
|
|
||||||
|
## [0.3.2] - 2026-06-30
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Fix optional dependency handling for Redis and MinIO adapters
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- fef9552 chore: release v0.3.2 [skip ci]
|
||||||
|
- e9b2189 Merge pull request '[patch] Fix optional dependency handling for Redis and MinIO adapters' (#19) from cursor/fix-optional-dependencies into main
|
||||||
|
- f595390 Fix optional-deps import test for full-suite execution.
|
||||||
|
- 1f02195 Delegate top-level adapter imports to adapters package.
|
||||||
|
- f092ca2 Fix trailing whitespace in LICENSE and .gitignore.
|
||||||
|
- c6eed43 Fix optional dependency handling for Redis and MinIO adapters.
|
||||||
|
- 9e4b1e2 Merge pull request 'Avoid testcontainers Redis deprecation warning in integration tests.' (#18) from fix/redis-testcontainer-deprecation into main
|
||||||
|
- 0096df7 Avoid testcontainers Redis deprecation warning in integration tests.
|
||||||
|
- 3073bfb Merge pull request 'Fix PR coverage comment posting in CI.' (#17) from fix/coverage-pr-comment into main
|
||||||
|
- 30a1de3 Fix PR coverage comment posting in CI.
|
||||||
|
- 8465957 Merge pull request '[patch] update dependencies' (#16) from renovate/auto-deps-update into main
|
||||||
|
- 3ecb9c3 chore: retrigger CI
|
||||||
|
- 0f31170 Fix mypy errors from minio 7.2.20 type stubs.
|
||||||
|
- e3cab5e chore(deps): update dependencies [automated]
|
||||||
|
- f428a97 Merge pull request 'Fix dependency bot PR creation for Gitea.' (#15) from cursor/fix-dependency-bot-gitea-pr into main
|
||||||
|
- a3b9912 Fix dependency bot PR creation for Gitea.
|
||||||
|
- abaa078 Merge pull request 'Fix dependency update bot to use uv lock --upgrade.' (#14) from cursor/fix-dependency-update-bot-uv-lock into main
|
||||||
|
- 9864f0c Fix dependency update bot to use uv lock --upgrade.
|
||||||
|
- e39b467 Merge pull request 'Automate releases from PR titles and clean up CI workflows.' (#13) from cursor/pr-title-release-automation into main
|
||||||
|
- 85c1a64 Sync uv.lock with pyproject.toml version 0.3.1.
|
||||||
|
- a8491c0 Automate releases from PR titles and clean up CI workflows.
|
||||||
|
|
||||||
|
## [0.3.1] - 2026-06-28
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Add public repository interfaces and subclassable adapter CRUD API.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 2bb361e Merge pull request 'Add public repository interfaces and subclassable adapter CRUD API.' (#11) from cursor/public-repository-api into main
|
||||||
|
- c5169df Use pre-commit for Prettier in CI and document local hook setup.
|
||||||
|
- 497533f Fix mypy context manager typing for adapter subclasses.
|
||||||
|
- e98fd3b Add public repository interfaces and subclassable adapter CRUD API.
|
||||||
|
- f4280f1 Merge pull request 'Remove non-existent Mongo adapter from README.' (#10) from cleanup/remove-mongo-adapter-mentions into main
|
||||||
|
- d9b9fa4 updated package settings
|
||||||
|
- 9d25f7c updated package settings
|
||||||
|
- 1635815 updated package settings
|
||||||
|
- 933c4ee updated package settings
|
||||||
|
- 2b0d98f updated package settings
|
||||||
|
- 729fa48 Remove non-existent Mongo adapter from README.
|
||||||
|
|
||||||
|
## [0.2.3] - 2025-09-18
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
added support for content type when putting object in minio
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- fa011be Merge pull request 'added support for content type when putting object in minio' (#9) from add-support-for-content-type-in-minio into main
|
||||||
|
- 2e93309 ruff format fix
|
||||||
|
- 3ab8da9 added support for content type when putting object in minio
|
||||||
|
|
||||||
|
## [0.2.2] - 2025-09-17
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
moved import of optional packages into classes
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 1dd604d Merge pull request 'moved import of optional packages into classes' (#6) from make-optional-parts-independent into main
|
||||||
|
- e5c4025 added graceful handling of missing optional packages
|
||||||
|
- 396cf83 code quality fixes
|
||||||
|
- 538b26b mypy fixed
|
||||||
|
- ee6acc6 moved import of optional packages into classes
|
||||||
|
|
||||||
|
## [0.2.1] - 2025-09-15
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
exposed minio adapter
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 0ca07c0 Merge pull request 'exposed minio adapter' (#5) from expose-minio-adapter into main
|
||||||
|
- 7446e88 added py.typed mypy config file
|
||||||
|
- d5f437f ruff format fix
|
||||||
|
- 7beca7c exposed minio adapter
|
||||||
|
|
||||||
|
## [0.2.0] - 2025-09-15
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
add-minio-adapter
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 65d8051 Merge pull request 'add-minio-adapter' (#4) from add-minio-adapter into main
|
||||||
|
- f5af303 auto install all extra packages
|
||||||
|
- 3260a3f added new optional package to ci
|
||||||
|
- c7bced4 ruff lint fixes
|
||||||
|
- 800d702 pyupgrade
|
||||||
|
- 5a72507 ruff format
|
||||||
|
- c45ba3c mypy fixes
|
||||||
|
- a3b5443 added minio adapter and tests
|
||||||
|
- cc99fe5 minor corrections
|
||||||
|
- 93fffe1 added optional group minio
|
||||||
|
|
||||||
|
## [0.1.0] - 2025-09-14
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
Release v0.1.0
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- 2eafd73 Merge pull request 'added redis adapter' (#1) from collect-code-from-other-repos into main
|
||||||
|
- 1cc047d removed unused bound ports for test
|
||||||
|
- c7cc2a3 added optional package redis
|
||||||
|
- b5d0297 pyupgrade fixed imports
|
||||||
|
- 2fa633a ruff fixed formatting
|
||||||
|
- 0e47db2 fuxed mypy issues
|
||||||
|
- 0948799 added integration tests
|
||||||
|
- 524bd0f fixed connection error handling
|
||||||
|
- 969c2c8 installed more pytest-cov
|
||||||
|
- 7caee87 added installing optional redis group
|
||||||
|
- 918535e added test coverage and upload step
|
||||||
|
- 5d7afe7 ruff format corrections
|
||||||
|
- 59a943e added redis types stub and fixed return type
|
||||||
|
- 1abcae4 added redis adapter
|
||||||
|
- c789e33 initial commit
|
||||||
|
- 56cad42 Initial commit
|
||||||
@@ -1,17 +1,333 @@
|
|||||||
# python-repositories
|
# python-repositories
|
||||||
|
|
||||||
Various python repository interfaces exposed as a python package.
|
Unified repository interfaces and technology-specific adapters for Python projects.
|
||||||
|
|
||||||
|
Subclass an adapter in your own repository to add domain-specific methods while reusing connection management and CRUD operations.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
| Layer | Responsibility |
|
||||||
|
| ---------------- | ---------------------------------------------------------------------------------------------------- |
|
||||||
|
| **Interfaces** | Abstract contracts for connection, context, CRUD, and queues |
|
||||||
|
| **Adapters** | Technology-specific base classes (`RedisAdapter`, `MinioAdapter`, `PostgresAdapter`, queue adapters) |
|
||||||
|
| **Your project** | Subclass an adapter and add domain methods |
|
||||||
|
|
||||||
|
Each public interface is a `@runtime_checkable` `Protocol` with `@abstractmethod` members. **Subclass an adapter** when you need connection management and shared behavior — explicit subclasses get runtime instantiation guards and inherited default methods (e.g. `scan_keys`). **Type-annotate against an interface** when you want loose coupling — any object with the right methods satisfies the contract for mypy and `isinstance()` checks, without inheriting from this package.
|
||||||
|
|
||||||
|
Connection adapters expose `connect()`, `disconnect()`, and `is_connected()`. The latter verifies backend reachability with a cached health probe (default TTL: 1 second). Subclasses may override `health_check_ttl_seconds`. `connect()` is idempotent: calling it while already connected and healthy is a no-op.
|
||||||
|
|
||||||
|
## Future direction
|
||||||
|
|
||||||
|
The current API is synchronous. Async repository interfaces and adapters may be added in a future release; existing sync usage would remain supported.
|
||||||
|
|
||||||
## Optional dependencies
|
## Optional dependencies
|
||||||
|
|
||||||
This package supports interacting with multiple different backends:
|
Repository **interfaces** import with the base package. Networked **adapters** (`RedisAdapter`, `MinioAdapter`, `PostgresAdapter`) require the matching extra; importing one without its extra raises `ImportError` with install instructions. Queue adapters (`MemoryQueueAdapter`, `FileBackedQueueAdapter`) need no extra.
|
||||||
|
|
||||||
- Redis
|
Install with the extras you need:
|
||||||
- Mongo
|
|
||||||
- MinIO
|
|
||||||
|
|
||||||
To add support for a specific backend install this package with one or more of these optional packages:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv add python-repositories[redis, mongo, minio]
|
uv add python-repositories[redis]
|
||||||
|
uv add python-repositories[minio]
|
||||||
|
uv add python-repositories[postgres]
|
||||||
|
uv add python-repositories[redis,minio,postgres]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Redis (`JsonRepositoryInterface`)
|
||||||
|
|
||||||
|
Requires Redis with the RedisJSON module (e.g. redis-stack).
|
||||||
|
|
||||||
|
| Environment variable | Description |
|
||||||
|
| -------------------- | ---------------------------------------------------- |
|
||||||
|
| `REDIS_URI` | Redis connection URL (e.g. `redis://localhost:6379`) |
|
||||||
|
|
||||||
|
For key discovery:
|
||||||
|
|
||||||
|
- `list_keys(pattern)` is simple and returns a `list[str]`, but it uses Redis `KEYS` and may block on large datasets.
|
||||||
|
- `scan_keys(pattern, *, count=None)` is preferred for production use and yields keys incrementally via Redis `SCAN`.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```python
|
||||||
|
for key in repo.scan_keys("user:*"):
|
||||||
|
print(key)
|
||||||
|
```
|
||||||
|
|
||||||
|
### MinIO (`ObjectRepositoryInterface`)
|
||||||
|
|
||||||
|
| Environment variable | Description |
|
||||||
|
| -------------------------------- | ------------------------------------------------------------------------ |
|
||||||
|
| `MINIO_ENDPOINT` | MinIO server endpoint |
|
||||||
|
| `MINIO_ACCESS_KEY` | Access key |
|
||||||
|
| `MINIO_SECRET_KEY` | Secret key |
|
||||||
|
| `MINIO_BUCKET` | Bucket name |
|
||||||
|
| `MINIO_SECURE` | Use HTTPS (`true`/`false`; default: `true`) |
|
||||||
|
| `MINIO_CREATE_BUCKET_IF_MISSING` | Auto-create `MINIO_BUCKET` on connect (`true`/`false`; default: `false`) |
|
||||||
|
|
||||||
|
For production, it is recommended to leave `MINIO_CREATE_BUCKET_IF_MISSING` unset so that `connect()` fails fast if the expected bucket is missing. For local development, you will often want `MINIO_SECURE=false` and `MINIO_CREATE_BUCKET_IF_MISSING=true`.
|
||||||
|
|
||||||
|
### Postgres (`TableRepositoryInterface`)
|
||||||
|
|
||||||
|
Requires PostgreSQL 10 or later; tested against PostgreSQL 16 in CI. Tables are owned by your migrations — the adapter verifies the configured table exists on connect.
|
||||||
|
|
||||||
|
| Environment variable | Description |
|
||||||
|
| ---------------------- | --------------------------------------- |
|
||||||
|
| `POSTGRES_URI` | PostgreSQL connection URL |
|
||||||
|
| `POSTGRES_TABLE` | Table name for CRUD operations |
|
||||||
|
| `POSTGRES_PRIMARY_KEY` | Primary key column name (default: `id`) |
|
||||||
|
|
||||||
|
### File-backed queue (`QueueRepositoryInterface`)
|
||||||
|
|
||||||
|
In-process and disk-backed FIFO queues for buffering dict items. No optional extra required.
|
||||||
|
|
||||||
|
| Adapter | Role |
|
||||||
|
| ------------------------ | --------------------------------------------------------------------------------- |
|
||||||
|
| `MemoryQueueAdapter` | Thread-safe in-memory buffer with optional dedup and age eviction |
|
||||||
|
| `FileBackedQueueAdapter` | Mirrors memory to a JSONL file; replays on `connect()`, compacts on dequeue/evict |
|
||||||
|
|
||||||
|
| Environment variable | Description |
|
||||||
|
| -------------------------- | ----------------------------------------- |
|
||||||
|
| `FILE_QUEUE_PATH` | Path to the JSONL queue file |
|
||||||
|
| `FILE_QUEUE_MAX_AGE_HOURS` | Retention window in hours (default: `24`) |
|
||||||
|
|
||||||
|
Dedup keys and the age field name are domain-specific: pass them when constructing `FileQueueConfig` (or as kwargs to `FileQueueConfig.from_env(...)`). Callers choose the file path and item schema.
|
||||||
|
|
||||||
|
```python
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from python_repositories import FileBackedQueueAdapter, FileQueueConfig
|
||||||
|
|
||||||
|
config = FileQueueConfig(
|
||||||
|
path=Path("/var/lib/my-service/queue/items.jsonl"),
|
||||||
|
max_age_hours=24,
|
||||||
|
dedup_keys=("id",),
|
||||||
|
age_key="created_at",
|
||||||
|
)
|
||||||
|
with FileBackedQueueAdapter(config=config) as queue:
|
||||||
|
queue.enqueue([{"id": 1, "created_at": "2024-01-01T00:00:00+00:00"}])
|
||||||
|
batch = queue.dequeue_batch(max_items=100)
|
||||||
|
```
|
||||||
|
|
||||||
|
Copy [`.env.example`](.env.example) to `.env` for local development. `RedisConfig.from_env()`, `MinioConfig.from_env()`, `PostgresConfig.from_env()`, and `FileQueueConfig.from_env()` load `.env` automatically when resolving configuration from the environment.
|
||||||
|
|
||||||
|
## Configuration injection
|
||||||
|
|
||||||
|
Adapters accept optional `config` and `client` keyword arguments for explicit setup and testing. Queue adapters accept `config` (and an optional injected `memory` buffer for `FileBackedQueueAdapter`):
|
||||||
|
|
||||||
|
```python
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from python_repositories import (
|
||||||
|
FileBackedQueueAdapter,
|
||||||
|
FileQueueConfig,
|
||||||
|
MinioAdapter,
|
||||||
|
MinioConfig,
|
||||||
|
PostgresAdapter,
|
||||||
|
PostgresConfig,
|
||||||
|
RedisAdapter,
|
||||||
|
RedisConfig,
|
||||||
|
)
|
||||||
|
|
||||||
|
redis = RedisAdapter(config=RedisConfig(uri="redis://localhost:6379"))
|
||||||
|
minio = MinioAdapter(
|
||||||
|
config=MinioConfig(
|
||||||
|
endpoint="localhost:9000",
|
||||||
|
access_key="minioadmin",
|
||||||
|
secret_key="minioadmin",
|
||||||
|
bucket="my-bucket",
|
||||||
|
secure=False,
|
||||||
|
# create_bucket_if_missing=True, # convenient for local dev
|
||||||
|
)
|
||||||
|
)
|
||||||
|
postgres = PostgresAdapter(
|
||||||
|
config=PostgresConfig(
|
||||||
|
uri="postgresql://localhost/mydb",
|
||||||
|
table="users",
|
||||||
|
primary_key="user_id",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
queue = FileBackedQueueAdapter(
|
||||||
|
config=FileQueueConfig(
|
||||||
|
path=Path("/var/lib/my-service/queue/items.jsonl"),
|
||||||
|
dedup_keys=("id",),
|
||||||
|
age_key="created_at",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
When both `config` and `client` are provided, `connect()` skips client creation (the caller owns the client lifecycle). `config` is required whenever `client` is injected.
|
||||||
|
|
||||||
|
Load a `.env` file explicitly:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from python_repositories import load_dotenv
|
||||||
|
|
||||||
|
load_dotenv() # optional — from_env() also loads .env by default
|
||||||
|
```
|
||||||
|
|
||||||
|
Calling `RedisAdapter()`, `MinioAdapter()`, or `PostgresAdapter()` with no arguments still loads configuration from environment variables (and `.env` if present).
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
### JSON documents with Redis
|
||||||
|
|
||||||
|
```python
|
||||||
|
from python_repositories.examples.user_json_repository import UserJsonRepository
|
||||||
|
|
||||||
|
with UserJsonRepository() as repo:
|
||||||
|
repo.save_user("alice", {"name": "Alice", "email": "[email protected]"})
|
||||||
|
user = repo.get_user("alice")
|
||||||
|
repo.delete_user("alice")
|
||||||
|
```
|
||||||
|
|
||||||
|
### Binary objects with MinIO
|
||||||
|
|
||||||
|
```python
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
from python_repositories.examples.artifact_object_repository import (
|
||||||
|
ArtifactObjectRepository,
|
||||||
|
)
|
||||||
|
|
||||||
|
with ArtifactObjectRepository() as repo:
|
||||||
|
repo.store_artifact("report-1", BytesIO(b"pdf bytes here"))
|
||||||
|
data = repo.get_artifact("report-1")
|
||||||
|
```
|
||||||
|
|
||||||
|
### Relational rows with Postgres
|
||||||
|
|
||||||
|
```python
|
||||||
|
from python_repositories.examples.user_table_repository import UserTableRepository
|
||||||
|
|
||||||
|
with UserTableRepository() as repo:
|
||||||
|
repo.save_user("alice", {"name": "Alice", "email": "[email protected]"})
|
||||||
|
user = repo.get_user("alice")
|
||||||
|
repo.delete_user("alice")
|
||||||
|
```
|
||||||
|
|
||||||
|
### Subclassing in your own project
|
||||||
|
|
||||||
|
```python
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from python_repositories import RedisAdapter
|
||||||
|
|
||||||
|
class UserRepository(RedisAdapter):
|
||||||
|
def _key(self, user_id: str) -> str:
|
||||||
|
return f"user:{user_id}"
|
||||||
|
|
||||||
|
def get_user(self, user_id: str) -> dict[str, Any] | None:
|
||||||
|
return self.get(self._key(user_id))
|
||||||
|
|
||||||
|
def save_user(self, user_id: str, user: dict[str, Any]) -> None:
|
||||||
|
self.set(self._key(user_id), user)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Public API
|
||||||
|
|
||||||
|
```python
|
||||||
|
from python_repositories import (
|
||||||
|
ConnectionAwareInterface,
|
||||||
|
ContextAwareInterface,
|
||||||
|
FileBackedQueueAdapter,
|
||||||
|
FileQueueConfig,
|
||||||
|
JsonRepositoryInterface,
|
||||||
|
MemoryQueueAdapter,
|
||||||
|
MinioAdapter,
|
||||||
|
MinioConfig,
|
||||||
|
ObjectRepositoryInterface,
|
||||||
|
PostgresAdapter,
|
||||||
|
PostgresConfig,
|
||||||
|
QueueRepositoryInterface,
|
||||||
|
RedisAdapter,
|
||||||
|
RedisConfig,
|
||||||
|
TableRepositoryInterface,
|
||||||
|
load_dotenv,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv sync --all-extras
|
||||||
|
uv run pre-commit install # once per clone — runs hooks on git commit
|
||||||
|
uv run pytest tests/unit/ -v # fast, no Docker
|
||||||
|
uv run pytest -m "not integration" -v # all non-Docker tests
|
||||||
|
uv run pytest tests/integration/redis/ -v # Redis container only
|
||||||
|
uv run pytest tests/integration/minio/ -v # MinIO container only
|
||||||
|
uv run pytest tests/integration/postgres/ -v # Postgres container only
|
||||||
|
uv run pytest -v # full suite (requires Docker)
|
||||||
|
```
|
||||||
|
|
||||||
|
Integration tests are marked with `@pytest.mark.integration` and require Docker (testcontainers). Backend-specific markers (`needs_redis`, `needs_minio`, `needs_postgres`) let you run only the containers a test module needs. Run unit tests alone for quick local feedback.
|
||||||
|
|
||||||
|
### Test organization
|
||||||
|
|
||||||
|
Unit tests live in `tests/unit/` and follow a one-to-one naming convention: `<module>_test.py` tests `python_repositories/<module path>.py`. Examples:
|
||||||
|
|
||||||
|
- `json_repository_interface.py` → `tests/unit/json_repository_interface_test.py`
|
||||||
|
- `redis_adapter.py` → `tests/unit/redis_adapter_test.py`
|
||||||
|
|
||||||
|
Add new tests to the existing file for that module rather than creating cross-cutting test files. Shared fixtures belong in `tests/conftest.py`; module-specific helpers may live in the matching test file.
|
||||||
|
|
||||||
|
### CI base image
|
||||||
|
|
||||||
|
Gitea Actions jobs use a pre-built image (`python-repositories-ci`) with Python 3.12,
|
||||||
|
`uv`, and locked dependencies baked in. The image is rebuilt nightly and when
|
||||||
|
`uv.lock` changes; see [`docs/ci-image.md`](docs/ci-image.md) for runner setup and
|
||||||
|
bootstrap order.
|
||||||
|
|
||||||
|
CI runs unit and integration tests in parallel with coverage, then merges `.coverage` artifacts in a follow-up job (via [christopherhx/gitea-\*-artifact@v4](https://github.com/christopherHX/gitea-upload-artifact) for Gitea 1.26 compatibility). Combined coverage must meet the floor in [`fail_under` in `pyproject.toml`](pyproject.toml#L45-L48); enforcement happens after merging unit and integration coverage, not on unit-only runs.
|
||||||
|
|
||||||
|
To check coverage locally (requires Docker for the full suite):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run pytest --cov=python_repositories --cov-report=
|
||||||
|
uv run coverage report
|
||||||
|
```
|
||||||
|
|
||||||
|
`pre-commit` is included in the dev dependency group. `uv sync` installs the CLI, but git does not run hooks until you install them with `pre-commit install` (one time per clone). After that, commits run the checks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml) (ruff, mypy, pyupgrade, prettier, and general file hygiene).
|
||||||
|
|
||||||
|
To run all hooks manually without committing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run pre-commit run --all-files
|
||||||
|
```
|
||||||
|
|
||||||
|
## Releases
|
||||||
|
|
||||||
|
Releases are automated when a pull request is merged to `main`. CI reads the **merged PR title** to decide whether and how to bump the version.
|
||||||
|
|
||||||
|
- **Version history:** [`CHANGELOG.md`](CHANGELOG.md)
|
||||||
|
- **Gitea releases:** [releases page](https://gitea.lille-vemmelund.dk/LilleVemmelund/python-repositories/releases)
|
||||||
|
|
||||||
|
### How it works
|
||||||
|
|
||||||
|
1. Open a PR targeting `main` (see [`.gitea/PULL_REQUEST_TEMPLATE.md`](.gitea/PULL_REQUEST_TEMPLATE.md)).
|
||||||
|
2. If the PR changes files under `python_repositories/`, the title **must** start with a version bump prefix (enforced by CI).
|
||||||
|
3. On merge, [`release.yml`](.gitea/workflows/release.yml) bumps [`pyproject.toml`](pyproject.toml) and syncs [`uv.lock`](uv.lock), updates [`CHANGELOG.md`](CHANGELOG.md), commits, tags `vX.Y.Z`, creates a Gitea release with auto-generated notes, pushes the tag, and publishes the package to the Gitea Package Registry.
|
||||||
|
4. [`publish.yml`](.gitea/workflows/publish.yml) handles manual `v*.*.*` tag pushes only (automated releases publish from `release.yml` because release commits use `[skip ci]`, which suppresses tag-push workflows).
|
||||||
|
|
||||||
|
Docs-, CI-, and test-only PRs do not need a prefix and will not trigger a release.
|
||||||
|
|
||||||
|
### PR title prefixes
|
||||||
|
|
||||||
|
| Prefix | Bump | Example |
|
||||||
|
| ------------------------- | ---------- | --------------------- |
|
||||||
|
| `[patch]` or `[fix]` | patch | `1.2.3` → `1.2.4` |
|
||||||
|
| `[minor]` or `[feat]` | minor | `1.2.3` → `1.3.0` |
|
||||||
|
| `[major]` or `[breaking]` | major | `1.2.3` → `2.0.0` |
|
||||||
|
| _(none)_ | no release | docs / CI / deps only |
|
||||||
|
|
||||||
|
Example titles:
|
||||||
|
|
||||||
|
- `[minor] Add public repository interfaces and subclassable adapter CRUD API`
|
||||||
|
- `[patch] Fix mypy context manager typing for adapter subclasses`
|
||||||
|
|
||||||
|
### Release notes
|
||||||
|
|
||||||
|
Release notes are generated from commits since the previous tag (see [`scripts/ci/generate-release-notes.sh`](scripts/ci/generate-release-notes.sh)). The same content is appended to [`CHANGELOG.md`](CHANGELOG.md) on each release (see [`scripts/ci/update-changelog.sh`](scripts/ci/update-changelog.sh)).
|
||||||
|
|
||||||
|
### Manual release
|
||||||
|
|
||||||
|
You can still push a `v*.*.*` tag manually; `publish.yml` will build and publish. The current released version is in [`pyproject.toml`](pyproject.toml) (and on the latest `v*.*.*` tag).
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
# CI job image for Gitea Actions (act_runner requires Node.js in job containers).
|
||||||
|
FROM node:20-bookworm
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
ca-certificates \
|
||||||
|
curl \
|
||||||
|
git \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=docker:27-cli /usr/local/bin/docker /usr/local/bin/docker
|
||||||
|
COPY --from=ghcr.io/astral-sh/uv:0.7.0 /uv /usr/local/bin/uv
|
||||||
|
|
||||||
|
ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python \
|
||||||
|
UV_LINK_MODE=copy
|
||||||
|
|
||||||
|
COPY .python-version /tmp/.python-version
|
||||||
|
RUN uv python install "$(cat /tmp/.python-version)"
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY pyproject.toml uv.lock .python-version ./
|
||||||
|
|
||||||
|
ENV PATH="/app/.venv/bin:${PATH}"
|
||||||
|
|
||||||
|
RUN --mount=type=secret,id=uv_token \
|
||||||
|
UV_INDEX_GITEA_USERNAME=ci-bot \
|
||||||
|
UV_INDEX_GITEA_PASSWORD="$(cat /run/secrets/uv_token)" \
|
||||||
|
uv sync --all-extras --frozen --no-install-project
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
# CI base image
|
||||||
|
|
||||||
|
This repository uses a **per-repo Docker image** for Gitea Actions jobs instead of
|
||||||
|
installing Python and `uv` on every run. Infrastructure images (redis, minio, etc.)
|
||||||
|
are cached cluster-wide via Harbor (see homelab-platform
|
||||||
|
[`docs/harbor-registry-mirror.md`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/docs/harbor-registry-mirror.md)).
|
||||||
|
|
||||||
|
## Image contents
|
||||||
|
|
||||||
|
Built from [`docker/ci/Dockerfile`](../docker/ci/Dockerfile):
|
||||||
|
|
||||||
|
- Node.js 20 (required by act_runner job containers)
|
||||||
|
- Python 3.12 (installed via `uv python install` from [`.python-version`](../.python-version)) and pinned `uv` 0.7.0
|
||||||
|
- Docker CLI (integration tests via testcontainers)
|
||||||
|
- Dev dependencies from `uv.lock` (`uv sync --all-extras --no-install-project`)
|
||||||
|
|
||||||
|
Published to the Gitea container registry (owner segment must be lowercase for Docker):
|
||||||
|
|
||||||
|
- `gitea.lille-vemmelund.dk/lillevemmelund/python-repositories-ci:latest`
|
||||||
|
- `gitea.lille-vemmelund.dk/lillevemmelund/python-repositories-ci:YYYYMMDDHHmm` (timestamped rollback tag)
|
||||||
|
|
||||||
|
## Rebuild triggers
|
||||||
|
|
||||||
|
[`ci-image.yml`](../.gitea/workflows/ci-image.yml) runs on:
|
||||||
|
|
||||||
|
- Nightly cron (`0 2 * * *` UTC)
|
||||||
|
- Manual `workflow_dispatch`
|
||||||
|
- Push to `main` when `pyproject.toml`, `uv.lock`, or `docker/ci/**` change
|
||||||
|
|
||||||
|
## Workflow usage
|
||||||
|
|
||||||
|
Python jobs use `runs-on: python-repositories-ci` and a fast incremental sync:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
runs-on: python-repositories-ci
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Sync dependencies
|
||||||
|
env:
|
||||||
|
UV_LINK_MODE: copy
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
```
|
||||||
|
|
||||||
|
`ci-image.yml` uses `runs-on: ubuntu-latest` so it can bootstrap before the custom
|
||||||
|
image exists.
|
||||||
|
|
||||||
|
## Registry authentication
|
||||||
|
|
||||||
|
act_runner pulls the job image **before any workflow step runs**, so a `docker
|
||||||
|
login` step inside a job cannot authenticate that pull. Authentication must be
|
||||||
|
configured on the runner (or via a job-level `container.credentials` block — not
|
||||||
|
used here).
|
||||||
|
|
||||||
|
### k8s Gitea runners (automatic)
|
||||||
|
|
||||||
|
The homelab-platform runners mount Gitea registry credentials automatically:
|
||||||
|
|
||||||
|
- Secret: `gitea-runners/gitea-registry-dockerconfig` (created by
|
||||||
|
[`scripts/create-gitea-registry-secret.sh`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/scripts/create-gitea-registry-secret.sh))
|
||||||
|
- Mounted at `/root/.docker/config.json` on the **runner** container (not DinD)
|
||||||
|
- Configured in
|
||||||
|
[`platform/gitea-runners/values.yaml`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/platform/gitea-runners/values.yaml)
|
||||||
|
|
||||||
|
No manual `docker login` is required on the in-cluster runners once the secret
|
||||||
|
exists. To create or rotate credentials:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export KUBECONFIG=/path/to/homelab-cluster/talos/_out/kubeconfig
|
||||||
|
export GITEA_REGISTRY_USERNAME='ci-bot'
|
||||||
|
export GITEA_REGISTRY_PASSWORD='personal-access-token-with-read-package'
|
||||||
|
bash scripts/create-gitea-registry-secret.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Then restart runner pods so they pick up the updated secret. See homelab-platform
|
||||||
|
[`docs/gitea-actions-runners.md`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/docs/gitea-actions-runners.md)
|
||||||
|
for full runner setup.
|
||||||
|
|
||||||
|
Verify on a running pod:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
kubectl -n gitea-runners exec homelab-cluster-gitea-runner-0 -c runner -- \
|
||||||
|
test -f /root/.docker/config.json && echo "registry auth mounted"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Standalone runners (e.g. Unraid `homelab`)
|
||||||
|
|
||||||
|
The 4th runner is outside the k8s cluster and does **not** get the automatic
|
||||||
|
mount. Configure registry auth manually on that host:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker login gitea.lille-vemmelund.dk -u ci-bot -p <token>
|
||||||
|
```
|
||||||
|
|
||||||
|
Also add the `python-repositories-ci` label to that runner's act_runner config.
|
||||||
|
|
||||||
|
### `ci-image.yml` push login
|
||||||
|
|
||||||
|
The build workflow still runs `docker login` before `docker push`. That step
|
||||||
|
authenticates **DinD inside the job** for pushing the image to Gitea — a different
|
||||||
|
code path from act_runner pulling the job container.
|
||||||
|
|
||||||
|
## Runner label
|
||||||
|
|
||||||
|
Jobs use the `python-repositories-ci` act_runner label, configured in
|
||||||
|
homelab-platform
|
||||||
|
[`platform/gitea-runners/values.yaml`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/platform/gitea-runners/values.yaml):
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
python-repositories-ci:docker://gitea.lille-vemmelund.dk/lillevemmelund/python-repositories-ci:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
After label changes, roll runner pods so they re-register with Gitea.
|
||||||
|
|
||||||
|
## Bootstrap order
|
||||||
|
|
||||||
|
1. Ensure homelab-platform runners have `gitea-registry-dockerconfig` and the
|
||||||
|
`python-repositories-ci` label (see homelab-platform docs).
|
||||||
|
2. Merge `ci-image.yml`, `docker/ci/Dockerfile`, and workflow migrations to `main`.
|
||||||
|
3. Seed the registry with a first image (see below).
|
||||||
|
4. Confirm a test workflow job starts on `python-repositories-ci`.
|
||||||
|
|
||||||
|
Until step 3 completes, jobs targeting `python-repositories-ci` will fail because
|
||||||
|
the image does not exist in the Gitea registry yet.
|
||||||
|
|
||||||
|
### Seed the image
|
||||||
|
|
||||||
|
**After merge to `main`:** Actions → **Build CI Image** → **Run workflow**.
|
||||||
|
|
||||||
|
**Before merge (e.g. PR branch):** the workflow file is not on `main` yet — build
|
||||||
|
and push locally with [`scripts/ci/build-ci-image.sh`](../scripts/ci/build-ci-image.sh):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||||
|
bash scripts/ci/build-ci-image.sh --push
|
||||||
|
```
|
||||||
|
|
||||||
|
Run from the repo root on the branch you want to test. Runners pull
|
||||||
|
`gitea.lille-vemmelund.dk/lillevemmelund/python-repositories-ci:latest` from the registry;
|
||||||
|
they do not care which git branch built it.
|
||||||
|
|
||||||
|
Override registry settings if needed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export REGISTRY=gitea.lille-vemmelund.dk
|
||||||
|
export REGISTRY_USER=ci-bot
|
||||||
|
export CI_RUNNER_TOKEN='...'
|
||||||
|
bash scripts/ci/build-ci-image.sh --push
|
||||||
|
```
|
||||||
|
|
||||||
|
## Local build
|
||||||
|
|
||||||
|
Build only (no registry login or push):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||||
|
bash scripts/ci/build-ci-image.sh --local
|
||||||
|
```
|
||||||
|
|
||||||
|
Or manually:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
echo "$CI_RUNNER_TOKEN" > /tmp/uv_token
|
||||||
|
docker build -f docker/ci/Dockerfile \
|
||||||
|
--secret id=uv_token,src=/tmp/uv_token \
|
||||||
|
-t python-repositories-ci:local .
|
||||||
|
rm -f /tmp/uv_token
|
||||||
|
```
|
||||||
+52
-9
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "python-repositories"
|
name = "python-repositories"
|
||||||
version = "0.1.0"
|
version = "2.2.0"
|
||||||
description = "Various python repository interfaces exposed as a python package."
|
description = "Various python repository interfaces exposed as a python package."
|
||||||
authors = [
|
authors = [
|
||||||
{ name = "Brian Bjarke Jensen", email = "[email protected]" }
|
{ name = "Brian Bjarke Jensen", email = "[email protected]" }
|
||||||
@@ -10,14 +10,21 @@ requires-python = ">=3.12"
|
|||||||
license = { text = "MIT" }
|
license = { text = "MIT" }
|
||||||
classifiers = [
|
classifiers = [
|
||||||
"Programming Language :: Python :: 3",
|
"Programming Language :: Python :: 3",
|
||||||
|
"Programming Language :: Python :: 3.12",
|
||||||
"License :: OSI Approved :: MIT License",
|
"License :: OSI Approved :: MIT License",
|
||||||
"Operating System :: OS Independent",
|
"Operating System :: OS Independent",
|
||||||
]
|
]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"python-dotenv>=1.0.0",
|
||||||
"python-utils>=0.1.0",
|
"python-utils>=0.1.0",
|
||||||
"structlog>=25.4.0",
|
"structlog>=25.4.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[project.urls]
|
||||||
|
Repository = "https://gitea.lille-vemmelund.dk/LilleVemmelund/python-repositories"
|
||||||
|
Changelog = "https://gitea.lille-vemmelund.dk/LilleVemmelund/python-repositories/src/branch/main/CHANGELOG.md"
|
||||||
|
Releases = "https://gitea.lille-vemmelund.dk/LilleVemmelund/python-repositories/releases"
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
redis = [
|
redis = [
|
||||||
"redis>=6.4.0",
|
"redis>=6.4.0",
|
||||||
@@ -25,22 +32,59 @@ redis = [
|
|||||||
minio = [
|
minio = [
|
||||||
"minio>=7.2.16",
|
"minio>=7.2.16",
|
||||||
]
|
]
|
||||||
|
postgres = [
|
||||||
|
"psycopg[binary]>=3.2.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[build-system]
|
||||||
|
requires = ["hatchling"]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
testpaths = ["tests"]
|
||||||
|
addopts = "--import-mode=importlib"
|
||||||
|
markers = [
|
||||||
|
"integration: tests requiring Docker containers (deselect with '-m \"not integration\"')",
|
||||||
|
"needs_redis: integration test requiring a Redis container",
|
||||||
|
"needs_minio: integration test requiring a MinIO container",
|
||||||
|
"needs_postgres: integration test requiring a Postgres container",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.coverage.run]
|
||||||
|
source = ["python_repositories"]
|
||||||
|
|
||||||
|
[tool.coverage.report]
|
||||||
|
fail_under = 100
|
||||||
|
show_missing = true
|
||||||
|
precision = 2
|
||||||
|
|
||||||
[tool.uv.sources]
|
[tool.uv.sources]
|
||||||
python-utils = { index = "gitea" }
|
python-utils = { index = "gitea" }
|
||||||
|
|
||||||
[[tool.uv.index]]
|
[[tool.uv.index]]
|
||||||
name = "threadripper-proxpi-cache"
|
name = "threadripper-proxpi-cache"
|
||||||
url = "http://10.0.0.2:5001/index/"
|
url = "https://proxpi.lille-vemmelund.dk/index/"
|
||||||
default = true
|
default = true
|
||||||
|
|
||||||
[[tool.uv.index]]
|
[[tool.uv.index]]
|
||||||
name = "gitea"
|
name = "gitea"
|
||||||
url = "https://gitea.gt-proj.com/api/packages/brian/pypi/simple/"
|
url = "https://gitea.lille-vemmelund.dk/api/packages/brian/pypi/simple/"
|
||||||
explicit = true
|
explicit = true
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
line-length = 88
|
||||||
|
target-version = "py312"
|
||||||
|
|
||||||
|
[tool.ruff.lint]
|
||||||
|
extend-select = ["I"]
|
||||||
|
|
||||||
|
[tool.ruff.lint.isort]
|
||||||
|
known-first-party = ["python_repositories", "tests"]
|
||||||
|
combine-as-imports = true
|
||||||
|
force-sort-within-sections = true
|
||||||
|
|
||||||
[tool.mypy]
|
[tool.mypy]
|
||||||
python_version = "3.10"
|
python_version = "3.12"
|
||||||
warn_return_any = true # nudge to use stricter types
|
warn_return_any = true # nudge to use stricter types
|
||||||
warn_unused_configs = true # nudge to remove unused configs
|
warn_unused_configs = true # nudge to remove unused configs
|
||||||
disallow_untyped_defs = true # disallow untyped function definitions
|
disallow_untyped_defs = true # disallow untyped function definitions
|
||||||
@@ -66,13 +110,12 @@ ignore_missing_imports = true
|
|||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = [
|
dev = [
|
||||||
"mypy>=1.17.1",
|
"mypy>=2.1.0",
|
||||||
"pre-commit>=4.3.0",
|
"pre-commit>=4.6.0",
|
||||||
"pytest>=8.4.1",
|
"pytest>=8.4.1",
|
||||||
"pytest-cov>=7.0.0",
|
"pytest-cov>=7.0.0",
|
||||||
"pyupgrade>=3.20.0",
|
"pyupgrade>=3.21.2",
|
||||||
"ruff>=0.12.11",
|
"ruff>=0.15.20",
|
||||||
"safety>=3.6.0",
|
"safety>=3.6.0",
|
||||||
"testcontainers>=4.13.0",
|
"testcontainers>=4.13.0",
|
||||||
"types-redis>=4.6.0.20241004",
|
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,6 +1,60 @@
|
|||||||
"""python_repositories: Unified repository interfaces and adapters."""
|
"""python_repositories: Unified repository interfaces and adapters."""
|
||||||
|
|
||||||
from . import adapters
|
from __future__ import annotations
|
||||||
from . import interfaces
|
|
||||||
|
|
||||||
__all__ = ["adapters", "interfaces"]
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
# Interfaces are always available; they have no optional backend dependencies.
|
||||||
|
from . import adapters
|
||||||
|
from .config import (
|
||||||
|
FileQueueConfig,
|
||||||
|
MinioConfig,
|
||||||
|
PostgresConfig,
|
||||||
|
RedisConfig,
|
||||||
|
load_dotenv,
|
||||||
|
)
|
||||||
|
from .interfaces import (
|
||||||
|
ConnectionAwareInterface,
|
||||||
|
ContextAwareInterface,
|
||||||
|
JsonRepositoryInterface,
|
||||||
|
ObjectRepositoryInterface,
|
||||||
|
QueueRepositoryInterface,
|
||||||
|
TableRepositoryInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Adapters are imported only for static type checkers; runtime loading is delegated below.
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from .adapters.file_backed_queue_adapter import (
|
||||||
|
FileBackedQueueAdapter as FileBackedQueueAdapter,
|
||||||
|
)
|
||||||
|
from .adapters.memory_queue_adapter import MemoryQueueAdapter as MemoryQueueAdapter
|
||||||
|
from .adapters.minio_adapter import MinioAdapter as MinioAdapter
|
||||||
|
from .adapters.postgres_adapter import PostgresAdapter as PostgresAdapter
|
||||||
|
from .adapters.redis_adapter import RedisAdapter as RedisAdapter
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ConnectionAwareInterface",
|
||||||
|
"ContextAwareInterface",
|
||||||
|
"FileQueueConfig",
|
||||||
|
"JsonRepositoryInterface",
|
||||||
|
"MinioConfig",
|
||||||
|
"ObjectRepositoryInterface",
|
||||||
|
"PostgresConfig",
|
||||||
|
"QueueRepositoryInterface",
|
||||||
|
"RedisConfig",
|
||||||
|
"TableRepositoryInterface",
|
||||||
|
"load_dotenv",
|
||||||
|
*adapters.__all__,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def __getattr__(name: str) -> object:
|
||||||
|
"""Delegate adapter lookups to adapters; lazy loading is defined there."""
|
||||||
|
if name in adapters.__all__:
|
||||||
|
return getattr(adapters, name)
|
||||||
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def __dir__() -> list[str]:
|
||||||
|
"""Expose lazy adapter names in tab completion and dir()."""
|
||||||
|
return sorted(__all__)
|
||||||
|
|||||||
@@ -4,10 +4,51 @@ Adapters for various backend repositories (e.g., Redis, Minio).
|
|||||||
This module exposes concrete implementations for repository interfaces.
|
This module exposes concrete implementations for repository interfaces.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .redis_adapter import RedisAdapter
|
from __future__ import annotations
|
||||||
from .minio_adapter import MinioAdapter
|
|
||||||
|
import importlib
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
# Adapters are imported only for static type checkers; runtime loading is deferred below.
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from .file_backed_queue_adapter import FileBackedQueueAdapter
|
||||||
|
from .memory_queue_adapter import MemoryQueueAdapter
|
||||||
|
from .minio_adapter import MinioAdapter
|
||||||
|
from .postgres_adapter import PostgresAdapter
|
||||||
|
from .redis_adapter import RedisAdapter
|
||||||
|
|
||||||
|
# Map public adapter names to their defining module and class.
|
||||||
|
# Each adapter module fails fast with an install hint if its extra is missing.
|
||||||
|
# When adding a new adapter, update this dict and __all__ only.
|
||||||
|
_LAZY_EXPORTS = {
|
||||||
|
"RedisAdapter": (".redis_adapter", "RedisAdapter"),
|
||||||
|
"MinioAdapter": (".minio_adapter", "MinioAdapter"),
|
||||||
|
"PostgresAdapter": (".postgres_adapter", "PostgresAdapter"),
|
||||||
|
"MemoryQueueAdapter": (".memory_queue_adapter", "MemoryQueueAdapter"),
|
||||||
|
"FileBackedQueueAdapter": (
|
||||||
|
".file_backed_queue_adapter",
|
||||||
|
"FileBackedQueueAdapter",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"RedisAdapter",
|
"RedisAdapter",
|
||||||
"MinioAdapter",
|
"MinioAdapter",
|
||||||
|
"PostgresAdapter",
|
||||||
|
"MemoryQueueAdapter",
|
||||||
|
"FileBackedQueueAdapter",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def __getattr__(name: str) -> object:
|
||||||
|
"""Load an adapter on first access so the base package installs without backend clients."""
|
||||||
|
if name in _LAZY_EXPORTS:
|
||||||
|
module_path, attr = _LAZY_EXPORTS[name]
|
||||||
|
module = importlib.import_module(module_path, __package__)
|
||||||
|
return getattr(module, attr)
|
||||||
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def __dir__() -> list[str]:
|
||||||
|
"""Expose lazy adapter names in tab completion and dir()."""
|
||||||
|
return sorted(__all__)
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
"""Shared connection lifecycle behavior for repository adapters."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from abc import abstractmethod
|
||||||
|
import time
|
||||||
|
from typing import Self
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
|
||||||
|
from python_repositories.interfaces import (
|
||||||
|
ConnectionAwareInterface,
|
||||||
|
ContextAwareInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectionAwareAdapter(ConnectionAwareInterface, ContextAwareInterface):
|
||||||
|
"""Base adapter with context-manager and TTL-cached connection health checks."""
|
||||||
|
|
||||||
|
health_check_ttl_seconds: float = 1.0
|
||||||
|
connection_name: str = ""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.logger = structlog.get_logger(self.__class__.__name__)
|
||||||
|
self._client_injected = False
|
||||||
|
self._health_check_at: float | None = None
|
||||||
|
self._health_check_ok: bool = False
|
||||||
|
|
||||||
|
def __enter__(self) -> Self:
|
||||||
|
"""Enter the context."""
|
||||||
|
self.connect()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(
|
||||||
|
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
||||||
|
) -> None:
|
||||||
|
"""Exit the context."""
|
||||||
|
ctx_info = {"exc_type": exc_type, "exc_val": exc_val, "exc_tb": exc_tb}
|
||||||
|
if any(
|
||||||
|
(
|
||||||
|
exc_type is not None,
|
||||||
|
exc_val is not None,
|
||||||
|
exc_tb is not None,
|
||||||
|
),
|
||||||
|
):
|
||||||
|
self.logger.error("Error while exiting context", **ctx_info)
|
||||||
|
self.disconnect()
|
||||||
|
|
||||||
|
def _invalidate_health_cache(self) -> None:
|
||||||
|
self._health_check_at = None
|
||||||
|
self._health_check_ok = False
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def _is_client_ready(self) -> bool:
|
||||||
|
"""Return True when internal state is sufficient for a probe."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def _probe_connection(self) -> bool:
|
||||||
|
"""Backend-specific liveness check; called only when client is ready."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def _validate_injected_client(self) -> None:
|
||||||
|
"""Verify an injected client is reachable; raise ConnectionError on failure."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def _establish_connection(self) -> None:
|
||||||
|
"""Create a backend client and set internal connection state."""
|
||||||
|
|
||||||
|
def connect(self) -> None:
|
||||||
|
"""Connect to the backend; idempotent when already connected and healthy."""
|
||||||
|
if self._client_injected:
|
||||||
|
self._validate_injected_client()
|
||||||
|
self._invalidate_health_cache()
|
||||||
|
return
|
||||||
|
if self._is_client_ready() and self.is_connected():
|
||||||
|
self.logger.info("Already connected", connection_name=self.connection_name)
|
||||||
|
return
|
||||||
|
self.disconnect()
|
||||||
|
self._establish_connection()
|
||||||
|
self._invalidate_health_cache()
|
||||||
|
|
||||||
|
def is_connected(self) -> bool:
|
||||||
|
"""Check if connected to the backend."""
|
||||||
|
if not self._is_client_ready():
|
||||||
|
return False
|
||||||
|
now = time.monotonic()
|
||||||
|
if self._health_check_at is not None:
|
||||||
|
seconds_since_last_health_check = now - self._health_check_at
|
||||||
|
cache_is_fresh = (
|
||||||
|
seconds_since_last_health_check < self.health_check_ttl_seconds
|
||||||
|
)
|
||||||
|
if cache_is_fresh:
|
||||||
|
return self._health_check_ok
|
||||||
|
result = self._probe_connection()
|
||||||
|
self._health_check_at = now
|
||||||
|
self._health_check_ok = result
|
||||||
|
self.logger.debug("Connection status", connected=result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _require_connected(self) -> None:
|
||||||
|
if not self.is_connected():
|
||||||
|
raise ConnectionError(f"Not connected to {self.connection_name}")
|
||||||
@@ -0,0 +1,215 @@
|
|||||||
|
"""File-backed queue adapter: in-memory hot buffer mirrored to JSONL on disk."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import threading
|
||||||
|
from typing import Any, Self, TextIO
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
|
||||||
|
from python_repositories.adapters.memory_queue_adapter import MemoryQueueAdapter
|
||||||
|
from python_repositories.config.file_queue_config import FileQueueConfig
|
||||||
|
from python_repositories.interfaces.queue_repository_interface import (
|
||||||
|
QueueRepositoryInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FileBackedQueueAdapter(QueueRepositoryInterface):
|
||||||
|
"""Queue that mirrors an in-memory buffer to an append-friendly JSONL file.
|
||||||
|
|
||||||
|
Does not extend ``ConnectionAwareAdapter``. ``connect()`` replays the JSONL
|
||||||
|
file into memory; ``disconnect()`` flushes any open handle.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
config: FileQueueConfig | None = None,
|
||||||
|
memory: MemoryQueueAdapter | None = None,
|
||||||
|
) -> None:
|
||||||
|
if config is None:
|
||||||
|
config = FileQueueConfig.from_env()
|
||||||
|
if memory is None:
|
||||||
|
memory = MemoryQueueAdapter(
|
||||||
|
dedup_keys=config.dedup_keys,
|
||||||
|
age_key=config.age_key,
|
||||||
|
)
|
||||||
|
self._config = config
|
||||||
|
self._memory = memory
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self._connected = False
|
||||||
|
self._append_handle: TextIO | None = None
|
||||||
|
self.logger = structlog.get_logger(self.__class__.__name__)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _json_default(value: object) -> str:
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
return value.isoformat()
|
||||||
|
raise TypeError(f"Object of type {type(value)!r} is not JSON serializable")
|
||||||
|
|
||||||
|
def __enter__(self) -> Self:
|
||||||
|
self.connect()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(
|
||||||
|
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
||||||
|
) -> None:
|
||||||
|
del exc_type, exc_val, exc_tb
|
||||||
|
self.disconnect()
|
||||||
|
|
||||||
|
def connect(self) -> None:
|
||||||
|
"""Create parent directories and replay the JSONL file into memory."""
|
||||||
|
with self._lock:
|
||||||
|
if self._connected:
|
||||||
|
return
|
||||||
|
path = self._config.path
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._memory.clear()
|
||||||
|
if path.is_file():
|
||||||
|
self._replay_file(path)
|
||||||
|
self._append_handle = path.open("a", encoding="utf-8")
|
||||||
|
self._connected = True
|
||||||
|
|
||||||
|
def disconnect(self) -> None:
|
||||||
|
"""Flush and close the append handle if open."""
|
||||||
|
with self._lock:
|
||||||
|
if self._append_handle is not None:
|
||||||
|
self._append_handle.flush()
|
||||||
|
try:
|
||||||
|
os.fsync(self._append_handle.fileno())
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
self._append_handle.close()
|
||||||
|
self._append_handle = None
|
||||||
|
self._connected = False
|
||||||
|
|
||||||
|
def is_connected(self) -> bool:
|
||||||
|
"""Return whether ``connect()`` has completed successfully."""
|
||||||
|
return self._connected
|
||||||
|
|
||||||
|
def enqueue(self, items: list[dict[str, Any]]) -> None:
|
||||||
|
"""Append items to memory and the JSONL file; optionally age-evict."""
|
||||||
|
self._require_connected()
|
||||||
|
with self._lock:
|
||||||
|
added = self._memory.enqueue_and_return_added(items)
|
||||||
|
if added:
|
||||||
|
self._append_items(added)
|
||||||
|
if self._config.age_key is not None:
|
||||||
|
cutoff = datetime.now(UTC) - timedelta(hours=self._config.max_age_hours)
|
||||||
|
if self._memory.evict_older_than(cutoff):
|
||||||
|
self._compact()
|
||||||
|
|
||||||
|
def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
|
||||||
|
"""Dequeue from memory and compact the JSONL file to match."""
|
||||||
|
self._require_connected()
|
||||||
|
with self._lock:
|
||||||
|
batch = self._memory.dequeue_batch(max_items=max_items)
|
||||||
|
if batch:
|
||||||
|
self._compact()
|
||||||
|
return batch
|
||||||
|
|
||||||
|
def size(self) -> int:
|
||||||
|
"""Return the number of items currently in the in-memory buffer."""
|
||||||
|
self._require_connected()
|
||||||
|
return self._memory.size()
|
||||||
|
|
||||||
|
def evict_older_than(self, cutoff: datetime) -> int:
|
||||||
|
"""Evict aged items from memory and compact the JSONL file."""
|
||||||
|
self._require_connected()
|
||||||
|
with self._lock:
|
||||||
|
removed = self._memory.evict_older_than(cutoff)
|
||||||
|
if removed:
|
||||||
|
self._compact()
|
||||||
|
return removed
|
||||||
|
|
||||||
|
def _require_connected(self) -> None:
|
||||||
|
if not self._connected:
|
||||||
|
raise RuntimeError("FileBackedQueueAdapter is not connected")
|
||||||
|
|
||||||
|
def _replay_file(self, path: Path) -> None:
|
||||||
|
cutoff: datetime | None = None
|
||||||
|
if self._config.age_key is not None:
|
||||||
|
cutoff = datetime.now(UTC) - timedelta(hours=self._config.max_age_hours)
|
||||||
|
with path.open(encoding="utf-8") as handle:
|
||||||
|
for line_number, line in enumerate(handle, start=1):
|
||||||
|
stripped = line.strip()
|
||||||
|
if not stripped:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
payload = json.loads(stripped)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
self.logger.warning(
|
||||||
|
"Skipping corrupt JSONL line",
|
||||||
|
path=str(path),
|
||||||
|
line_number=line_number,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
self.logger.warning(
|
||||||
|
"Skipping non-object JSONL line",
|
||||||
|
path=str(path),
|
||||||
|
line_number=line_number,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
item: dict[str, Any] = payload
|
||||||
|
if cutoff is not None and self._config.age_key is not None:
|
||||||
|
age_value = item.get(self._config.age_key)
|
||||||
|
if age_value is None:
|
||||||
|
self.logger.warning(
|
||||||
|
"Skipping item missing age key on replay",
|
||||||
|
path=str(path),
|
||||||
|
line_number=line_number,
|
||||||
|
age_key=self._config.age_key,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if MemoryQueueAdapter._parse_age(age_value) < cutoff:
|
||||||
|
continue
|
||||||
|
except ValueError:
|
||||||
|
self.logger.warning(
|
||||||
|
"Skipping item with invalid age on replay",
|
||||||
|
path=str(path),
|
||||||
|
line_number=line_number,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
self._memory.enqueue_and_return_added([item])
|
||||||
|
except ValueError as exc:
|
||||||
|
self.logger.warning(
|
||||||
|
"Skipping invalid item on replay",
|
||||||
|
path=str(path),
|
||||||
|
line_number=line_number,
|
||||||
|
error=str(exc),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _append_items(self, items: list[dict[str, Any]]) -> None:
|
||||||
|
if self._append_handle is None:
|
||||||
|
raise RuntimeError("append handle is not open")
|
||||||
|
for item in items:
|
||||||
|
self._append_handle.write(
|
||||||
|
json.dumps(item, default=self._json_default, separators=(",", ":"))
|
||||||
|
)
|
||||||
|
self._append_handle.write("\n")
|
||||||
|
self._append_handle.flush()
|
||||||
|
|
||||||
|
def _compact(self) -> None:
|
||||||
|
"""Rewrite the JSONL file from the current in-memory snapshot."""
|
||||||
|
path = self._config.path
|
||||||
|
tmp_path = path.with_suffix(path.suffix + ".tmp")
|
||||||
|
if self._append_handle is not None:
|
||||||
|
self._append_handle.flush()
|
||||||
|
self._append_handle.close()
|
||||||
|
self._append_handle = None
|
||||||
|
with tmp_path.open("w", encoding="utf-8") as handle:
|
||||||
|
for item in self._memory.snapshot():
|
||||||
|
handle.write(
|
||||||
|
json.dumps(item, default=self._json_default, separators=(",", ":"))
|
||||||
|
)
|
||||||
|
handle.write("\n")
|
||||||
|
handle.flush()
|
||||||
|
tmp_path.replace(path)
|
||||||
|
self._append_handle = path.open("a", encoding="utf-8")
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
"""In-memory queue adapter with optional dedup and age-based eviction."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import OrderedDict
|
||||||
|
from collections.abc import Hashable
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import threading
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from python_repositories.interfaces.queue_repository_interface import (
|
||||||
|
QueueRepositoryInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MemoryQueueAdapter(QueueRepositoryInterface):
|
||||||
|
"""Thread-safe in-process FIFO queue with optional deduplication."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
dedup_keys: tuple[str, ...] = (),
|
||||||
|
age_key: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._dedup_keys = dedup_keys
|
||||||
|
self._age_key = age_key
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self._items: OrderedDict[Hashable, dict[str, Any]] = OrderedDict()
|
||||||
|
self._seq = 0
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_age(value: object) -> datetime:
|
||||||
|
"""Normalize an age field to a timezone-aware UTC datetime."""
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
if value.tzinfo is None:
|
||||||
|
return value.replace(tzinfo=UTC)
|
||||||
|
return value.astimezone(UTC)
|
||||||
|
if isinstance(value, str):
|
||||||
|
normalized = value.replace("Z", "+00:00")
|
||||||
|
parsed = datetime.fromisoformat(normalized)
|
||||||
|
if parsed.tzinfo is None:
|
||||||
|
return parsed.replace(tzinfo=UTC)
|
||||||
|
return parsed.astimezone(UTC)
|
||||||
|
raise ValueError(
|
||||||
|
f"age value must be datetime or ISO-8601 str, got {type(value)!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def enqueue(self, items: list[dict[str, Any]]) -> None:
|
||||||
|
"""Append items; skip duplicates when ``dedup_keys`` is configured."""
|
||||||
|
self.enqueue_and_return_added(items)
|
||||||
|
|
||||||
|
def enqueue_and_return_added(
|
||||||
|
self, items: list[dict[str, Any]]
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Enqueue items and return the subset that was newly stored."""
|
||||||
|
if not items:
|
||||||
|
return []
|
||||||
|
added: list[dict[str, Any]] = []
|
||||||
|
with self._lock:
|
||||||
|
for raw in items:
|
||||||
|
item = self._normalize_item(raw)
|
||||||
|
key = self._make_key(item)
|
||||||
|
if self._dedup_keys and key in self._items:
|
||||||
|
continue
|
||||||
|
self._items[key] = item
|
||||||
|
added.append(item)
|
||||||
|
return added
|
||||||
|
|
||||||
|
def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
|
||||||
|
"""Remove and return up to ``max_items`` items in FIFO order."""
|
||||||
|
if max_items < 0:
|
||||||
|
raise ValueError("max_items must be >= 0")
|
||||||
|
with self._lock:
|
||||||
|
batch: list[dict[str, Any]] = []
|
||||||
|
for _ in range(min(max_items, len(self._items))):
|
||||||
|
_key, item = self._items.popitem(last=False)
|
||||||
|
batch.append(item)
|
||||||
|
return batch
|
||||||
|
|
||||||
|
def size(self) -> int:
|
||||||
|
"""Return the number of items currently in the queue."""
|
||||||
|
with self._lock:
|
||||||
|
return len(self._items)
|
||||||
|
|
||||||
|
def evict_older_than(self, cutoff: datetime) -> int:
|
||||||
|
"""Remove items whose age field is strictly older than ``cutoff``."""
|
||||||
|
if self._age_key is None:
|
||||||
|
return 0
|
||||||
|
cutoff_utc = self._parse_age(cutoff)
|
||||||
|
removed = 0
|
||||||
|
with self._lock:
|
||||||
|
to_remove = [
|
||||||
|
key
|
||||||
|
for key, item in self._items.items()
|
||||||
|
if self._parse_age(item[self._age_key]) < cutoff_utc
|
||||||
|
]
|
||||||
|
for key in to_remove:
|
||||||
|
del self._items[key]
|
||||||
|
removed += 1
|
||||||
|
return removed
|
||||||
|
|
||||||
|
def clear(self) -> None:
|
||||||
|
"""Remove all items from the queue."""
|
||||||
|
with self._lock:
|
||||||
|
self._items.clear()
|
||||||
|
|
||||||
|
def snapshot(self) -> list[dict[str, Any]]:
|
||||||
|
"""Return a shallow copy of queued items in FIFO order."""
|
||||||
|
with self._lock:
|
||||||
|
return [dict(item) for item in self._items.values()]
|
||||||
|
|
||||||
|
def _normalize_item(self, raw: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
item = dict(raw)
|
||||||
|
if self._dedup_keys:
|
||||||
|
missing = [key for key in self._dedup_keys if key not in item]
|
||||||
|
if missing:
|
||||||
|
raise ValueError(f"item missing dedup key(s): {missing}")
|
||||||
|
if self._age_key is not None:
|
||||||
|
if self._age_key not in item:
|
||||||
|
raise ValueError(f"item missing age key: {self._age_key!r}")
|
||||||
|
item[self._age_key] = self._parse_age(item[self._age_key])
|
||||||
|
return item
|
||||||
|
|
||||||
|
def _make_key(self, item: dict[str, Any]) -> Hashable:
|
||||||
|
if not self._dedup_keys:
|
||||||
|
self._seq += 1
|
||||||
|
return self._seq
|
||||||
|
return tuple(item[key] for key in self._dedup_keys)
|
||||||
@@ -1,128 +1,135 @@
|
|||||||
"""Definition of MinioAdapter class."""
|
"""Definition of MinioAdapter class."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
import os
|
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from importlib.util import find_spec
|
|
||||||
import structlog
|
|
||||||
from python_utils import check_env
|
|
||||||
|
|
||||||
from python_repositories.interfaces import (
|
from python_repositories.adapters.connection_aware_adapter import (
|
||||||
ContextAwareInterface,
|
ConnectionAwareAdapter,
|
||||||
ConnectionAwareInterface,
|
|
||||||
)
|
)
|
||||||
|
from python_repositories.config import MinioConfig
|
||||||
|
from python_repositories.interfaces import ObjectRepositoryInterface
|
||||||
|
|
||||||
# Handle optional dependencies
|
try:
|
||||||
if find_spec("minio") is not None:
|
|
||||||
import minio
|
import minio
|
||||||
|
except ImportError as exc:
|
||||||
|
raise ImportError(
|
||||||
|
"MinIO support requires the minio extra. "
|
||||||
|
"Install with: pip install python-repositories[minio]"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
class MinioAdapter(
|
class MinioAdapter(ConnectionAwareAdapter, ObjectRepositoryInterface):
|
||||||
ContextAwareInterface,
|
|
||||||
ConnectionAwareInterface,
|
|
||||||
):
|
|
||||||
"""Minio adapter exposing basic CRUD functionality."""
|
"""Minio adapter exposing basic CRUD functionality."""
|
||||||
|
|
||||||
endpoint_env_var_name: str = "MINIO_ENDPOINT"
|
endpoint_env_var_name: str = "MINIO_ENDPOINT"
|
||||||
access_key_env_var_name: str = "MINIO_ACCESS_KEY"
|
access_key_env_var_name: str = "MINIO_ACCESS_KEY"
|
||||||
secret_key_env_var_name: str = "MINIO_SECRET_KEY"
|
secret_key_env_var_name: str = "MINIO_SECRET_KEY"
|
||||||
bucket_env_var_name: str = "MINIO_BUCKET"
|
bucket_env_var_name: str = "MINIO_BUCKET"
|
||||||
|
secure_env_var_name: str = "MINIO_SECURE"
|
||||||
|
create_bucket_if_missing_env_var_name: str = "MINIO_CREATE_BUCKET_IF_MISSING"
|
||||||
chunk_size: int = 5 * 2**20 # 5 MiB
|
chunk_size: int = 5 * 2**20 # 5 MiB
|
||||||
|
connection_name: str = "Minio"
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(
|
||||||
# Setup logger
|
self,
|
||||||
self.logger = structlog.get_logger(
|
*,
|
||||||
self.__class__.__name__,
|
config: MinioConfig | None = None,
|
||||||
)
|
client: minio.Minio | None = None,
|
||||||
# Check environment variables
|
) -> None:
|
||||||
check_env(
|
super().__init__()
|
||||||
{
|
if client is not None and config is None:
|
||||||
|
raise ValueError("config is required when client is provided")
|
||||||
|
if config is None:
|
||||||
|
config = MinioConfig.from_env(
|
||||||
self.endpoint_env_var_name,
|
self.endpoint_env_var_name,
|
||||||
self.access_key_env_var_name,
|
self.access_key_env_var_name,
|
||||||
self.secret_key_env_var_name,
|
self.secret_key_env_var_name,
|
||||||
self.bucket_env_var_name,
|
self.bucket_env_var_name,
|
||||||
},
|
self.secure_env_var_name,
|
||||||
)
|
self.create_bucket_if_missing_env_var_name,
|
||||||
# Prepare internal variables
|
)
|
||||||
self._client: minio.Minio | None = None
|
self._config = config
|
||||||
self._bucket_name: str | None = None
|
self._client_injected = client is not None
|
||||||
|
self._client: minio.Minio | None = client
|
||||||
|
self._bucket_name: str | None = config.bucket if client is not None else None
|
||||||
|
if self._client_injected:
|
||||||
|
self._invalidate_health_cache()
|
||||||
|
|
||||||
def __enter__(self) -> MinioAdapter:
|
def _is_client_ready(self) -> bool:
|
||||||
"""Enter the context."""
|
return self._client is not None and self._bucket_name is not None
|
||||||
self.connect()
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __exit__(
|
def _validate_injected_client(self) -> None:
|
||||||
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
if self._client is None:
|
||||||
) -> None:
|
|
||||||
"""Exit the context."""
|
|
||||||
ctx_info = {"exc_type": exc_type, "exc_val": exc_val, "exc_tb": exc_tb}
|
|
||||||
if any(
|
|
||||||
(
|
|
||||||
exc_type is not None,
|
|
||||||
exc_val is not None,
|
|
||||||
exc_tb is not None,
|
|
||||||
),
|
|
||||||
):
|
|
||||||
self.logger.error("Error while exiting context", **ctx_info)
|
|
||||||
self.disconnect()
|
|
||||||
|
|
||||||
def connect(self) -> None:
|
|
||||||
"""Connect to the Minio server."""
|
|
||||||
# Stop if already connected
|
|
||||||
if self.is_connected:
|
|
||||||
self.logger.info("Already connected to Minio")
|
|
||||||
return
|
return
|
||||||
# Prepare arguments
|
try:
|
||||||
endpoint = str(os.getenv(self.endpoint_env_var_name))
|
_ = self._client.list_buckets()
|
||||||
access_key = str(os.getenv(self.access_key_env_var_name))
|
except Exception as exc: # pylint: disable=broad-except
|
||||||
secret_key = str(os.getenv(self.secret_key_env_var_name))
|
raise ConnectionError(
|
||||||
bucket = str(os.getenv(self.bucket_env_var_name))
|
f"Could not connect to Minio at {self._config.endpoint}"
|
||||||
# Connect client
|
) from exc
|
||||||
|
|
||||||
|
def _establish_connection(self) -> None:
|
||||||
|
endpoint = self._config.endpoint
|
||||||
|
access_key = self._config.access_key
|
||||||
|
secret_key = self._config.secret_key
|
||||||
|
bucket = self._config.bucket
|
||||||
client = minio.Minio(
|
client = minio.Minio(
|
||||||
endpoint=endpoint,
|
endpoint=endpoint,
|
||||||
access_key=access_key,
|
access_key=access_key,
|
||||||
secret_key=secret_key,
|
secret_key=secret_key,
|
||||||
secure=False,
|
secure=self._config.secure,
|
||||||
)
|
)
|
||||||
# Test the connection by listing buckets (will raise if connection fails)
|
|
||||||
try:
|
try:
|
||||||
_ = client.list_buckets()
|
_ = client.list_buckets()
|
||||||
except Exception as exc: # pylint: disable=broad-except
|
except Exception as exc: # pylint: disable=broad-except
|
||||||
raise ConnectionError(f"Could not connect to Minio at {endpoint}") from exc
|
raise ConnectionError(f"Could not connect to Minio at {endpoint}") from exc
|
||||||
# Ensure bucket exists
|
|
||||||
if not client.bucket_exists(bucket):
|
if not client.bucket_exists(bucket):
|
||||||
self.logger.info(f"Creating bucket '{bucket}'")
|
if not self._config.create_bucket_if_missing:
|
||||||
|
raise ConnectionError(
|
||||||
|
f"Bucket '{bucket}' does not exist on Minio at {endpoint}"
|
||||||
|
)
|
||||||
|
self.logger.info("Creating bucket", bucket=bucket)
|
||||||
client.make_bucket(bucket)
|
client.make_bucket(bucket)
|
||||||
# Persist information
|
|
||||||
self._client = client
|
self._client = client
|
||||||
self._bucket_name = bucket
|
self._bucket_name = bucket
|
||||||
|
|
||||||
def disconnect(self) -> None:
|
def disconnect(self) -> None:
|
||||||
"""Disconnect from the Minio server."""
|
"""Disconnect from the Minio server."""
|
||||||
# Close connection
|
# N.B. Minio client does not have a close method, but we include this for symmetry
|
||||||
# N.B. Minio client does not have a close method, but we include this for symmetry with other adapters
|
|
||||||
# Reset client
|
|
||||||
self._client = None
|
self._client = None
|
||||||
self._bucket_name = None
|
self._bucket_name = None
|
||||||
|
self._invalidate_health_cache()
|
||||||
|
|
||||||
@property
|
def _probe_connection(self) -> bool:
|
||||||
def is_connected(self) -> bool:
|
assert self._client is not None and self._bucket_name is not None
|
||||||
"""Check if connected to Minio server."""
|
try:
|
||||||
res = bool(isinstance(self._client, minio.Minio))
|
return bool(self._client.bucket_exists(self._bucket_name))
|
||||||
self.logger.debug(res)
|
except Exception: # pylint: disable=broad-except
|
||||||
return res
|
return False
|
||||||
|
|
||||||
def _put(self, object_name: str, data: BytesIO) -> None:
|
def put(
|
||||||
"""Put an object into the Minio bucket."""
|
self,
|
||||||
|
object_name: str,
|
||||||
|
data: BytesIO,
|
||||||
|
content_type: str = "application/octet-stream",
|
||||||
|
) -> None:
|
||||||
|
"""Put an object into the Minio bucket.
|
||||||
|
|
||||||
|
Accepts zero-byte ``BytesIO``. A zero-byte object is returned by
|
||||||
|
``get()`` as an empty buffer, not ``None``. Use ``delete()`` to remove
|
||||||
|
an object entirely.
|
||||||
|
"""
|
||||||
# Check input
|
# Check input
|
||||||
if not isinstance(object_name, str) or len(object_name) == 0:
|
if not isinstance(object_name, str) or len(object_name) == 0:
|
||||||
raise ValueError("object_name must be a non-empty string")
|
raise ValueError("object_name must be a non-empty string")
|
||||||
if not isinstance(data, BytesIO) or data.getbuffer().nbytes == 0:
|
if not isinstance(data, BytesIO):
|
||||||
raise ValueError("data must be a non-empty BytesIO object")
|
raise ValueError("data must be a BytesIO object")
|
||||||
|
if not isinstance(content_type, str) or len(content_type) == 0:
|
||||||
|
raise ValueError("content_type must be a non-empty string")
|
||||||
# Check connection
|
# Check connection
|
||||||
if self._client is None or not self.is_connected:
|
self._require_connected()
|
||||||
raise ConnectionError("Not connected to Minio")
|
assert self._client is not None and self._bucket_name is not None
|
||||||
# Prepare buffer for reading
|
# Prepare buffer for reading
|
||||||
num_bytes = data.getbuffer().nbytes
|
num_bytes = data.getbuffer().nbytes
|
||||||
data.seek(0)
|
data.seek(0)
|
||||||
@@ -134,21 +141,28 @@ class MinioAdapter(
|
|||||||
data=data,
|
data=data,
|
||||||
length=num_bytes,
|
length=num_bytes,
|
||||||
part_size=self.chunk_size,
|
part_size=self.chunk_size,
|
||||||
|
content_type=content_type,
|
||||||
)
|
)
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Put object '{object_name}' into bucket '{self._bucket_name}'"
|
"Put object", object_name=object_name, bucket=self._bucket_name
|
||||||
)
|
)
|
||||||
|
|
||||||
def _get(self, object_name: str) -> BytesIO | None:
|
def get(self, object_name: str) -> BytesIO | None:
|
||||||
"""Get an object from the Minio bucket."""
|
"""Get an object from the Minio bucket.
|
||||||
|
|
||||||
|
Returns ``None`` when the object does not exist. Returns an empty
|
||||||
|
``BytesIO`` for a zero-byte object. Use ``value is not None`` to test
|
||||||
|
existence.
|
||||||
|
"""
|
||||||
# Check input
|
# Check input
|
||||||
if not isinstance(object_name, str) or len(object_name) == 0:
|
if not isinstance(object_name, str) or len(object_name) == 0:
|
||||||
raise ValueError("object_name must be a non-empty string")
|
raise ValueError("object_name must be a non-empty string")
|
||||||
# Check connection
|
# Check connection
|
||||||
if self._client is None or not self.is_connected:
|
self._require_connected()
|
||||||
raise ConnectionError("Not connected to Minio")
|
assert self._client is not None and self._bucket_name is not None
|
||||||
# Get data from bucket
|
# Get data from bucket
|
||||||
# N.B. bucket name is set when connecting
|
# N.B. bucket name is set when connecting
|
||||||
|
response = None
|
||||||
try:
|
try:
|
||||||
response = self._client.get_object(
|
response = self._client.get_object(
|
||||||
bucket_name=self._bucket_name,
|
bucket_name=self._bucket_name,
|
||||||
@@ -160,28 +174,31 @@ class MinioAdapter(
|
|||||||
buffer.write(chunk)
|
buffer.write(chunk)
|
||||||
buffer.seek(0)
|
buffer.seek(0)
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Got object '{object_name}' from bucket '{self._bucket_name}'"
|
"Got object", object_name=object_name, bucket=self._bucket_name
|
||||||
)
|
)
|
||||||
return buffer
|
return buffer
|
||||||
except minio.S3Error as exc:
|
except minio.S3Error as exc:
|
||||||
if exc.code == "NoSuchKey":
|
if exc.code == "NoSuchKey":
|
||||||
self.logger.warning(
|
self.logger.warning(
|
||||||
f"Object '{object_name}' not found in bucket '{self._bucket_name}'"
|
"Object not found",
|
||||||
|
object_name=object_name,
|
||||||
|
bucket=self._bucket_name,
|
||||||
)
|
)
|
||||||
else:
|
return None
|
||||||
self.logger.error(repr(exc))
|
raise
|
||||||
except Exception as exc: # pylint: disable=broad-except
|
finally:
|
||||||
self.logger.error(repr(exc))
|
if response is not None:
|
||||||
return None
|
response.close()
|
||||||
|
response.release_conn()
|
||||||
|
|
||||||
def _delete(self, object_name: str) -> None:
|
def delete(self, object_name: str) -> None:
|
||||||
"""Delete an object from the Minio bucket."""
|
"""Delete an object from the Minio bucket."""
|
||||||
# Check input
|
# Check input
|
||||||
if not isinstance(object_name, str) or len(object_name) == 0:
|
if not isinstance(object_name, str) or len(object_name) == 0:
|
||||||
raise ValueError("object_name must be a non-empty string")
|
raise ValueError("object_name must be a non-empty string")
|
||||||
# Check connection
|
# Check connection
|
||||||
if self._client is None or not self.is_connected:
|
self._require_connected()
|
||||||
raise ConnectionError("Not connected to Minio")
|
assert self._client is not None and self._bucket_name is not None
|
||||||
# Delete object from bucket
|
# Delete object from bucket
|
||||||
# N.B. bucket name is set when connecting
|
# N.B. bucket name is set when connecting
|
||||||
self._client.remove_object(
|
self._client.remove_object(
|
||||||
@@ -189,18 +206,18 @@ class MinioAdapter(
|
|||||||
object_name=object_name,
|
object_name=object_name,
|
||||||
)
|
)
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Deleted object '{object_name}' from bucket '{self._bucket_name}'"
|
"Deleted object", object_name=object_name, bucket=self._bucket_name
|
||||||
)
|
)
|
||||||
|
|
||||||
def _list_objects(self, prefix: str = "") -> list[str]:
|
def list_objects(self, prefix: str = "") -> list[str]:
|
||||||
"""List objects in the Minio bucket with an optional prefix."""
|
"""List objects in the Minio bucket with an optional prefix."""
|
||||||
# Check input
|
# Check input
|
||||||
if not isinstance(prefix, str):
|
if not isinstance(prefix, str):
|
||||||
raise ValueError("prefix must be a string")
|
raise ValueError("prefix must be a string")
|
||||||
# Check connection
|
# Check connection
|
||||||
# N.B. bucket name is set when connecting
|
# N.B. bucket name is set when connecting
|
||||||
if self._client is None or not self.is_connected:
|
self._require_connected()
|
||||||
raise ConnectionError("Not connected to Minio")
|
assert self._client is not None and self._bucket_name is not None
|
||||||
# List objects in bucket
|
# List objects in bucket
|
||||||
objects = self._client.list_objects(
|
objects = self._client.list_objects(
|
||||||
bucket_name=self._bucket_name,
|
bucket_name=self._bucket_name,
|
||||||
@@ -211,6 +228,9 @@ class MinioAdapter(
|
|||||||
obj.object_name for obj in objects if obj.object_name is not None
|
obj.object_name for obj in objects if obj.object_name is not None
|
||||||
]
|
]
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Listed {len(object_names)} object(s) in bucket '{self._bucket_name}' with prefix '{prefix}'"
|
"Listed objects",
|
||||||
|
count=len(object_names),
|
||||||
|
bucket=self._bucket_name,
|
||||||
|
prefix=prefix,
|
||||||
)
|
)
|
||||||
return object_names
|
return object_names
|
||||||
|
|||||||
@@ -0,0 +1,230 @@
|
|||||||
|
"""Definition of PostgresAdapter class."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from python_repositories.adapters.connection_aware_adapter import (
|
||||||
|
ConnectionAwareAdapter,
|
||||||
|
)
|
||||||
|
from python_repositories.config import PostgresConfig
|
||||||
|
from python_repositories.interfaces import TableRepositoryInterface
|
||||||
|
|
||||||
|
try:
|
||||||
|
import psycopg
|
||||||
|
from psycopg import sql
|
||||||
|
from psycopg.errors import UndefinedTable
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
except ImportError as exc:
|
||||||
|
raise ImportError(
|
||||||
|
"Postgres support requires the postgres extra. "
|
||||||
|
"Install with: pip install python-repositories[postgres]"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
class PostgresAdapter(ConnectionAwareAdapter, TableRepositoryInterface):
|
||||||
|
"""Postgres adapter exposing basic table CRUD functionality."""
|
||||||
|
|
||||||
|
uri_env_var_name: str = "POSTGRES_URI"
|
||||||
|
table_env_var_name: str = "POSTGRES_TABLE"
|
||||||
|
primary_key_env_var_name: str = "POSTGRES_PRIMARY_KEY"
|
||||||
|
connection_name: str = "Postgres"
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
config: PostgresConfig | None = None,
|
||||||
|
client: psycopg.Connection[Any] | None = None,
|
||||||
|
) -> None:
|
||||||
|
super().__init__()
|
||||||
|
if client is not None and config is None:
|
||||||
|
raise ValueError("config is required when client is provided")
|
||||||
|
if config is None:
|
||||||
|
config = PostgresConfig.from_env(
|
||||||
|
self.uri_env_var_name,
|
||||||
|
self.table_env_var_name,
|
||||||
|
self.primary_key_env_var_name,
|
||||||
|
)
|
||||||
|
self._config = config
|
||||||
|
self._client_injected = client is not None
|
||||||
|
self._client: psycopg.Connection[Any] | None = client
|
||||||
|
if self._client_injected:
|
||||||
|
self._invalidate_health_cache()
|
||||||
|
|
||||||
|
def _is_client_ready(self) -> bool:
|
||||||
|
return self._client is not None
|
||||||
|
|
||||||
|
def _table_identifier(self) -> sql.Identifier:
|
||||||
|
return sql.Identifier(self._config.table)
|
||||||
|
|
||||||
|
def _primary_key_identifier(self) -> sql.Identifier:
|
||||||
|
return sql.Identifier(self._config.primary_key)
|
||||||
|
|
||||||
|
def _validate_injected_client(self) -> None:
|
||||||
|
if self._client is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self._run_connection_probe()
|
||||||
|
except (psycopg.Error, ConnectionError) as exc:
|
||||||
|
raise ConnectionError(
|
||||||
|
f"Could not connect to Postgres at {self._config.uri}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def _establish_connection(self) -> None:
|
||||||
|
uri = self._config.uri
|
||||||
|
try:
|
||||||
|
client = psycopg.connect(
|
||||||
|
uri,
|
||||||
|
row_factory=dict_row,
|
||||||
|
connect_timeout=10,
|
||||||
|
autocommit=True,
|
||||||
|
)
|
||||||
|
self._client = client
|
||||||
|
self._run_connection_probe()
|
||||||
|
except ConnectionError:
|
||||||
|
self._client = None
|
||||||
|
raise
|
||||||
|
except psycopg.Error as exc:
|
||||||
|
self._client = None
|
||||||
|
raise ConnectionError(f"Could not connect to Postgres at {uri}") from exc
|
||||||
|
|
||||||
|
def _run_connection_probe(self) -> None:
|
||||||
|
assert self._client is not None
|
||||||
|
with self._client.cursor() as cursor:
|
||||||
|
cursor.execute("SELECT 1")
|
||||||
|
try:
|
||||||
|
cursor.execute(
|
||||||
|
sql.SQL("SELECT 1 FROM {} LIMIT 0").format(self._table_identifier())
|
||||||
|
)
|
||||||
|
except UndefinedTable as exc:
|
||||||
|
raise ConnectionError(
|
||||||
|
f"Table '{self._config.table}' does not exist on Postgres at "
|
||||||
|
f"{self._config.uri}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def disconnect(self) -> None:
|
||||||
|
"""Disconnect from the Postgres server."""
|
||||||
|
if self._client is not None and not self._client_injected:
|
||||||
|
self._client.close()
|
||||||
|
self._client = None
|
||||||
|
self._invalidate_health_cache()
|
||||||
|
|
||||||
|
def _probe_connection(self) -> bool:
|
||||||
|
assert self._client is not None
|
||||||
|
try:
|
||||||
|
self._run_connection_probe()
|
||||||
|
except (psycopg.Error, ConnectionError):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _validate_pk(self, pk: Any) -> None:
|
||||||
|
if pk is None:
|
||||||
|
raise ValueError("Primary key must not be None")
|
||||||
|
|
||||||
|
def _validate_row(self, row: dict[str, Any]) -> None:
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
raise ValueError("Row must be a dictionary")
|
||||||
|
if self._config.primary_key not in row:
|
||||||
|
raise ValueError(
|
||||||
|
f"Row must include primary key column '{self._config.primary_key}'"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _validate_limit(self, limit: int | None) -> None:
|
||||||
|
if limit is not None and (not isinstance(limit, int) or limit < 0):
|
||||||
|
raise ValueError("Limit must be a non-negative integer or None")
|
||||||
|
|
||||||
|
def fetch_one(self, pk: Any) -> dict[str, Any] | None:
|
||||||
|
"""Fetch a single row by primary key."""
|
||||||
|
self._validate_pk(pk)
|
||||||
|
self._require_connected()
|
||||||
|
assert self._client is not None
|
||||||
|
query = sql.SQL("SELECT * FROM {} WHERE {} = %s").format(
|
||||||
|
self._table_identifier(),
|
||||||
|
self._primary_key_identifier(),
|
||||||
|
)
|
||||||
|
with self._client.cursor() as cursor:
|
||||||
|
cursor.execute(query, (pk,))
|
||||||
|
row = cursor.fetchone()
|
||||||
|
self.logger.debug("Fetched row", pk=pk, found=row is not None)
|
||||||
|
return row
|
||||||
|
|
||||||
|
def fetch_all(self, *, limit: int | None = None) -> list[dict[str, Any]]:
|
||||||
|
"""Fetch all rows from the configured table."""
|
||||||
|
self._validate_limit(limit)
|
||||||
|
self._require_connected()
|
||||||
|
assert self._client is not None
|
||||||
|
query = sql.SQL("SELECT * FROM {}").format(self._table_identifier())
|
||||||
|
params: tuple[Any, ...] = ()
|
||||||
|
if limit is not None:
|
||||||
|
query = sql.Composed([query, sql.SQL(" LIMIT %s")])
|
||||||
|
params = (limit,)
|
||||||
|
with self._client.cursor() as cursor:
|
||||||
|
cursor.execute(query, params)
|
||||||
|
rows = cursor.fetchall()
|
||||||
|
self.logger.debug("Fetched rows", count=len(rows), limit=limit)
|
||||||
|
return list(rows)
|
||||||
|
|
||||||
|
def upsert(self, row: dict[str, Any]) -> None:
|
||||||
|
"""Insert or update a row by primary key."""
|
||||||
|
self._validate_row(row)
|
||||||
|
self._require_connected()
|
||||||
|
assert self._client is not None
|
||||||
|
pk_col = self._config.primary_key
|
||||||
|
columns = list(row.keys())
|
||||||
|
identifiers = [sql.Identifier(column) for column in columns]
|
||||||
|
placeholders = sql.SQL(", ").join(sql.Placeholder() * len(columns))
|
||||||
|
column_list = sql.SQL(", ").join(identifiers)
|
||||||
|
update_columns = [column for column in columns if column != pk_col]
|
||||||
|
on_conflict: sql.Composed | sql.SQL
|
||||||
|
if update_columns:
|
||||||
|
update_assignments = sql.SQL(", ").join(
|
||||||
|
sql.SQL("{} = EXCLUDED.{}").format(
|
||||||
|
sql.Identifier(column),
|
||||||
|
sql.Identifier(column),
|
||||||
|
)
|
||||||
|
for column in update_columns
|
||||||
|
)
|
||||||
|
on_conflict = sql.SQL("DO UPDATE SET {}").format(update_assignments)
|
||||||
|
else:
|
||||||
|
on_conflict = sql.SQL("DO NOTHING")
|
||||||
|
query = sql.SQL("INSERT INTO {} ({}) VALUES ({}) ON CONFLICT ({}) {}").format(
|
||||||
|
self._table_identifier(),
|
||||||
|
column_list,
|
||||||
|
placeholders,
|
||||||
|
self._primary_key_identifier(),
|
||||||
|
on_conflict,
|
||||||
|
)
|
||||||
|
with self._client.cursor() as cursor:
|
||||||
|
cursor.execute(query, tuple(row[column] for column in columns))
|
||||||
|
self.logger.debug("Upserted row", pk=row[pk_col], columns=columns)
|
||||||
|
|
||||||
|
def delete(self, pk: Any) -> None:
|
||||||
|
"""Delete a row by primary key."""
|
||||||
|
self._validate_pk(pk)
|
||||||
|
self._require_connected()
|
||||||
|
assert self._client is not None
|
||||||
|
query = sql.SQL("DELETE FROM {} WHERE {} = %s").format(
|
||||||
|
self._table_identifier(),
|
||||||
|
self._primary_key_identifier(),
|
||||||
|
)
|
||||||
|
with self._client.cursor() as cursor:
|
||||||
|
cursor.execute(query, (pk,))
|
||||||
|
self.logger.debug("Deleted row", pk=pk)
|
||||||
|
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
sql_text: str,
|
||||||
|
params: tuple[Any, ...] = (),
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Run a read SQL statement and return rows as dicts."""
|
||||||
|
if not isinstance(sql_text, str) or len(sql_text) == 0:
|
||||||
|
raise ValueError("SQL must be a non-empty string")
|
||||||
|
if not isinstance(params, tuple):
|
||||||
|
raise ValueError("Params must be a tuple")
|
||||||
|
self._require_connected()
|
||||||
|
assert self._client is not None
|
||||||
|
with self._client.cursor() as cursor:
|
||||||
|
cursor.execute(sql_text, params)
|
||||||
|
rows = cursor.fetchall()
|
||||||
|
self.logger.debug("Executed SQL", row_count=len(rows))
|
||||||
|
return list(rows)
|
||||||
@@ -1,70 +1,70 @@
|
|||||||
"""Definition of RedisAdapter class."""
|
"""Definition of RedisAdapter class."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
from typing import cast
|
|
||||||
from importlib.util import find_spec
|
|
||||||
import os
|
|
||||||
import structlog
|
|
||||||
|
|
||||||
from python_utils import check_env
|
from collections.abc import Iterator
|
||||||
|
from typing import Any, cast
|
||||||
|
|
||||||
from python_repositories.interfaces import (
|
from python_repositories.adapters.connection_aware_adapter import (
|
||||||
ContextAwareInterface,
|
ConnectionAwareAdapter,
|
||||||
ConnectionAwareInterface,
|
|
||||||
)
|
)
|
||||||
|
from python_repositories.config import RedisConfig
|
||||||
|
from python_repositories.interfaces import JsonRepositoryInterface
|
||||||
|
|
||||||
# Handle optional dependencies
|
try:
|
||||||
if find_spec("redis") is not None:
|
|
||||||
import redis
|
import redis
|
||||||
from redis.commands.json.path import Path as RedisPath
|
from redis.commands.json.path import Path as RedisPath
|
||||||
|
except ImportError as exc:
|
||||||
|
raise ImportError(
|
||||||
|
"Redis support requires the redis extra. "
|
||||||
|
"Install with: pip install python-repositories[redis]"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
class RedisAdapter(
|
class RedisAdapter(ConnectionAwareAdapter, JsonRepositoryInterface):
|
||||||
ContextAwareInterface,
|
|
||||||
ConnectionAwareInterface,
|
|
||||||
):
|
|
||||||
"""Redis adapter exposing basic CRUD functionality."""
|
"""Redis adapter exposing basic CRUD functionality."""
|
||||||
|
|
||||||
uri_env_var_name: str = "REDIS_URI"
|
uri_env_var_name: str = "REDIS_URI"
|
||||||
path: str = "." # JSON root path, updated in __init__
|
path: str = "." # JSON root path, updated in __init__
|
||||||
encoding: str = "UTF-8"
|
encoding: str = "UTF-8"
|
||||||
|
connection_name: str = "Redis"
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(
|
||||||
# Setup logger
|
self,
|
||||||
self.logger = structlog.get_logger(
|
*,
|
||||||
self.__class__.__name__,
|
config: RedisConfig | None = None,
|
||||||
)
|
client: redis.Redis | None = None,
|
||||||
# Check environment variables
|
|
||||||
check_env(self.uri_env_var_name)
|
|
||||||
# Prepare internal variables
|
|
||||||
self._client: redis.Redis | None = None
|
|
||||||
self.path: str = RedisPath.root_path()
|
|
||||||
|
|
||||||
def __enter__(self) -> RedisAdapter:
|
|
||||||
"""Enter the context."""
|
|
||||||
self.connect()
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __exit__(
|
|
||||||
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Exit the context."""
|
super().__init__()
|
||||||
ctx_info = {"exc_type": exc_type, "exc_val": exc_val, "exc_tb": exc_tb}
|
if client is not None and config is None:
|
||||||
if any(
|
raise ValueError("config is required when client is provided")
|
||||||
(
|
if config is None:
|
||||||
exc_type is not None,
|
config = RedisConfig.from_env(self.uri_env_var_name)
|
||||||
exc_val is not None,
|
self._config = config
|
||||||
exc_tb is not None,
|
self._client_injected = client is not None
|
||||||
),
|
self._client: redis.Redis | None = client
|
||||||
):
|
self.path: str = RedisPath.root_path()
|
||||||
self.logger.error("Error while exiting context", **ctx_info)
|
if self._client_injected:
|
||||||
self.disconnect()
|
self._invalidate_health_cache()
|
||||||
|
|
||||||
def connect(self) -> None:
|
def _is_client_ready(self) -> bool:
|
||||||
"""Connect to the Redis server."""
|
return self._client is not None
|
||||||
# Prepare arguments
|
|
||||||
uri = str(os.getenv(self.uri_env_var_name))
|
def _validate_injected_client(self) -> None:
|
||||||
# Connect client
|
if self._client is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
if not self._client.ping():
|
||||||
|
raise ConnectionError(
|
||||||
|
f"Could not connect to Redis at {self._config.uri}"
|
||||||
|
)
|
||||||
|
except (redis.ConnectionError, redis.TimeoutError) as exc:
|
||||||
|
raise ConnectionError(
|
||||||
|
f"Could not connect to Redis at {self._config.uri}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def _establish_connection(self) -> None:
|
||||||
|
uri = self._config.uri
|
||||||
try:
|
try:
|
||||||
client = redis.Redis.from_url(
|
client = redis.Redis.from_url(
|
||||||
url=uri,
|
url=uri,
|
||||||
@@ -74,79 +74,115 @@ class RedisAdapter(
|
|||||||
raise ConnectionError(f"Could not connect to Redis at {uri}")
|
raise ConnectionError(f"Could not connect to Redis at {uri}")
|
||||||
except (redis.ConnectionError, redis.TimeoutError) as exc:
|
except (redis.ConnectionError, redis.TimeoutError) as exc:
|
||||||
raise ConnectionError(f"Could not connect to Redis at {uri}") from exc
|
raise ConnectionError(f"Could not connect to Redis at {uri}") from exc
|
||||||
# Persist client
|
|
||||||
self._client = client
|
self._client = client
|
||||||
|
|
||||||
def disconnect(self) -> None:
|
def disconnect(self) -> None:
|
||||||
"""Disconnect from the Redis server."""
|
"""Disconnect from the Redis server."""
|
||||||
# Close connection
|
if self._client is not None and not self._client_injected:
|
||||||
if self._client is not None:
|
|
||||||
self._client.close()
|
self._client.close()
|
||||||
# Reset client
|
|
||||||
self._client = None
|
self._client = None
|
||||||
|
self._invalidate_health_cache()
|
||||||
|
|
||||||
@property
|
def _probe_connection(self) -> bool:
|
||||||
def is_connected(self) -> bool:
|
assert self._client is not None
|
||||||
"""Check if connected to Redis server."""
|
try:
|
||||||
res = bool(isinstance(self._client, redis.Redis))
|
return bool(self._client.ping())
|
||||||
self.logger.debug(res)
|
except (redis.ConnectionError, redis.TimeoutError):
|
||||||
return res
|
return False
|
||||||
|
|
||||||
def _set(self, key: str, data: dict) -> None:
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
"""Set a JSON object in Redis."""
|
"""Set a JSON object in Redis.
|
||||||
|
|
||||||
|
Accepts any dict, including ``{}``. An empty dict creates a key that
|
||||||
|
``get()`` returns as ``{}``, not ``None``. Use ``delete()`` to remove a
|
||||||
|
key entirely.
|
||||||
|
"""
|
||||||
# Check input
|
# Check input
|
||||||
if not isinstance(key, str) or len(key) == 0:
|
if not isinstance(key, str) or len(key) == 0:
|
||||||
raise ValueError("Key must be a non-empty string")
|
raise ValueError("Key must be a non-empty string")
|
||||||
if not isinstance(data, dict) or len(data) == 0:
|
if not isinstance(data, dict):
|
||||||
raise ValueError("Data must be a non-empty dictionary")
|
raise ValueError("Data must be a dictionary")
|
||||||
# Check connection
|
# Check connection
|
||||||
if self._client is None or not self.is_connected:
|
self._require_connected()
|
||||||
raise ConnectionError("Not connected to Redis")
|
assert self._client is not None
|
||||||
# Set data
|
# Set data
|
||||||
self._client.json().set(key, self.path, data)
|
self._client.json().set(key, self.path, data)
|
||||||
self.logger.debug(f"Set {key} to {data}")
|
self.logger.debug("Set key", key=key, data_keys=list(data.keys()))
|
||||||
|
|
||||||
def _get(self, key: str) -> dict | None:
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
"""Get a JSON object from Redis."""
|
"""Get a JSON object from Redis.
|
||||||
|
|
||||||
|
Returns ``None`` if the key is absent. Returns ``{}`` if the key exists
|
||||||
|
with an empty JSON object. Use ``value is not None`` to test existence;
|
||||||
|
avoid truthiness checks (``{}`` is falsy).
|
||||||
|
"""
|
||||||
# Check input
|
# Check input
|
||||||
if not isinstance(key, str) or len(key) == 0:
|
if not isinstance(key, str) or len(key) == 0:
|
||||||
raise ValueError("Key must be a non-empty string")
|
raise ValueError("Key must be a non-empty string")
|
||||||
# Check connection
|
# Check connection
|
||||||
if self._client is None or not self.is_connected:
|
self._require_connected()
|
||||||
raise ConnectionError("Not connected to Redis")
|
assert self._client is not None
|
||||||
# Get data
|
# Get data
|
||||||
data = cast(
|
data = cast(
|
||||||
dict | None,
|
dict[str, Any] | None,
|
||||||
self._client.json().get(key),
|
self._client.json().get(key),
|
||||||
)
|
)
|
||||||
self.logger.debug(f"Got {data} from {key}")
|
self.logger.debug("Got value", key=key, found=data is not None)
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def _delete(self, key: str) -> None:
|
def delete(self, key: str) -> None:
|
||||||
"""Delete data from Redis."""
|
"""Delete data from Redis."""
|
||||||
# Check input
|
# Check input
|
||||||
if not isinstance(key, str) or len(key) == 0:
|
if not isinstance(key, str) or len(key) == 0:
|
||||||
raise ValueError("Key must be a non-empty string")
|
raise ValueError("Key must be a non-empty string")
|
||||||
# Check connection
|
# Check connection
|
||||||
if self._client is None or not self.is_connected:
|
self._require_connected()
|
||||||
raise ConnectionError("Not connected to Redis")
|
assert self._client is not None
|
||||||
# Delete data
|
# Delete data
|
||||||
self._client.json().delete(key)
|
self._client.json().delete(key)
|
||||||
self.logger.debug(f"Deleted {key}")
|
self.logger.debug("Deleted key", key=key)
|
||||||
|
|
||||||
def _list_keys(self, pattern: str) -> list[str]:
|
def _validate_pattern(self, pattern: str) -> None:
|
||||||
"""List keys in Redis matching a pattern."""
|
|
||||||
# Check input
|
|
||||||
if not isinstance(pattern, str) or len(pattern) == 0:
|
if not isinstance(pattern, str) or len(pattern) == 0:
|
||||||
raise ValueError("Pattern must be a non-empty string")
|
raise ValueError("Pattern must be a non-empty string")
|
||||||
|
|
||||||
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
|
"""List keys in Redis using KEYS; may block on large datasets."""
|
||||||
|
self._validate_pattern(pattern)
|
||||||
# Check connection
|
# Check connection
|
||||||
if self._client is None or not self.is_connected:
|
self._require_connected()
|
||||||
raise ConnectionError("Not connected to Redis")
|
assert self._client is not None
|
||||||
# List keys
|
# List keys
|
||||||
keys_raw = cast(
|
keys_raw = cast(
|
||||||
list[bytes],
|
list[bytes],
|
||||||
self._client.keys(pattern),
|
self._client.keys(pattern),
|
||||||
)
|
)
|
||||||
keys: list[str] = [key.decode(self.encoding) for key in keys_raw]
|
keys: list[str] = [key.decode(self.encoding) for key in keys_raw]
|
||||||
self.logger.debug(f"Got {keys} matching {pattern}")
|
self.logger.debug("Listed keys", pattern=pattern, count=len(keys))
|
||||||
return keys
|
return keys
|
||||||
|
|
||||||
|
def scan_keys(
|
||||||
|
self,
|
||||||
|
pattern: str,
|
||||||
|
*,
|
||||||
|
count: int | None = None,
|
||||||
|
) -> Iterator[str]:
|
||||||
|
"""Yield keys in Redis using SCAN to avoid blocking large datasets."""
|
||||||
|
self._validate_pattern(pattern)
|
||||||
|
self._require_connected()
|
||||||
|
assert self._client is not None
|
||||||
|
client = self._client
|
||||||
|
|
||||||
|
def _decode(key: bytes | str) -> str:
|
||||||
|
return key if isinstance(key, str) else key.decode(self.encoding)
|
||||||
|
|
||||||
|
def _iter() -> Iterator[str]:
|
||||||
|
scan_iter = (
|
||||||
|
client.scan_iter(match=pattern, count=count)
|
||||||
|
if count is not None
|
||||||
|
else client.scan_iter(match=pattern)
|
||||||
|
)
|
||||||
|
for key_raw in scan_iter:
|
||||||
|
yield _decode(key_raw)
|
||||||
|
|
||||||
|
return _iter()
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
from .dotenv_loader import load_dotenv as load_dotenv
|
||||||
|
from .file_queue_config import FileQueueConfig as FileQueueConfig
|
||||||
|
from .minio_config import MinioConfig as MinioConfig
|
||||||
|
from .postgres_config import PostgresConfig as PostgresConfig
|
||||||
|
from .redis_config import RedisConfig as RedisConfig
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"FileQueueConfig",
|
||||||
|
"MinioConfig",
|
||||||
|
"PostgresConfig",
|
||||||
|
"RedisConfig",
|
||||||
|
"load_dotenv",
|
||||||
|
]
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
"""Load environment variables from a .env file."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from dotenv import load_dotenv as _load_dotenv
|
||||||
|
|
||||||
|
|
||||||
|
def load_dotenv(path: str | Path | None = None) -> bool:
|
||||||
|
"""Load .env into os.environ. Idempotent; returns True if a file was loaded."""
|
||||||
|
if path is None:
|
||||||
|
return bool(_load_dotenv())
|
||||||
|
return bool(_load_dotenv(path))
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
"""Parse boolean values from environment variables."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
_TRUTHY = frozenset({"1", "true", "yes", "on"})
|
||||||
|
_FALSY = frozenset({"0", "false", "no", "off"})
|
||||||
|
|
||||||
|
|
||||||
|
def env_bool(name: str, default: bool) -> bool:
|
||||||
|
"""Parse an environment variable as a boolean value."""
|
||||||
|
raw = os.getenv(name)
|
||||||
|
if raw is None:
|
||||||
|
return default
|
||||||
|
normalized = raw.strip().lower()
|
||||||
|
if normalized in _TRUTHY:
|
||||||
|
return True
|
||||||
|
if normalized in _FALSY:
|
||||||
|
return False
|
||||||
|
raise ValueError(f"Invalid boolean value for {name}: {raw!r}")
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
"""File-backed queue configuration."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from python_utils import check_env
|
||||||
|
|
||||||
|
from python_repositories.config.dotenv_loader import load_dotenv
|
||||||
|
|
||||||
|
_DEFAULT_MAX_AGE_HOURS = 24
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FileQueueConfig:
|
||||||
|
"""Configuration for a JSONL file-backed queue.
|
||||||
|
|
||||||
|
Path and key schema are owned by the caller. This package does not assume
|
||||||
|
any particular directory layout or item field names.
|
||||||
|
"""
|
||||||
|
|
||||||
|
path: Path
|
||||||
|
max_age_hours: int = _DEFAULT_MAX_AGE_HOURS
|
||||||
|
dedup_keys: tuple[str, ...] = ()
|
||||||
|
age_key: str | None = None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(
|
||||||
|
cls,
|
||||||
|
path_env_var_name: str = "FILE_QUEUE_PATH",
|
||||||
|
*,
|
||||||
|
max_age_hours_env_var_name: str = "FILE_QUEUE_MAX_AGE_HOURS",
|
||||||
|
dedup_keys: tuple[str, ...] = (),
|
||||||
|
age_key: str | None = None,
|
||||||
|
use_dotenv: bool = True,
|
||||||
|
) -> FileQueueConfig:
|
||||||
|
"""Load path and optional max age from environment variables.
|
||||||
|
|
||||||
|
``dedup_keys`` and ``age_key`` are domain-specific and must be passed
|
||||||
|
explicitly; they are not read from the environment.
|
||||||
|
"""
|
||||||
|
if use_dotenv:
|
||||||
|
load_dotenv()
|
||||||
|
check_env(path_env_var_name)
|
||||||
|
raw_max_age = os.getenv(max_age_hours_env_var_name)
|
||||||
|
max_age_hours = (
|
||||||
|
_DEFAULT_MAX_AGE_HOURS if raw_max_age is None else int(raw_max_age)
|
||||||
|
)
|
||||||
|
return cls(
|
||||||
|
path=Path(str(os.getenv(path_env_var_name))),
|
||||||
|
max_age_hours=max_age_hours,
|
||||||
|
dedup_keys=dedup_keys,
|
||||||
|
age_key=age_key,
|
||||||
|
)
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"""MinIO connection configuration."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import os
|
||||||
|
|
||||||
|
from python_utils import check_env
|
||||||
|
|
||||||
|
from python_repositories.config.dotenv_loader import load_dotenv
|
||||||
|
from python_repositories.config.env_bool import env_bool
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class MinioConfig:
|
||||||
|
"""Configuration for connecting to MinIO."""
|
||||||
|
|
||||||
|
endpoint: str
|
||||||
|
access_key: str
|
||||||
|
secret_key: str
|
||||||
|
bucket: str
|
||||||
|
secure: bool = True
|
||||||
|
create_bucket_if_missing: bool = False
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(
|
||||||
|
cls,
|
||||||
|
endpoint_env_var_name: str = "MINIO_ENDPOINT",
|
||||||
|
access_key_env_var_name: str = "MINIO_ACCESS_KEY",
|
||||||
|
secret_key_env_var_name: str = "MINIO_SECRET_KEY",
|
||||||
|
bucket_env_var_name: str = "MINIO_BUCKET",
|
||||||
|
secure_env_var_name: str = "MINIO_SECURE",
|
||||||
|
create_bucket_if_missing_env_var_name: str = ("MINIO_CREATE_BUCKET_IF_MISSING"),
|
||||||
|
*,
|
||||||
|
use_dotenv: bool = True,
|
||||||
|
) -> MinioConfig:
|
||||||
|
"""Load configuration from environment variables."""
|
||||||
|
if use_dotenv:
|
||||||
|
load_dotenv()
|
||||||
|
env_var_names = {
|
||||||
|
endpoint_env_var_name,
|
||||||
|
access_key_env_var_name,
|
||||||
|
secret_key_env_var_name,
|
||||||
|
bucket_env_var_name,
|
||||||
|
}
|
||||||
|
check_env(env_var_names)
|
||||||
|
return cls(
|
||||||
|
endpoint=str(os.getenv(endpoint_env_var_name)),
|
||||||
|
access_key=str(os.getenv(access_key_env_var_name)),
|
||||||
|
secret_key=str(os.getenv(secret_key_env_var_name)),
|
||||||
|
bucket=str(os.getenv(bucket_env_var_name)),
|
||||||
|
secure=env_bool(secure_env_var_name, default=True),
|
||||||
|
create_bucket_if_missing=env_bool(
|
||||||
|
create_bucket_if_missing_env_var_name,
|
||||||
|
default=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
"""PostgreSQL connection configuration."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import os
|
||||||
|
|
||||||
|
from python_utils import check_env
|
||||||
|
|
||||||
|
from python_repositories.config.dotenv_loader import load_dotenv
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PostgresConfig:
|
||||||
|
"""Configuration for connecting to PostgreSQL."""
|
||||||
|
|
||||||
|
uri: str
|
||||||
|
table: str
|
||||||
|
primary_key: str = "id"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(
|
||||||
|
cls,
|
||||||
|
uri_env_var_name: str = "POSTGRES_URI",
|
||||||
|
table_env_var_name: str = "POSTGRES_TABLE",
|
||||||
|
primary_key_env_var_name: str = "POSTGRES_PRIMARY_KEY",
|
||||||
|
*,
|
||||||
|
use_dotenv: bool = True,
|
||||||
|
) -> PostgresConfig:
|
||||||
|
"""Load configuration from environment variables."""
|
||||||
|
if use_dotenv:
|
||||||
|
load_dotenv()
|
||||||
|
env_var_names = {uri_env_var_name, table_env_var_name}
|
||||||
|
check_env(env_var_names)
|
||||||
|
primary_key = os.getenv(primary_key_env_var_name)
|
||||||
|
if primary_key is None or primary_key == "":
|
||||||
|
primary_key = "id"
|
||||||
|
return cls(
|
||||||
|
uri=str(os.getenv(uri_env_var_name)),
|
||||||
|
table=str(os.getenv(table_env_var_name)),
|
||||||
|
primary_key=primary_key,
|
||||||
|
)
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
"""Redis connection configuration."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import os
|
||||||
|
|
||||||
|
from python_utils import check_env
|
||||||
|
|
||||||
|
from python_repositories.config.dotenv_loader import load_dotenv
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RedisConfig:
|
||||||
|
"""Configuration for connecting to Redis."""
|
||||||
|
|
||||||
|
uri: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(
|
||||||
|
cls,
|
||||||
|
uri_env_var_name: str = "REDIS_URI",
|
||||||
|
*,
|
||||||
|
use_dotenv: bool = True,
|
||||||
|
) -> RedisConfig:
|
||||||
|
"""Load configuration from environment variables."""
|
||||||
|
if use_dotenv:
|
||||||
|
load_dotenv()
|
||||||
|
check_env(uri_env_var_name)
|
||||||
|
return cls(uri=str(os.getenv(uri_env_var_name)))
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Example domain repositories built on technology adapters."""
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
"""Example domain repository backed by MinIO objects."""
|
||||||
|
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||||
|
|
||||||
|
|
||||||
|
class ArtifactObjectRepository(MinioAdapter):
|
||||||
|
"""Example: domain repository backed by MinIO objects."""
|
||||||
|
|
||||||
|
def _object_name(self, artifact_id: str) -> str:
|
||||||
|
return f"artifacts/{artifact_id}"
|
||||||
|
|
||||||
|
def get_artifact(self, artifact_id: str) -> BytesIO | None:
|
||||||
|
return self.get(self._object_name(artifact_id))
|
||||||
|
|
||||||
|
def store_artifact(
|
||||||
|
self,
|
||||||
|
artifact_id: str,
|
||||||
|
data: BytesIO,
|
||||||
|
content_type: str = "application/octet-stream",
|
||||||
|
) -> None:
|
||||||
|
self.put(self._object_name(artifact_id), data, content_type)
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
"""Example domain repository backed by Redis JSON."""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||||
|
|
||||||
|
|
||||||
|
class UserJsonRepository(RedisAdapter):
|
||||||
|
"""Example: domain repository backed by Redis JSON."""
|
||||||
|
|
||||||
|
def _key(self, user_id: str) -> str:
|
||||||
|
return f"user:{user_id}"
|
||||||
|
|
||||||
|
def get_user(self, user_id: str) -> dict[str, Any] | None:
|
||||||
|
return self.get(self._key(user_id))
|
||||||
|
|
||||||
|
def save_user(self, user_id: str, user: dict[str, Any]) -> None:
|
||||||
|
self.set(self._key(user_id), user)
|
||||||
|
|
||||||
|
def delete_user(self, user_id: str) -> None:
|
||||||
|
self.delete(self._key(user_id))
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
"""Example domain repository backed by Postgres tables."""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from python_repositories.adapters.postgres_adapter import PostgresAdapter
|
||||||
|
|
||||||
|
|
||||||
|
class UserTableRepository(PostgresAdapter):
|
||||||
|
"""Example: domain repository backed by a Postgres table."""
|
||||||
|
|
||||||
|
def get_user(self, user_id: str) -> dict[str, Any] | None:
|
||||||
|
return self.fetch_one(user_id)
|
||||||
|
|
||||||
|
def save_user(self, user_id: str, user: dict[str, Any]) -> None:
|
||||||
|
row = {self._config.primary_key: user_id, **user}
|
||||||
|
self.upsert(row)
|
||||||
|
|
||||||
|
def delete_user(self, user_id: str) -> None:
|
||||||
|
self.delete(user_id)
|
||||||
@@ -2,8 +2,24 @@ from .connection_aware_interface import (
|
|||||||
ConnectionAwareInterface as ConnectionAwareInterface,
|
ConnectionAwareInterface as ConnectionAwareInterface,
|
||||||
)
|
)
|
||||||
from .context_aware_interface import ContextAwareInterface as ContextAwareInterface
|
from .context_aware_interface import ContextAwareInterface as ContextAwareInterface
|
||||||
|
from .json_repository_interface import (
|
||||||
|
JsonRepositoryInterface as JsonRepositoryInterface,
|
||||||
|
)
|
||||||
|
from .object_repository_interface import (
|
||||||
|
ObjectRepositoryInterface as ObjectRepositoryInterface,
|
||||||
|
)
|
||||||
|
from .queue_repository_interface import (
|
||||||
|
QueueRepositoryInterface as QueueRepositoryInterface,
|
||||||
|
)
|
||||||
|
from .table_repository_interface import (
|
||||||
|
TableRepositoryInterface as TableRepositoryInterface,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"ConnectionAwareInterface",
|
"ConnectionAwareInterface",
|
||||||
"ContextAwareInterface",
|
"ContextAwareInterface",
|
||||||
|
"JsonRepositoryInterface",
|
||||||
|
"ObjectRepositoryInterface",
|
||||||
|
"QueueRepositoryInterface",
|
||||||
|
"TableRepositoryInterface",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,23 +1,31 @@
|
|||||||
"""Definition of ConnectionAwareInterface abstract base class."""
|
"""Definition of ConnectionAwareInterface protocol and abstract base class."""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import abstractmethod
|
||||||
|
from typing import Protocol, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
class ConnectionAwareInterface(ABC):
|
@runtime_checkable
|
||||||
|
class ConnectionAwareInterface(Protocol):
|
||||||
"""Interface that defines connection-related methods."""
|
"""Interface that defines connection-related methods."""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def connect(self) -> None:
|
def connect(self) -> None:
|
||||||
"""Connect to resource."""
|
"""Connect to resource.
|
||||||
raise NotImplementedError()
|
|
||||||
|
Implementations should be idempotent: calling connect while already
|
||||||
|
connected and healthy is a no-op.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def disconnect(self) -> None:
|
def disconnect(self) -> None:
|
||||||
"""Disconnect from resource."""
|
"""Disconnect from resource."""
|
||||||
raise NotImplementedError()
|
...
|
||||||
|
|
||||||
@property
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def is_connected(self) -> bool:
|
def is_connected(self) -> bool:
|
||||||
"""Check if connected to resource."""
|
"""Return whether the adapter has an active, reachable connection.
|
||||||
raise NotImplementedError()
|
|
||||||
|
Implementations may perform a cached network probe to verify liveness.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|||||||
@@ -1,21 +1,23 @@
|
|||||||
"""Definition of ConnectionAwareInterface abstract base class."""
|
"""Definition of ContextAwareInterface protocol and abstract base class."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import abstractmethod
|
||||||
|
from typing import Protocol, Self, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
class ContextAwareInterface(ABC):
|
@runtime_checkable
|
||||||
"""Interface that defined context-related methods."""
|
class ContextAwareInterface(Protocol):
|
||||||
|
"""Interface that defines context-related methods."""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def __enter__(self) -> ContextAwareInterface:
|
def __enter__(self) -> Self:
|
||||||
"""Enter the context."""
|
"""Enter the context."""
|
||||||
raise NotImplementedError()
|
...
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def __exit__(
|
def __exit__(
|
||||||
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Exit the context."""
|
"""Exit the context."""
|
||||||
raise NotImplementedError()
|
...
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
"""Definition of JsonRepositoryInterface protocol and abstract base class."""
|
||||||
|
|
||||||
|
from abc import abstractmethod
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from typing import Any, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class JsonRepositoryInterface(Protocol):
|
||||||
|
"""Interface that defines JSON document CRUD methods."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
|
"""Get a JSON object by key.
|
||||||
|
|
||||||
|
Returns ``None`` when the key is absent. Returns ``{}`` when the key
|
||||||
|
exists with an empty JSON object. Use ``value is not None`` to test
|
||||||
|
existence; avoid truthiness checks (``{}`` is falsy).
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
|
"""Set a JSON object by key.
|
||||||
|
|
||||||
|
Accepts any dict, including ``{}``. An empty dict creates a key that
|
||||||
|
``get()`` returns as ``{}``, not ``None``. Use ``delete()`` to remove a
|
||||||
|
key entirely.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def delete(self, key: str) -> None:
|
||||||
|
"""Delete a JSON object by key, removing it entirely."""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
|
"""List keys matching a glob pattern."""
|
||||||
|
...
|
||||||
|
|
||||||
|
def scan_keys(
|
||||||
|
self,
|
||||||
|
pattern: str,
|
||||||
|
*,
|
||||||
|
count: int | None = None,
|
||||||
|
) -> Iterator[str]:
|
||||||
|
"""Yield keys matching a glob pattern incrementally."""
|
||||||
|
del count
|
||||||
|
yield from self.list_keys(pattern)
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
"""Definition of ObjectRepositoryInterface protocol and abstract base class."""
|
||||||
|
|
||||||
|
from abc import abstractmethod
|
||||||
|
from io import BytesIO
|
||||||
|
from typing import Protocol, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class ObjectRepositoryInterface(Protocol):
|
||||||
|
"""Interface that defines binary object CRUD methods."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get(self, object_name: str) -> BytesIO | None:
|
||||||
|
"""Get an object by name.
|
||||||
|
|
||||||
|
Returns ``None`` when the object does not exist. Returns an empty
|
||||||
|
``BytesIO`` for a zero-byte object. Use ``value is not None`` to test
|
||||||
|
existence. Raises ConnectionError when not connected. Other backend
|
||||||
|
errors propagate to the caller.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def put(
|
||||||
|
self,
|
||||||
|
object_name: str,
|
||||||
|
data: BytesIO,
|
||||||
|
content_type: str = "application/octet-stream",
|
||||||
|
) -> None:
|
||||||
|
"""Put an object by name.
|
||||||
|
|
||||||
|
Accepts zero-byte ``BytesIO``. A zero-byte object is returned by
|
||||||
|
``get()`` as an empty buffer, not ``None``. Use ``delete()`` to remove
|
||||||
|
an object entirely.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def delete(self, object_name: str) -> None:
|
||||||
|
"""Delete an object by name, removing it entirely."""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def list_objects(self, prefix: str = "") -> list[str]:
|
||||||
|
"""List object names with an optional prefix."""
|
||||||
|
...
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
"""Definition of QueueRepositoryInterface protocol."""
|
||||||
|
|
||||||
|
from abc import abstractmethod
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class QueueRepositoryInterface(Protocol):
|
||||||
|
"""Interface that defines buffered queue enqueue/dequeue methods."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def enqueue(self, items: list[dict[str, Any]]) -> None:
|
||||||
|
"""Append items to the queue.
|
||||||
|
|
||||||
|
Duplicates may be skipped when the implementation is configured with
|
||||||
|
dedup keys. An empty list is a no-op.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
|
||||||
|
"""Remove and return up to ``max_items`` items in FIFO order.
|
||||||
|
|
||||||
|
Returns an empty list when the queue is empty.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def size(self) -> int:
|
||||||
|
"""Return the number of items currently in the queue."""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def evict_older_than(self, cutoff: datetime) -> int:
|
||||||
|
"""Remove items older than ``cutoff`` and return how many were removed.
|
||||||
|
|
||||||
|
Age is determined by an implementation-specific item field. When no age
|
||||||
|
field is configured, this is a no-op that returns ``0``.
|
||||||
|
"""
|
||||||
|
...
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
"""Definition of TableRepositoryInterface protocol."""
|
||||||
|
|
||||||
|
from abc import abstractmethod
|
||||||
|
from typing import Any, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class TableRepositoryInterface(Protocol):
|
||||||
|
"""Interface that defines relational table CRUD methods."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def fetch_one(self, pk: Any) -> dict[str, Any] | None:
|
||||||
|
"""Fetch a single row by primary key.
|
||||||
|
|
||||||
|
Returns ``None`` when no row matches. Use ``value is not None`` to test
|
||||||
|
existence; avoid truthiness checks.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def fetch_all(self, *, limit: int | None = None) -> list[dict[str, Any]]:
|
||||||
|
"""Fetch all rows from the configured table.
|
||||||
|
|
||||||
|
Returns an empty list when the table has no rows.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def upsert(self, row: dict[str, Any]) -> None:
|
||||||
|
"""Insert or update a row by primary key.
|
||||||
|
|
||||||
|
``row`` must include the configured primary key column. Raises
|
||||||
|
``ValueError`` for a non-dict row or a missing primary key.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def delete(self, pk: Any) -> None:
|
||||||
|
"""Delete a row by primary key.
|
||||||
|
|
||||||
|
Idempotent: no error when the row is already absent.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
sql: str,
|
||||||
|
params: tuple[Any, ...] = (),
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Run a read SQL statement and return rows as dicts.
|
||||||
|
|
||||||
|
Escape hatch for joins, filters, and other queries in subclasses.
|
||||||
|
"""
|
||||||
|
...
|
||||||
Executable
+64
@@ -0,0 +1,64 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=common.sh
|
||||||
|
source "${SCRIPT_DIR}/common.sh"
|
||||||
|
|
||||||
|
CHANGELOG="CHANGELOG.md"
|
||||||
|
|
||||||
|
# Best-effort summary for historical releases without stored PR titles.
|
||||||
|
summary_for_range() {
|
||||||
|
local from_ref="$1"
|
||||||
|
local to_ref="$2"
|
||||||
|
local subject
|
||||||
|
|
||||||
|
while IFS= read -r subject; do
|
||||||
|
[[ -z "$subject" ]] && continue
|
||||||
|
if echo "$subject" | grep -qE '^chore: release v'; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if echo "$subject" | grep -qE "^Merge pull request '"; then
|
||||||
|
local title
|
||||||
|
title=$(extract_pr_title_from_merge_commit "$subject")
|
||||||
|
if [[ -n "$title" ]]; then
|
||||||
|
strip_bump_prefix "$title"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
echo "$subject"
|
||||||
|
return 0
|
||||||
|
done < <(git log "${from_ref}..${to_ref}" --pretty=format:'%s' 2>/dev/null || true)
|
||||||
|
|
||||||
|
echo "Release ${to_ref}"
|
||||||
|
}
|
||||||
|
|
||||||
|
TAGS=()
|
||||||
|
while IFS= read -r tag; do
|
||||||
|
TAGS+=("$tag")
|
||||||
|
done < <(git tag -l 'v*' --sort=version:refname)
|
||||||
|
if [[ ${#TAGS[@]} -eq 0 ]]; then
|
||||||
|
echo "No version tags found." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
changelog_header > "$CHANGELOG"
|
||||||
|
|
||||||
|
for ((i = ${#TAGS[@]} - 1; i >= 0; i--)); do
|
||||||
|
TAG="${TAGS[$i]}"
|
||||||
|
VERSION="${TAG#v}"
|
||||||
|
DATE=$(git log -1 --format=%cs "$TAG" 2>/dev/null || date -u +%Y-%m-%d)
|
||||||
|
|
||||||
|
if [[ $i -eq 0 ]]; then
|
||||||
|
PREV_TAG=""
|
||||||
|
else
|
||||||
|
PREV_TAG="${TAGS[$((i - 1))]}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
SUMMARY=$(summary_for_range "$PREV_TAG" "$TAG")
|
||||||
|
COMMITS=$(commits_between "$PREV_TAG" "$TAG")
|
||||||
|
SECTION=$(changelog_section "$VERSION" "$DATE" "$SUMMARY" "$COMMITS")
|
||||||
|
printf '\n%s\n' "$SECTION" >> "$CHANGELOG"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Generated ${CHANGELOG} from ${#TAGS[@]} tags." >&2
|
||||||
Executable
+78
@@ -0,0 +1,78 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build and optionally push the Gitea Actions CI base image locally.
|
||||||
|
#
|
||||||
|
# Use when ci-image.yml is not yet on main (e.g. bootstrapping a PR branch) or
|
||||||
|
# when you need to rebuild without waiting for the nightly workflow.
|
||||||
|
#
|
||||||
|
# Build only (smoke test):
|
||||||
|
# export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||||
|
# bash scripts/ci/build-ci-image.sh --local
|
||||||
|
#
|
||||||
|
# Build and push to Gitea (unblocks python-repositories-ci jobs):
|
||||||
|
# export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||||
|
# bash scripts/ci/build-ci-image.sh --push
|
||||||
|
#
|
||||||
|
# Optional overrides:
|
||||||
|
# REGISTRY=gitea.lille-vemmelund.dk
|
||||||
|
# REGISTRY_USER=ci-bot
|
||||||
|
# IMAGE_OWNER=lillevemmelund
|
||||||
|
# IMAGE_NAME=python-repositories-ci
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||||
|
|
||||||
|
REGISTRY="${REGISTRY:-gitea.lille-vemmelund.dk}"
|
||||||
|
REGISTRY_USER="${REGISTRY_USER:-ci-bot}"
|
||||||
|
IMAGE_OWNER="${IMAGE_OWNER:-lillevemmelund}"
|
||||||
|
IMAGE_NAME="${IMAGE_NAME:-python-repositories-ci}"
|
||||||
|
IMAGE="${IMAGE:-${REGISTRY}/${IMAGE_OWNER}/${IMAGE_NAME}}"
|
||||||
|
|
||||||
|
MODE="push"
|
||||||
|
if [[ "${1:-}" == "--local" ]]; then
|
||||||
|
MODE="local"
|
||||||
|
elif [[ "${1:-}" == "--push" || -z "${1:-}" ]]; then
|
||||||
|
MODE="push"
|
||||||
|
elif [[ -n "${1:-}" ]]; then
|
||||||
|
echo "Usage: build-ci-image.sh [--local | --push]" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
: "${CI_RUNNER_TOKEN:?CI_RUNNER_TOKEN is required (ci-bot token with read/write package access)}"
|
||||||
|
|
||||||
|
TOKEN_FILE="$(mktemp)"
|
||||||
|
cleanup() {
|
||||||
|
rm -f "$TOKEN_FILE"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
printf '%s' "$CI_RUNNER_TOKEN" >"$TOKEN_FILE"
|
||||||
|
|
||||||
|
cd "$REPO_ROOT"
|
||||||
|
|
||||||
|
if [[ "$MODE" == "local" ]]; then
|
||||||
|
echo "=== Building local CI image (no push): python-repositories-ci:local ==="
|
||||||
|
docker build --network=host -f docker/ci/Dockerfile \
|
||||||
|
--secret "id=uv_token,src=${TOKEN_FILE}" \
|
||||||
|
-t python-repositories-ci:local \
|
||||||
|
.
|
||||||
|
echo "Built python-repositories-ci:local"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "=== Logging in to ${REGISTRY} as ${REGISTRY_USER} ==="
|
||||||
|
echo "$CI_RUNNER_TOKEN" | docker login "$REGISTRY" -u "$REGISTRY_USER" --password-stdin
|
||||||
|
|
||||||
|
echo "=== Building ${IMAGE}:latest ==="
|
||||||
|
docker build --network=host -f docker/ci/Dockerfile \
|
||||||
|
--secret "id=uv_token,src=${TOKEN_FILE}" \
|
||||||
|
-t "${IMAGE}:latest" \
|
||||||
|
.
|
||||||
|
|
||||||
|
STAMP="$(date -u +%Y%m%d%H%M)"
|
||||||
|
echo "=== Pushing ${IMAGE}:latest and ${IMAGE}:${STAMP} ==="
|
||||||
|
docker tag "${IMAGE}:latest" "${IMAGE}:${STAMP}"
|
||||||
|
docker push "${IMAGE}:latest"
|
||||||
|
docker push "${IMAGE}:${STAMP}"
|
||||||
|
|
||||||
|
echo "Done. Re-run failing CI jobs — runners pull ${IMAGE}:latest"
|
||||||
Executable
+44
@@ -0,0 +1,44 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=common.sh
|
||||||
|
source "${SCRIPT_DIR}/common.sh"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "Usage: bump-version.sh <major|minor|patch>" >&2
|
||||||
|
echo " bump-version.sh --from-tag <vX.Y.Z>" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ $# -lt 1 ]]; then
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$1" == "--from-tag" ]]; then
|
||||||
|
if [[ $# -ne 2 ]]; then
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
VERSION="${2#v}"
|
||||||
|
else
|
||||||
|
BUMP_TYPE="$1"
|
||||||
|
CURRENT=$(latest_tag_version)
|
||||||
|
VERSION=$(bump_semver "$CURRENT" "$BUMP_TYPE")
|
||||||
|
fi
|
||||||
|
|
||||||
|
set_pyproject_version "$VERSION"
|
||||||
|
|
||||||
|
if command -v uv >/dev/null 2>&1; then
|
||||||
|
uv lock
|
||||||
|
else
|
||||||
|
echo "uv not found; uv.lock was not updated" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||||
|
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "version=${VERSION}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Updated pyproject.toml and uv.lock to version ${VERSION}" >&2
|
||||||
Executable
+41
@@ -0,0 +1,41 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=common.sh
|
||||||
|
source "${SCRIPT_DIR}/common.sh"
|
||||||
|
|
||||||
|
PR_TITLE="${1:-${PR_TITLE:-}}"
|
||||||
|
|
||||||
|
if [[ -z "$PR_TITLE" ]]; then
|
||||||
|
echo "Usage: check-pr-title.sh <pr-title> (changed files on stdin)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
CHANGED_FILES=$(cat)
|
||||||
|
|
||||||
|
if ! has_source_changes "$CHANGED_FILES"; then
|
||||||
|
echo "No changes under ${SOURCE_DIR}/ — PR title prefix not required."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if title_has_bump_prefix "$PR_TITLE"; then
|
||||||
|
echo "PR title has a valid version bump prefix."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat >&2 <<EOF
|
||||||
|
PR changes files under ${SOURCE_DIR}/ but the title lacks a version bump prefix.
|
||||||
|
|
||||||
|
Title: ${PR_TITLE}
|
||||||
|
|
||||||
|
When changing package source, start the PR title with one of:
|
||||||
|
[patch] or [fix] — bug fix
|
||||||
|
[minor] or [feat] — new feature
|
||||||
|
[major] or [breaking] — breaking change
|
||||||
|
|
||||||
|
Example: [patch] Fix connection retry in RedisAdapter
|
||||||
|
|
||||||
|
See .gitea/PULL_REQUEST_TEMPLATE.md for details.
|
||||||
|
EOF
|
||||||
|
exit 1
|
||||||
Executable
+146
@@ -0,0 +1,146 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Shared helpers for release and PR title CI scripts.
|
||||||
|
|
||||||
|
SOURCE_DIR="python_repositories"
|
||||||
|
|
||||||
|
# Returns bump type (major|minor|patch) or empty if no valid prefix.
|
||||||
|
bump_type_from_title() {
|
||||||
|
local title="$1"
|
||||||
|
local lower
|
||||||
|
lower=$(echo "$title" | tr '[:upper:]' '[:lower:]')
|
||||||
|
|
||||||
|
if echo "$lower" | grep -qE '^\[(major|breaking)\]'; then
|
||||||
|
echo "major"
|
||||||
|
elif echo "$lower" | grep -qE '^\[(minor|feat)\]'; then
|
||||||
|
echo "minor"
|
||||||
|
elif echo "$lower" | grep -qE '^\[(patch|fix)\]'; then
|
||||||
|
echo "patch"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
title_has_bump_prefix() {
|
||||||
|
[[ -n "$(bump_type_from_title "$1")" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
strip_bump_prefix() {
|
||||||
|
local title="$1"
|
||||||
|
echo "$title" | sed -E 's/^\[(patch|fix|minor|feat|major|breaking)\][[:space:]]*//i'
|
||||||
|
}
|
||||||
|
|
||||||
|
extract_pr_title_from_merge_commit() {
|
||||||
|
local msg="$1"
|
||||||
|
echo "$msg" | sed -n "s/^Merge pull request '\(.*\)' (#.*/\1/p"
|
||||||
|
}
|
||||||
|
|
||||||
|
changed_source_files() {
|
||||||
|
grep -E "^${SOURCE_DIR}/" || true
|
||||||
|
}
|
||||||
|
|
||||||
|
has_source_changes() {
|
||||||
|
local files="$1"
|
||||||
|
echo "$files" | changed_source_files | grep -q .
|
||||||
|
}
|
||||||
|
|
||||||
|
read_pyproject_version() {
|
||||||
|
sed -n 's/^version = "\(.*\)"/\1/p' pyproject.toml | head -1
|
||||||
|
}
|
||||||
|
|
||||||
|
latest_tag_version() {
|
||||||
|
local tag
|
||||||
|
tag=$(git describe --tags --abbrev=0 2>/dev/null || true)
|
||||||
|
if [[ -n "$tag" ]]; then
|
||||||
|
echo "${tag#v}"
|
||||||
|
else
|
||||||
|
read_pyproject_version
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
set_pyproject_version() {
|
||||||
|
local version="$1"
|
||||||
|
sed -i.bak -E "s/^version = \".*\"/version = \"${version}\"/" pyproject.toml
|
||||||
|
rm -f pyproject.toml.bak
|
||||||
|
}
|
||||||
|
|
||||||
|
# Returns markdown bullet list of commits between two refs (exclusive..inclusive).
|
||||||
|
# Optional third argument limits entries when from_ref is empty (release notes only).
|
||||||
|
commits_between() {
|
||||||
|
local from_ref="$1"
|
||||||
|
local to_ref="${2:-HEAD}"
|
||||||
|
local limit="${3:-}"
|
||||||
|
|
||||||
|
if [[ -z "$from_ref" ]]; then
|
||||||
|
if [[ -n "$limit" ]]; then
|
||||||
|
git log "${to_ref}" --pretty=format:'- %h %s' -"${limit}" 2>/dev/null || true
|
||||||
|
else
|
||||||
|
git log "${to_ref}" --pretty=format:'- %h %s' 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
git log "${from_ref}..${to_ref}" --pretty=format:'- %h %s' 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
commits_since() {
|
||||||
|
commits_between "$1" HEAD
|
||||||
|
}
|
||||||
|
|
||||||
|
changelog_header() {
|
||||||
|
cat <<'EOF'
|
||||||
|
# Changelog
|
||||||
|
|
||||||
|
All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||||
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
|
## [Unreleased]
|
||||||
|
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
changelog_section() {
|
||||||
|
local version="$1"
|
||||||
|
local date="$2"
|
||||||
|
local summary="$3"
|
||||||
|
local commits="$4"
|
||||||
|
|
||||||
|
# Blank lines after ATX headings match Prettier v4 markdown formatting
|
||||||
|
# (see mirrors-prettier in .pre-commit-config.yaml).
|
||||||
|
cat <<EOF
|
||||||
|
## [${version}] - ${date}
|
||||||
|
|
||||||
|
### Summary
|
||||||
|
|
||||||
|
${summary}
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
${commits:-- (no commits recorded)}
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
bump_semver() {
|
||||||
|
local current="$1"
|
||||||
|
local bump_type="$2"
|
||||||
|
local major minor patch
|
||||||
|
|
||||||
|
IFS=. read -r major minor patch <<< "$current"
|
||||||
|
case "$bump_type" in
|
||||||
|
major)
|
||||||
|
major=$((major + 1))
|
||||||
|
minor=0
|
||||||
|
patch=0
|
||||||
|
;;
|
||||||
|
minor)
|
||||||
|
minor=$((minor + 1))
|
||||||
|
patch=0
|
||||||
|
;;
|
||||||
|
patch)
|
||||||
|
patch=$((patch + 1))
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown bump type: $bump_type" >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
echo "${major}.${minor}.${patch}"
|
||||||
|
}
|
||||||
Executable
+70
@@ -0,0 +1,70 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
: "${API_URL:?API_URL is required}"
|
||||||
|
: "${REPO_OWNER:?REPO_OWNER is required}"
|
||||||
|
: "${REPO_NAME:?REPO_NAME is required}"
|
||||||
|
: "${CI_RUNNER_TOKEN:?CI_RUNNER_TOKEN is required}"
|
||||||
|
: "${GITHUB_SERVER_URL:?GITHUB_SERVER_URL is required}"
|
||||||
|
: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}"
|
||||||
|
|
||||||
|
BRANCH="${BRANCH:-renovate/auto-deps-update}"
|
||||||
|
BASE_BRANCH="${BASE_BRANCH:-main}"
|
||||||
|
LOCKFILE="${LOCKFILE:-uv.lock}"
|
||||||
|
PRE_COMMIT_CONFIG="${PRE_COMMIT_CONFIG:-.pre-commit-config.yaml}"
|
||||||
|
|
||||||
|
if git diff --quiet "$LOCKFILE" && git diff --quiet "$PRE_COMMIT_CONFIG"; then
|
||||||
|
echo "No dependency updates available."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
git config user.name "CI Bot"
|
||||||
|
git config user.email "[email protected]"
|
||||||
|
git add "$LOCKFILE" "$PRE_COMMIT_CONFIG"
|
||||||
|
git commit -m "chore(deps): update dependencies [automated]"
|
||||||
|
|
||||||
|
git remote set-url origin "https://x-access-token:${CI_RUNNER_TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
|
||||||
|
git push origin "HEAD:${BRANCH}"
|
||||||
|
|
||||||
|
if curl -s "${API_URL}/repos/${REPO_OWNER}/${REPO_NAME}/pulls?state=open" \
|
||||||
|
-H "Authorization: token ${CI_RUNNER_TOKEN}" \
|
||||||
|
| BRANCH="$BRANCH" python3 -c '
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
branch = os.environ["BRANCH"]
|
||||||
|
prs = json.load(sys.stdin)
|
||||||
|
sys.exit(0 if any(pr.get("head", {}).get("ref") == branch for pr in prs) else 1)
|
||||||
|
'; then
|
||||||
|
echo "Pull request already exists; branch push updates it."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
PAYLOAD=$(BRANCH="$BRANCH" BASE_BRANCH="$BASE_BRANCH" python3 -c '
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"title": "chore(deps): update dependencies",
|
||||||
|
"body": (
|
||||||
|
"This PR was created automatically to update dependencies.\n\n"
|
||||||
|
"- `uv.lock`: upgraded Python dependencies\n"
|
||||||
|
"- `.pre-commit-config.yaml`: updated remote hook revs via "
|
||||||
|
"`pre-commit autoupdate`"
|
||||||
|
),
|
||||||
|
"head": os.environ["BRANCH"],
|
||||||
|
"base": os.environ["BASE_BRANCH"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
')
|
||||||
|
|
||||||
|
curl -sf -X POST "${API_URL}/repos/${REPO_OWNER}/${REPO_NAME}/pulls" \
|
||||||
|
-H "Authorization: token ${CI_RUNNER_TOKEN}" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "$PAYLOAD"
|
||||||
|
|
||||||
|
echo "Created pull request."
|
||||||
Executable
+16
@@ -0,0 +1,16 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Fetch the current PR title from the Gitea API.
|
||||||
|
# Requires: API_URL, REPO_OWNER, REPO_NAME, PR_NUMBER, CI_RUNNER_TOKEN
|
||||||
|
|
||||||
|
: "${API_URL:?API_URL is required}"
|
||||||
|
: "${REPO_OWNER:?REPO_OWNER is required}"
|
||||||
|
: "${REPO_NAME:?REPO_NAME is required}"
|
||||||
|
: "${PR_NUMBER:?PR_NUMBER is required}"
|
||||||
|
: "${CI_RUNNER_TOKEN:?CI_RUNNER_TOKEN is required}"
|
||||||
|
|
||||||
|
curl -sf \
|
||||||
|
"${API_URL}/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PR_NUMBER}" \
|
||||||
|
-H "Authorization: token ${CI_RUNNER_TOKEN}" \
|
||||||
|
| python3 -c 'import json, sys; print(json.load(sys.stdin)["title"])'
|
||||||
Executable
+54
@@ -0,0 +1,54 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=common.sh
|
||||||
|
source "${SCRIPT_DIR}/common.sh"
|
||||||
|
|
||||||
|
NEW_TAG="${1:-}"
|
||||||
|
SUMMARY="${2:-}"
|
||||||
|
PREV_TAG="${3:-}"
|
||||||
|
|
||||||
|
if [[ -z "$NEW_TAG" ]]; then
|
||||||
|
echo "Usage: generate-release-notes.sh <new-tag> [summary] [prev-tag]" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$PREV_TAG" ]]; then
|
||||||
|
PREV_TAG=$(git describe --tags --abbrev=0 "${NEW_TAG}^" 2>/dev/null || git describe --tags --abbrev=0 2>/dev/null || true)
|
||||||
|
fi
|
||||||
|
|
||||||
|
NOTES="## Summary"
|
||||||
|
if [[ -n "$SUMMARY" ]]; then
|
||||||
|
NOTES="${NOTES}
|
||||||
|
|
||||||
|
$(strip_bump_prefix "$SUMMARY")"
|
||||||
|
else
|
||||||
|
NOTES="${NOTES}
|
||||||
|
|
||||||
|
Automated release ${NEW_TAG}."
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$PREV_TAG" ]]; then
|
||||||
|
NOTES="${NOTES}
|
||||||
|
|
||||||
|
## Changes since ${PREV_TAG}
|
||||||
|
|
||||||
|
$(commits_since "$PREV_TAG")"
|
||||||
|
else
|
||||||
|
NOTES="${NOTES}
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
$(commits_between "" HEAD 20)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||||
|
{
|
||||||
|
echo 'body<<EOF'
|
||||||
|
echo "$NOTES"
|
||||||
|
echo 'EOF'
|
||||||
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "$NOTES"
|
||||||
|
fi
|
||||||
Executable
+36
@@ -0,0 +1,36 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=common.sh
|
||||||
|
source "${SCRIPT_DIR}/common.sh"
|
||||||
|
|
||||||
|
COMMIT_MSG="${1:-${COMMIT_MSG:-}}"
|
||||||
|
|
||||||
|
if [[ -z "$COMMIT_MSG" ]]; then
|
||||||
|
echo "Usage: parse-merge-commit.sh <commit-message>" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
PR_TITLE=$(extract_pr_title_from_merge_commit "$COMMIT_MSG")
|
||||||
|
|
||||||
|
if [[ -z "$PR_TITLE" ]]; then
|
||||||
|
bump="skip"
|
||||||
|
else
|
||||||
|
bump_type=$(bump_type_from_title "$PR_TITLE")
|
||||||
|
if [[ -n "$bump_type" ]]; then
|
||||||
|
bump="$bump_type"
|
||||||
|
else
|
||||||
|
bump="skip"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||||
|
{
|
||||||
|
echo "pr_title=${PR_TITLE}"
|
||||||
|
echo "bump=${bump}"
|
||||||
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "pr_title=${PR_TITLE}"
|
||||||
|
echo "bump=${bump}"
|
||||||
|
fi
|
||||||
Executable
+63
@@ -0,0 +1,63 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=common.sh
|
||||||
|
source "${SCRIPT_DIR}/common.sh"
|
||||||
|
|
||||||
|
VERSION="${1:-}"
|
||||||
|
SUMMARY="${2:-}"
|
||||||
|
PREV_TAG="${3:-}"
|
||||||
|
DATE="${4:-$(date -u +%Y-%m-%d)}"
|
||||||
|
CHANGELOG="CHANGELOG.md"
|
||||||
|
|
||||||
|
if [[ -z "$VERSION" ]]; then
|
||||||
|
echo "Usage: update-changelog.sh <version> <summary> <prev-tag> [date]" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "$CHANGELOG" ]] && grep -qF "## [${VERSION}]" "$CHANGELOG"; then
|
||||||
|
echo "CHANGELOG already contains version ${VERSION}; skipping." >&2
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$SUMMARY" ]]; then
|
||||||
|
SUMMARY_TEXT=$(strip_bump_prefix "$SUMMARY")
|
||||||
|
else
|
||||||
|
SUMMARY_TEXT="Automated release v${VERSION}."
|
||||||
|
fi
|
||||||
|
|
||||||
|
COMMITS=$(commits_since "$PREV_TAG")
|
||||||
|
SECTION=$(changelog_section "$VERSION" "$DATE" "$SUMMARY_TEXT" "$COMMITS")
|
||||||
|
|
||||||
|
if [[ ! -f "$CHANGELOG" ]]; then
|
||||||
|
changelog_header > "$CHANGELOG"
|
||||||
|
printf '\n%s\n' "$SECTION" >> "$CHANGELOG"
|
||||||
|
else
|
||||||
|
UNRELEASED_MARKER='## [Unreleased]'
|
||||||
|
if ! grep -qF "$UNRELEASED_MARKER" "$CHANGELOG"; then
|
||||||
|
echo "CHANGELOG.md exists but is missing ${UNRELEASED_MARKER}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
TMP=$(mktemp)
|
||||||
|
SECTION_FILE=$(mktemp)
|
||||||
|
printf '%s\n' "$SECTION" > "$SECTION_FILE"
|
||||||
|
|
||||||
|
awk -v section_file="$SECTION_FILE" '
|
||||||
|
/^## \[Unreleased\]/ {
|
||||||
|
print
|
||||||
|
print ""
|
||||||
|
while ((getline line < section_file) > 0) {
|
||||||
|
print line
|
||||||
|
}
|
||||||
|
close(section_file)
|
||||||
|
next
|
||||||
|
}
|
||||||
|
{ print }
|
||||||
|
' "$CHANGELOG" > "$TMP"
|
||||||
|
rm -f "$SECTION_FILE"
|
||||||
|
mv "$TMP" "$CHANGELOG"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Updated ${CHANGELOG} with version ${VERSION}." >&2
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
"""Shared test configuration constants."""
|
||||||
|
|
||||||
|
from python_repositories.config import MinioConfig, PostgresConfig, RedisConfig
|
||||||
|
|
||||||
|
TEST_REDIS_CONFIG = RedisConfig(uri="redis://localhost:6379")
|
||||||
|
TEST_MINIO_CONFIG = MinioConfig(
|
||||||
|
endpoint="localhost:9000",
|
||||||
|
access_key="minioadmin",
|
||||||
|
secret_key="minioadmin",
|
||||||
|
bucket="test-bucket",
|
||||||
|
secure=False,
|
||||||
|
)
|
||||||
|
TEST_POSTGRES_CONFIG = PostgresConfig(
|
||||||
|
uri="postgresql://localhost/mydb",
|
||||||
|
table="test_items",
|
||||||
|
primary_key="id",
|
||||||
|
)
|
||||||
@@ -1,25 +1,14 @@
|
|||||||
"""Integration tests configuration."""
|
"""Integration tests configuration."""
|
||||||
|
|
||||||
import os
|
|
||||||
from collections.abc import Generator
|
|
||||||
import pytest
|
|
||||||
import redis
|
|
||||||
import structlog
|
|
||||||
import logging
|
import logging
|
||||||
from minio import Minio
|
|
||||||
|
|
||||||
from testcontainers.redis import RedisContainer
|
import pytest
|
||||||
from testcontainers.minio import MinioContainer
|
import structlog
|
||||||
|
|
||||||
MINIO_ACCESS_KEY = "minioadmin"
|
|
||||||
MINIO_SECRET_KEY = "minioadmin"
|
|
||||||
MINIO_BUCKET = "test-bucket"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session", autouse=True)
|
@pytest.fixture(scope="session", autouse=True)
|
||||||
def configure_logging() -> None:
|
def configure_logging() -> None:
|
||||||
"""Configure logging for the test session."""
|
"""Configure logging for the test session."""
|
||||||
# Configure structlog
|
|
||||||
structlog.configure(
|
structlog.configure(
|
||||||
processors=[
|
processors=[
|
||||||
structlog.stdlib.filter_by_level,
|
structlog.stdlib.filter_by_level,
|
||||||
@@ -32,110 +21,4 @@ def configure_logging() -> None:
|
|||||||
wrapper_class=structlog.stdlib.BoundLogger,
|
wrapper_class=structlog.stdlib.BoundLogger,
|
||||||
cache_logger_on_first_use=True,
|
cache_logger_on_first_use=True,
|
||||||
)
|
)
|
||||||
# Set up basic logging configuration
|
|
||||||
logging.basicConfig(level=logging.ERROR)
|
logging.basicConfig(level=logging.ERROR)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
|
||||||
def redis_container() -> Generator[str]:
|
|
||||||
"""Set up a Redis container for testing and yield the Redis URI."""
|
|
||||||
# Start container
|
|
||||||
container = RedisContainer(
|
|
||||||
image="redis/redis-stack:7.2.0-v0",
|
|
||||||
)
|
|
||||||
container.start()
|
|
||||||
# Set environment variable for Redis URI
|
|
||||||
redis_host = container.get_container_host_ip()
|
|
||||||
redis_port = container.get_exposed_port(6379)
|
|
||||||
redis_uri = f"redis://{redis_host}:{redis_port}"
|
|
||||||
|
|
||||||
yield redis_uri
|
|
||||||
|
|
||||||
# Stop container
|
|
||||||
container.stop()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
|
||||||
def minio_container() -> Generator[dict[str, str]]:
|
|
||||||
"""Set up a Minio container for testing and yield the Minio URI."""
|
|
||||||
# Start container
|
|
||||||
container = MinioContainer(
|
|
||||||
image="minio/minio:latest",
|
|
||||||
access_key=MINIO_ACCESS_KEY,
|
|
||||||
secret_key=MINIO_SECRET_KEY,
|
|
||||||
)
|
|
||||||
container.start()
|
|
||||||
# Build environment variables dictionary
|
|
||||||
minio_host = container.get_container_host_ip()
|
|
||||||
minio_port = container.get_exposed_port(9000)
|
|
||||||
minio_endpoint = f"{minio_host}:{minio_port}"
|
|
||||||
env_vars = {
|
|
||||||
"MINIO_ENDPOINT": minio_endpoint,
|
|
||||||
"MINIO_ACCESS_KEY": MINIO_ACCESS_KEY,
|
|
||||||
"MINIO_SECRET_KEY": MINIO_SECRET_KEY,
|
|
||||||
"MINIO_BUCKET": MINIO_BUCKET,
|
|
||||||
}
|
|
||||||
|
|
||||||
yield env_vars
|
|
||||||
|
|
||||||
# Stop container
|
|
||||||
container.stop()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session", autouse=True)
|
|
||||||
def set_environment_variables(
|
|
||||||
redis_container: str,
|
|
||||||
minio_container: dict[str, str],
|
|
||||||
) -> Generator[dict[str, str]]:
|
|
||||||
"""Set environment variables needed for tests."""
|
|
||||||
# Build environment variables dictionary
|
|
||||||
env_vars = {"REDIS_URI": redis_container}
|
|
||||||
env_vars.update(minio_container)
|
|
||||||
# Set environment variables
|
|
||||||
for key, value in env_vars.items():
|
|
||||||
os.environ[key] = value
|
|
||||||
|
|
||||||
yield env_vars
|
|
||||||
|
|
||||||
# Cleanup
|
|
||||||
for key in env_vars:
|
|
||||||
_ = os.environ.pop(key, default=None)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
|
||||||
def raw_redis_client(redis_container: str) -> Generator[redis.Redis]:
|
|
||||||
"""Provide a raw Redis client connected to the test Redis container."""
|
|
||||||
# Connect client
|
|
||||||
client = redis.Redis.from_url(
|
|
||||||
url=redis_container,
|
|
||||||
socket_connect_timeout=10,
|
|
||||||
)
|
|
||||||
|
|
||||||
yield client
|
|
||||||
|
|
||||||
# Cleanup
|
|
||||||
client.flushall()
|
|
||||||
client.close()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
|
||||||
def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio]:
|
|
||||||
"""Provide a raw Minio client connected to the test Minio container."""
|
|
||||||
# Connect client
|
|
||||||
client = Minio(
|
|
||||||
endpoint=minio_container["MINIO_ENDPOINT"],
|
|
||||||
access_key=minio_container["MINIO_ACCESS_KEY"],
|
|
||||||
secret_key=minio_container["MINIO_SECRET_KEY"],
|
|
||||||
secure=False,
|
|
||||||
)
|
|
||||||
# Ensure bucket exists
|
|
||||||
bucket_name = minio_container["MINIO_BUCKET"]
|
|
||||||
if not client.bucket_exists(bucket_name):
|
|
||||||
client.make_bucket(bucket_name)
|
|
||||||
|
|
||||||
yield client
|
|
||||||
|
|
||||||
# Cleanup
|
|
||||||
objects = client.list_objects(bucket_name, recursive=True)
|
|
||||||
for obj in objects:
|
|
||||||
client.remove_object(bucket_name, obj.object_name)
|
|
||||||
|
|||||||
@@ -1,119 +0,0 @@
|
|||||||
"""Integration tests for ConnectionAwareInterface."""
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from python_repositories.interfaces.connection_aware_interface import (
|
|
||||||
ConnectionAwareInterface,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_instantiation_fails_when_connect_not_implemented() -> None:
|
|
||||||
"""Test that instantiation fails if connect is not implemented."""
|
|
||||||
|
|
||||||
class Incomplete(ConnectionAwareInterface):
|
|
||||||
"""A class that does not implement connect."""
|
|
||||||
|
|
||||||
def disconnect(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_connected(self) -> bool:
|
|
||||||
return False
|
|
||||||
|
|
||||||
with pytest.raises(TypeError):
|
|
||||||
_ = Incomplete() # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
def test_instantiation_fails_when_disconnect_not_implemented() -> None:
|
|
||||||
"""Test that instantiation fails if disconnect is not implemented."""
|
|
||||||
|
|
||||||
class Incomplete(ConnectionAwareInterface):
|
|
||||||
"""A class that does not implement disconnect."""
|
|
||||||
|
|
||||||
def connect(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_connected(self) -> bool:
|
|
||||||
return False
|
|
||||||
|
|
||||||
with pytest.raises(TypeError):
|
|
||||||
_ = Incomplete() # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
def test_instantiation_fails_when_is_connected_not_implemented() -> None:
|
|
||||||
"""Test that instantiation fails if is_connected is not implemented."""
|
|
||||||
|
|
||||||
class Incomplete(ConnectionAwareInterface):
|
|
||||||
"""A class that does not implement is_connected."""
|
|
||||||
|
|
||||||
def connect(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def disconnect(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
with pytest.raises(TypeError):
|
|
||||||
_ = Incomplete() # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
def test_connect_raises_not_implemented_if_not_overwritten() -> None:
|
|
||||||
"""Test that connect raises NotImplementedError if not implemented."""
|
|
||||||
|
|
||||||
class Incomplete(ConnectionAwareInterface):
|
|
||||||
"""A class that does not implement connect."""
|
|
||||||
|
|
||||||
def connect(self) -> None:
|
|
||||||
super().connect() # type: ignore
|
|
||||||
|
|
||||||
def disconnect(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_connected(self) -> bool:
|
|
||||||
return False
|
|
||||||
|
|
||||||
instance = Incomplete()
|
|
||||||
with pytest.raises(NotImplementedError):
|
|
||||||
instance.connect()
|
|
||||||
|
|
||||||
|
|
||||||
def test_disconnect_raises_not_implemented_if_not_overwritten() -> None:
|
|
||||||
"""Test that disconnect raises NotImplementedError if not implemented."""
|
|
||||||
|
|
||||||
class Incomplete(ConnectionAwareInterface):
|
|
||||||
"""A class that does not implement disconnect."""
|
|
||||||
|
|
||||||
def connect(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def disconnect(self) -> None:
|
|
||||||
super().disconnect() # type: ignore
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_connected(self) -> bool:
|
|
||||||
return False
|
|
||||||
|
|
||||||
instance = Incomplete()
|
|
||||||
with pytest.raises(NotImplementedError):
|
|
||||||
instance.disconnect()
|
|
||||||
|
|
||||||
|
|
||||||
def test_is_connected_raises_not_implemented_if_not_overwritten() -> None:
|
|
||||||
"""Test that is_connected raises NotImplementedError if not implemented."""
|
|
||||||
|
|
||||||
class Incomplete(ConnectionAwareInterface):
|
|
||||||
"""A class that does not implement is_connected."""
|
|
||||||
|
|
||||||
def connect(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def disconnect(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_connected(self) -> bool:
|
|
||||||
return super().is_connected # type: ignore
|
|
||||||
|
|
||||||
instance = Incomplete()
|
|
||||||
with pytest.raises(NotImplementedError):
|
|
||||||
_ = instance.is_connected
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
"""Integration tests for ContextAwareInterface."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
import pytest
|
|
||||||
from python_repositories.interfaces.context_aware_interface import ContextAwareInterface
|
|
||||||
|
|
||||||
|
|
||||||
def test_instantiation_fails_when_enter_not_implemented() -> None:
|
|
||||||
"""Test that instantiation fails if __enter__ is not implemented."""
|
|
||||||
|
|
||||||
class Incomplete(ContextAwareInterface):
|
|
||||||
"""A class that does not implement __enter__."""
|
|
||||||
|
|
||||||
def __exit__(
|
|
||||||
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
|
||||||
) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
with pytest.raises(TypeError):
|
|
||||||
_ = Incomplete() # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
def test_instantiation_fails_when_exit_not_implemented() -> None:
|
|
||||||
"""Test that instantiation fails if __exit__ is not implemented."""
|
|
||||||
|
|
||||||
class Incomplete(ContextAwareInterface):
|
|
||||||
"""A class that does not implement __exit__."""
|
|
||||||
|
|
||||||
def __enter__(self) -> Incomplete:
|
|
||||||
return self
|
|
||||||
|
|
||||||
with pytest.raises(TypeError):
|
|
||||||
_ = Incomplete() # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
def test_enter_raises_not_implemented_if_not_overwritten() -> None:
|
|
||||||
"""Test that __enter__ raises NotImplementedError if not implemented."""
|
|
||||||
|
|
||||||
class Incomplete(ContextAwareInterface):
|
|
||||||
"""A class that does not implement __enter__."""
|
|
||||||
|
|
||||||
def __enter__(self) -> Incomplete:
|
|
||||||
super().__enter__() # type: ignore
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __exit__(
|
|
||||||
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
|
||||||
) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
instance = Incomplete()
|
|
||||||
with pytest.raises(NotImplementedError):
|
|
||||||
instance.__enter__()
|
|
||||||
|
|
||||||
|
|
||||||
def test_exit_raises_not_implemented_if_not_overwritten() -> None:
|
|
||||||
"""Test that __exit__ raises NotImplementedError if not implemented."""
|
|
||||||
|
|
||||||
class Incomplete(ContextAwareInterface):
|
|
||||||
"""A class that does not implement __exit__."""
|
|
||||||
|
|
||||||
def __enter__(self) -> ContextAwareInterface:
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __exit__(
|
|
||||||
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
|
||||||
) -> None:
|
|
||||||
super().__exit__(exc_type, exc_val, exc_tb) # type: ignore
|
|
||||||
|
|
||||||
instance = Incomplete()
|
|
||||||
with pytest.raises(NotImplementedError):
|
|
||||||
instance.__exit__(None, None, None)
|
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
"""Examples integration test fixtures."""
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
import pytest
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from python_repositories.config import MinioConfig, PostgresConfig, RedisConfig
|
||||||
|
from tests.integration.minio._containers import (
|
||||||
|
minio_config_from_env,
|
||||||
|
minio_env,
|
||||||
|
raw_minio_client_from_env,
|
||||||
|
)
|
||||||
|
from tests.integration.postgres._containers import (
|
||||||
|
postgres_config_from_container,
|
||||||
|
postgres_uri,
|
||||||
|
raw_postgres_client_from_container,
|
||||||
|
)
|
||||||
|
from tests.integration.redis._containers import (
|
||||||
|
raw_redis_client_from_container,
|
||||||
|
redis_config_from_container,
|
||||||
|
redis_uri,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def redis_container() -> Generator[str, None, None]:
|
||||||
|
"""Set up a Redis container for testing and yield the Redis URI."""
|
||||||
|
yield from redis_uri()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def redis_config(redis_container: str) -> RedisConfig:
|
||||||
|
"""Provide RedisConfig built from the test container."""
|
||||||
|
return redis_config_from_container(redis_container)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def raw_redis_client(redis_container: str) -> Generator[redis.Redis, None, None]:
|
||||||
|
"""Provide a raw Redis client connected to the test Redis container."""
|
||||||
|
yield from raw_redis_client_from_container(redis_container)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def minio_container() -> Generator[dict[str, str], None, None]:
|
||||||
|
"""Set up a Minio container for testing and yield connection settings."""
|
||||||
|
yield from minio_env()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def minio_config(minio_container: dict[str, str]) -> MinioConfig:
|
||||||
|
"""Provide MinioConfig built from the test container."""
|
||||||
|
return minio_config_from_env(minio_container)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio, None, None]:
|
||||||
|
"""Provide a raw Minio client connected to the test Minio container."""
|
||||||
|
yield from raw_minio_client_from_env(minio_container)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def postgres_container() -> Generator[str, None, None]:
|
||||||
|
"""Set up a Postgres container for testing and yield the Postgres URI."""
|
||||||
|
yield from postgres_uri()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def postgres_config(postgres_container: str) -> PostgresConfig:
|
||||||
|
"""Provide PostgresConfig built from the test container."""
|
||||||
|
return postgres_config_from_container(postgres_container)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def raw_postgres_client(
|
||||||
|
postgres_container: str,
|
||||||
|
) -> Generator[object, None, None]:
|
||||||
|
"""Provide a raw Postgres client connected to the test Postgres container."""
|
||||||
|
yield from raw_postgres_client_from_container(postgres_container)
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
"""Integration tests for example domain repositories."""
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
from io import BytesIO
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
import psycopg
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.examples.artifact_object_repository import (
|
||||||
|
ArtifactObjectRepository,
|
||||||
|
)
|
||||||
|
from python_repositories.examples.user_json_repository import UserJsonRepository
|
||||||
|
from python_repositories.examples.user_table_repository import UserTableRepository
|
||||||
|
from tests.integration.postgres._containers import TEST_TABLE
|
||||||
|
|
||||||
|
pytestmark = [
|
||||||
|
pytest.mark.integration,
|
||||||
|
pytest.mark.needs_redis,
|
||||||
|
pytest.mark.needs_minio,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def set_example_env(
|
||||||
|
redis_container: str,
|
||||||
|
minio_container: dict[str, str],
|
||||||
|
raw_minio_client: Minio,
|
||||||
|
) -> Generator[None, None, None]:
|
||||||
|
"""Set env vars so example repositories can use from_env() defaults."""
|
||||||
|
_ = raw_minio_client
|
||||||
|
env_vars = {"REDIS_URI": redis_container, **minio_container}
|
||||||
|
for key, value in env_vars.items():
|
||||||
|
os.environ[key] = value
|
||||||
|
yield
|
||||||
|
for key in env_vars:
|
||||||
|
_ = os.environ.pop(key, default=None)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def set_postgres_example_env(
|
||||||
|
postgres_container: str,
|
||||||
|
raw_postgres_client: psycopg.Connection,
|
||||||
|
) -> Generator[None, None, None]:
|
||||||
|
"""Set Postgres env vars for example repositories using from_env() defaults."""
|
||||||
|
env_vars = {
|
||||||
|
"POSTGRES_URI": postgres_container,
|
||||||
|
"POSTGRES_TABLE": TEST_TABLE,
|
||||||
|
"POSTGRES_PRIMARY_KEY": "id",
|
||||||
|
}
|
||||||
|
for key, value in env_vars.items():
|
||||||
|
os.environ[key] = value
|
||||||
|
yield
|
||||||
|
for key in env_vars:
|
||||||
|
_ = os.environ.pop(key, default=None)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def user_data() -> Generator[dict[str, str], None, None]:
|
||||||
|
"""Provide sample user data for tests."""
|
||||||
|
yield {"name": "Alice", "email": "[email protected]"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def artifact_data() -> Generator[BytesIO, None, None]:
|
||||||
|
"""Provide sample artifact data for tests."""
|
||||||
|
yield BytesIO(random.randbytes(2**20))
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_json_repository_save_and_get(
|
||||||
|
user_data: dict[str, str],
|
||||||
|
) -> None:
|
||||||
|
"""Test that UserJsonRepository can save and retrieve a user."""
|
||||||
|
with UserJsonRepository() as repo:
|
||||||
|
repo.save_user("alice", user_data)
|
||||||
|
assert repo.get_user("alice") == user_data
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_json_repository_delete(
|
||||||
|
user_data: dict[str, str],
|
||||||
|
) -> None:
|
||||||
|
"""Test that UserJsonRepository can delete a user."""
|
||||||
|
with UserJsonRepository() as repo:
|
||||||
|
repo.save_user("alice", user_data)
|
||||||
|
repo.delete_user("alice")
|
||||||
|
assert repo.get_user("alice") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_artifact_object_repository_store_and_get(
|
||||||
|
artifact_data: BytesIO,
|
||||||
|
) -> None:
|
||||||
|
"""Test that ArtifactObjectRepository can store and retrieve an artifact."""
|
||||||
|
with ArtifactObjectRepository() as repo:
|
||||||
|
repo.store_artifact("report-1", artifact_data)
|
||||||
|
received = repo.get_artifact("report-1")
|
||||||
|
assert received is not None
|
||||||
|
artifact_data.seek(0)
|
||||||
|
assert received.read() == artifact_data.read()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.needs_postgres
|
||||||
|
def test_user_table_repository_save_and_get(
|
||||||
|
user_data: dict[str, str],
|
||||||
|
) -> None:
|
||||||
|
"""Test that UserTableRepository can save and retrieve a user."""
|
||||||
|
with UserTableRepository() as repo:
|
||||||
|
repo.save_user("alice", user_data)
|
||||||
|
user = repo.get_user("alice")
|
||||||
|
assert user is not None
|
||||||
|
assert user["name"] == user_data["name"]
|
||||||
|
assert user["email"] == user_data["email"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.needs_postgres
|
||||||
|
def test_user_table_repository_delete(
|
||||||
|
user_data: dict[str, str],
|
||||||
|
) -> None:
|
||||||
|
"""Test that UserTableRepository can delete a user."""
|
||||||
|
with UserTableRepository() as repo:
|
||||||
|
repo.save_user("alice", user_data)
|
||||||
|
repo.delete_user("alice")
|
||||||
|
assert repo.get_user("alice") is None
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
"""MinIO container session helpers for integration tests."""
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
from testcontainers.minio import MinioContainer
|
||||||
|
|
||||||
|
from python_repositories.config import MinioConfig
|
||||||
|
|
||||||
|
MINIO_ACCESS_KEY = "minioadmin"
|
||||||
|
MINIO_SECRET_KEY = "minioadmin"
|
||||||
|
MINIO_BUCKET = "test-bucket"
|
||||||
|
|
||||||
|
_minio_env: dict[str, str] | None = None
|
||||||
|
_minio_container: MinioContainer | None = None
|
||||||
|
_raw_minio_client: Minio | None = None
|
||||||
|
_minio_env_refs = 0
|
||||||
|
_raw_minio_client_refs = 0
|
||||||
|
|
||||||
|
|
||||||
|
def minio_env() -> Generator[dict[str, str], None, None]:
|
||||||
|
"""Yield session-scoped MinIO connection settings, starting the container once."""
|
||||||
|
global _minio_env, _minio_container, _minio_env_refs
|
||||||
|
if _minio_env is None:
|
||||||
|
_minio_container = MinioContainer(
|
||||||
|
image="minio/minio:latest",
|
||||||
|
access_key=MINIO_ACCESS_KEY,
|
||||||
|
secret_key=MINIO_SECRET_KEY,
|
||||||
|
)
|
||||||
|
_minio_container.start()
|
||||||
|
minio_host = _minio_container.get_container_host_ip()
|
||||||
|
minio_port = _minio_container.get_exposed_port(9000)
|
||||||
|
minio_endpoint = f"{minio_host}:{minio_port}"
|
||||||
|
_minio_env = {
|
||||||
|
"MINIO_ENDPOINT": minio_endpoint,
|
||||||
|
"MINIO_ACCESS_KEY": MINIO_ACCESS_KEY,
|
||||||
|
"MINIO_SECRET_KEY": MINIO_SECRET_KEY,
|
||||||
|
"MINIO_BUCKET": MINIO_BUCKET,
|
||||||
|
"MINIO_SECURE": "false",
|
||||||
|
}
|
||||||
|
|
||||||
|
_minio_env_refs += 1
|
||||||
|
yield _minio_env
|
||||||
|
_minio_env_refs -= 1
|
||||||
|
|
||||||
|
if _minio_env_refs == 0 and _minio_container is not None:
|
||||||
|
_minio_container.stop()
|
||||||
|
_minio_container = None
|
||||||
|
_minio_env = None
|
||||||
|
|
||||||
|
|
||||||
|
def minio_config_from_env(env: dict[str, str]) -> MinioConfig:
|
||||||
|
"""Build MinioConfig from container environment settings."""
|
||||||
|
return MinioConfig(
|
||||||
|
endpoint=env["MINIO_ENDPOINT"],
|
||||||
|
access_key=env["MINIO_ACCESS_KEY"],
|
||||||
|
secret_key=env["MINIO_SECRET_KEY"],
|
||||||
|
bucket=env["MINIO_BUCKET"],
|
||||||
|
secure=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def raw_minio_client_from_env(env: dict[str, str]) -> Generator[Minio, None, None]:
|
||||||
|
"""Yield a session-scoped raw MinIO client, reusing one client per session."""
|
||||||
|
global _raw_minio_client, _raw_minio_client_refs
|
||||||
|
if _raw_minio_client is None:
|
||||||
|
_raw_minio_client = Minio(
|
||||||
|
endpoint=env["MINIO_ENDPOINT"],
|
||||||
|
access_key=env["MINIO_ACCESS_KEY"],
|
||||||
|
secret_key=env["MINIO_SECRET_KEY"],
|
||||||
|
secure=False,
|
||||||
|
)
|
||||||
|
bucket_name = env["MINIO_BUCKET"]
|
||||||
|
if not _raw_minio_client.bucket_exists(bucket_name):
|
||||||
|
_raw_minio_client.make_bucket(bucket_name)
|
||||||
|
|
||||||
|
_raw_minio_client_refs += 1
|
||||||
|
yield _raw_minio_client
|
||||||
|
_raw_minio_client_refs -= 1
|
||||||
|
|
||||||
|
if _raw_minio_client_refs == 0 and _raw_minio_client is not None:
|
||||||
|
bucket_name = env["MINIO_BUCKET"]
|
||||||
|
objects = _raw_minio_client.list_objects(bucket_name, recursive=True)
|
||||||
|
for obj in objects:
|
||||||
|
_raw_minio_client.remove_object(bucket_name, obj.object_name)
|
||||||
|
_raw_minio_client = None
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
"""MinIO integration test fixtures."""
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.config import MinioConfig
|
||||||
|
from tests.integration.minio._containers import (
|
||||||
|
minio_config_from_env,
|
||||||
|
minio_env,
|
||||||
|
raw_minio_client_from_env,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def minio_container() -> Generator[dict[str, str], None, None]:
|
||||||
|
"""Set up a Minio container for testing and yield connection settings."""
|
||||||
|
yield from minio_env()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def minio_config(minio_container: dict[str, str]) -> MinioConfig:
|
||||||
|
"""Provide MinioConfig built from the test container."""
|
||||||
|
return minio_config_from_env(minio_container)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio, None, None]:
|
||||||
|
"""Provide a raw Minio client connected to the test Minio container."""
|
||||||
|
yield from raw_minio_client_from_env(minio_container)
|
||||||
+194
-207
@@ -1,18 +1,21 @@
|
|||||||
# pylint: disable=protected-access
|
|
||||||
# The above line disables pylint's protected member access warnings for this file,
|
|
||||||
# allowing tests to access MinioAdapter's internal methods as needed for integration testing.
|
|
||||||
"""Integration tests for the MinioAdapter."""
|
"""Integration tests for the MinioAdapter."""
|
||||||
|
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
import pytest
|
from dataclasses import replace
|
||||||
from unittest.mock import MagicMock
|
|
||||||
from minio import Minio
|
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
import random
|
|
||||||
import os
|
|
||||||
import logging
|
import logging
|
||||||
from minio import S3Error
|
import random
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from minio import Minio, S3Error
|
||||||
|
import pytest
|
||||||
|
from urllib3.response import BaseHTTPResponse
|
||||||
|
|
||||||
from python_repositories.adapters.minio_adapter import MinioAdapter
|
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||||
|
from python_repositories.config import MinioConfig
|
||||||
|
from tests.conftest import TEST_MINIO_CONFIG
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.integration, pytest.mark.needs_minio]
|
||||||
|
|
||||||
|
|
||||||
def same_data(
|
def same_data(
|
||||||
@@ -20,15 +23,10 @@ def same_data(
|
|||||||
data_b: BytesIO,
|
data_b: BytesIO,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Check if two BytesIO-objects contain the same data."""
|
"""Check if two BytesIO-objects contain the same data."""
|
||||||
assert isinstance(data_a, BytesIO)
|
|
||||||
assert isinstance(data_b, BytesIO)
|
|
||||||
# prepare for being read
|
|
||||||
data_a.seek(0)
|
data_a.seek(0)
|
||||||
data_b.seek(0)
|
data_b.seek(0)
|
||||||
# convert to bytes
|
|
||||||
data_a_bytes = data_a.read()
|
data_a_bytes = data_a.read()
|
||||||
data_b_bytes = data_b.read()
|
data_b_bytes = data_b.read()
|
||||||
# compare size
|
|
||||||
if len(data_a_bytes) != len(data_b_bytes):
|
if len(data_a_bytes) != len(data_b_bytes):
|
||||||
logging.error(
|
logging.error(
|
||||||
"data has different length: %s and %s",
|
"data has different length: %s and %s",
|
||||||
@@ -36,7 +34,6 @@ def same_data(
|
|||||||
len(data_b_bytes),
|
len(data_b_bytes),
|
||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
# compare content
|
|
||||||
if data_a_bytes != data_b_bytes:
|
if data_a_bytes != data_b_bytes:
|
||||||
logging.error("data has different bytes")
|
logging.error("data has different bytes")
|
||||||
return False
|
return False
|
||||||
@@ -44,22 +41,21 @@ def same_data(
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def data() -> Generator[BytesIO]:
|
def data() -> Generator[BytesIO, None, None]:
|
||||||
"""Provide a sample data bytes for tests."""
|
"""Provide a sample data bytes for tests."""
|
||||||
# Generate random bytes
|
random_bytes = random.randbytes(2**21)
|
||||||
random_bytes = random.randbytes(2**21) # 2 MiB
|
|
||||||
yield BytesIO(random_bytes)
|
yield BytesIO(random_bytes)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def data_in_minio(
|
def data_in_minio(
|
||||||
raw_minio_client: Minio,
|
raw_minio_client: Minio,
|
||||||
|
minio_config: MinioConfig,
|
||||||
data: BytesIO,
|
data: BytesIO,
|
||||||
) -> Generator[tuple[str, BytesIO]]:
|
) -> Generator[tuple[str, BytesIO], None, None]:
|
||||||
"""Fixture to set up a known value in Minio before each test."""
|
"""Fixture to set up a known value in Minio before each test."""
|
||||||
object_name = "test_object"
|
object_name = "test_object"
|
||||||
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
bucket_name = minio_config.bucket
|
||||||
# Upload object
|
|
||||||
num_bytes = data.getbuffer().nbytes
|
num_bytes = data.getbuffer().nbytes
|
||||||
data.seek(0)
|
data.seek(0)
|
||||||
raw_minio_client.put_object(
|
raw_minio_client.put_object(
|
||||||
@@ -69,19 +65,17 @@ def data_in_minio(
|
|||||||
length=num_bytes,
|
length=num_bytes,
|
||||||
part_size=MinioAdapter.chunk_size,
|
part_size=MinioAdapter.chunk_size,
|
||||||
)
|
)
|
||||||
# Reset data for reading in tests
|
|
||||||
data.seek(0)
|
data.seek(0)
|
||||||
|
|
||||||
yield object_name, data
|
yield object_name, data
|
||||||
|
|
||||||
# Cleanup
|
|
||||||
raw_minio_client.remove_object(bucket_name, object_name)
|
raw_minio_client.remove_object(bucket_name, object_name)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def minio_adapter() -> Generator[MinioAdapter]:
|
def minio_adapter(minio_config: MinioConfig) -> Generator[MinioAdapter, None, None]:
|
||||||
"""Fixture to provide a connected MinioAdapter instance."""
|
"""Fixture to provide a connected MinioAdapter instance."""
|
||||||
adapter = MinioAdapter()
|
adapter = MinioAdapter(config=minio_config)
|
||||||
adapter.connect()
|
adapter.connect()
|
||||||
yield adapter
|
yield adapter
|
||||||
adapter.disconnect()
|
adapter.disconnect()
|
||||||
@@ -90,10 +84,10 @@ def minio_adapter() -> Generator[MinioAdapter]:
|
|||||||
@pytest.fixture(scope="function", autouse=True)
|
@pytest.fixture(scope="function", autouse=True)
|
||||||
def clear_minio(
|
def clear_minio(
|
||||||
raw_minio_client: Minio,
|
raw_minio_client: Minio,
|
||||||
|
minio_config: MinioConfig,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Fixture to clear all Minio objects before each test."""
|
"""Fixture to clear all Minio objects before each test."""
|
||||||
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
bucket_name = minio_config.bucket
|
||||||
# Clear all objects before each test
|
|
||||||
objects = raw_minio_client.list_objects(bucket_name, recursive=True)
|
objects = raw_minio_client.list_objects(bucket_name, recursive=True)
|
||||||
for obj in objects:
|
for obj in objects:
|
||||||
if not obj.object_name:
|
if not obj.object_name:
|
||||||
@@ -101,24 +95,6 @@ def clear_minio(
|
|||||||
raw_minio_client.remove_object(bucket_name, obj.object_name)
|
raw_minio_client.remove_object(bucket_name, obj.object_name)
|
||||||
|
|
||||||
|
|
||||||
def test_should_adhere_to_interface() -> None:
|
|
||||||
"""Test that the MinioAdapter adheres to the expected interface."""
|
|
||||||
# Instantiation fails if interface not adhered to
|
|
||||||
_ = MinioAdapter()
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_have_logger_when_instantiated() -> None:
|
|
||||||
"""Test that the MinioAdapter has a logger when instantiated."""
|
|
||||||
adapter = MinioAdapter()
|
|
||||||
assert hasattr(adapter, "logger")
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_not_be_connected_when_instantiated() -> None:
|
|
||||||
"""Test that the MinioAdapter is not connected when instantiated."""
|
|
||||||
adapter = MinioAdapter()
|
|
||||||
assert not adapter.is_connected
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_info_when_already_connected(
|
def test_should_log_info_when_already_connected(
|
||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
caplog: pytest.LogCaptureFixture,
|
caplog: pytest.LogCaptureFixture,
|
||||||
@@ -126,55 +102,76 @@ def test_should_log_info_when_already_connected(
|
|||||||
"""Test that the MinioAdapter logs info when connect is called while already connected."""
|
"""Test that the MinioAdapter logs info when connect is called while already connected."""
|
||||||
with caplog.at_level(logging.INFO):
|
with caplog.at_level(logging.INFO):
|
||||||
minio_adapter.connect()
|
minio_adapter.connect()
|
||||||
assert "Already connected to Minio" in caplog.text
|
assert "Already connected" in caplog.text
|
||||||
|
assert "Minio" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_when_unable_to_connect(
|
def test_should_raise_connection_error_when_unable_to_connect() -> None:
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
|
||||||
) -> None:
|
|
||||||
"""Test that the MinioAdapter raises a ConnectionError when unable to connect."""
|
"""Test that the MinioAdapter raises a ConnectionError when unable to connect."""
|
||||||
# Arrange
|
config = MinioConfig(
|
||||||
monkeypatch.setenv("MINIO_ENDPOINT", "invalid_uri")
|
endpoint="invalid_uri",
|
||||||
adapter = MinioAdapter()
|
access_key="minioadmin",
|
||||||
|
secret_key="minioadmin",
|
||||||
|
bucket="test-bucket",
|
||||||
|
)
|
||||||
|
adapter = MinioAdapter(config=config)
|
||||||
with pytest.raises(ConnectionError):
|
with pytest.raises(ConnectionError):
|
||||||
adapter.connect()
|
adapter.connect()
|
||||||
assert not adapter.is_connected
|
assert not adapter.is_connected()
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_info_when_creating_expected_bucket(
|
def test_connect_raises_when_bucket_missing(
|
||||||
raw_minio_client: Minio,
|
raw_minio_client: Minio,
|
||||||
|
minio_config: MinioConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that connect fails when the configured bucket is missing."""
|
||||||
|
bucket_name = minio_config.bucket
|
||||||
|
raw_minio_client.remove_bucket(bucket_name)
|
||||||
|
adapter = MinioAdapter(config=minio_config)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with pytest.raises(ConnectionError, match="does not exist"):
|
||||||
|
adapter.connect()
|
||||||
|
assert not adapter.is_connected()
|
||||||
|
finally:
|
||||||
|
raw_minio_client.make_bucket(bucket_name)
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_creates_bucket_when_create_bucket_if_missing_enabled(
|
||||||
|
raw_minio_client: Minio,
|
||||||
|
minio_config: MinioConfig,
|
||||||
caplog: pytest.LogCaptureFixture,
|
caplog: pytest.LogCaptureFixture,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter logs info when creating the expected bucket."""
|
"""Test that connect can create the configured bucket when enabled."""
|
||||||
# Arrange
|
bucket_name = minio_config.bucket
|
||||||
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
|
||||||
raw_minio_client.remove_bucket(bucket_name)
|
raw_minio_client.remove_bucket(bucket_name)
|
||||||
adapter = MinioAdapter()
|
config = replace(minio_config, create_bucket_if_missing=True)
|
||||||
# Act
|
adapter = MinioAdapter(config=config)
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
with caplog.at_level(logging.INFO):
|
||||||
adapter.connect()
|
adapter.connect()
|
||||||
# Assert
|
|
||||||
assert f"Creating bucket '{bucket_name}'" in caplog.text
|
assert "Creating bucket" in caplog.text
|
||||||
|
assert bucket_name in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_error_on_exception_during_exit(
|
def test_should_log_error_on_exception_during_exit(
|
||||||
minio_container: dict[str, str],
|
minio_config: MinioConfig,
|
||||||
caplog: pytest.LogCaptureFixture,
|
caplog: pytest.LogCaptureFixture,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter logs an error if an exception occurs during __exit__."""
|
"""Test that the MinioAdapter logs an error if an exception occurs during __exit__."""
|
||||||
try:
|
try:
|
||||||
with MinioAdapter() as adapter:
|
with MinioAdapter(config=minio_config) as adapter:
|
||||||
assert adapter.is_connected
|
assert adapter.is_connected()
|
||||||
raise ValueError("Simulated error")
|
raise ValueError("Simulated error")
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass # Expected
|
pass
|
||||||
# Assert error was logged
|
|
||||||
assert "Error while exiting context" in caplog.text
|
assert "Error while exiting context" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_should_have_context_manager() -> None:
|
def test_should_have_context_manager(minio_config: MinioConfig) -> None:
|
||||||
"""Test that the MinioAdapter can be used as a context manager."""
|
"""Test that the MinioAdapter can be used as a context manager."""
|
||||||
with MinioAdapter() as adapter:
|
with MinioAdapter(config=minio_config) as adapter:
|
||||||
assert adapter._client is not None
|
assert adapter._client is not None
|
||||||
assert adapter._client is None
|
assert adapter._client is None
|
||||||
|
|
||||||
@@ -184,11 +181,8 @@ def test_should_get_data(
|
|||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter can get data from a bucket."""
|
"""Test that the MinioAdapter can get data from a bucket."""
|
||||||
# Arrange
|
|
||||||
object_name, expected_data = data_in_minio
|
object_name, expected_data = data_in_minio
|
||||||
# Act
|
received_data = minio_adapter.get(object_name)
|
||||||
received_data = minio_adapter._get(object_name)
|
|
||||||
# Assert
|
|
||||||
assert received_data is not None
|
assert received_data is not None
|
||||||
assert same_data(expected_data, received_data)
|
assert same_data(expected_data, received_data)
|
||||||
|
|
||||||
@@ -197,11 +191,7 @@ def test_should_get_none_for_nonexistent_object(
|
|||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter returns None for a nonexistent object."""
|
"""Test that the MinioAdapter returns None for a nonexistent object."""
|
||||||
# Arrange
|
received_data = minio_adapter.get("nonexistent_object")
|
||||||
object_name = "nonexistent_object"
|
|
||||||
# Act
|
|
||||||
received_data = minio_adapter._get(object_name)
|
|
||||||
# Assert
|
|
||||||
assert received_data is None
|
assert received_data is None
|
||||||
|
|
||||||
|
|
||||||
@@ -209,120 +199,95 @@ def test_should_raise_value_error_on_invalid_get_object_name(
|
|||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter raises ValueError when getting with an invalid object name."""
|
"""Test that the MinioAdapter raises ValueError when getting with an invalid object name."""
|
||||||
# Arrange
|
|
||||||
invalid_object_names = ["", 123, None]
|
invalid_object_names = ["", 123, None]
|
||||||
# Act & Assert
|
|
||||||
for object_name in invalid_object_names:
|
for object_name in invalid_object_names:
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
minio_adapter._get(object_name) # type: ignore
|
minio_adapter.get(object_name) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_on_get_when_not_connected(
|
def test_should_raise_connection_error_on_get_when_not_connected(
|
||||||
minio_adapter: MinioAdapter,
|
minio_config: MinioConfig,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter raises ConnectionError when getting while not connected."""
|
"""Test that the MinioAdapter raises ConnectionError when getting while not connected."""
|
||||||
# Arrange
|
adapter = MinioAdapter(config=minio_config)
|
||||||
adapter = MinioAdapter() # not connected
|
|
||||||
# Act & Assert
|
|
||||||
with pytest.raises(ConnectionError):
|
with pytest.raises(ConnectionError):
|
||||||
adapter._get("some_object")
|
adapter.get("some_object")
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_warning_when_getting_nonexistent_object(
|
def test_should_log_warning_when_getting_nonexistent_object(
|
||||||
caplog: pytest.LogCaptureFixture,
|
caplog: pytest.LogCaptureFixture,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter logs a warning when getting a nonexistent object."""
|
"""Test that the MinioAdapter logs a warning when getting a nonexistent object."""
|
||||||
# Arrange
|
mock_client = MagicMock(spec=Minio)
|
||||||
adapter = MinioAdapter()
|
mock_client.bucket_exists.return_value = True
|
||||||
adapter._client = MagicMock(spec=Minio)
|
mock_client.get_object.side_effect = S3Error(
|
||||||
adapter._client.get_object.side_effect = S3Error(
|
MagicMock(spec=BaseHTTPResponse),
|
||||||
code="NoSuchKey",
|
"NoSuchKey",
|
||||||
message="",
|
"",
|
||||||
resource="",
|
"",
|
||||||
request_id="",
|
"",
|
||||||
host_id="",
|
"",
|
||||||
response="",
|
|
||||||
bucket_name="test-bucket",
|
bucket_name="test-bucket",
|
||||||
object_name="missing-object",
|
object_name="missing-object",
|
||||||
)
|
)
|
||||||
adapter._bucket_name = "test-bucket"
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
object_name = "missing-object"
|
object_name = "missing-object"
|
||||||
# Act
|
|
||||||
with caplog.at_level("WARNING"):
|
with caplog.at_level("WARNING"):
|
||||||
result = adapter._get(object_name)
|
result = adapter.get(object_name)
|
||||||
# Assert
|
|
||||||
assert result is None
|
assert result is None
|
||||||
assert (
|
assert "Object not found" in caplog.text
|
||||||
f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'"
|
assert object_name in caplog.text
|
||||||
in caplog.text
|
bucket_name = adapter._bucket_name
|
||||||
)
|
assert bucket_name is not None
|
||||||
|
assert bucket_name in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
|
def test_should_reraise_s3error_other_than_no_such_key() -> None:
|
||||||
caplog: pytest.LogCaptureFixture,
|
"""Test that the MinioAdapter re-raises unhandled S3 errors."""
|
||||||
) -> None:
|
mock_client = MagicMock(spec=Minio)
|
||||||
"""Test that the MinioAdapter logs an error when getting a nonexistent object."""
|
mock_client.bucket_exists.return_value = True
|
||||||
# Arrange
|
|
||||||
adapter = MinioAdapter()
|
|
||||||
adapter._client = MagicMock(spec=Minio)
|
|
||||||
other_s3error = S3Error(
|
other_s3error = S3Error(
|
||||||
code="UnhandledError",
|
MagicMock(spec=BaseHTTPResponse),
|
||||||
message="",
|
"UnhandledError",
|
||||||
resource="",
|
"",
|
||||||
request_id="",
|
"",
|
||||||
host_id="",
|
"",
|
||||||
response="",
|
"",
|
||||||
bucket_name="test-bucket",
|
bucket_name="test-bucket",
|
||||||
object_name="missing-object",
|
object_name="missing-object",
|
||||||
)
|
)
|
||||||
adapter._client.get_object.side_effect = other_s3error
|
mock_client.get_object.side_effect = other_s3error
|
||||||
adapter._bucket_name = "test-bucket"
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
object_name = "missing-object"
|
with pytest.raises(S3Error) as exc_info:
|
||||||
# Act
|
adapter.get("missing-object")
|
||||||
with caplog.at_level("ERROR"):
|
assert exc_info.value.code == "UnhandledError"
|
||||||
result = adapter._get(object_name)
|
|
||||||
# Assert
|
|
||||||
assert result is None
|
|
||||||
assert repr(other_s3error) in caplog.text
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_error_when_getting_with_general_exception(
|
def test_should_reraise_general_exception() -> None:
|
||||||
caplog: pytest.LogCaptureFixture,
|
"""Test that the MinioAdapter re-raises general exceptions during get."""
|
||||||
) -> None:
|
mock_client = MagicMock(spec=Minio)
|
||||||
"""Test that the MinioAdapter logs an error when getting a nonexistent object."""
|
mock_client.bucket_exists.return_value = True
|
||||||
# Arrange
|
|
||||||
adapter = MinioAdapter()
|
|
||||||
adapter._client = MagicMock(spec=Minio)
|
|
||||||
general_exception = Exception("General failure")
|
general_exception = Exception("General failure")
|
||||||
adapter._client.get_object.side_effect = general_exception
|
mock_client.get_object.side_effect = general_exception
|
||||||
adapter._bucket_name = "test-bucket"
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
object_name = "missing-object"
|
with pytest.raises(Exception, match="General failure"):
|
||||||
# Act
|
adapter.get("missing-object")
|
||||||
with caplog.at_level("ERROR"):
|
|
||||||
result = adapter._get(object_name)
|
|
||||||
# Assert
|
|
||||||
assert result is None
|
|
||||||
assert repr(general_exception) in caplog.text
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_put_data(
|
def test_should_put_data(
|
||||||
data: BytesIO,
|
data: BytesIO,
|
||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
|
minio_config: MinioConfig,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter can put data into a bucket."""
|
"""Test that the MinioAdapter can put data into a bucket."""
|
||||||
# Arrange
|
|
||||||
object_name = "new_test_object"
|
object_name = "new_test_object"
|
||||||
received_data = minio_adapter._get(object_name)
|
received_data = minio_adapter.get(object_name)
|
||||||
assert received_data is None # ensure object does not exist yet
|
assert received_data is None
|
||||||
# Act
|
minio_adapter.put(object_name, data)
|
||||||
minio_adapter._put(object_name, data)
|
received_data = minio_adapter.get(object_name)
|
||||||
# Assert
|
|
||||||
received_data = minio_adapter._get(object_name)
|
|
||||||
assert received_data is not None
|
assert received_data is not None
|
||||||
assert same_data(data, received_data)
|
assert same_data(data, received_data)
|
||||||
# Cleanup
|
minio_adapter._client.remove_object(minio_config.bucket, object_name) # type: ignore[union-attr]
|
||||||
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
|
||||||
minio_adapter._client.remove_object(bucket_name, object_name) # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_update_data(
|
def test_should_update_data(
|
||||||
@@ -330,16 +295,13 @@ def test_should_update_data(
|
|||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter can update data in a bucket."""
|
"""Test that the MinioAdapter can update data in a bucket."""
|
||||||
# Arrange
|
|
||||||
object_name, _ = data_in_minio
|
object_name, _ = data_in_minio
|
||||||
new_data = BytesIO(random.randbytes(2**21)) # 2 MiB
|
new_data = BytesIO(random.randbytes(2**21))
|
||||||
received_data = minio_adapter._get(object_name)
|
received_data = minio_adapter.get(object_name)
|
||||||
assert received_data is not None
|
assert received_data is not None
|
||||||
assert not same_data(received_data, new_data)
|
assert not same_data(received_data, new_data)
|
||||||
# Act
|
minio_adapter.put(object_name, new_data)
|
||||||
minio_adapter._put(object_name, new_data)
|
received_data = minio_adapter.get(object_name)
|
||||||
# Assert
|
|
||||||
received_data = minio_adapter._get(object_name)
|
|
||||||
assert received_data is not None
|
assert received_data is not None
|
||||||
assert same_data(new_data, received_data)
|
assert same_data(new_data, received_data)
|
||||||
|
|
||||||
@@ -349,37 +311,81 @@ def test_should_raise_value_error_on_invalid_put_object_name(
|
|||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter raises ValueError when putting with an invalid object name."""
|
"""Test that the MinioAdapter raises ValueError when putting with an invalid object name."""
|
||||||
# Arrange
|
|
||||||
invalid_object_names = ["", 123, None]
|
invalid_object_names = ["", 123, None]
|
||||||
# Act & Assert
|
|
||||||
for object_name in invalid_object_names:
|
for object_name in invalid_object_names:
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
minio_adapter._put(object_name, data) # type: ignore
|
minio_adapter.put(object_name, data) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_value_error_on_invalid_put_data(
|
def test_should_raise_value_error_on_invalid_put_data(
|
||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter raises ValueError when putting with invalid data."""
|
"""Test that the MinioAdapter raises ValueError when putting with invalid data."""
|
||||||
# Arrange
|
|
||||||
object_name = "valid_object_name"
|
object_name = "valid_object_name"
|
||||||
invalid_data = ["not_bytesio", 123, None]
|
invalid_data = ["not_bytesio", 123, None]
|
||||||
# Act & Assert
|
for invalid in invalid_data:
|
||||||
for data in invalid_data:
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
minio_adapter._put(object_name, data) # type: ignore
|
minio_adapter.put(object_name, invalid) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_on_put_when_not_connected(
|
def test_should_put_and_get_empty_bytesio(
|
||||||
|
minio_adapter: MinioAdapter,
|
||||||
|
minio_config: MinioConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that the MinioAdapter can put and get a zero-byte object."""
|
||||||
|
object_name = "empty_object"
|
||||||
|
empty_data = BytesIO()
|
||||||
|
assert minio_adapter.get(object_name) is None
|
||||||
|
minio_adapter.put(object_name, empty_data)
|
||||||
|
received_data = minio_adapter.get(object_name)
|
||||||
|
assert received_data is not None
|
||||||
|
assert same_data(empty_data, received_data)
|
||||||
|
minio_adapter._client.remove_object(minio_config.bucket, object_name) # type: ignore[union-attr]
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_distinguish_missing_object_from_empty_object(
|
||||||
|
minio_adapter: MinioAdapter,
|
||||||
|
minio_config: MinioConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that missing objects and zero-byte objects are distinguishable."""
|
||||||
|
object_name = "empty_object"
|
||||||
|
minio_adapter.put(object_name, BytesIO())
|
||||||
|
assert minio_adapter.get("other_object") is None
|
||||||
|
minio_adapter.delete(object_name)
|
||||||
|
assert minio_adapter.get(object_name) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_list_zero_byte_object(
|
||||||
|
minio_adapter: MinioAdapter,
|
||||||
|
minio_config: MinioConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that a zero-byte object appears in object listings."""
|
||||||
|
object_name = "empty_object"
|
||||||
|
minio_adapter.put(object_name, BytesIO())
|
||||||
|
assert object_name in minio_adapter.list_objects()
|
||||||
|
minio_adapter._client.remove_object(minio_config.bucket, object_name) # type: ignore[union-attr]
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_value_error_on_invalid_put_content_type(
|
||||||
data: BytesIO,
|
data: BytesIO,
|
||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that the MinioAdapter raises ValueError when putting with an invalid content type."""
|
||||||
|
object_name = "valid_object_name"
|
||||||
|
invalid_content_types = ["", 123, None]
|
||||||
|
for content_type in invalid_content_types:
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
minio_adapter.put(object_name, data, content_type) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_on_put_when_not_connected(
|
||||||
|
minio_config: MinioConfig,
|
||||||
|
data: BytesIO,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter raises ConnectionError when putting while not connected."""
|
"""Test that the MinioAdapter raises ConnectionError when putting while not connected."""
|
||||||
# Arrange
|
adapter = MinioAdapter(config=minio_config)
|
||||||
adapter = MinioAdapter() # not connected
|
|
||||||
# Act & Assert
|
|
||||||
with pytest.raises(ConnectionError):
|
with pytest.raises(ConnectionError):
|
||||||
adapter._put("some_object", data)
|
adapter.put("some_object", data)
|
||||||
|
|
||||||
|
|
||||||
def test_should_delete_object(
|
def test_should_delete_object(
|
||||||
@@ -387,38 +393,30 @@ def test_should_delete_object(
|
|||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter can delete an object from a bucket."""
|
"""Test that the MinioAdapter can delete an object from a bucket."""
|
||||||
# Arrange
|
|
||||||
object_name, _ = data_in_minio
|
object_name, _ = data_in_minio
|
||||||
received_data = minio_adapter._get(object_name)
|
received_data = minio_adapter.get(object_name)
|
||||||
assert received_data is not None # ensure object exists
|
assert received_data is not None
|
||||||
# Act
|
minio_adapter.delete(object_name)
|
||||||
minio_adapter._delete(object_name)
|
assert minio_adapter.get(object_name) is None
|
||||||
# Assert
|
|
||||||
received_data = minio_adapter._get(object_name)
|
|
||||||
assert received_data is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_value_error_on_invalid_delete_object_name(
|
def test_should_raise_value_error_on_invalid_delete_object_name(
|
||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter raises ValueError when deleting with an invalid object name."""
|
"""Test that the MinioAdapter raises ValueError when deleting with an invalid object name."""
|
||||||
# Arrange
|
|
||||||
invalid_object_names = ["", 123, None]
|
invalid_object_names = ["", 123, None]
|
||||||
# Act & Assert
|
|
||||||
for object_name in invalid_object_names:
|
for object_name in invalid_object_names:
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
minio_adapter._delete(object_name) # type: ignore
|
minio_adapter.delete(object_name) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_on_delete_when_not_connected(
|
def test_should_raise_connection_error_on_delete_when_not_connected(
|
||||||
minio_adapter: MinioAdapter,
|
minio_config: MinioConfig,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter raises ConnectionError when deleting while not connected."""
|
"""Test that the MinioAdapter raises ConnectionError when deleting while not connected."""
|
||||||
# Arrange
|
adapter = MinioAdapter(config=minio_config)
|
||||||
adapter = MinioAdapter() # not connected
|
|
||||||
# Act & Assert
|
|
||||||
with pytest.raises(ConnectionError):
|
with pytest.raises(ConnectionError):
|
||||||
adapter._delete("some_object")
|
adapter.delete("some_object")
|
||||||
|
|
||||||
|
|
||||||
def test_should_list_objects(
|
def test_should_list_objects(
|
||||||
@@ -426,14 +424,11 @@ def test_should_list_objects(
|
|||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter can list objects in a bucket."""
|
"""Test that the MinioAdapter can list objects in a bucket."""
|
||||||
# Arrange
|
|
||||||
object_name, _ = data_in_minio
|
object_name, _ = data_in_minio
|
||||||
new_data = BytesIO(random.randbytes(2**21)) # 2 MiB
|
new_data = BytesIO(random.randbytes(2**21))
|
||||||
new_data_name = "another_test_object"
|
new_data_name = "another_test_object"
|
||||||
minio_adapter._put(new_data_name, new_data)
|
minio_adapter.put(new_data_name, new_data)
|
||||||
# Act
|
objects = minio_adapter.list_objects()
|
||||||
objects = minio_adapter._list_objects()
|
|
||||||
# Assert
|
|
||||||
assert isinstance(objects, list)
|
assert isinstance(objects, list)
|
||||||
assert len(objects) == 2
|
assert len(objects) == 2
|
||||||
assert object_name in objects
|
assert object_name in objects
|
||||||
@@ -445,15 +440,12 @@ def test_should_list_objects_with_prefix(
|
|||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter can list objects in a bucket with a prefix."""
|
"""Test that the MinioAdapter can list objects in a bucket with a prefix."""
|
||||||
# Arrange
|
|
||||||
object_name, _ = data_in_minio
|
object_name, _ = data_in_minio
|
||||||
new_data = BytesIO(random.randbytes(2**21)) # 2 MiB
|
new_data = BytesIO(random.randbytes(2**21))
|
||||||
new_data_name = "prefix_test_object"
|
new_data_name = "prefix_test_object"
|
||||||
minio_adapter._put(new_data_name, new_data)
|
minio_adapter.put(new_data_name, new_data)
|
||||||
prefix = "prefix_"
|
prefix = "prefix_"
|
||||||
# Act
|
objects = minio_adapter.list_objects(prefix)
|
||||||
objects = minio_adapter._list_objects(prefix)
|
|
||||||
# Assert
|
|
||||||
assert isinstance(objects, list)
|
assert isinstance(objects, list)
|
||||||
assert len(objects) == 1
|
assert len(objects) == 1
|
||||||
assert new_data_name in objects
|
assert new_data_name in objects
|
||||||
@@ -464,25 +456,20 @@ def test_should_raise_value_error_on_invalid_list_objects_prefix(
|
|||||||
minio_adapter: MinioAdapter,
|
minio_adapter: MinioAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter raises ValueError when listing with an invalid prefix."""
|
"""Test that the MinioAdapter raises ValueError when listing with an invalid prefix."""
|
||||||
# Arrange
|
|
||||||
invalid_prefixes = [123, None]
|
invalid_prefixes = [123, None]
|
||||||
# Act & Assert
|
|
||||||
for prefix in invalid_prefixes:
|
for prefix in invalid_prefixes:
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
minio_adapter._list_objects(prefix) # type: ignore
|
minio_adapter.list_objects(prefix) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_on_list_objects_when_not_connected(
|
def test_should_raise_connection_error_on_list_objects_when_not_connected(
|
||||||
minio_adapter: MinioAdapter,
|
minio_config: MinioConfig,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the MinioAdapter raises ConnectionError when listing while not connected."""
|
"""Test that the MinioAdapter raises ConnectionError when listing while not connected."""
|
||||||
# Arrange
|
adapter = MinioAdapter(config=minio_config)
|
||||||
adapter = MinioAdapter() # not connected
|
|
||||||
# Act & Assert
|
|
||||||
with pytest.raises(ConnectionError):
|
with pytest.raises(ConnectionError):
|
||||||
adapter._list_objects()
|
adapter.list_objects()
|
||||||
|
|
||||||
|
|
||||||
# allows local debugging by running file as script
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
pytest.main(["-s", "-v", __file__])
|
pytest.main(["-s", "-v", __file__])
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
"""Postgres container session helpers for integration tests."""
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
from testcontainers.postgres import PostgresContainer
|
||||||
|
|
||||||
|
from python_repositories.config import PostgresConfig
|
||||||
|
|
||||||
|
POSTGRES_IMAGE = "postgres:16"
|
||||||
|
TEST_TABLE = "test_items"
|
||||||
|
|
||||||
|
|
||||||
|
_postgres_uri: str | None = None
|
||||||
|
_postgres_container: PostgresContainer | None = None
|
||||||
|
_raw_postgres_client: psycopg.Connection | None = None
|
||||||
|
_postgres_uri_refs = 0
|
||||||
|
_raw_postgres_client_refs = 0
|
||||||
|
|
||||||
|
|
||||||
|
def postgres_uri() -> Generator[str, None, None]:
|
||||||
|
"""Yield a session-scoped Postgres URI, starting the container once."""
|
||||||
|
global _postgres_uri, _postgres_container, _postgres_uri_refs
|
||||||
|
if _postgres_uri is None:
|
||||||
|
_postgres_container = PostgresContainer(POSTGRES_IMAGE, driver=None)
|
||||||
|
_postgres_container.start()
|
||||||
|
_postgres_uri = _postgres_container.get_connection_url()
|
||||||
|
|
||||||
|
_postgres_uri_refs += 1
|
||||||
|
yield _postgres_uri
|
||||||
|
_postgres_uri_refs -= 1
|
||||||
|
|
||||||
|
if _postgres_uri_refs == 0 and _postgres_container is not None:
|
||||||
|
_postgres_container.stop()
|
||||||
|
_postgres_container = None
|
||||||
|
_postgres_uri = None
|
||||||
|
|
||||||
|
|
||||||
|
def postgres_config_from_container(uri: str) -> PostgresConfig:
|
||||||
|
"""Build PostgresConfig from a container URI."""
|
||||||
|
return PostgresConfig(uri=uri, table=TEST_TABLE, primary_key="id")
|
||||||
|
|
||||||
|
|
||||||
|
def raw_postgres_client_from_container(
|
||||||
|
uri: str,
|
||||||
|
) -> Generator[psycopg.Connection, None, None]:
|
||||||
|
"""Yield a session-scoped raw Postgres client, reusing one client per session."""
|
||||||
|
global _raw_postgres_client, _raw_postgres_client_refs
|
||||||
|
if _raw_postgres_client is None:
|
||||||
|
_raw_postgres_client = psycopg.connect(
|
||||||
|
uri,
|
||||||
|
row_factory=dict_row, # type: ignore[arg-type]
|
||||||
|
autocommit=True,
|
||||||
|
)
|
||||||
|
with _raw_postgres_client.cursor() as cursor:
|
||||||
|
cursor.execute(
|
||||||
|
f"""
|
||||||
|
CREATE TABLE IF NOT EXISTS {TEST_TABLE} (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
value INTEGER,
|
||||||
|
email TEXT
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
_raw_postgres_client_refs += 1
|
||||||
|
yield _raw_postgres_client
|
||||||
|
_raw_postgres_client_refs -= 1
|
||||||
|
|
||||||
|
if _raw_postgres_client_refs == 0 and _raw_postgres_client is not None:
|
||||||
|
with _raw_postgres_client.cursor() as cursor:
|
||||||
|
cursor.execute(f"TRUNCATE TABLE {TEST_TABLE}")
|
||||||
|
_raw_postgres_client.close()
|
||||||
|
_raw_postgres_client = None
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
"""Postgres integration test fixtures."""
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.config import PostgresConfig
|
||||||
|
from tests.integration.postgres._containers import (
|
||||||
|
postgres_config_from_container,
|
||||||
|
postgres_uri,
|
||||||
|
raw_postgres_client_from_container,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def postgres_container() -> Generator[str, None, None]:
|
||||||
|
"""Set up a Postgres container for testing and yield the Postgres URI."""
|
||||||
|
yield from postgres_uri()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def postgres_config(postgres_container: str) -> PostgresConfig:
|
||||||
|
"""Provide PostgresConfig built from the test container."""
|
||||||
|
return postgres_config_from_container(postgres_container)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def raw_postgres_client(
|
||||||
|
postgres_container: str,
|
||||||
|
) -> Generator[psycopg.Connection, None, None]:
|
||||||
|
"""Provide a raw Postgres client connected to the test container."""
|
||||||
|
yield from raw_postgres_client_from_container(postgres_container)
|
||||||
@@ -0,0 +1,318 @@
|
|||||||
|
"""Integration tests for the PostgresAdapter."""
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.adapters.postgres_adapter import PostgresAdapter
|
||||||
|
from python_repositories.config import PostgresConfig
|
||||||
|
from tests.integration.postgres._containers import TEST_TABLE
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.integration, pytest.mark.needs_postgres]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def row() -> Generator[dict[str, object], None, None]:
|
||||||
|
"""Provide a sample row for tests."""
|
||||||
|
yield {"id": "test-id", "name": "Alice", "value": 42}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="function")
|
||||||
|
def row_in_postgres(
|
||||||
|
raw_postgres_client: psycopg.Connection,
|
||||||
|
row: dict[str, object],
|
||||||
|
) -> Generator[tuple[str, dict[str, object]], None, None]:
|
||||||
|
"""Fixture to set up a known row in Postgres before each test."""
|
||||||
|
with raw_postgres_client.cursor() as cursor:
|
||||||
|
cursor.execute(
|
||||||
|
f"INSERT INTO {TEST_TABLE} (id, name, value) VALUES (%s, %s, %s)",
|
||||||
|
(row["id"], row["name"], row["value"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
yield str(row["id"]), row
|
||||||
|
|
||||||
|
with raw_postgres_client.cursor() as cursor:
|
||||||
|
cursor.execute(f"DELETE FROM {TEST_TABLE} WHERE id = %s", (row["id"],))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def postgres_adapter(
|
||||||
|
postgres_config: PostgresConfig,
|
||||||
|
raw_postgres_client: psycopg.Connection,
|
||||||
|
) -> Generator[PostgresAdapter, None, None]:
|
||||||
|
"""Fixture to provide a connected PostgresAdapter instance."""
|
||||||
|
_ = raw_postgres_client # ensure test table exists before connect probe
|
||||||
|
adapter = PostgresAdapter(config=postgres_config)
|
||||||
|
adapter.connect()
|
||||||
|
yield adapter
|
||||||
|
adapter.disconnect()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="function", autouse=True)
|
||||||
|
def clear_postgres(raw_postgres_client: psycopg.Connection) -> None:
|
||||||
|
"""Fixture to clear all rows before each test."""
|
||||||
|
with raw_postgres_client.cursor() as cursor:
|
||||||
|
cursor.execute(f"TRUNCATE TABLE {TEST_TABLE}")
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_log_info_when_already_connected(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter logs info when connect is called while connected."""
|
||||||
|
with caplog.at_level(logging.INFO):
|
||||||
|
postgres_adapter.connect()
|
||||||
|
assert "Already connected" in caplog.text
|
||||||
|
assert "Postgres" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_when_unable_to_connect() -> None:
|
||||||
|
"""Test that PostgresAdapter raises ConnectionError when unable to connect."""
|
||||||
|
adapter = PostgresAdapter(
|
||||||
|
config=PostgresConfig(
|
||||||
|
uri="postgresql://invalid:5432/nodb",
|
||||||
|
table=TEST_TABLE,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
adapter.connect()
|
||||||
|
assert adapter._client is None
|
||||||
|
assert not adapter.is_connected()
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_when_table_missing(
|
||||||
|
postgres_config: PostgresConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter raises ConnectionError when the table is missing."""
|
||||||
|
config = PostgresConfig(
|
||||||
|
uri=postgres_config.uri,
|
||||||
|
table="missing_table",
|
||||||
|
primary_key="id",
|
||||||
|
)
|
||||||
|
adapter = PostgresAdapter(config=config)
|
||||||
|
with pytest.raises(ConnectionError, match="does not exist"):
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_log_error_on_exception_during_exit(
|
||||||
|
postgres_config: PostgresConfig,
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter logs an error when an exception occurs during exit."""
|
||||||
|
try:
|
||||||
|
with PostgresAdapter(config=postgres_config) as adapter:
|
||||||
|
assert adapter.is_connected()
|
||||||
|
raise ValueError("Simulated error")
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
assert "Error while exiting context" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_have_context_manager(postgres_config: PostgresConfig) -> None:
|
||||||
|
"""Test that PostgresAdapter can be used as a context manager."""
|
||||||
|
with PostgresAdapter(config=postgres_config) as adapter:
|
||||||
|
assert adapter._client is not None
|
||||||
|
assert adapter._client is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_fetch_one(
|
||||||
|
row_in_postgres: tuple[str, dict[str, object]],
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter can fetch a row by primary key."""
|
||||||
|
pk, row = row_in_postgres
|
||||||
|
fetched = postgres_adapter.fetch_one(pk)
|
||||||
|
assert fetched is not None
|
||||||
|
assert fetched["id"] == row["id"]
|
||||||
|
assert fetched["name"] == row["name"]
|
||||||
|
assert fetched["value"] == row["value"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_fetch_none_for_missing_row(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that fetching a non-existent row returns None."""
|
||||||
|
assert postgres_adapter.fetch_one("missing-id") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_value_error_on_invalid_fetch_one_pk(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter raises ValueError for an invalid primary key."""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
postgres_adapter.fetch_one(None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_on_fetch_one_when_not_connected(
|
||||||
|
postgres_config: PostgresConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that fetch_one raises ConnectionError when not connected."""
|
||||||
|
adapter = PostgresAdapter(config=postgres_config)
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
adapter.fetch_one("some-id")
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_upsert_row(
|
||||||
|
row: dict[str, object],
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter can insert a row."""
|
||||||
|
assert postgres_adapter.fetch_one(row["id"]) is None
|
||||||
|
postgres_adapter.upsert(row)
|
||||||
|
fetched = postgres_adapter.fetch_one(row["id"])
|
||||||
|
assert fetched is not None
|
||||||
|
assert fetched["name"] == row["name"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_update_row(
|
||||||
|
row_in_postgres: tuple[str, dict[str, object]],
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter can update an existing row."""
|
||||||
|
pk, _ = row_in_postgres
|
||||||
|
new_row = {"id": pk, "name": "Bob", "value": 99}
|
||||||
|
postgres_adapter.upsert(new_row)
|
||||||
|
fetched = postgres_adapter.fetch_one(pk)
|
||||||
|
assert fetched is not None
|
||||||
|
assert fetched["name"] == "Bob"
|
||||||
|
assert fetched["value"] == 99
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_value_error_on_invalid_upsert_row(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter raises ValueError for invalid upsert data."""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
postgres_adapter.upsert({"name": "Alice"}) # missing primary key
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_value_error_on_non_dict_upsert_row(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter raises ValueError when upsert row is not a dict."""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
postgres_adapter.upsert("not-a-dict") # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_on_upsert_when_not_connected(
|
||||||
|
postgres_config: PostgresConfig,
|
||||||
|
row: dict[str, object],
|
||||||
|
) -> None:
|
||||||
|
"""Test that upsert raises ConnectionError when not connected."""
|
||||||
|
adapter = PostgresAdapter(config=postgres_config)
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
adapter.upsert(row)
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_fetch_all_rows(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter can fetch all rows."""
|
||||||
|
postgres_adapter.upsert({"id": "a", "name": "Alice", "value": 1})
|
||||||
|
postgres_adapter.upsert({"id": "b", "name": "Bob", "value": 2})
|
||||||
|
rows = postgres_adapter.fetch_all()
|
||||||
|
assert len(rows) == 2
|
||||||
|
ids = {row["id"] for row in rows}
|
||||||
|
assert ids == {"a", "b"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_fetch_all_with_limit(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter respects fetch_all limit."""
|
||||||
|
postgres_adapter.upsert({"id": "a", "name": "Alice", "value": 1})
|
||||||
|
postgres_adapter.upsert({"id": "b", "name": "Bob", "value": 2})
|
||||||
|
rows = postgres_adapter.fetch_all(limit=1)
|
||||||
|
assert len(rows) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_return_empty_list_for_empty_table(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that fetch_all returns an empty list for an empty table."""
|
||||||
|
assert postgres_adapter.fetch_all() == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_value_error_on_invalid_fetch_all_limit(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter raises ValueError for an invalid limit."""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
postgres_adapter.fetch_all(limit=-1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_on_fetch_all_when_not_connected(
|
||||||
|
postgres_config: PostgresConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that fetch_all raises ConnectionError when not connected."""
|
||||||
|
adapter = PostgresAdapter(config=postgres_config)
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
adapter.fetch_all()
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_delete_row(
|
||||||
|
row_in_postgres: tuple[str, dict[str, object]],
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that deleting a row removes it from the table."""
|
||||||
|
pk, _ = row_in_postgres
|
||||||
|
assert postgres_adapter.fetch_one(pk) is not None
|
||||||
|
postgres_adapter.delete(pk)
|
||||||
|
assert postgres_adapter.fetch_one(pk) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_delete_idempotently(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that deleting a missing row does not raise."""
|
||||||
|
postgres_adapter.delete("missing-id")
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_value_error_on_invalid_delete_pk(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter raises ValueError for an invalid delete pk."""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
postgres_adapter.delete(None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_on_delete_when_not_connected(
|
||||||
|
postgres_config: PostgresConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that delete raises ConnectionError when not connected."""
|
||||||
|
adapter = PostgresAdapter(config=postgres_config)
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
adapter.delete("some-id")
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_execute_sql(
|
||||||
|
row_in_postgres: tuple[str, dict[str, object]],
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter can execute read SQL."""
|
||||||
|
pk, _ = row_in_postgres
|
||||||
|
rows = postgres_adapter.execute(
|
||||||
|
f"SELECT * FROM {TEST_TABLE} WHERE id = %s",
|
||||||
|
(pk,),
|
||||||
|
)
|
||||||
|
assert len(rows) == 1
|
||||||
|
assert rows[0]["id"] == pk
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_value_error_on_invalid_execute_sql(
|
||||||
|
postgres_adapter: PostgresAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that PostgresAdapter raises ValueError for invalid SQL."""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
postgres_adapter.execute("")
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_on_execute_when_not_connected(
|
||||||
|
postgres_config: PostgresConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that execute raises ConnectionError when not connected."""
|
||||||
|
adapter = PostgresAdapter(config=postgres_config)
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
adapter.execute("SELECT 1")
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
"""Redis container session helpers for integration tests."""
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
from typing import Any, cast
|
||||||
|
|
||||||
|
import redis
|
||||||
|
from testcontainers.core.container import DockerContainer
|
||||||
|
from testcontainers.core.waiting_utils import WaitStrategy, WaitStrategyTarget
|
||||||
|
|
||||||
|
from python_repositories.config import RedisConfig
|
||||||
|
|
||||||
|
REDIS_PORT = 6379
|
||||||
|
|
||||||
|
|
||||||
|
class _RedisPingWaitStrategy(WaitStrategy):
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self.with_transient_exceptions(redis.exceptions.ConnectionError)
|
||||||
|
|
||||||
|
def wait_until_ready(self, container: WaitStrategyTarget) -> None:
|
||||||
|
redis_container = cast("RedisTestContainer", container)
|
||||||
|
if not self._poll(lambda: redis_container.get_client().ping()):
|
||||||
|
raise redis.exceptions.ConnectionError("Could not connect to Redis")
|
||||||
|
|
||||||
|
|
||||||
|
class RedisTestContainer(DockerContainer):
|
||||||
|
"""Redis container using wait strategies instead of the deprecated decorator."""
|
||||||
|
|
||||||
|
def __init__(self, image: str, port: int = REDIS_PORT) -> None:
|
||||||
|
super().__init__(image, _wait_strategy=_RedisPingWaitStrategy())
|
||||||
|
self.port = port
|
||||||
|
self.with_exposed_ports(self.port)
|
||||||
|
|
||||||
|
def get_client(self, **kwargs: Any) -> redis.Redis:
|
||||||
|
return redis.Redis(
|
||||||
|
host=self.get_container_host_ip(),
|
||||||
|
port=self.get_exposed_port(self.port),
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_redis_uri: str | None = None
|
||||||
|
_redis_container: RedisTestContainer | None = None
|
||||||
|
_raw_redis_client: redis.Redis | None = None
|
||||||
|
_redis_uri_refs = 0
|
||||||
|
_raw_redis_client_refs = 0
|
||||||
|
|
||||||
|
|
||||||
|
def redis_uri() -> Generator[str, None, None]:
|
||||||
|
"""Yield a session-scoped Redis URI, starting the container once."""
|
||||||
|
global _redis_uri, _redis_container, _redis_uri_refs
|
||||||
|
if _redis_uri is None:
|
||||||
|
_redis_container = RedisTestContainer(image="redis/redis-stack:7.2.0-v0")
|
||||||
|
_redis_container.start()
|
||||||
|
redis_host = _redis_container.get_container_host_ip()
|
||||||
|
redis_port = _redis_container.get_exposed_port(REDIS_PORT)
|
||||||
|
_redis_uri = f"redis://{redis_host}:{redis_port}"
|
||||||
|
|
||||||
|
_redis_uri_refs += 1
|
||||||
|
yield _redis_uri
|
||||||
|
_redis_uri_refs -= 1
|
||||||
|
|
||||||
|
if _redis_uri_refs == 0 and _redis_container is not None:
|
||||||
|
_redis_container.stop()
|
||||||
|
_redis_container = None
|
||||||
|
_redis_uri = None
|
||||||
|
|
||||||
|
|
||||||
|
def redis_config_from_container(uri: str) -> RedisConfig:
|
||||||
|
"""Build RedisConfig from a container URI."""
|
||||||
|
return RedisConfig(uri=uri)
|
||||||
|
|
||||||
|
|
||||||
|
def raw_redis_client_from_container(uri: str) -> Generator[redis.Redis, None, None]:
|
||||||
|
"""Yield a session-scoped raw Redis client, reusing one client per session."""
|
||||||
|
global _raw_redis_client, _raw_redis_client_refs
|
||||||
|
if _raw_redis_client is None:
|
||||||
|
_raw_redis_client = redis.Redis.from_url(
|
||||||
|
url=uri,
|
||||||
|
socket_connect_timeout=10,
|
||||||
|
)
|
||||||
|
|
||||||
|
_raw_redis_client_refs += 1
|
||||||
|
yield _raw_redis_client
|
||||||
|
_raw_redis_client_refs -= 1
|
||||||
|
|
||||||
|
if _raw_redis_client_refs == 0 and _raw_redis_client is not None:
|
||||||
|
_raw_redis_client.flushall()
|
||||||
|
_raw_redis_client.close()
|
||||||
|
_raw_redis_client = None
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
"""Redis integration test fixtures."""
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from python_repositories.config import RedisConfig
|
||||||
|
from tests.integration.redis._containers import (
|
||||||
|
raw_redis_client_from_container,
|
||||||
|
redis_config_from_container,
|
||||||
|
redis_uri,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def redis_container() -> Generator[str, None, None]:
|
||||||
|
"""Set up a Redis container for testing and yield the Redis URI."""
|
||||||
|
yield from redis_uri()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def redis_config(redis_container: str) -> RedisConfig:
|
||||||
|
"""Provide RedisConfig built from the test container."""
|
||||||
|
return redis_config_from_container(redis_container)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def raw_redis_client(redis_container: str) -> Generator[redis.Redis, None, None]:
|
||||||
|
"""Provide a raw Redis client connected to the test Redis container."""
|
||||||
|
yield from raw_redis_client_from_container(redis_container)
|
||||||
+127
-120
@@ -1,17 +1,20 @@
|
|||||||
# pylint: disable=protected-access
|
|
||||||
# The above line disables pylint's protected member access warnings for this file,
|
|
||||||
# allowing tests to access RedisAdapter's internal methods as needed for integration testing.
|
|
||||||
"""Integration tests for the RedisAdapter."""
|
"""Integration tests for the RedisAdapter."""
|
||||||
|
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
|
import logging
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import redis
|
import redis
|
||||||
from redis.commands.json.path import Path as RedisPath
|
from redis.commands.json.path import Path as RedisPath
|
||||||
|
|
||||||
from python_repositories.adapters.redis_adapter import RedisAdapter
|
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||||
|
from python_repositories.config import RedisConfig
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.integration, pytest.mark.needs_redis]
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def data() -> Generator[dict[str, str]]:
|
def data() -> Generator[dict[str, str], None, None]:
|
||||||
"""Provide a sample data dictionary for tests."""
|
"""Provide a sample data dictionary for tests."""
|
||||||
yield {"foo": "bar"}
|
yield {"foo": "bar"}
|
||||||
|
|
||||||
@@ -20,7 +23,7 @@ def data() -> Generator[dict[str, str]]:
|
|||||||
def data_in_redis(
|
def data_in_redis(
|
||||||
raw_redis_client: redis.Redis,
|
raw_redis_client: redis.Redis,
|
||||||
data: dict[str, str],
|
data: dict[str, str],
|
||||||
) -> Generator[tuple[str, dict[str, str]]]:
|
) -> Generator[tuple[str, dict[str, str]], None, None]:
|
||||||
"""Fixture to set up a known value in Redis before each test."""
|
"""Fixture to set up a known value in Redis before each test."""
|
||||||
key = "test_key"
|
key = "test_key"
|
||||||
path = RedisPath.root_path()
|
path = RedisPath.root_path()
|
||||||
@@ -28,14 +31,13 @@ def data_in_redis(
|
|||||||
|
|
||||||
yield key, data
|
yield key, data
|
||||||
|
|
||||||
# Cleanup
|
|
||||||
raw_redis_client.delete(key)
|
raw_redis_client.delete(key)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def redis_adapter(redis_container: str) -> Generator[RedisAdapter]:
|
def redis_adapter(redis_config: RedisConfig) -> Generator[RedisAdapter, None, None]:
|
||||||
"""Fixture to provide a connected RedisAdapter instance."""
|
"""Fixture to provide a connected RedisAdapter instance."""
|
||||||
adapter = RedisAdapter()
|
adapter = RedisAdapter(config=redis_config)
|
||||||
adapter.connect()
|
adapter.connect()
|
||||||
yield adapter
|
yield adapter
|
||||||
adapter.disconnect()
|
adapter.disconnect()
|
||||||
@@ -44,84 +46,64 @@ def redis_adapter(redis_container: str) -> Generator[RedisAdapter]:
|
|||||||
@pytest.fixture(scope="function", autouse=True)
|
@pytest.fixture(scope="function", autouse=True)
|
||||||
def clear_redis(raw_redis_client: redis.Redis) -> None:
|
def clear_redis(raw_redis_client: redis.Redis) -> None:
|
||||||
"""Fixture to clear all Redis keys before each test."""
|
"""Fixture to clear all Redis keys before each test."""
|
||||||
# Clear all keys before each test
|
|
||||||
raw_redis_client.flushall()
|
raw_redis_client.flushall()
|
||||||
|
|
||||||
|
|
||||||
def test_should_adhere_to_interface(redis_container: str) -> None:
|
def test_should_log_info_when_already_connected(
|
||||||
"""Test that the RedisAdapter adheres to the expected interface."""
|
redis_adapter: RedisAdapter,
|
||||||
# Instantiation fails if interface not adhered to
|
caplog: pytest.LogCaptureFixture,
|
||||||
_ = RedisAdapter()
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_have_logger_when_instantiated(redis_container: str) -> None:
|
|
||||||
"""Test that the RedisAdapter has a logger when instantiated."""
|
|
||||||
adapter = RedisAdapter()
|
|
||||||
assert hasattr(adapter, "logger")
|
|
||||||
assert adapter.logger is not None
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_not_be_connected_when_instantiated(redis_container: str) -> None:
|
|
||||||
"""Test that the RedisAdapter is not connected when instantiated."""
|
|
||||||
adapter = RedisAdapter()
|
|
||||||
assert adapter._client is None
|
|
||||||
assert not adapter.is_connected
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_when_unable_to_connect(
|
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
|
"""Test that the RedisAdapter logs info when connect is called while already connected."""
|
||||||
|
with caplog.at_level(logging.INFO):
|
||||||
|
redis_adapter.connect()
|
||||||
|
assert "Already connected" in caplog.text
|
||||||
|
assert "Redis" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_when_unable_to_connect() -> None:
|
||||||
"""Test that the RedisAdapter raises ConnectionError when unable to connect."""
|
"""Test that the RedisAdapter raises ConnectionError when unable to connect."""
|
||||||
# Arrange
|
adapter = RedisAdapter(config=RedisConfig(uri="redis://invalid:6379"))
|
||||||
monkeypatch.setenv("REDIS_URI", "redis://invalid:6379")
|
|
||||||
adapter = RedisAdapter()
|
|
||||||
# Act & Assert
|
|
||||||
with pytest.raises(ConnectionError):
|
with pytest.raises(ConnectionError):
|
||||||
adapter.connect()
|
adapter.connect()
|
||||||
assert adapter._client is None
|
assert adapter._client is None
|
||||||
assert not adapter.is_connected
|
assert not adapter.is_connected()
|
||||||
|
|
||||||
|
|
||||||
def test_connect_raises_connection_error_when_unable_to_ping(
|
def test_connect_raises_connection_error_when_unable_to_ping(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter raises ConnectionError when ping fails."""
|
"""Test that the RedisAdapter raises ConnectionError when ping fails."""
|
||||||
# Set an invalid URI
|
|
||||||
monkeypatch.setenv("REDIS_URI", "redis://invalid:6379")
|
|
||||||
|
|
||||||
# Monkeypatch redis.Redis.from_url to return a mock client
|
|
||||||
class MockRedis:
|
class MockRedis:
|
||||||
"""A mock Redis client that simulates a failed ping."""
|
"""A mock Redis client that simulates a failed ping."""
|
||||||
|
|
||||||
def ping(self) -> bool:
|
def ping(self) -> bool:
|
||||||
"""Simulate a failed ping."""
|
return False
|
||||||
return False # Simulate failed ping
|
|
||||||
|
|
||||||
monkeypatch.setattr("redis.Redis.from_url", lambda *a, **kw: MockRedis())
|
monkeypatch.setattr("redis.Redis.from_url", lambda *a, **kw: MockRedis())
|
||||||
|
|
||||||
adapter = RedisAdapter()
|
adapter = RedisAdapter(config=RedisConfig(uri="redis://invalid:6379"))
|
||||||
with pytest.raises(ConnectionError, match="Could not connect to Redis"):
|
with pytest.raises(ConnectionError, match="Could not connect to Redis"):
|
||||||
adapter.connect()
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_error_on_exception_during_exit(
|
def test_should_log_error_on_exception_during_exit(
|
||||||
redis_container: str,
|
redis_config: RedisConfig,
|
||||||
caplog: pytest.LogCaptureFixture,
|
caplog: pytest.LogCaptureFixture,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter logs an error when an exception occurs during context exit."""
|
"""Test that the RedisAdapter logs an error when an exception occurs during context exit."""
|
||||||
try:
|
try:
|
||||||
with RedisAdapter() as adapter:
|
with RedisAdapter(config=redis_config) as adapter:
|
||||||
assert adapter.is_connected
|
assert adapter.is_connected()
|
||||||
raise ValueError("Simulated error")
|
raise ValueError("Simulated error")
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass # Expected
|
pass
|
||||||
# Assert error was logged
|
|
||||||
assert "Error while exiting context" in caplog.text
|
assert "Error while exiting context" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_should_have_context_manager(redis_container: str) -> None:
|
def test_should_have_context_manager(redis_config: RedisConfig) -> None:
|
||||||
"""Test that the RedisAdapter can be used as a context manager."""
|
"""Test that the RedisAdapter can be used as a context manager."""
|
||||||
with RedisAdapter() as adapter:
|
with RedisAdapter(config=redis_config) as adapter:
|
||||||
assert adapter._client is not None
|
assert adapter._client is not None
|
||||||
assert adapter._client is None
|
assert adapter._client is None
|
||||||
|
|
||||||
@@ -131,11 +113,8 @@ def test_should_get_value(
|
|||||||
redis_adapter: RedisAdapter,
|
redis_adapter: RedisAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter can get a value."""
|
"""Test that the RedisAdapter can get a value."""
|
||||||
# Arrange
|
|
||||||
key, data = data_in_redis
|
key, data = data_in_redis
|
||||||
# Act
|
value = redis_adapter.get(key)
|
||||||
value = redis_adapter._get(key)
|
|
||||||
# Assert
|
|
||||||
assert value is not None
|
assert value is not None
|
||||||
assert value == data
|
assert value == data
|
||||||
|
|
||||||
@@ -144,9 +123,7 @@ def test_should_get_none_for_missing_key(
|
|||||||
redis_adapter: RedisAdapter,
|
redis_adapter: RedisAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that getting a non-existent key returns None."""
|
"""Test that getting a non-existent key returns None."""
|
||||||
# Act
|
value = redis_adapter.get("nonexistent_key")
|
||||||
value = redis_adapter._get("nonexistent_key")
|
|
||||||
# Assert
|
|
||||||
assert value is None
|
assert value is None
|
||||||
|
|
||||||
|
|
||||||
@@ -154,23 +131,19 @@ def test_should_raise_value_error_on_invalid_get_key(
|
|||||||
redis_adapter: RedisAdapter,
|
redis_adapter: RedisAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter raises ValueError when getting with an invalid key."""
|
"""Test that the RedisAdapter raises ValueError when getting with an invalid key."""
|
||||||
# Arrange
|
|
||||||
invalid_keys = ["", 123, None]
|
invalid_keys = ["", 123, None]
|
||||||
# Act & Assert
|
|
||||||
for key in invalid_keys:
|
for key in invalid_keys:
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
redis_adapter._get(key) # type: ignore
|
redis_adapter.get(key) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_on_get_when_not_connected(
|
def test_should_raise_connection_error_on_get_when_not_connected(
|
||||||
redis_adapter: RedisAdapter,
|
redis_config: RedisConfig,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter raises ConnectionError when getting while not connected."""
|
"""Test that the RedisAdapter raises ConnectionError when getting while not connected."""
|
||||||
# Arrange
|
adapter = RedisAdapter(config=redis_config)
|
||||||
adapter = RedisAdapter() # not connected
|
|
||||||
# Act & Assert
|
|
||||||
with pytest.raises(ConnectionError):
|
with pytest.raises(ConnectionError):
|
||||||
adapter._get("some_key")
|
adapter.get("some_key")
|
||||||
|
|
||||||
|
|
||||||
def test_should_set_value(
|
def test_should_set_value(
|
||||||
@@ -178,14 +151,11 @@ def test_should_set_value(
|
|||||||
redis_adapter: RedisAdapter,
|
redis_adapter: RedisAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter can set a value."""
|
"""Test that the RedisAdapter can set a value."""
|
||||||
# Arrange
|
|
||||||
key = "test_key"
|
key = "test_key"
|
||||||
received_data = redis_adapter._get(key)
|
received_data = redis_adapter.get(key)
|
||||||
assert received_data is None # Ensure key does not exist
|
assert received_data is None
|
||||||
# Act
|
redis_adapter.set(key, data)
|
||||||
redis_adapter._set(key, data)
|
received_data = redis_adapter.get(key)
|
||||||
# Assert
|
|
||||||
received_data = redis_adapter._get(key)
|
|
||||||
assert received_data is not None
|
assert received_data is not None
|
||||||
assert received_data == data
|
assert received_data == data
|
||||||
|
|
||||||
@@ -195,16 +165,13 @@ def test_should_update_value(
|
|||||||
redis_adapter: RedisAdapter,
|
redis_adapter: RedisAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter can update an existing value."""
|
"""Test that the RedisAdapter can update an existing value."""
|
||||||
# Arrange
|
|
||||||
key, _ = data_in_redis
|
key, _ = data_in_redis
|
||||||
new_data = {"new_key": "new_value"}
|
new_data = {"new_key": "new_value"}
|
||||||
received_data = redis_adapter._get(key)
|
received_data = redis_adapter.get(key)
|
||||||
assert received_data is not None
|
assert received_data is not None
|
||||||
assert received_data != new_data
|
assert received_data != new_data
|
||||||
# Act
|
redis_adapter.set(key, new_data)
|
||||||
redis_adapter._set(key, new_data)
|
assert redis_adapter.get(key) == new_data
|
||||||
# Assert
|
|
||||||
assert redis_adapter._get(key) == new_data
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_value_error_on_invalid_set_key(
|
def test_should_raise_value_error_on_invalid_set_key(
|
||||||
@@ -212,38 +179,64 @@ def test_should_raise_value_error_on_invalid_set_key(
|
|||||||
data: dict[str, str],
|
data: dict[str, str],
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter raises ValueError when setting with an invalid key."""
|
"""Test that the RedisAdapter raises ValueError when setting with an invalid key."""
|
||||||
# Arrange
|
|
||||||
invalid_keys = ["", 123, None]
|
invalid_keys = ["", 123, None]
|
||||||
# Act & Assert
|
|
||||||
for key in invalid_keys:
|
for key in invalid_keys:
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
redis_adapter._set(key, data) # type: ignore
|
redis_adapter.set(key, data) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_value_error_on_invalid_set_data(
|
def test_should_raise_value_error_on_invalid_set_data(
|
||||||
redis_adapter: RedisAdapter,
|
redis_adapter: RedisAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter raises ValueError when setting with invalid data."""
|
"""Test that the RedisAdapter raises ValueError when setting with invalid data."""
|
||||||
# Arrange
|
|
||||||
key = "test_key"
|
key = "test_key"
|
||||||
invalid_data = ["", 123, None, [], {}]
|
invalid_data = ["", 123, None, []]
|
||||||
# Act & Assert
|
|
||||||
for data in invalid_data:
|
for data in invalid_data:
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
redis_adapter._set(key, data) # type: ignore
|
redis_adapter.set(key, data) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_set_and_get_empty_dict(
|
||||||
|
redis_adapter: RedisAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that the RedisAdapter can set and get an empty dict."""
|
||||||
|
key = "empty_key"
|
||||||
|
assert redis_adapter.get(key) is None
|
||||||
|
redis_adapter.set(key, {})
|
||||||
|
value = redis_adapter.get(key)
|
||||||
|
assert value is not None
|
||||||
|
assert value == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_distinguish_missing_key_from_empty_dict(
|
||||||
|
redis_adapter: RedisAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that missing keys and empty dicts are distinguishable."""
|
||||||
|
key = "empty_key"
|
||||||
|
redis_adapter.set(key, {})
|
||||||
|
assert redis_adapter.get("other_key") is None
|
||||||
|
redis_adapter.delete(key)
|
||||||
|
assert redis_adapter.get(key) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_list_empty_dict_key(
|
||||||
|
redis_adapter: RedisAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that a key with an empty dict appears in key listings."""
|
||||||
|
key = "empty_key"
|
||||||
|
redis_adapter.set(key, {})
|
||||||
|
assert key in redis_adapter.list_keys(key)
|
||||||
|
assert key in list(redis_adapter.scan_keys(key))
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_on_set_when_not_connected(
|
def test_should_raise_connection_error_on_set_when_not_connected(
|
||||||
redis_adapter: RedisAdapter,
|
redis_config: RedisConfig,
|
||||||
data: dict[str, str],
|
data: dict[str, str],
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter raises ConnectionError when setting while not connected."""
|
"""Test that the RedisAdapter raises ConnectionError when setting while not connected."""
|
||||||
# Arrange
|
adapter = RedisAdapter(config=redis_config)
|
||||||
adapter = RedisAdapter()
|
|
||||||
key = "test_key"
|
|
||||||
# Act & Assert
|
|
||||||
with pytest.raises(ConnectionError):
|
with pytest.raises(ConnectionError):
|
||||||
adapter._set(key, data)
|
adapter.set("test_key", data)
|
||||||
|
|
||||||
|
|
||||||
def test_should_delete_key(
|
def test_should_delete_key(
|
||||||
@@ -251,49 +244,39 @@ def test_should_delete_key(
|
|||||||
redis_adapter: RedisAdapter,
|
redis_adapter: RedisAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that deleting a key removes it from Redis."""
|
"""Test that deleting a key removes it from Redis."""
|
||||||
# Arrange
|
|
||||||
key, _ = data_in_redis
|
key, _ = data_in_redis
|
||||||
received_data = redis_adapter._get(key)
|
received_data = redis_adapter.get(key)
|
||||||
assert received_data is not None # Ensure key exists
|
assert received_data is not None
|
||||||
# Act
|
redis_adapter.delete(key)
|
||||||
redis_adapter._delete(key)
|
assert redis_adapter.get(key) is None
|
||||||
# Assert
|
|
||||||
assert redis_adapter._get(key) is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_value_error_on_invalid_delete_key(
|
def test_should_raise_value_error_on_invalid_delete_key(
|
||||||
redis_adapter: RedisAdapter,
|
redis_adapter: RedisAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter raises ValueError when deleting with an invalid key."""
|
"""Test that the RedisAdapter raises ValueError when deleting with an invalid key."""
|
||||||
# Arrange
|
|
||||||
invalid_keys = ["", 123, None]
|
invalid_keys = ["", 123, None]
|
||||||
# Act & Assert
|
|
||||||
for key in invalid_keys:
|
for key in invalid_keys:
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
redis_adapter._delete(key) # type: ignore
|
redis_adapter.delete(key) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_on_delete_when_not_connected(
|
def test_should_raise_connection_error_on_delete_when_not_connected(
|
||||||
redis_adapter: RedisAdapter,
|
redis_config: RedisConfig,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter raises ConnectionError when deleting while not connected."""
|
"""Test that the RedisAdapter raises ConnectionError when deleting while not connected."""
|
||||||
# Arrange
|
adapter = RedisAdapter(config=redis_config)
|
||||||
adapter = RedisAdapter()
|
|
||||||
# Act & Assert
|
|
||||||
with pytest.raises(ConnectionError):
|
with pytest.raises(ConnectionError):
|
||||||
adapter._delete("some_key")
|
adapter.delete("some_key")
|
||||||
|
|
||||||
|
|
||||||
def test_should_list_keys(
|
def test_should_list_keys(
|
||||||
redis_adapter: RedisAdapter,
|
redis_adapter: RedisAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test listing keys matching a pattern returns correct keys."""
|
"""Test listing keys matching a pattern returns correct keys."""
|
||||||
# Arrange
|
redis_adapter.set("key1", {"a": 1})
|
||||||
redis_adapter._set("key1", {"a": 1})
|
redis_adapter.set("key2", {"b": 2})
|
||||||
redis_adapter._set("key2", {"b": 2})
|
keys = redis_adapter.list_keys("key*")
|
||||||
# Act
|
|
||||||
keys = redis_adapter._list_keys("key*")
|
|
||||||
# Assert
|
|
||||||
assert set(keys) == {"key1", "key2"}
|
assert set(keys) == {"key1", "key2"}
|
||||||
|
|
||||||
|
|
||||||
@@ -301,25 +284,49 @@ def test_should_raise_value_error_on_invalid_list_keys_pattern(
|
|||||||
redis_adapter: RedisAdapter,
|
redis_adapter: RedisAdapter,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter raises ValueError when listing keys with an invalid pattern."""
|
"""Test that the RedisAdapter raises ValueError when listing keys with an invalid pattern."""
|
||||||
# Arrange
|
|
||||||
invalid_patterns = ["", 123, None]
|
invalid_patterns = ["", 123, None]
|
||||||
# Act & Assert
|
|
||||||
for pattern in invalid_patterns:
|
for pattern in invalid_patterns:
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
redis_adapter._list_keys(pattern) # type: ignore
|
redis_adapter.list_keys(pattern) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_on_list_keys_when_not_connected(
|
def test_should_raise_connection_error_on_list_keys_when_not_connected(
|
||||||
redis_adapter: RedisAdapter,
|
redis_config: RedisConfig,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Test that the RedisAdapter raises ConnectionError when listing keys while not connected."""
|
"""Test that the RedisAdapter raises ConnectionError when listing keys while not connected."""
|
||||||
# Arrange
|
adapter = RedisAdapter(config=redis_config)
|
||||||
adapter = RedisAdapter()
|
|
||||||
# Act & Assert
|
|
||||||
with pytest.raises(ConnectionError):
|
with pytest.raises(ConnectionError):
|
||||||
adapter._list_keys("some_pattern")
|
adapter.list_keys("some_pattern")
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_scan_keys(
|
||||||
|
redis_adapter: RedisAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test scanning keys matching a pattern returns correct keys."""
|
||||||
|
redis_adapter.set("key1", {"a": 1})
|
||||||
|
redis_adapter.set("key2", {"b": 2})
|
||||||
|
keys = set(redis_adapter.scan_keys("key*"))
|
||||||
|
assert keys == {"key1", "key2"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_value_error_on_invalid_scan_keys_pattern(
|
||||||
|
redis_adapter: RedisAdapter,
|
||||||
|
) -> None:
|
||||||
|
"""Test that the RedisAdapter raises ValueError when scanning keys with an invalid pattern."""
|
||||||
|
invalid_patterns = ["", 123, None]
|
||||||
|
for pattern in invalid_patterns:
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
list(redis_adapter.scan_keys(pattern)) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_raise_connection_error_on_scan_keys_when_not_connected(
|
||||||
|
redis_config: RedisConfig,
|
||||||
|
) -> None:
|
||||||
|
"""Test that the RedisAdapter raises ConnectionError when scanning keys while not connected."""
|
||||||
|
adapter = RedisAdapter(config=redis_config)
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
list(adapter.scan_keys("some_pattern"))
|
||||||
|
|
||||||
|
|
||||||
# allows local debugging by running file as script
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
pytest.main(["-s", "-v", __file__])
|
pytest.main(["-s", "-v", __file__])
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
"""Unit tests for lazy adapter loading in adapters subpackage."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import builtins
|
||||||
|
from collections.abc import Callable, Mapping, Sequence
|
||||||
|
import importlib
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from types import ModuleType
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import python_repositories
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def _block_backend_import(blocked_prefix: str) -> Callable[..., ModuleType]:
|
||||||
|
real_import = builtins.__import__
|
||||||
|
|
||||||
|
def fake_import(
|
||||||
|
name: str,
|
||||||
|
globals: Mapping[str, object] | None = None,
|
||||||
|
locals: Mapping[str, object] | None = None,
|
||||||
|
fromlist: Sequence[str] = (),
|
||||||
|
level: int = 0,
|
||||||
|
) -> ModuleType:
|
||||||
|
if name == blocked_prefix or name.startswith(f"{blocked_prefix}."):
|
||||||
|
raise ImportError(f"No module named '{name}'")
|
||||||
|
return real_import(name, globals, locals, fromlist, level)
|
||||||
|
|
||||||
|
return fake_import
|
||||||
|
|
||||||
|
|
||||||
|
def test_base_import_does_not_load_adapters() -> None:
|
||||||
|
"""Base package import does not eagerly load backend adapter modules."""
|
||||||
|
script = """
|
||||||
|
import sys
|
||||||
|
from python_repositories import JsonRepositoryInterface
|
||||||
|
|
||||||
|
assert JsonRepositoryInterface is not None
|
||||||
|
assert "python_repositories.adapters.redis_adapter" not in sys.modules
|
||||||
|
assert "python_repositories.adapters.minio_adapter" not in sys.modules
|
||||||
|
assert "python_repositories.adapters.postgres_adapter" not in sys.modules
|
||||||
|
assert "python_repositories.adapters.memory_queue_adapter" not in sys.modules
|
||||||
|
assert "python_repositories.adapters.file_backed_queue_adapter" not in sys.modules
|
||||||
|
"""
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, "-c", script],
|
||||||
|
cwd=_ROOT,
|
||||||
|
env={**os.environ, "PYTHONPATH": str(_ROOT)},
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stderr or result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_lazy_adapter_load_succeeds_when_extra_present() -> None:
|
||||||
|
"""Adapters load when their optional dependencies are installed."""
|
||||||
|
from python_repositories import (
|
||||||
|
FileBackedQueueAdapter,
|
||||||
|
MemoryQueueAdapter,
|
||||||
|
MinioAdapter,
|
||||||
|
PostgresAdapter,
|
||||||
|
RedisAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert RedisAdapter.__name__ == "RedisAdapter"
|
||||||
|
assert MinioAdapter.__name__ == "MinioAdapter"
|
||||||
|
assert PostgresAdapter.__name__ == "PostgresAdapter"
|
||||||
|
assert MemoryQueueAdapter.__name__ == "MemoryQueueAdapter"
|
||||||
|
assert FileBackedQueueAdapter.__name__ == "FileBackedQueueAdapter"
|
||||||
|
|
||||||
|
|
||||||
|
def test_redis_adapter_import_error_without_extra() -> None:
|
||||||
|
"""Missing redis extra raises ImportError with install hint."""
|
||||||
|
import python_repositories.adapters.redis_adapter as redis_adapter_module
|
||||||
|
|
||||||
|
with patch.object(builtins, "__import__", new=_block_backend_import("redis")):
|
||||||
|
with pytest.raises(ImportError, match=r"python-repositories\[redis\]"):
|
||||||
|
importlib.reload(redis_adapter_module)
|
||||||
|
|
||||||
|
importlib.reload(redis_adapter_module)
|
||||||
|
|
||||||
|
|
||||||
|
def test_minio_adapter_import_error_without_extra() -> None:
|
||||||
|
"""Missing minio extra raises ImportError with install hint."""
|
||||||
|
import python_repositories.adapters.minio_adapter as minio_adapter_module
|
||||||
|
|
||||||
|
with patch.object(builtins, "__import__", new=_block_backend_import("minio")):
|
||||||
|
with pytest.raises(ImportError, match=r"python-repositories\[minio\]"):
|
||||||
|
importlib.reload(minio_adapter_module)
|
||||||
|
|
||||||
|
importlib.reload(minio_adapter_module)
|
||||||
|
|
||||||
|
|
||||||
|
def test_postgres_adapter_import_error_without_extra() -> None:
|
||||||
|
"""Missing postgres extra raises ImportError with install hint."""
|
||||||
|
import python_repositories.adapters.postgres_adapter as postgres_adapter_module
|
||||||
|
|
||||||
|
with patch.object(builtins, "__import__", new=_block_backend_import("psycopg")):
|
||||||
|
with pytest.raises(ImportError, match=r"python-repositories\[postgres\]"):
|
||||||
|
importlib.reload(postgres_adapter_module)
|
||||||
|
|
||||||
|
importlib.reload(postgres_adapter_module)
|
||||||
|
|
||||||
|
|
||||||
|
def test_top_level_lazy_import_propagates_redis_import_error() -> None:
|
||||||
|
"""Top-level RedisAdapter access surfaces adapter import errors."""
|
||||||
|
with patch(
|
||||||
|
"importlib.import_module",
|
||||||
|
side_effect=ImportError(
|
||||||
|
"Redis support requires the redis extra. "
|
||||||
|
"Install with: pip install python-repositories[redis]"
|
||||||
|
),
|
||||||
|
):
|
||||||
|
with pytest.raises(ImportError, match=r"python-repositories\[redis\]"):
|
||||||
|
_ = python_repositories.RedisAdapter
|
||||||
|
|
||||||
|
|
||||||
|
def test_top_level_lazy_import_propagates_minio_import_error() -> None:
|
||||||
|
"""Top-level MinioAdapter access surfaces adapter import errors."""
|
||||||
|
with patch(
|
||||||
|
"importlib.import_module",
|
||||||
|
side_effect=ImportError(
|
||||||
|
"MinIO support requires the minio extra. "
|
||||||
|
"Install with: pip install python-repositories[minio]"
|
||||||
|
),
|
||||||
|
):
|
||||||
|
with pytest.raises(ImportError, match=r"python-repositories\[minio\]"):
|
||||||
|
_ = python_repositories.MinioAdapter
|
||||||
|
|
||||||
|
|
||||||
|
def test_top_level_lazy_import_propagates_postgres_import_error() -> None:
|
||||||
|
"""Top-level PostgresAdapter access surfaces adapter import errors."""
|
||||||
|
with patch(
|
||||||
|
"importlib.import_module",
|
||||||
|
side_effect=ImportError(
|
||||||
|
"Postgres support requires the postgres extra. "
|
||||||
|
"Install with: pip install python-repositories[postgres]"
|
||||||
|
),
|
||||||
|
):
|
||||||
|
with pytest.raises(ImportError, match=r"python-repositories\[postgres\]"):
|
||||||
|
_ = python_repositories.PostgresAdapter
|
||||||
|
|
||||||
|
|
||||||
|
def test_adapters_subpackage_lazy_import_succeeds() -> None:
|
||||||
|
"""Adapter subpackage imports delegate to the same lazy loader."""
|
||||||
|
from python_repositories.adapters import RedisAdapter
|
||||||
|
|
||||||
|
assert RedisAdapter.__name__ == "RedisAdapter"
|
||||||
|
|
||||||
|
|
||||||
|
def test_adapters_dir_exposes_lazy_exports() -> None:
|
||||||
|
"""dir(adapters) includes lazy adapter names for tab completion."""
|
||||||
|
import python_repositories.adapters as adapters
|
||||||
|
|
||||||
|
assert {
|
||||||
|
"RedisAdapter",
|
||||||
|
"MinioAdapter",
|
||||||
|
"PostgresAdapter",
|
||||||
|
"MemoryQueueAdapter",
|
||||||
|
"FileBackedQueueAdapter",
|
||||||
|
}.issubset(set(dir(adapters)))
|
||||||
|
|
||||||
|
|
||||||
|
def test_adapters_getattr_raises_for_unknown() -> None:
|
||||||
|
"""Unknown adapter names raise AttributeError."""
|
||||||
|
import python_repositories.adapters as adapters
|
||||||
|
|
||||||
|
with pytest.raises(AttributeError, match="has no attribute 'NoSuchAdapter'"):
|
||||||
|
_ = adapters.NoSuchAdapter
|
||||||
|
|
||||||
|
|
||||||
|
def test_top_level_dir_exposes_lazy_exports() -> None:
|
||||||
|
"""dir(python_repositories) includes lazy adapter names for tab completion."""
|
||||||
|
assert "RedisAdapter" in dir(python_repositories)
|
||||||
|
assert "MinioAdapter" in dir(python_repositories)
|
||||||
|
assert "PostgresAdapter" in dir(python_repositories)
|
||||||
|
assert "MemoryQueueAdapter" in dir(python_repositories)
|
||||||
|
assert "FileBackedQueueAdapter" in dir(python_repositories)
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""Unit test fixtures for mocked adapters."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
import psycopg
|
||||||
|
import pytest
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||||
|
from python_repositories.adapters.postgres_adapter import PostgresAdapter
|
||||||
|
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||||
|
from tests.conftest import TEST_MINIO_CONFIG, TEST_POSTGRES_CONFIG, TEST_REDIS_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def redis_adapter() -> RedisAdapter:
|
||||||
|
"""Provide a RedisAdapter with an injected mock client."""
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
return RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def minio_adapter() -> MinioAdapter:
|
||||||
|
"""Provide a MinioAdapter with an injected mock client."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
return MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def postgres_adapter() -> PostgresAdapter:
|
||||||
|
"""Provide a PostgresAdapter with an injected mock client."""
|
||||||
|
mock_client = MagicMock(spec=psycopg.Connection)
|
||||||
|
mock_cursor = MagicMock()
|
||||||
|
mock_cursor.__enter__ = MagicMock(return_value=mock_cursor)
|
||||||
|
mock_cursor.__exit__ = MagicMock(return_value=False)
|
||||||
|
mock_client.cursor.return_value = mock_cursor
|
||||||
|
return PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
@@ -0,0 +1,236 @@
|
|||||||
|
"""Unit tests for TTL-cached connection health checks on ConnectionAwareAdapter."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import cast
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||||
|
from python_repositories.adapters.postgres_adapter import PostgresAdapter
|
||||||
|
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||||
|
from tests.conftest import TEST_MINIO_CONFIG, TEST_POSTGRES_CONFIG, TEST_REDIS_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
class TestRedisConnectionHealth:
|
||||||
|
def test_not_connected_when_no_client(self) -> None:
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
assert not adapter.is_connected()
|
||||||
|
|
||||||
|
def test_connected_when_probe_succeeds(self, redis_adapter: RedisAdapter) -> None:
|
||||||
|
mock_client = cast(MagicMock, redis_adapter._client)
|
||||||
|
mock_client.ping.return_value = True
|
||||||
|
|
||||||
|
assert redis_adapter.is_connected()
|
||||||
|
mock_client.ping.assert_called_once()
|
||||||
|
|
||||||
|
def test_stale_connection_when_probe_fails(
|
||||||
|
self, redis_adapter: RedisAdapter
|
||||||
|
) -> None:
|
||||||
|
cast(MagicMock, redis_adapter._client).ping.side_effect = redis.ConnectionError(
|
||||||
|
"connection lost"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not redis_adapter.is_connected()
|
||||||
|
|
||||||
|
def test_cache_hit_avoids_second_probe(self, redis_adapter: RedisAdapter) -> None:
|
||||||
|
mock_client = cast(MagicMock, redis_adapter._client)
|
||||||
|
mock_client.ping.return_value = True
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
return_value=100.0,
|
||||||
|
):
|
||||||
|
assert redis_adapter.is_connected()
|
||||||
|
assert redis_adapter.is_connected()
|
||||||
|
|
||||||
|
mock_client.ping.assert_called_once()
|
||||||
|
|
||||||
|
def test_cache_miss_runs_probe_again(self, redis_adapter: RedisAdapter) -> None:
|
||||||
|
mock_client = cast(MagicMock, redis_adapter._client)
|
||||||
|
mock_client.ping.return_value = True
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
side_effect=[100.0, 102.0],
|
||||||
|
):
|
||||||
|
assert redis_adapter.is_connected()
|
||||||
|
assert redis_adapter.is_connected()
|
||||||
|
|
||||||
|
assert mock_client.ping.call_count == 2
|
||||||
|
|
||||||
|
def test_disconnect_clears_cache(self, redis_adapter: RedisAdapter) -> None:
|
||||||
|
mock_client = cast(MagicMock, redis_adapter._client)
|
||||||
|
mock_client.ping.return_value = True
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
return_value=100.0,
|
||||||
|
):
|
||||||
|
assert redis_adapter.is_connected()
|
||||||
|
|
||||||
|
redis_adapter.disconnect()
|
||||||
|
reinjected = RedisAdapter(
|
||||||
|
config=redis_adapter._config,
|
||||||
|
client=mock_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
return_value=100.0,
|
||||||
|
):
|
||||||
|
assert reinjected.is_connected()
|
||||||
|
|
||||||
|
assert mock_client.ping.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
class TestMinioConnectionHealth:
|
||||||
|
def test_not_connected_when_no_client(self) -> None:
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||||
|
assert not adapter.is_connected()
|
||||||
|
|
||||||
|
def test_not_connected_when_bucket_name_missing(
|
||||||
|
self, minio_adapter: MinioAdapter
|
||||||
|
) -> None:
|
||||||
|
minio_adapter._bucket_name = None
|
||||||
|
|
||||||
|
assert not minio_adapter.is_connected()
|
||||||
|
|
||||||
|
def test_connected_when_probe_succeeds(self, minio_adapter: MinioAdapter) -> None:
|
||||||
|
mock_client = cast(MagicMock, minio_adapter._client)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
|
||||||
|
assert minio_adapter.is_connected()
|
||||||
|
mock_client.bucket_exists.assert_called_once_with("test-bucket")
|
||||||
|
|
||||||
|
def test_stale_connection_when_probe_fails(
|
||||||
|
self, minio_adapter: MinioAdapter
|
||||||
|
) -> None:
|
||||||
|
cast(MagicMock, minio_adapter._client).bucket_exists.side_effect = Exception(
|
||||||
|
"connection lost"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not minio_adapter.is_connected()
|
||||||
|
|
||||||
|
def test_cache_hit_avoids_second_probe(self, minio_adapter: MinioAdapter) -> None:
|
||||||
|
mock_client = cast(MagicMock, minio_adapter._client)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
return_value=100.0,
|
||||||
|
):
|
||||||
|
assert minio_adapter.is_connected()
|
||||||
|
assert minio_adapter.is_connected()
|
||||||
|
|
||||||
|
mock_client.bucket_exists.assert_called_once()
|
||||||
|
|
||||||
|
def test_cache_miss_runs_probe_again(self, minio_adapter: MinioAdapter) -> None:
|
||||||
|
mock_client = cast(MagicMock, minio_adapter._client)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
side_effect=[100.0, 102.0],
|
||||||
|
):
|
||||||
|
assert minio_adapter.is_connected()
|
||||||
|
assert minio_adapter.is_connected()
|
||||||
|
|
||||||
|
assert mock_client.bucket_exists.call_count == 2
|
||||||
|
|
||||||
|
def test_disconnect_clears_cache(self, minio_adapter: MinioAdapter) -> None:
|
||||||
|
mock_client = cast(MagicMock, minio_adapter._client)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
return_value=100.0,
|
||||||
|
):
|
||||||
|
assert minio_adapter.is_connected()
|
||||||
|
|
||||||
|
minio_adapter.disconnect()
|
||||||
|
reinjected = MinioAdapter(
|
||||||
|
config=minio_adapter._config,
|
||||||
|
client=mock_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
return_value=100.0,
|
||||||
|
):
|
||||||
|
assert reinjected.is_connected()
|
||||||
|
|
||||||
|
assert mock_client.bucket_exists.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
class TestPostgresConnectionHealth:
|
||||||
|
def test_not_connected_when_no_client(self) -> None:
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG)
|
||||||
|
assert not adapter.is_connected()
|
||||||
|
|
||||||
|
def test_connected_when_probe_succeeds(
|
||||||
|
self, postgres_adapter: PostgresAdapter
|
||||||
|
) -> None:
|
||||||
|
assert postgres_adapter.is_connected()
|
||||||
|
cast(MagicMock, postgres_adapter._client).cursor.assert_called()
|
||||||
|
|
||||||
|
def test_stale_connection_when_probe_fails(
|
||||||
|
self, postgres_adapter: PostgresAdapter
|
||||||
|
) -> None:
|
||||||
|
mock_cursor = cast(MagicMock, postgres_adapter._client).cursor.return_value
|
||||||
|
mock_cursor.execute.side_effect = psycopg.OperationalError("connection lost")
|
||||||
|
|
||||||
|
assert not postgres_adapter.is_connected()
|
||||||
|
|
||||||
|
def test_cache_hit_avoids_second_probe(
|
||||||
|
self, postgres_adapter: PostgresAdapter
|
||||||
|
) -> None:
|
||||||
|
mock_client = cast(MagicMock, postgres_adapter._client)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
return_value=100.0,
|
||||||
|
):
|
||||||
|
assert postgres_adapter.is_connected()
|
||||||
|
assert postgres_adapter.is_connected()
|
||||||
|
|
||||||
|
assert mock_client.cursor.call_count == 1
|
||||||
|
|
||||||
|
def test_cache_miss_runs_probe_again(
|
||||||
|
self, postgres_adapter: PostgresAdapter
|
||||||
|
) -> None:
|
||||||
|
mock_client = cast(MagicMock, postgres_adapter._client)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
side_effect=[100.0, 102.0],
|
||||||
|
):
|
||||||
|
assert postgres_adapter.is_connected()
|
||||||
|
assert postgres_adapter.is_connected()
|
||||||
|
|
||||||
|
assert mock_client.cursor.call_count == 2
|
||||||
|
|
||||||
|
def test_disconnect_clears_cache(self, postgres_adapter: PostgresAdapter) -> None:
|
||||||
|
mock_client = cast(MagicMock, postgres_adapter._client)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
return_value=100.0,
|
||||||
|
):
|
||||||
|
assert postgres_adapter.is_connected()
|
||||||
|
|
||||||
|
postgres_adapter.disconnect()
|
||||||
|
reinjected = PostgresAdapter(
|
||||||
|
config=postgres_adapter._config,
|
||||||
|
client=mock_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||||
|
return_value=100.0,
|
||||||
|
):
|
||||||
|
assert reinjected.is_connected()
|
||||||
|
|
||||||
|
assert mock_client.cursor.call_count == 2
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
"""Unit tests for ConnectionAwareInterface."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.interfaces.connection_aware_interface import (
|
||||||
|
ConnectionAwareInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FakeConnection:
|
||||||
|
"""Plain class that satisfies ConnectionAwareInterface without inheritance."""
|
||||||
|
|
||||||
|
def connect(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def disconnect(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def is_connected(self) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def accepts_connection_aware(connection: ConnectionAwareInterface) -> None:
|
||||||
|
"""Type-checking hook for ConnectionAwareInterface structural subtyping."""
|
||||||
|
connection.is_connected()
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_connect_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if connect is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(ConnectionAwareInterface):
|
||||||
|
"""A class that does not implement connect."""
|
||||||
|
|
||||||
|
def disconnect(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def is_connected(self) -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_disconnect_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if disconnect is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(ConnectionAwareInterface):
|
||||||
|
"""A class that does not implement disconnect."""
|
||||||
|
|
||||||
|
def connect(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def is_connected(self) -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_is_connected_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if is_connected is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(ConnectionAwareInterface):
|
||||||
|
"""A class that does not implement is_connected."""
|
||||||
|
|
||||||
|
def connect(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def disconnect(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_structural_subtyping() -> None:
|
||||||
|
"""Test that a plain class satisfies ConnectionAwareInterface structurally."""
|
||||||
|
connection: ConnectionAwareInterface = FakeConnection()
|
||||||
|
accepts_connection_aware(connection)
|
||||||
|
assert isinstance(connection, ConnectionAwareInterface)
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
"""Unit tests for ContextAwareInterface."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.interfaces.context_aware_interface import ContextAwareInterface
|
||||||
|
|
||||||
|
|
||||||
|
class FakeContextManager:
|
||||||
|
"""Plain class that satisfies ContextAwareInterface without inheritance."""
|
||||||
|
|
||||||
|
def __enter__(self) -> FakeContextManager:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(
|
||||||
|
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
||||||
|
) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def accepts_context_aware(context: ContextAwareInterface) -> None:
|
||||||
|
"""Type-checking hook for ContextAwareInterface structural subtyping."""
|
||||||
|
with context:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_enter_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if __enter__ is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(ContextAwareInterface):
|
||||||
|
"""A class that does not implement __enter__."""
|
||||||
|
|
||||||
|
def __exit__(
|
||||||
|
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
||||||
|
) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_exit_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if __exit__ is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(ContextAwareInterface):
|
||||||
|
"""A class that does not implement __exit__."""
|
||||||
|
|
||||||
|
def __enter__(self) -> Incomplete:
|
||||||
|
return self
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_structural_subtyping() -> None:
|
||||||
|
"""Test that a plain class satisfies ContextAwareInterface structurally."""
|
||||||
|
context: ContextAwareInterface = FakeContextManager()
|
||||||
|
accepts_context_aware(context)
|
||||||
|
assert isinstance(context, ContextAwareInterface)
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
"""Unit tests for dotenv_loader."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.config.dotenv_loader import load_dotenv
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_dotenv_loads_file(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
env_file = tmp_path / ".env"
|
||||||
|
env_file.write_text("DOTENV_TEST_VAR=loaded_value\n")
|
||||||
|
monkeypatch.delenv("DOTENV_TEST_VAR", raising=False)
|
||||||
|
loaded = load_dotenv(env_file)
|
||||||
|
assert loaded is True
|
||||||
|
assert os.getenv("DOTENV_TEST_VAR") == "loaded_value"
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_dotenv_returns_false_for_missing_file(tmp_path: Path) -> None:
|
||||||
|
missing = tmp_path / "missing.env"
|
||||||
|
assert load_dotenv(missing) is False
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
"""Unit tests for env_bool."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.config.env_bool import env_bool
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("value", "expected"),
|
||||||
|
[
|
||||||
|
("true", True),
|
||||||
|
("1", True),
|
||||||
|
("yes", True),
|
||||||
|
("on", True),
|
||||||
|
("false", False),
|
||||||
|
("0", False),
|
||||||
|
("no", False),
|
||||||
|
("off", False),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_env_bool_parses_truthy_and_falsy(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
value: str,
|
||||||
|
expected: bool,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setenv("TEST_BOOL", value)
|
||||||
|
assert env_bool("TEST_BOOL", default=not expected) is expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_env_bool_returns_default_when_unset(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.delenv("TEST_BOOL", raising=False)
|
||||||
|
assert env_bool("TEST_BOOL", default=True) is True
|
||||||
|
assert env_bool("TEST_BOOL", default=False) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_env_bool_raises_for_invalid_value(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setenv("TEST_BOOL", "not-a-bool")
|
||||||
|
with pytest.raises(ValueError, match="TEST_BOOL"):
|
||||||
|
env_bool("TEST_BOOL", default=False)
|
||||||
@@ -0,0 +1,239 @@
|
|||||||
|
"""Unit tests for FileBackedQueueAdapter."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.adapters.file_backed_queue_adapter import (
|
||||||
|
FileBackedQueueAdapter,
|
||||||
|
)
|
||||||
|
from python_repositories.adapters.memory_queue_adapter import MemoryQueueAdapter
|
||||||
|
from python_repositories.config.file_queue_config import FileQueueConfig
|
||||||
|
from python_repositories.interfaces.queue_repository_interface import (
|
||||||
|
QueueRepositoryInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _config(path: Path, **kwargs: object) -> FileQueueConfig:
|
||||||
|
return FileQueueConfig(path=path, **kwargs) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_implements_interface() -> None:
|
||||||
|
assert issubclass(FileBackedQueueAdapter, QueueRepositoryInterface)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ops_require_connect(tmp_path: Path) -> None:
|
||||||
|
queue = FileBackedQueueAdapter(config=_config(tmp_path / "q.jsonl"))
|
||||||
|
with pytest.raises(RuntimeError, match="not connected"):
|
||||||
|
queue.enqueue([{"id": 1}])
|
||||||
|
with pytest.raises(RuntimeError, match="not connected"):
|
||||||
|
queue.dequeue_batch()
|
||||||
|
with pytest.raises(RuntimeError, match="not connected"):
|
||||||
|
queue.size()
|
||||||
|
with pytest.raises(RuntimeError, match="not connected"):
|
||||||
|
queue.evict_older_than(datetime.now(UTC))
|
||||||
|
|
||||||
|
|
||||||
|
def test_enqueue_dequeue_persists_and_compacts(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
with FileBackedQueueAdapter(config=_config(path, dedup_keys=("id",))) as queue:
|
||||||
|
queue.enqueue([{"id": 1}, {"id": 2}, {"id": 1}])
|
||||||
|
assert queue.size() == 2
|
||||||
|
assert path.is_file()
|
||||||
|
lines = path.read_text(encoding="utf-8").strip().splitlines()
|
||||||
|
assert len(lines) == 2
|
||||||
|
batch = queue.dequeue_batch(max_items=1)
|
||||||
|
assert batch == [{"id": 1}]
|
||||||
|
remaining = path.read_text(encoding="utf-8").strip().splitlines()
|
||||||
|
assert len(remaining) == 1
|
||||||
|
assert queue.size() == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_replay_on_connect(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
config = _config(path, dedup_keys=("id",), age_key="created_at", max_age_hours=24)
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
with FileBackedQueueAdapter(config=config) as queue:
|
||||||
|
queue.enqueue(
|
||||||
|
[
|
||||||
|
{"id": 1, "created_at": now - timedelta(hours=1)},
|
||||||
|
{"id": 2, "created_at": now - timedelta(minutes=10)},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
with FileBackedQueueAdapter(config=config) as queue:
|
||||||
|
assert queue.size() == 2
|
||||||
|
assert [item["id"] for item in queue.dequeue_batch(max_items=10)] == [1, 2]
|
||||||
|
|
||||||
|
|
||||||
|
def test_replay_filters_by_max_age(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
path.write_text(
|
||||||
|
"\n".join(
|
||||||
|
[
|
||||||
|
f'{{"id": "old", "created_at": "{(now - timedelta(hours=30)).isoformat()}"}}',
|
||||||
|
f'{{"id": "new", "created_at": "{(now - timedelta(hours=1)).isoformat()}"}}',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
+ "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
config = _config(path, dedup_keys=("id",), age_key="created_at", max_age_hours=24)
|
||||||
|
with FileBackedQueueAdapter(config=config) as queue:
|
||||||
|
assert queue.size() == 1
|
||||||
|
assert queue.dequeue_batch(max_items=10)[0]["id"] == "new"
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_skips_corrupt_and_invalid_lines(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
path.write_text(
|
||||||
|
"\n".join(
|
||||||
|
[
|
||||||
|
"",
|
||||||
|
"not-json",
|
||||||
|
"[1, 2]",
|
||||||
|
'{"id": 1}',
|
||||||
|
'{"name": "missing-id"}',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
+ "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
config = _config(path, dedup_keys=("id",))
|
||||||
|
with FileBackedQueueAdapter(config=config) as queue:
|
||||||
|
assert queue.size() == 1
|
||||||
|
assert queue.dequeue_batch(max_items=10) == [{"id": 1}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_skips_missing_or_invalid_age_on_replay(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
path.write_text(
|
||||||
|
"\n".join(
|
||||||
|
[
|
||||||
|
'{"id": 1}',
|
||||||
|
'{"id": 2, "created_at": "not-a-date"}',
|
||||||
|
f'{{"id": 3, "created_at": "{now.isoformat()}"}}',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
+ "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
config = _config(path, dedup_keys=("id",), age_key="created_at")
|
||||||
|
with FileBackedQueueAdapter(config=config) as queue:
|
||||||
|
assert queue.size() == 1
|
||||||
|
assert queue.dequeue_batch(max_items=10)[0]["id"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_is_idempotent(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
queue = FileBackedQueueAdapter(config=_config(path))
|
||||||
|
queue.connect()
|
||||||
|
queue.enqueue([{"id": 1}])
|
||||||
|
queue.connect()
|
||||||
|
assert queue.size() == 1
|
||||||
|
queue.disconnect()
|
||||||
|
assert not queue.is_connected()
|
||||||
|
|
||||||
|
|
||||||
|
def test_evict_compacts_file(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
config = _config(path, dedup_keys=("id",), age_key="created_at", max_age_hours=24)
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
with FileBackedQueueAdapter(config=config) as queue:
|
||||||
|
queue.enqueue(
|
||||||
|
[
|
||||||
|
{"id": 1, "created_at": now - timedelta(hours=2)},
|
||||||
|
{"id": 2, "created_at": now - timedelta(minutes=5)},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
removed = queue.evict_older_than(now - timedelta(hours=1))
|
||||||
|
assert removed == 1
|
||||||
|
assert queue.size() == 1
|
||||||
|
lines = [line for line in path.read_text(encoding="utf-8").splitlines() if line]
|
||||||
|
assert len(lines) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_auto_evict_on_enqueue(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
config = _config(path, dedup_keys=("id",), age_key="created_at", max_age_hours=1)
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
with FileBackedQueueAdapter(config=config) as queue:
|
||||||
|
queue.enqueue([{"id": 1, "created_at": now - timedelta(hours=2)}])
|
||||||
|
# Item is enqueued then immediately age-evicted.
|
||||||
|
assert queue.size() == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_when_config_omitted(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
path = tmp_path / "from-env.jsonl"
|
||||||
|
monkeypatch.setenv("FILE_QUEUE_PATH", str(path))
|
||||||
|
with FileBackedQueueAdapter() as queue:
|
||||||
|
queue.enqueue([{"id": 1}])
|
||||||
|
assert queue.size() == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_injected_memory(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
memory = MemoryQueueAdapter(dedup_keys=("id",))
|
||||||
|
config = _config(path, dedup_keys=("id",))
|
||||||
|
with FileBackedQueueAdapter(config=config, memory=memory) as queue:
|
||||||
|
queue.enqueue([{"id": 1}])
|
||||||
|
assert memory.size() == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_enqueue_and_dequeue_skip_compact(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
with FileBackedQueueAdapter(config=_config(path)) as queue:
|
||||||
|
queue.enqueue([])
|
||||||
|
assert queue.dequeue_batch(max_items=10) == []
|
||||||
|
assert queue.evict_older_than(datetime.now(UTC)) == 0
|
||||||
|
assert not path.exists() or path.read_text(encoding="utf-8") == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconnect_replays_after_disconnect(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
queue = FileBackedQueueAdapter(config=_config(path, dedup_keys=("id",)))
|
||||||
|
queue.connect()
|
||||||
|
queue.enqueue([{"id": 1}])
|
||||||
|
queue.disconnect()
|
||||||
|
queue.connect()
|
||||||
|
assert queue.size() == 1
|
||||||
|
queue.disconnect()
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_default_rejects_unsupported() -> None:
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
FileBackedQueueAdapter._json_default(object())
|
||||||
|
|
||||||
|
|
||||||
|
def test_disconnect_ignores_fsync_oserror(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
queue = FileBackedQueueAdapter(config=_config(path))
|
||||||
|
queue.connect()
|
||||||
|
|
||||||
|
def boom(_fd: int) -> None:
|
||||||
|
raise OSError("fsync failed")
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"python_repositories.adapters.file_backed_queue_adapter.os.fsync",
|
||||||
|
boom,
|
||||||
|
)
|
||||||
|
queue.disconnect()
|
||||||
|
assert not queue.is_connected()
|
||||||
|
|
||||||
|
|
||||||
|
def test_append_items_requires_open_handle(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "items.jsonl"
|
||||||
|
queue = FileBackedQueueAdapter(config=_config(path))
|
||||||
|
queue.connect()
|
||||||
|
queue._append_handle = None # noqa: SLF001
|
||||||
|
with pytest.raises(RuntimeError, match="append handle is not open"):
|
||||||
|
queue._append_items([{"id": 1}]) # noqa: SLF001
|
||||||
|
queue._connected = False # noqa: SLF001
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
"""Unit tests for FileQueueConfig."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.config import FileQueueConfig
|
||||||
|
|
||||||
|
|
||||||
|
def test_defaults() -> None:
|
||||||
|
config = FileQueueConfig(path=Path("/tmp/queue.jsonl"))
|
||||||
|
assert config.path == Path("/tmp/queue.jsonl")
|
||||||
|
assert config.max_age_hours == 24
|
||||||
|
assert config.dedup_keys == ()
|
||||||
|
assert config.age_key is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_loads_path(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("FILE_QUEUE_PATH", "/data/items.jsonl")
|
||||||
|
monkeypatch.delenv("FILE_QUEUE_MAX_AGE_HOURS", raising=False)
|
||||||
|
config = FileQueueConfig.from_env(use_dotenv=False)
|
||||||
|
assert config.path == Path("/data/items.jsonl")
|
||||||
|
assert config.max_age_hours == 24
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_loads_max_age_hours(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("FILE_QUEUE_PATH", "/data/items.jsonl")
|
||||||
|
monkeypatch.setenv("FILE_QUEUE_MAX_AGE_HOURS", "48")
|
||||||
|
config = FileQueueConfig.from_env(use_dotenv=False)
|
||||||
|
assert config.max_age_hours == 48
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_raises_when_path_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.delenv("FILE_QUEUE_PATH", raising=False)
|
||||||
|
with pytest.raises(Exception):
|
||||||
|
FileQueueConfig.from_env(use_dotenv=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_respects_custom_env_var_names(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setenv("CUSTOM_QUEUE_PATH", "/custom/q.jsonl")
|
||||||
|
monkeypatch.setenv("CUSTOM_MAX_AGE", "12")
|
||||||
|
config = FileQueueConfig.from_env(
|
||||||
|
"CUSTOM_QUEUE_PATH",
|
||||||
|
max_age_hours_env_var_name="CUSTOM_MAX_AGE",
|
||||||
|
dedup_keys=("id",),
|
||||||
|
age_key="created_at",
|
||||||
|
use_dotenv=False,
|
||||||
|
)
|
||||||
|
assert config.path == Path("/custom/q.jsonl")
|
||||||
|
assert config.max_age_hours == 12
|
||||||
|
assert config.dedup_keys == ("id",)
|
||||||
|
assert config.age_key == "created_at"
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
"""Unit tests for JsonRepositoryInterface."""
|
||||||
|
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.interfaces.json_repository_interface import (
|
||||||
|
JsonRepositoryInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class InMemoryJsonRepo:
|
||||||
|
"""Plain class that satisfies JsonRepositoryInterface without inheritance."""
|
||||||
|
|
||||||
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, key: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def scan_keys(
|
||||||
|
self,
|
||||||
|
pattern: str,
|
||||||
|
*,
|
||||||
|
count: int | None = None,
|
||||||
|
) -> Iterator[str]:
|
||||||
|
del count
|
||||||
|
yield from self.list_keys(pattern)
|
||||||
|
|
||||||
|
|
||||||
|
def accepts_json_repo(repo: JsonRepositoryInterface) -> None:
|
||||||
|
"""Type-checking hook for JsonRepositoryInterface structural subtyping."""
|
||||||
|
repo.get("key")
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_get_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if get is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(JsonRepositoryInterface):
|
||||||
|
"""A class that does not implement get."""
|
||||||
|
|
||||||
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, key: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_set_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if set is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(JsonRepositoryInterface):
|
||||||
|
"""A class that does not implement set."""
|
||||||
|
|
||||||
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def delete(self, key: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_delete_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if delete is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(JsonRepositoryInterface):
|
||||||
|
"""A class that does not implement delete."""
|
||||||
|
|
||||||
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_list_keys_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if list_keys is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(JsonRepositoryInterface):
|
||||||
|
"""A class that does not implement list_keys."""
|
||||||
|
|
||||||
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, key: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_keys_defaults_to_list_keys() -> None:
|
||||||
|
"""Test that the default scan_keys implementation delegates to list_keys."""
|
||||||
|
|
||||||
|
class Complete(JsonRepositoryInterface):
|
||||||
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, key: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
|
return [f"{pattern}-1", f"{pattern}-2"]
|
||||||
|
|
||||||
|
repository = Complete()
|
||||||
|
|
||||||
|
assert list(repository.scan_keys("user")) == ["user-1", "user-2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_structural_subtyping() -> None:
|
||||||
|
"""Test that a plain class satisfies JsonRepositoryInterface structurally."""
|
||||||
|
repo: JsonRepositoryInterface = InMemoryJsonRepo()
|
||||||
|
accepts_json_repo(repo)
|
||||||
|
assert isinstance(repo, JsonRepositoryInterface)
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
"""Unit tests for MemoryQueueAdapter."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
import threading
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.adapters.memory_queue_adapter import MemoryQueueAdapter
|
||||||
|
from python_repositories.interfaces.queue_repository_interface import (
|
||||||
|
QueueRepositoryInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_implements_interface() -> None:
|
||||||
|
assert issubclass(MemoryQueueAdapter, QueueRepositoryInterface)
|
||||||
|
|
||||||
|
|
||||||
|
def test_fifo_without_dedup() -> None:
|
||||||
|
queue = MemoryQueueAdapter()
|
||||||
|
queue.enqueue([{"id": 1}, {"id": 2}, {"id": 3}])
|
||||||
|
assert queue.size() == 3
|
||||||
|
assert queue.dequeue_batch(max_items=2) == [{"id": 1}, {"id": 2}]
|
||||||
|
assert queue.dequeue_batch(max_items=10) == [{"id": 3}]
|
||||||
|
assert queue.dequeue_batch() == []
|
||||||
|
assert queue.size() == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_enqueue_is_noop() -> None:
|
||||||
|
queue = MemoryQueueAdapter()
|
||||||
|
queue.enqueue([])
|
||||||
|
assert queue.size() == 0
|
||||||
|
assert queue.enqueue_and_return_added([]) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_dedup_keeps_first() -> None:
|
||||||
|
queue = MemoryQueueAdapter(dedup_keys=("id",))
|
||||||
|
queue.enqueue([{"id": 1, "v": "a"}, {"id": 1, "v": "b"}, {"id": 2, "v": "c"}])
|
||||||
|
assert queue.size() == 2
|
||||||
|
assert queue.dequeue_batch(max_items=10) == [
|
||||||
|
{"id": 1, "v": "a"},
|
||||||
|
{"id": 2, "v": "c"},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_dedup_key_raises() -> None:
|
||||||
|
queue = MemoryQueueAdapter(dedup_keys=("id",))
|
||||||
|
with pytest.raises(ValueError, match="missing dedup key"):
|
||||||
|
queue.enqueue([{"name": "x"}])
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_age_key_raises() -> None:
|
||||||
|
queue = MemoryQueueAdapter(age_key="created_at")
|
||||||
|
with pytest.raises(ValueError, match="missing age key"):
|
||||||
|
queue.enqueue([{"id": 1}])
|
||||||
|
|
||||||
|
|
||||||
|
def test_evict_older_than() -> None:
|
||||||
|
queue = MemoryQueueAdapter(age_key="created_at", dedup_keys=("id",))
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
queue.enqueue(
|
||||||
|
[
|
||||||
|
{"id": 1, "created_at": now - timedelta(hours=2)},
|
||||||
|
{"id": 2, "created_at": now - timedelta(minutes=30)},
|
||||||
|
{"id": 3, "created_at": (now - timedelta(hours=3)).isoformat()},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
removed = queue.evict_older_than(now - timedelta(hours=1))
|
||||||
|
assert removed == 2
|
||||||
|
assert queue.size() == 1
|
||||||
|
remaining = queue.dequeue_batch(max_items=10)
|
||||||
|
assert remaining[0]["id"] == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_evict_without_age_key_is_noop() -> None:
|
||||||
|
queue = MemoryQueueAdapter()
|
||||||
|
queue.enqueue([{"id": 1}])
|
||||||
|
assert queue.evict_older_than(datetime.now(UTC)) == 0
|
||||||
|
assert queue.size() == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_age_rejects_unsupported_type() -> None:
|
||||||
|
with pytest.raises(ValueError, match="age value must be"):
|
||||||
|
MemoryQueueAdapter._parse_age(123)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_age_aware_datetime() -> None:
|
||||||
|
from datetime import timezone
|
||||||
|
|
||||||
|
eastern = timezone(timedelta(hours=-5))
|
||||||
|
aware = datetime(2024, 1, 1, 12, 0, 0, tzinfo=eastern)
|
||||||
|
assert MemoryQueueAdapter._parse_age(aware) == datetime(
|
||||||
|
2024, 1, 1, 17, 0, 0, tzinfo=UTC
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_age_naive_datetime() -> None:
|
||||||
|
naive = datetime(2024, 1, 1, 12, 0, 0)
|
||||||
|
assert MemoryQueueAdapter._parse_age(naive) == datetime(
|
||||||
|
2024, 1, 1, 12, 0, 0, tzinfo=UTC
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_age_zulu_string() -> None:
|
||||||
|
assert MemoryQueueAdapter._parse_age("2024-01-01T00:00:00Z") == datetime(
|
||||||
|
2024, 1, 1, 0, 0, 0, tzinfo=UTC
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_age_naive_string() -> None:
|
||||||
|
assert MemoryQueueAdapter._parse_age("2024-01-01T00:00:00") == datetime(
|
||||||
|
2024, 1, 1, 0, 0, 0, tzinfo=UTC
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_dequeue_rejects_negative_max_items() -> None:
|
||||||
|
queue = MemoryQueueAdapter()
|
||||||
|
with pytest.raises(ValueError, match="max_items"):
|
||||||
|
queue.dequeue_batch(max_items=-1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_clear_and_snapshot() -> None:
|
||||||
|
queue = MemoryQueueAdapter()
|
||||||
|
queue.enqueue([{"id": 1}, {"id": 2}])
|
||||||
|
assert queue.snapshot() == [{"id": 1}, {"id": 2}]
|
||||||
|
queue.clear()
|
||||||
|
assert queue.size() == 0
|
||||||
|
assert queue.snapshot() == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_thread_safety_smoke() -> None:
|
||||||
|
queue = MemoryQueueAdapter(dedup_keys=("id",))
|
||||||
|
errors: list[BaseException] = []
|
||||||
|
|
||||||
|
def worker(start: int) -> None:
|
||||||
|
try:
|
||||||
|
for i in range(start, start + 50):
|
||||||
|
queue.enqueue([{"id": i}])
|
||||||
|
except BaseException as exc: # noqa: BLE001
|
||||||
|
errors.append(exc)
|
||||||
|
|
||||||
|
threads = [threading.Thread(target=worker, args=(i * 50,)) for i in range(4)]
|
||||||
|
for thread in threads:
|
||||||
|
thread.start()
|
||||||
|
for thread in threads:
|
||||||
|
thread.join()
|
||||||
|
assert errors == []
|
||||||
|
assert queue.size() == 200
|
||||||
@@ -0,0 +1,252 @@
|
|||||||
|
"""Unit tests for MinioAdapter instantiation and injection."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import replace
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from minio import Minio, S3Error
|
||||||
|
import pytest
|
||||||
|
from urllib3.response import BaseHTTPResponse
|
||||||
|
|
||||||
|
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||||
|
from python_repositories.interfaces import ObjectRepositoryInterface
|
||||||
|
from tests.conftest import TEST_MINIO_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_adhere_to_interface() -> None:
|
||||||
|
assert issubclass(MinioAdapter, ObjectRepositoryInterface)
|
||||||
|
_ = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_have_logger_when_instantiated() -> None:
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||||
|
assert hasattr(adapter, "logger")
|
||||||
|
assert adapter.logger is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_not_be_connected_when_instantiated() -> None:
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||||
|
assert not adapter.is_connected()
|
||||||
|
|
||||||
|
|
||||||
|
def test_constructs_with_injected_config_without_env(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
for var in (
|
||||||
|
"MINIO_ENDPOINT",
|
||||||
|
"MINIO_ACCESS_KEY",
|
||||||
|
"MINIO_SECRET_KEY",
|
||||||
|
"MINIO_BUCKET",
|
||||||
|
):
|
||||||
|
monkeypatch.delenv(var, raising=False)
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||||
|
assert adapter._config == TEST_MINIO_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
def test_injected_client_sets_bucket_name() -> None:
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
assert adapter._bucket_name == "test-bucket"
|
||||||
|
assert adapter._client is mock_client
|
||||||
|
|
||||||
|
|
||||||
|
def test_raises_when_client_provided_without_config() -> None:
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
with pytest.raises(ValueError, match="config is required"):
|
||||||
|
MinioAdapter(client=mock_client)
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_succeeds() -> None:
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.list_buckets.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_raises_on_failure() -> None:
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.list_buckets.side_effect = Exception("connection lost")
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError, match="Could not connect to Minio"):
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_skips_validation_when_client_cleared() -> None:
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
adapter.disconnect()
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.list_buckets.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_disconnects_before_reconnect(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
stale_client = MagicMock(spec=Minio)
|
||||||
|
new_client = MagicMock(spec=Minio)
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||||
|
adapter._client = stale_client
|
||||||
|
adapter._bucket_name = None
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||||
|
lambda *args, **kwargs: new_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
assert adapter._client is new_client
|
||||||
|
new_client.list_buckets.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_skips_reconnect_when_already_connected(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
stale_client = MagicMock(spec=Minio)
|
||||||
|
stale_client.bucket_exists.return_value = True
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||||
|
adapter._client = stale_client
|
||||||
|
adapter._bucket_name = TEST_MINIO_CONFIG.bucket
|
||||||
|
|
||||||
|
minio_ctor = MagicMock()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||||
|
minio_ctor,
|
||||||
|
)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
minio_ctor.assert_not_called()
|
||||||
|
assert adapter._client is stale_client
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_raises_when_bucket_missing(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = False
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||||
|
lambda *args, **kwargs: mock_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError, match="does not exist"):
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.make_bucket.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_creates_bucket_when_create_bucket_if_missing_enabled(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = False
|
||||||
|
config = replace(TEST_MINIO_CONFIG, create_bucket_if_missing=True)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||||
|
lambda *args, **kwargs: mock_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
adapter = MinioAdapter(config=config)
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.bucket_exists.assert_called_once_with(config.bucket)
|
||||||
|
mock_client.make_bucket.assert_called_once_with(config.bucket)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_closes_response_on_success() -> None:
|
||||||
|
"""get() must close and release the get_object HTTP response."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.read.side_effect = [b"data", b""]
|
||||||
|
mock_client.get_object.return_value = mock_response
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
result = adapter.get("some-object")
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result.read() == b"data"
|
||||||
|
mock_response.close.assert_called_once()
|
||||||
|
mock_response.release_conn.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_closes_response_when_read_fails() -> None:
|
||||||
|
"""get() must close and release the response even if read() raises."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.read.side_effect = OSError("connection reset")
|
||||||
|
mock_client.get_object.return_value = mock_response
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(OSError, match="connection reset"):
|
||||||
|
adapter.get("some-object")
|
||||||
|
|
||||||
|
mock_response.close.assert_called_once()
|
||||||
|
mock_response.release_conn.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_returns_none_for_no_such_key() -> None:
|
||||||
|
"""get() returns None when the object does not exist."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
mock_client.get_object.side_effect = S3Error(
|
||||||
|
MagicMock(spec=BaseHTTPResponse),
|
||||||
|
"NoSuchKey",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
bucket_name="test-bucket",
|
||||||
|
object_name="missing-object",
|
||||||
|
)
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
result = adapter.get("missing-object")
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_reraises_other_s3_errors() -> None:
|
||||||
|
"""get() re-raises S3 errors other than NoSuchKey."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
other_s3error = S3Error(
|
||||||
|
MagicMock(spec=BaseHTTPResponse),
|
||||||
|
"AccessDenied",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
bucket_name="test-bucket",
|
||||||
|
object_name="some-object",
|
||||||
|
)
|
||||||
|
mock_client.get_object.side_effect = other_s3error
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(S3Error) as exc_info:
|
||||||
|
adapter.get("some-object")
|
||||||
|
|
||||||
|
assert exc_info.value.code == "AccessDenied"
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_reraises_general_exception_from_get_object() -> None:
|
||||||
|
"""get() re-raises unexpected exceptions from get_object."""
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
mock_client.bucket_exists.return_value = True
|
||||||
|
mock_client.get_object.side_effect = Exception("General failure")
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(Exception, match="General failure"):
|
||||||
|
adapter.get("some-object")
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
"""Unit tests for MinioConfig."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.config import MinioConfig
|
||||||
|
|
||||||
|
|
||||||
|
def _set_required_minio_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("MINIO_ENDPOINT", "localhost:9000")
|
||||||
|
monkeypatch.setenv("MINIO_ACCESS_KEY", "access")
|
||||||
|
monkeypatch.setenv("MINIO_SECRET_KEY", "secret")
|
||||||
|
monkeypatch.setenv("MINIO_BUCKET", "my-bucket")
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_loads_all_fields(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_set_required_minio_env(monkeypatch)
|
||||||
|
config = MinioConfig.from_env(use_dotenv=False)
|
||||||
|
assert config.endpoint == "localhost:9000"
|
||||||
|
assert config.access_key == "access"
|
||||||
|
assert config.secret_key == "secret"
|
||||||
|
assert config.bucket == "my-bucket"
|
||||||
|
assert config.secure is True
|
||||||
|
assert config.create_bucket_if_missing is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_defaults_secure_to_true_when_unset(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
_set_required_minio_env(monkeypatch)
|
||||||
|
monkeypatch.delenv("MINIO_SECURE", raising=False)
|
||||||
|
config = MinioConfig.from_env(use_dotenv=False)
|
||||||
|
assert config.secure is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_reads_secure_from_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_set_required_minio_env(monkeypatch)
|
||||||
|
monkeypatch.setenv("MINIO_SECURE", "false")
|
||||||
|
assert MinioConfig.from_env(use_dotenv=False).secure is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_reads_create_bucket_if_missing_from_env(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
_set_required_minio_env(monkeypatch)
|
||||||
|
monkeypatch.setenv("MINIO_CREATE_BUCKET_IF_MISSING", "true")
|
||||||
|
assert MinioConfig.from_env(use_dotenv=False).create_bucket_if_missing is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_defaults_create_bucket_if_missing_to_false_when_unset(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
_set_required_minio_env(monkeypatch)
|
||||||
|
monkeypatch.delenv("MINIO_CREATE_BUCKET_IF_MISSING", raising=False)
|
||||||
|
config = MinioConfig.from_env(use_dotenv=False)
|
||||||
|
assert config.create_bucket_if_missing is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_raises_when_endpoint_missing(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.delenv("MINIO_ENDPOINT", raising=False)
|
||||||
|
monkeypatch.setenv("MINIO_ACCESS_KEY", "access")
|
||||||
|
monkeypatch.setenv("MINIO_SECRET_KEY", "secret")
|
||||||
|
monkeypatch.setenv("MINIO_BUCKET", "my-bucket")
|
||||||
|
with pytest.raises(Exception):
|
||||||
|
MinioConfig.from_env(use_dotenv=False)
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
"""Unit tests for ObjectRepositoryInterface."""
|
||||||
|
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.interfaces.object_repository_interface import (
|
||||||
|
ObjectRepositoryInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class InMemoryObjectRepo:
|
||||||
|
"""Plain class that satisfies ObjectRepositoryInterface without inheritance."""
|
||||||
|
|
||||||
|
def get(self, object_name: str) -> BytesIO | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def put(
|
||||||
|
self,
|
||||||
|
object_name: str,
|
||||||
|
data: BytesIO,
|
||||||
|
content_type: str = "application/octet-stream",
|
||||||
|
) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, object_name: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_objects(self, prefix: str = "") -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def accepts_object_repo(repo: ObjectRepositoryInterface) -> None:
|
||||||
|
"""Type-checking hook for ObjectRepositoryInterface structural subtyping."""
|
||||||
|
repo.get("object")
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_get_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if get is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(ObjectRepositoryInterface):
|
||||||
|
"""A class that does not implement get."""
|
||||||
|
|
||||||
|
def put(
|
||||||
|
self,
|
||||||
|
object_name: str,
|
||||||
|
data: BytesIO,
|
||||||
|
content_type: str = "application/octet-stream",
|
||||||
|
) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, object_name: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_objects(self, prefix: str = "") -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_put_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if put is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(ObjectRepositoryInterface):
|
||||||
|
"""A class that does not implement put."""
|
||||||
|
|
||||||
|
def get(self, object_name: str) -> BytesIO | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def delete(self, object_name: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_objects(self, prefix: str = "") -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_delete_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if delete is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(ObjectRepositoryInterface):
|
||||||
|
"""A class that does not implement delete."""
|
||||||
|
|
||||||
|
def get(self, object_name: str) -> BytesIO | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def put(
|
||||||
|
self,
|
||||||
|
object_name: str,
|
||||||
|
data: BytesIO,
|
||||||
|
content_type: str = "application/octet-stream",
|
||||||
|
) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_objects(self, prefix: str = "") -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_list_objects_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if list_objects is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(ObjectRepositoryInterface):
|
||||||
|
"""A class that does not implement list_objects."""
|
||||||
|
|
||||||
|
def get(self, object_name: str) -> BytesIO | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def put(
|
||||||
|
self,
|
||||||
|
object_name: str,
|
||||||
|
data: BytesIO,
|
||||||
|
content_type: str = "application/octet-stream",
|
||||||
|
) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, object_name: str) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_structural_subtyping() -> None:
|
||||||
|
"""Test that a plain class satisfies ObjectRepositoryInterface structurally."""
|
||||||
|
repo: ObjectRepositoryInterface = InMemoryObjectRepo()
|
||||||
|
accepts_object_repo(repo)
|
||||||
|
assert isinstance(repo, ObjectRepositoryInterface)
|
||||||
@@ -0,0 +1,237 @@
|
|||||||
|
"""Unit tests for PostgresAdapter instantiation and injection."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.adapters.postgres_adapter import PostgresAdapter
|
||||||
|
from python_repositories.interfaces import TableRepositoryInterface
|
||||||
|
from tests.conftest import TEST_POSTGRES_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_cursor() -> MagicMock:
|
||||||
|
mock_cursor = MagicMock()
|
||||||
|
mock_cursor.__enter__ = MagicMock(return_value=mock_cursor)
|
||||||
|
mock_cursor.__exit__ = MagicMock(return_value=False)
|
||||||
|
return mock_cursor
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_client(*, probe_raises: Exception | None = None) -> MagicMock:
|
||||||
|
mock_client = MagicMock(spec=psycopg.Connection)
|
||||||
|
mock_cursor = _mock_cursor()
|
||||||
|
if probe_raises is not None:
|
||||||
|
mock_cursor.execute.side_effect = probe_raises
|
||||||
|
mock_client.cursor.return_value = mock_cursor
|
||||||
|
return mock_client
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_adhere_to_interface() -> None:
|
||||||
|
assert issubclass(PostgresAdapter, TableRepositoryInterface)
|
||||||
|
_ = PostgresAdapter(config=TEST_POSTGRES_CONFIG)
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_have_logger_when_instantiated() -> None:
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG)
|
||||||
|
assert hasattr(adapter, "logger")
|
||||||
|
assert adapter.logger is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_not_be_connected_when_instantiated() -> None:
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG)
|
||||||
|
assert adapter._client is None
|
||||||
|
assert not adapter.is_connected()
|
||||||
|
|
||||||
|
|
||||||
|
def test_constructs_with_injected_config_without_env(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.delenv("POSTGRES_URI", raising=False)
|
||||||
|
monkeypatch.delenv("POSTGRES_TABLE", raising=False)
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG)
|
||||||
|
assert adapter._config == TEST_POSTGRES_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
def test_constructs_with_injected_client_without_env(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.delenv("POSTGRES_URI", raising=False)
|
||||||
|
monkeypatch.delenv("POSTGRES_TABLE", raising=False)
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
assert adapter._client is mock_client
|
||||||
|
assert adapter._client_injected is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_raises_when_client_provided_without_config() -> None:
|
||||||
|
mock_client = MagicMock(spec=psycopg.Connection)
|
||||||
|
with pytest.raises(ValueError, match="config is required"):
|
||||||
|
PostgresAdapter(client=mock_client)
|
||||||
|
|
||||||
|
|
||||||
|
def test_disconnect_does_not_close_injected_client() -> None:
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
adapter.disconnect()
|
||||||
|
mock_client.close.assert_not_called()
|
||||||
|
assert adapter._client is None
|
||||||
|
|
||||||
|
|
||||||
|
class CustomEnvPostgresAdapter(PostgresAdapter):
|
||||||
|
uri_env_var_name = "CUSTOM_POSTGRES_URI"
|
||||||
|
table_env_var_name = "CUSTOM_POSTGRES_TABLE"
|
||||||
|
primary_key_env_var_name = "CUSTOM_POSTGRES_PRIMARY_KEY"
|
||||||
|
|
||||||
|
|
||||||
|
def test_subclass_custom_env_var_names(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("CUSTOM_POSTGRES_URI", "postgresql://custom/mydb")
|
||||||
|
monkeypatch.setenv("CUSTOM_POSTGRES_TABLE", "items")
|
||||||
|
monkeypatch.setenv("CUSTOM_POSTGRES_PRIMARY_KEY", "item_id")
|
||||||
|
adapter = CustomEnvPostgresAdapter()
|
||||||
|
assert adapter._config.uri == "postgresql://custom/mydb"
|
||||||
|
assert adapter._config.table == "items"
|
||||||
|
assert adapter._config.primary_key == "item_id"
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_succeeds_when_probe_ok() -> None:
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
assert mock_client.cursor.return_value.execute.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_raises_on_probe_failure() -> None:
|
||||||
|
mock_client = _mock_client(probe_raises=psycopg.OperationalError("connection lost"))
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError, match="Could not connect to Postgres"):
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_skips_validation_when_client_cleared() -> None:
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
adapter.disconnect()
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.cursor.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_reconnects_when_existing_client_unhealthy(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
stale_client = _mock_client(
|
||||||
|
probe_raises=psycopg.OperationalError("connection lost")
|
||||||
|
)
|
||||||
|
new_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG)
|
||||||
|
adapter._client = stale_client
|
||||||
|
|
||||||
|
monkeypatch.setattr("psycopg.connect", lambda *args, **kwargs: new_client)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
stale_client.close.assert_called_once()
|
||||||
|
assert adapter._client is new_client
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_skips_reconnect_when_already_connected(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
healthy_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG)
|
||||||
|
adapter._client = healthy_client
|
||||||
|
|
||||||
|
connect = MagicMock()
|
||||||
|
monkeypatch.setattr("psycopg.connect", connect)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
healthy_client.close.assert_not_called()
|
||||||
|
connect.assert_not_called()
|
||||||
|
assert adapter._client is healthy_client
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_one_raises_value_error_on_invalid_pk() -> None:
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="Primary key must not be None"):
|
||||||
|
adapter.fetch_one(None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_one_raises_connection_error_when_not_connected() -> None:
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
adapter.fetch_one("some-id")
|
||||||
|
|
||||||
|
|
||||||
|
def test_upsert_raises_value_error_on_invalid_row() -> None:
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="Row must be a dictionary"):
|
||||||
|
adapter.upsert("not-a-dict") # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_upsert_raises_value_error_when_primary_key_missing() -> None:
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="Row must include primary key column 'id'"):
|
||||||
|
adapter.upsert({"name": "Alice"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_upsert_raises_connection_error_when_not_connected() -> None:
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
adapter.upsert({"id": "alice", "name": "Alice"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_all_raises_value_error_on_invalid_limit() -> None:
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="Limit must be a non-negative integer"):
|
||||||
|
adapter.fetch_all(limit=-1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execute_raises_value_error_on_invalid_sql() -> None:
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="SQL must be a non-empty string"):
|
||||||
|
adapter.execute("")
|
||||||
|
|
||||||
|
|
||||||
|
def test_execute_raises_value_error_on_invalid_params() -> None:
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="Params must be a tuple"):
|
||||||
|
adapter.execute("SELECT 1", []) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_upsert_with_primary_key_only_uses_do_nothing() -> None:
|
||||||
|
mock_client = _mock_client()
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
adapter.upsert({"id": "pk-only"})
|
||||||
|
|
||||||
|
mock_cursor = mock_client.cursor.return_value.__enter__.return_value
|
||||||
|
insert_call = mock_cursor.execute.call_args_list[-1]
|
||||||
|
assert insert_call[0][1] == ("pk-only",)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execute_raises_connection_error_when_not_connected() -> None:
|
||||||
|
adapter = PostgresAdapter(config=TEST_POSTGRES_CONFIG)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
adapter.execute("SELECT 1")
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
"""Unit tests for PostgresConfig."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.config import PostgresConfig
|
||||||
|
|
||||||
|
|
||||||
|
def _set_required_postgres_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("POSTGRES_URI", "postgresql://localhost/mydb")
|
||||||
|
monkeypatch.setenv("POSTGRES_TABLE", "users")
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_loads_all_fields(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_set_required_postgres_env(monkeypatch)
|
||||||
|
config = PostgresConfig.from_env(use_dotenv=False)
|
||||||
|
assert config.uri == "postgresql://localhost/mydb"
|
||||||
|
assert config.table == "users"
|
||||||
|
assert config.primary_key == "id"
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_reads_primary_key_from_env(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
_set_required_postgres_env(monkeypatch)
|
||||||
|
monkeypatch.setenv("POSTGRES_PRIMARY_KEY", "user_id")
|
||||||
|
config = PostgresConfig.from_env(use_dotenv=False)
|
||||||
|
assert config.primary_key == "user_id"
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_defaults_primary_key_to_id_when_unset(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
_set_required_postgres_env(monkeypatch)
|
||||||
|
monkeypatch.delenv("POSTGRES_PRIMARY_KEY", raising=False)
|
||||||
|
config = PostgresConfig.from_env(use_dotenv=False)
|
||||||
|
assert config.primary_key == "id"
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_raises_when_uri_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.delenv("POSTGRES_URI", raising=False)
|
||||||
|
monkeypatch.setenv("POSTGRES_TABLE", "users")
|
||||||
|
with pytest.raises(Exception):
|
||||||
|
PostgresConfig.from_env(use_dotenv=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_raises_when_table_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("POSTGRES_URI", "postgresql://localhost/mydb")
|
||||||
|
monkeypatch.delenv("POSTGRES_TABLE", raising=False)
|
||||||
|
with pytest.raises(Exception):
|
||||||
|
PostgresConfig.from_env(use_dotenv=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_respects_custom_env_var_names(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setenv("CUSTOM_POSTGRES_URI", "postgresql://custom/mydb")
|
||||||
|
monkeypatch.setenv("CUSTOM_POSTGRES_TABLE", "items")
|
||||||
|
monkeypatch.setenv("CUSTOM_POSTGRES_PRIMARY_KEY", "item_id")
|
||||||
|
config = PostgresConfig.from_env(
|
||||||
|
"CUSTOM_POSTGRES_URI",
|
||||||
|
"CUSTOM_POSTGRES_TABLE",
|
||||||
|
"CUSTOM_POSTGRES_PRIMARY_KEY",
|
||||||
|
use_dotenv=False,
|
||||||
|
)
|
||||||
|
assert config.uri == "postgresql://custom/mydb"
|
||||||
|
assert config.table == "items"
|
||||||
|
assert config.primary_key == "item_id"
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
"""Unit tests for QueueRepositoryInterface."""
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.interfaces.queue_repository_interface import (
|
||||||
|
QueueRepositoryInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class InMemoryQueueRepo:
|
||||||
|
"""Plain class that satisfies QueueRepositoryInterface without inheritance."""
|
||||||
|
|
||||||
|
def enqueue(self, items: list[dict[str, Any]]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
|
||||||
|
del max_items
|
||||||
|
return []
|
||||||
|
|
||||||
|
def size(self) -> int:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def evict_older_than(self, cutoff: datetime) -> int:
|
||||||
|
del cutoff
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def accepts_queue_repo(repo: QueueRepositoryInterface) -> None:
|
||||||
|
"""Type-checking hook for QueueRepositoryInterface structural subtyping."""
|
||||||
|
repo.size()
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_enqueue_not_implemented() -> None:
|
||||||
|
class Incomplete(QueueRepositoryInterface):
|
||||||
|
def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def size(self) -> int:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def evict_older_than(self, cutoff: datetime) -> int:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_dequeue_batch_not_implemented() -> None:
|
||||||
|
class Incomplete(QueueRepositoryInterface):
|
||||||
|
def enqueue(self, items: list[dict[str, Any]]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def size(self) -> int:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def evict_older_than(self, cutoff: datetime) -> int:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_size_not_implemented() -> None:
|
||||||
|
class Incomplete(QueueRepositoryInterface):
|
||||||
|
def enqueue(self, items: list[dict[str, Any]]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def evict_older_than(self, cutoff: datetime) -> int:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_evict_older_than_not_implemented() -> None:
|
||||||
|
class Incomplete(QueueRepositoryInterface):
|
||||||
|
def enqueue(self, items: list[dict[str, Any]]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def dequeue_batch(self, *, max_items: int = 1000) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def size(self) -> int:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_structural_subtyping() -> None:
|
||||||
|
repo: QueueRepositoryInterface = InMemoryQueueRepo()
|
||||||
|
accepts_queue_repo(repo)
|
||||||
|
assert isinstance(repo, QueueRepositoryInterface)
|
||||||
|
assert repo.evict_older_than(datetime.now(UTC)) == 0
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
"""Unit tests for RedisAdapter instantiation and injection."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||||
|
from python_repositories.interfaces import JsonRepositoryInterface
|
||||||
|
from tests.conftest import TEST_REDIS_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_adhere_to_interface() -> None:
|
||||||
|
assert issubclass(RedisAdapter, JsonRepositoryInterface)
|
||||||
|
_ = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_have_logger_when_instantiated() -> None:
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
assert hasattr(adapter, "logger")
|
||||||
|
assert adapter.logger is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_not_be_connected_when_instantiated() -> None:
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
assert adapter._client is None
|
||||||
|
assert not adapter.is_connected()
|
||||||
|
|
||||||
|
|
||||||
|
def test_constructs_with_injected_config_without_env(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.delenv("REDIS_URI", raising=False)
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
assert adapter._config == TEST_REDIS_CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
def test_constructs_with_injected_client_without_env(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.delenv("REDIS_URI", raising=False)
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
assert adapter._client is mock_client
|
||||||
|
assert adapter._client_injected is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_raises_when_client_provided_without_config() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
with pytest.raises(ValueError, match="config is required"):
|
||||||
|
RedisAdapter(client=mock_client)
|
||||||
|
|
||||||
|
|
||||||
|
def test_disconnect_does_not_close_injected_client() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
adapter.disconnect()
|
||||||
|
mock_client.close.assert_not_called()
|
||||||
|
assert adapter._client is None
|
||||||
|
|
||||||
|
|
||||||
|
class CustomEnvRedisAdapter(RedisAdapter):
|
||||||
|
uri_env_var_name = "CUSTOM_REDIS_URI"
|
||||||
|
|
||||||
|
|
||||||
|
def test_subclass_custom_env_var_name(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("CUSTOM_REDIS_URI", "redis://custom:6379")
|
||||||
|
adapter = CustomEnvRedisAdapter()
|
||||||
|
assert adapter._config.uri == "redis://custom:6379"
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_succeeds_when_ping_ok() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
mock_client.ping.return_value = True
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.ping.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_raises_when_ping_false() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
mock_client.ping.return_value = False
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError, match="Could not connect to Redis"):
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_raises_on_redis_error() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
mock_client.ping.side_effect = redis.ConnectionError("connection lost")
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError, match="Could not connect to Redis"):
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_skips_validation_when_client_cleared() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
adapter.disconnect()
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.ping.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_reconnects_when_existing_client_unhealthy(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
stale_client = MagicMock(spec=redis.Redis)
|
||||||
|
stale_client.ping.side_effect = redis.ConnectionError("connection lost")
|
||||||
|
new_client = MagicMock(spec=redis.Redis)
|
||||||
|
new_client.ping.return_value = True
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
adapter._client = stale_client
|
||||||
|
|
||||||
|
monkeypatch.setattr("redis.Redis.from_url", lambda *args, **kwargs: new_client)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
stale_client.close.assert_called_once()
|
||||||
|
assert adapter._client is new_client
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_skips_reconnect_when_already_connected(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
stale_client = MagicMock(spec=redis.Redis)
|
||||||
|
stale_client.ping.return_value = True
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
adapter._client = stale_client
|
||||||
|
|
||||||
|
from_url = MagicMock()
|
||||||
|
monkeypatch.setattr("redis.Redis.from_url", from_url)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
stale_client.close.assert_not_called()
|
||||||
|
from_url.assert_not_called()
|
||||||
|
assert adapter._client is stale_client
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_keys_yields_decoded_keys() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
mock_client.scan_iter.return_value = iter([b"key1", b"key2"])
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
keys = list(adapter.scan_keys("key*"))
|
||||||
|
|
||||||
|
assert keys == ["key1", "key2"]
|
||||||
|
mock_client.scan_iter.assert_called_once_with(match="key*")
|
||||||
|
mock_client.keys.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_keys_forwards_count() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
mock_client.scan_iter.return_value = iter([b"key1"])
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
keys = list(adapter.scan_keys("key*", count=50))
|
||||||
|
|
||||||
|
assert keys == ["key1"]
|
||||||
|
mock_client.scan_iter.assert_called_once_with(match="key*", count=50)
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_keys_raises_value_error_on_invalid_pattern() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
invalid_patterns = ["", 123, None]
|
||||||
|
for pattern in invalid_patterns:
|
||||||
|
with pytest.raises(ValueError, match="Pattern must be a non-empty string"):
|
||||||
|
adapter.list_keys(pattern) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_keys_raises_value_error_on_invalid_pattern() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
|
||||||
|
invalid_patterns = ["", 123, None]
|
||||||
|
for pattern in invalid_patterns:
|
||||||
|
with pytest.raises(ValueError, match="Pattern must be a non-empty string"):
|
||||||
|
list(adapter.scan_keys(pattern)) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_keys_raises_connection_error_when_not_connected() -> None:
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
|
||||||
|
with pytest.raises(ConnectionError):
|
||||||
|
list(adapter.scan_keys("key*"))
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
"""Unit tests for RedisConfig."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.config import RedisConfig
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_loads_uri(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("REDIS_URI", "redis://example:6379")
|
||||||
|
config = RedisConfig.from_env(use_dotenv=False)
|
||||||
|
assert config.uri == "redis://example:6379"
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_raises_when_uri_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.delenv("REDIS_URI", raising=False)
|
||||||
|
with pytest.raises(Exception):
|
||||||
|
RedisConfig.from_env(use_dotenv=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_from_env_respects_custom_env_var_name(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setenv("CUSTOM_REDIS_URI", "redis://custom:6379")
|
||||||
|
config = RedisConfig.from_env("CUSTOM_REDIS_URI", use_dotenv=False)
|
||||||
|
assert config.uri == "redis://custom:6379"
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
"""Unit tests for TableRepositoryInterface."""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_repositories.interfaces.table_repository_interface import (
|
||||||
|
TableRepositoryInterface,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class InMemoryTableRepo:
|
||||||
|
"""Plain class that satisfies TableRepositoryInterface without inheritance."""
|
||||||
|
|
||||||
|
def fetch_one(self, pk: Any) -> dict[str, Any] | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def fetch_all(self, *, limit: int | None = None) -> list[dict[str, Any]]:
|
||||||
|
del limit
|
||||||
|
return []
|
||||||
|
|
||||||
|
def upsert(self, row: dict[str, Any]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, pk: Any) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
sql: str,
|
||||||
|
params: tuple[Any, ...] = (),
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
del sql, params
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def accepts_table_repo(repo: TableRepositoryInterface) -> None:
|
||||||
|
"""Type-checking hook for TableRepositoryInterface structural subtyping."""
|
||||||
|
repo.fetch_one("pk")
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_fetch_one_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if fetch_one is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(TableRepositoryInterface):
|
||||||
|
def fetch_all(self, *, limit: int | None = None) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def upsert(self, row: dict[str, Any]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, pk: Any) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
sql: str,
|
||||||
|
params: tuple[Any, ...] = (),
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_fetch_all_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if fetch_all is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(TableRepositoryInterface):
|
||||||
|
def fetch_one(self, pk: Any) -> dict[str, Any] | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def upsert(self, row: dict[str, Any]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, pk: Any) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
sql: str,
|
||||||
|
params: tuple[Any, ...] = (),
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_upsert_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if upsert is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(TableRepositoryInterface):
|
||||||
|
def fetch_one(self, pk: Any) -> dict[str, Any] | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def fetch_all(self, *, limit: int | None = None) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def delete(self, pk: Any) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
sql: str,
|
||||||
|
params: tuple[Any, ...] = (),
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_delete_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if delete is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(TableRepositoryInterface):
|
||||||
|
def fetch_one(self, pk: Any) -> dict[str, Any] | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def fetch_all(self, *, limit: int | None = None) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def upsert(self, row: dict[str, Any]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
sql: str,
|
||||||
|
params: tuple[Any, ...] = (),
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_instantiation_fails_when_execute_not_implemented() -> None:
|
||||||
|
"""Test that instantiation fails if execute is not implemented."""
|
||||||
|
|
||||||
|
class Incomplete(TableRepositoryInterface):
|
||||||
|
def fetch_one(self, pk: Any) -> dict[str, Any] | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def fetch_all(self, *, limit: int | None = None) -> list[dict[str, Any]]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def upsert(self, row: dict[str, Any]) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete(self, pk: Any) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
with pytest.raises(TypeError):
|
||||||
|
_ = Incomplete() # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def test_structural_subtyping() -> None:
|
||||||
|
"""Test that a plain class satisfies TableRepositoryInterface structurally."""
|
||||||
|
repo: TableRepositoryInterface = InMemoryTableRepo()
|
||||||
|
accepts_table_repo(repo)
|
||||||
|
assert isinstance(repo, TableRepositoryInterface)
|
||||||
Reference in New Issue
Block a user