Compare commits
82
Commits
v0.3.1
..
aebddb60a2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aebddb60a2 | ||
|
|
fa2554ef5f | ||
|
|
b21fd247c5 | ||
|
|
441a21204d | ||
|
|
f2a8581de2 | ||
|
|
3b05260323 | ||
|
|
369295b8ae | ||
|
|
95c78596e7 | ||
|
|
42e885edd5 | ||
|
|
bd4794d1c0 | ||
|
|
3dc412cd15 | ||
|
|
b8d88e6703 | ||
|
|
67bb88fbb4 | ||
|
|
c2e6a96c5d | ||
|
|
547a1b3a90 | ||
|
|
39baf9badc | ||
|
|
4a5b3b631f | ||
|
|
1431f455c1 | ||
|
|
eafc717045 | ||
|
|
9a9b7b2985 | ||
|
|
5b9573d499 | ||
|
|
50444af982 | ||
|
|
2f19fcc972 | ||
|
|
3bd65895ec | ||
|
|
5311d49fa6 | ||
|
|
703bb9521f | ||
|
|
8e47ebf4c6 | ||
|
|
9a800bf553 | ||
|
|
8c34534187 | ||
|
|
b1210bddf2 | ||
|
|
39b104383c | ||
|
|
beb2b5128e | ||
|
|
ce062be411 | ||
|
|
a381d45650 | ||
|
|
5c8cd841b6 | ||
|
|
34632980ba | ||
|
|
5393efc6cd | ||
|
|
e39fc96ac1 | ||
|
|
c1f4826d78 | ||
|
|
0315a33a3b | ||
|
|
5d33ec6091 | ||
|
|
9538e4d26d | ||
|
|
14b8d2798c | ||
|
|
d3cbb65bbc | ||
|
|
7dbb1fc8ab | ||
|
|
918e5e6ba4 | ||
|
|
3aa91280ec | ||
|
|
7991dabdbc | ||
|
|
366831ac52 | ||
|
|
a98905daa7 | ||
|
|
7ed1b34233 | ||
|
|
3236f27610 | ||
|
|
ac487d91f8 | ||
|
|
a049ce327c | ||
|
|
7b4cf8421b | ||
|
|
b5a92e6222 | ||
|
|
7b950e0d93 | ||
|
|
fe0c477dff | ||
|
|
e7e0fc8c1d | ||
|
|
34fc046868 | ||
|
|
5149299c1b | ||
|
|
fef9552cbf | ||
|
|
e9b21895f4 | ||
|
|
f5953906c1 | ||
|
|
1f02195c27 | ||
|
|
f092ca2022 | ||
|
|
c6eed43d70 | ||
|
|
9e4b1e2c5e | ||
|
|
0096df78a5 | ||
|
|
3073bfbf50 | ||
|
|
30a1de32e8 | ||
|
|
8465957976 | ||
|
|
3ecb9c3136 | ||
|
|
0f31170209 | ||
|
|
e3cab5e0b6 | ||
|
|
f428a974f6 | ||
|
|
a3b9912107 | ||
|
|
abaa078471 | ||
|
|
9864f0cc9d | ||
|
|
e39b467b6b | ||
|
|
85c1a64e14 | ||
|
|
a8491c005c |
@@ -0,0 +1,16 @@
|
||||
.git
|
||||
.github
|
||||
.gitea
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
.venv
|
||||
__pycache__
|
||||
*.pyc
|
||||
.coverage
|
||||
coverage-unit
|
||||
coverage-integration
|
||||
coverage.txt
|
||||
dist
|
||||
build
|
||||
*.egg-info
|
||||
@@ -0,0 +1,10 @@
|
||||
# Redis (requires redis extra)
|
||||
REDIS_URI=redis://localhost:6379
|
||||
|
||||
# MinIO (requires minio extra)
|
||||
MINIO_ENDPOINT=localhost:9000
|
||||
MINIO_ACCESS_KEY=minioadmin
|
||||
MINIO_SECRET_KEY=minioadmin
|
||||
MINIO_BUCKET=my-bucket
|
||||
MINIO_SECURE=false
|
||||
MINIO_CREATE_BUCKET_IF_MISSING=true
|
||||
@@ -0,0 +1,21 @@
|
||||
## Version bump
|
||||
|
||||
If this PR changes files under `python_repositories/`, the **title must** start with one of:
|
||||
|
||||
- `[patch]` or `[fix]` — bug fix (`1.2.3` → `1.2.4`)
|
||||
- `[minor]` or `[feat]` — new feature (`1.2.3` → `1.3.0`)
|
||||
- `[major]` or `[breaking]` — breaking change (`1.2.3` → `2.0.0`)
|
||||
|
||||
Current version: see [`pyproject.toml`](pyproject.toml) on `main`.
|
||||
|
||||
Docs-, CI-, or test-only PRs do not need a prefix.
|
||||
|
||||
**Example title:** `[minor] Add streaming support to RedisAdapter`
|
||||
|
||||
## Summary
|
||||
|
||||
<!-- What changed and why -->
|
||||
|
||||
## Test plan
|
||||
|
||||
- [ ] Integration tests pass locally
|
||||
@@ -0,0 +1,52 @@
|
||||
# Build and publish the per-repo CI base image to the Gitea container registry.
|
||||
|
||||
name: Build CI Image
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 2 * * *"
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- pyproject.toml
|
||||
- uv.lock
|
||||
- docker/ci/**
|
||||
|
||||
env:
|
||||
REGISTRY: gitea.lille-vemmelund.dk
|
||||
REGISTRY_USER: ci-bot
|
||||
IMAGE: gitea.lille-vemmelund.dk/brian/python-repositories-ci
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Gitea container registry
|
||||
env:
|
||||
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: |
|
||||
echo "$CI_RUNNER_TOKEN" | docker login "$REGISTRY" -u "$REGISTRY_USER" --password-stdin
|
||||
|
||||
- name: Build CI image
|
||||
env:
|
||||
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: |
|
||||
echo "$CI_RUNNER_TOKEN" > /tmp/uv_token
|
||||
docker build -f docker/ci/Dockerfile \
|
||||
--secret id=uv_token,src=/tmp/uv_token \
|
||||
-t "${IMAGE}:latest" \
|
||||
.
|
||||
rm -f /tmp/uv_token
|
||||
|
||||
- name: Tag and push CI image
|
||||
run: |
|
||||
STAMP="$(date -u +%Y%m%d%H%M)"
|
||||
docker tag "${IMAGE}:latest" "${IMAGE}:${STAMP}"
|
||||
docker push "${IMAGE}:latest"
|
||||
docker push "${IMAGE}:${STAMP}"
|
||||
echo "Pushed ${IMAGE}:latest and ${IMAGE}:${STAMP}"
|
||||
@@ -8,23 +8,17 @@ on:
|
||||
|
||||
jobs:
|
||||
code-quality:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: python-repositories-ci
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
|
||||
- name: Install dependencies
|
||||
- name: Sync dependencies
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
run: uv sync --all-extras
|
||||
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: uv sync --all-extras --frozen
|
||||
|
||||
- name: Type check with mypy
|
||||
run: uv run mypy .
|
||||
@@ -36,7 +30,7 @@ jobs:
|
||||
run: uv run ruff format --check .
|
||||
|
||||
- name: Pyupgrade check
|
||||
run: uv run pyupgrade --py313-plus $(git ls-files '*.py') && git diff --exit-code
|
||||
run: uv run pyupgrade --py312-plus $(git ls-files '*.py') && git diff --exit-code
|
||||
|
||||
- name: Prettier format
|
||||
run: uv run pre-commit run prettier --all-files
|
||||
|
||||
@@ -7,30 +7,23 @@ on:
|
||||
|
||||
jobs:
|
||||
update-check:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: python-repositories-ci
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
token: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
|
||||
- name: Install uv and pip-review
|
||||
run: |
|
||||
pip install uv pip-review
|
||||
|
||||
- name: Update dependencies in pyproject.toml
|
||||
run: |
|
||||
uv pip update --all --group dev
|
||||
uv pip compile --all
|
||||
- name: Upgrade dependencies
|
||||
env:
|
||||
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: uv lock --upgrade
|
||||
|
||||
- name: Commit and push changes
|
||||
uses: peter-evans/create-pull-request@v6
|
||||
with:
|
||||
commit-message: "chore(deps): update dependencies [automated]"
|
||||
branch: "renovate/auto-deps-update"
|
||||
title: "chore(deps): update dependencies"
|
||||
body: "This PR was created automatically to update dependencies."
|
||||
token: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
env:
|
||||
API_URL: ${{ vars.API_URL }}
|
||||
REPO_OWNER: ${{ github.repository_owner }}
|
||||
REPO_NAME: ${{ github.event.repository.name }}
|
||||
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: scripts/ci/create-deps-update-pr.sh
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
name: PR Title Check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
check-title:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Check PR title when source files change
|
||||
env:
|
||||
API_URL: ${{ vars.API_URL }}
|
||||
REPO_OWNER: ${{ github.repository_owner }}
|
||||
REPO_NAME: ${{ github.event.repository.name }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: |
|
||||
git fetch origin "${{ github.base_ref }}"
|
||||
PR_TITLE=$(scripts/ci/fetch-pr-title.sh)
|
||||
echo "Live PR title: ${PR_TITLE}"
|
||||
git diff --name-only "origin/${{ github.base_ref }}...HEAD" \
|
||||
| scripts/ci/check-pr-title.sh "$PR_TITLE"
|
||||
@@ -9,29 +9,24 @@ on:
|
||||
|
||||
jobs:
|
||||
build-and-publish:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: python-repositories-ci
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
|
||||
- name: Update version in pyproject.toml to match tag
|
||||
run: |
|
||||
TAG_VERSION="${GITHUB_REF##*/}"
|
||||
VERSION="${TAG_VERSION#v}"
|
||||
sed -i.bak -E "s/^version = \".*\"/version = \"${VERSION}\"/" pyproject.toml
|
||||
rm pyproject.toml.bak
|
||||
env:
|
||||
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: scripts/ci/bump-version.sh --from-tag "${GITHUB_REF##*/}"
|
||||
|
||||
- name: Build package
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: |
|
||||
uv sync --no-dev
|
||||
uv sync --no-dev --frozen
|
||||
uv build
|
||||
|
||||
- name: Publish to Gitea Package Registry
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
name: Release on merge to main
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
release:
|
||||
if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
|
||||
runs-on: python-repositories-ci
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
|
||||
- name: Parse merge commit
|
||||
id: meta
|
||||
env:
|
||||
COMMIT_MSG: ${{ github.event.head_commit.message }}
|
||||
run: scripts/ci/parse-merge-commit.sh "$COMMIT_MSG"
|
||||
|
||||
- name: Bump version
|
||||
if: steps.meta.outputs.bump != 'skip'
|
||||
id: bump
|
||||
env:
|
||||
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: scripts/ci/bump-version.sh "${{ steps.meta.outputs.bump }}"
|
||||
|
||||
- name: Generate release notes
|
||||
if: steps.meta.outputs.bump != 'skip'
|
||||
id: notes
|
||||
env:
|
||||
PR_TITLE: ${{ steps.meta.outputs.pr_title }}
|
||||
run: |
|
||||
PREV_TAG=$(git describe --tags --abbrev=0)
|
||||
NEW_TAG="v${{ steps.bump.outputs.version }}"
|
||||
scripts/ci/generate-release-notes.sh "$NEW_TAG" "$PR_TITLE" "$PREV_TAG"
|
||||
|
||||
- name: Commit, tag, and push
|
||||
if: steps.meta.outputs.bump != 'skip'
|
||||
env:
|
||||
VERSION: ${{ steps.bump.outputs.version }}
|
||||
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: |
|
||||
git config user.name "CI Bot"
|
||||
git config user.email "[email protected]"
|
||||
git add pyproject.toml uv.lock
|
||||
git commit -m "chore: release v${VERSION} [skip ci]"
|
||||
git tag "v${VERSION}"
|
||||
git remote set-url origin "https://x-access-token:${CI_RUNNER_TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
|
||||
git push origin main
|
||||
git push origin "v${VERSION}"
|
||||
|
||||
- name: Create Gitea release
|
||||
if: steps.meta.outputs.bump != 'skip'
|
||||
env:
|
||||
API_URL: ${{ vars.API_URL }}
|
||||
REPO_OWNER: ${{ github.repository_owner }}
|
||||
REPO_NAME: ${{ github.event.repository.name }}
|
||||
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
VERSION: ${{ steps.bump.outputs.version }}
|
||||
PR_TITLE: ${{ steps.meta.outputs.pr_title }}
|
||||
RELEASE_BODY: ${{ steps.notes.outputs.body }}
|
||||
run: |
|
||||
SUMMARY=$(echo "$PR_TITLE" | sed -E 's/^\[(patch|fix|minor|feat|major|breaking)\][[:space:]]*//i')
|
||||
export RELEASE_NAME="v${VERSION} — ${SUMMARY}"
|
||||
PAYLOAD=$(python3 -c 'import json, os; print(json.dumps({"tag_name": "v" + os.environ["VERSION"], "name": os.environ["RELEASE_NAME"], "body": os.environ["RELEASE_BODY"]}))')
|
||||
curl -s -X POST \
|
||||
"${API_URL}/repos/${REPO_OWNER}/${REPO_NAME}/releases" \
|
||||
-H "Authorization: token ${CI_RUNNER_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$PAYLOAD"
|
||||
@@ -1,29 +0,0 @@
|
||||
name: Security Audit
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * 0" # Weekly
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
safety:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
run: uv sync
|
||||
|
||||
- name: Run safety check
|
||||
run: uv run safety check
|
||||
@@ -7,23 +7,17 @@ on:
|
||||
|
||||
jobs:
|
||||
safety:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: python-repositories-ci
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
|
||||
- name: Install dependencies
|
||||
- name: Sync dependencies
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
run: uv sync
|
||||
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: uv sync --frozen
|
||||
|
||||
- name: Run safety check
|
||||
run: uv run safety check
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
name: Sync pyproject.toml Version
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["Publish Python Package"]
|
||||
types:
|
||||
- completed
|
||||
|
||||
jobs:
|
||||
commit-version-update:
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
|
||||
- name: Configure git
|
||||
run: |
|
||||
git config user.name "CI Bot"
|
||||
git config user.email "[email protected]"
|
||||
|
||||
- name: Update version in pyproject.toml to match tag
|
||||
run: |
|
||||
TAG_VERSION="${{ github.event.workflow_run.head_branch }}"
|
||||
VERSION="${TAG_VERSION#v}"
|
||||
sed -i.bak -E "s/^version = \".*\"/version = \"${VERSION}\"/" pyproject.toml
|
||||
rm pyproject.toml.bak
|
||||
|
||||
- name: Commit and push changes
|
||||
run: |
|
||||
git add pyproject.toml
|
||||
git commit -m "chore: update version in pyproject.toml to ${VERSION} [skip ci]" || echo "No changes to commit"
|
||||
git push
|
||||
+117
-19
@@ -7,31 +7,118 @@ on:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
unit-tests:
|
||||
runs-on: python-repositories-ci
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
|
||||
- name: Install dependencies
|
||||
- name: Sync dependencies
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
run: uv sync --all-extras
|
||||
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: uv sync --all-extras --frozen
|
||||
|
||||
- name: Run pytest
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# --cov-fail-under=0: partial coverage only; floor is checked in coverage-report.
|
||||
uv run pytest tests/unit/ -m "not integration" \
|
||||
--cov=python_repositories \
|
||||
--cov-report= \
|
||||
--cov-fail-under=0
|
||||
|
||||
- name: Upload unit coverage
|
||||
uses: https://github.com/christopherHX/gitea-upload-artifact@v4
|
||||
with:
|
||||
name: coverage-unit
|
||||
path: .coverage
|
||||
retention-days: 1
|
||||
compression-level: 0
|
||||
|
||||
integration-tests:
|
||||
runs-on: python-repositories-ci
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Sync dependencies
|
||||
env:
|
||||
PYTHONPATH: .
|
||||
run: uv run pytest --cov=python_repositories --cov-report=term-missing > coverage.txt
|
||||
UV_LINK_MODE: copy
|
||||
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: uv sync --all-extras --frozen
|
||||
|
||||
- name: Verify Docker
|
||||
run: docker info
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
# --cov-fail-under=0: partial coverage only; floor is checked in coverage-report.
|
||||
uv run pytest -m integration \
|
||||
--cov=python_repositories \
|
||||
--cov-report= \
|
||||
--cov-fail-under=0
|
||||
|
||||
- name: Upload integration coverage
|
||||
uses: https://github.com/christopherHX/gitea-upload-artifact@v4
|
||||
with:
|
||||
name: coverage-integration
|
||||
path: .coverage
|
||||
retention-days: 1
|
||||
compression-level: 0
|
||||
|
||||
coverage-report:
|
||||
# Merges unit + integration coverage and enforces fail_under from pyproject.toml.
|
||||
needs: [unit-tests, integration-tests]
|
||||
runs-on: python-repositories-ci
|
||||
if: github.event_name == 'pull_request' || github.event_name == 'push'
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Sync dependencies
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: uv sync --all-extras --frozen
|
||||
|
||||
- name: Download unit coverage
|
||||
uses: https://github.com/christopherHX/gitea-download-artifact@v4
|
||||
with:
|
||||
name: coverage-unit
|
||||
path: coverage-unit
|
||||
|
||||
- name: Download integration coverage
|
||||
uses: https://github.com/christopherHX/gitea-download-artifact@v4
|
||||
with:
|
||||
name: coverage-integration
|
||||
path: coverage-integration
|
||||
|
||||
- name: Combine coverage report
|
||||
run: |
|
||||
# --fail-under=0 so the full report is always written before the floor check.
|
||||
uv run coverage combine coverage-unit/.coverage coverage-integration/.coverage
|
||||
uv run coverage report -m --include='python_repositories/*' --fail-under=0 > coverage.txt
|
||||
|
||||
- name: Show coverage report
|
||||
run: cat coverage.txt
|
||||
|
||||
- name: Enforce coverage floor
|
||||
run: |
|
||||
# Reads fail_under from pyproject.toml; only combined coverage is evaluated here.
|
||||
FAIL_UNDER=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['tool']['coverage']['report']['fail_under'])")
|
||||
echo "Checking combined coverage against ${FAIL_UNDER}% floor..."
|
||||
if uv run coverage report --fail-under="$FAIL_UNDER" --include='python_repositories/*'; then
|
||||
echo "Coverage floor met."
|
||||
else
|
||||
echo "::error::Combined coverage is below the ${FAIL_UNDER}% floor"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Post coverage summary to PR
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
API_URL: ${{ vars.API_URL }}
|
||||
REPO_OWNER: ${{ github.repository_owner }}
|
||||
@@ -39,9 +126,20 @@ jobs:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: |
|
||||
COVERAGE=$(cat coverage.txt)
|
||||
COMMENT_BODY="**Test Coverage Report:**\n\`\`\`\n$COVERAGE\n\`\`\`"
|
||||
curl -s -X POST "$API_URL/repos/$REPO_OWNER/$REPO_NAME/issues/$PR_NUMBER/comments" \
|
||||
PAYLOAD=$(python3 -c '
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
coverage = pathlib.Path("coverage.txt").read_text()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"body": f"**Test Coverage Report:**\n```\n{coverage}\n```",
|
||||
}
|
||||
)
|
||||
)
|
||||
')
|
||||
curl -sf -X POST "$API_URL/repos/$REPO_OWNER/$REPO_NAME/issues/$PR_NUMBER/comments" \
|
||||
-H "Authorization: token $CI_RUNNER_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"body\": \"$COMMENT_BODY\"}"
|
||||
-d "$PAYLOAD"
|
||||
|
||||
@@ -173,4 +173,3 @@ cython_debug/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
|
||||
@@ -24,13 +24,15 @@ repos:
|
||||
rev: v1.8.0
|
||||
hooks:
|
||||
- id: mypy
|
||||
additional_dependencies:
|
||||
- types-redis
|
||||
|
||||
# Python syntax modernization with pyupgrade
|
||||
- repo: https://github.com/asottile/pyupgrade
|
||||
rev: v3.20.0
|
||||
hooks:
|
||||
- id: pyupgrade
|
||||
args: ["--py311-plus"]
|
||||
args: ["--py312-plus"]
|
||||
|
||||
# Formatting for Markdown, JSON, and YAML with Prettier
|
||||
- repo: https://github.com/pre-commit/mirrors-prettier
|
||||
|
||||
@@ -2,17 +2,17 @@ MIT License
|
||||
|
||||
Copyright (c) 2025 brian
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial
|
||||
portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
||||
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
|
||||
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
||||
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
|
||||
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
||||
USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
@@ -12,8 +12,12 @@ Subclass an adapter in your own repository to add domain-specific methods while
|
||||
| **Adapters** | Technology-specific base classes (`RedisAdapter`, `MinioAdapter`) |
|
||||
| **Your project** | Subclass an adapter and add domain methods |
|
||||
|
||||
Connection adapters expose `connect()`, `disconnect()`, and `is_connected()`. The latter verifies backend reachability with a cached health probe (default TTL: 1 second). Subclasses may override `health_check_ttl_seconds`. `connect()` is idempotent: calling it while already connected and healthy is a no-op.
|
||||
|
||||
## Optional dependencies
|
||||
|
||||
Repository **interfaces** import with the base package. **Adapters** require the matching extra; importing an adapter without its extra raises `ImportError` with install instructions.
|
||||
|
||||
Install with the extras you need:
|
||||
|
||||
```bash
|
||||
@@ -30,14 +34,64 @@ Requires Redis with the RedisJSON module (e.g. redis-stack).
|
||||
| -------------------- | ---------------------------------------------------- |
|
||||
| `REDIS_URI` | Redis connection URL (e.g. `redis://localhost:6379`) |
|
||||
|
||||
For key discovery:
|
||||
|
||||
- `list_keys(pattern)` is simple and returns a `list[str]`, but it uses Redis `KEYS` and may block on large datasets.
|
||||
- `scan_keys(pattern, *, count=None)` is preferred for production use and yields keys incrementally via Redis `SCAN`.
|
||||
|
||||
Example:
|
||||
|
||||
```python
|
||||
for key in repo.scan_keys("user:*"):
|
||||
print(key)
|
||||
```
|
||||
|
||||
### MinIO (`ObjectRepositoryInterface`)
|
||||
|
||||
| Environment variable | Description |
|
||||
| -------------------- | ------------------------------------------- |
|
||||
| `MINIO_ENDPOINT` | MinIO server endpoint |
|
||||
| `MINIO_ACCESS_KEY` | Access key |
|
||||
| `MINIO_SECRET_KEY` | Secret key |
|
||||
| `MINIO_BUCKET` | Bucket name (created on connect if missing) |
|
||||
| Environment variable | Description |
|
||||
| -------------------------------- | ------------------------------------------------------------------------ |
|
||||
| `MINIO_ENDPOINT` | MinIO server endpoint |
|
||||
| `MINIO_ACCESS_KEY` | Access key |
|
||||
| `MINIO_SECRET_KEY` | Secret key |
|
||||
| `MINIO_BUCKET` | Bucket name |
|
||||
| `MINIO_SECURE` | Use HTTPS (`true`/`false`; default: `true`) |
|
||||
| `MINIO_CREATE_BUCKET_IF_MISSING` | Auto-create `MINIO_BUCKET` on connect (`true`/`false`; default: `false`) |
|
||||
|
||||
Copy [`.env.example`](.env.example) to `.env` for local development. `RedisConfig.from_env()` and `MinioConfig.from_env()` load `.env` automatically when resolving configuration from the environment.
|
||||
|
||||
For production, it is recommended to leave `MINIO_CREATE_BUCKET_IF_MISSING` unset so that `connect()` fails fast if the expected bucket is missing. For local development, you will often want `MINIO_SECURE=false` and `MINIO_CREATE_BUCKET_IF_MISSING=true`.
|
||||
|
||||
## Configuration injection
|
||||
|
||||
Adapters accept optional `config` and `client` keyword arguments for explicit setup and testing:
|
||||
|
||||
```python
|
||||
from python_repositories import RedisAdapter, RedisConfig, MinioAdapter, MinioConfig
|
||||
|
||||
redis = RedisAdapter(config=RedisConfig(uri="redis://localhost:6379"))
|
||||
minio = MinioAdapter(
|
||||
config=MinioConfig(
|
||||
endpoint="localhost:9000",
|
||||
access_key="minioadmin",
|
||||
secret_key="minioadmin",
|
||||
bucket="my-bucket",
|
||||
secure=False,
|
||||
# create_bucket_if_missing=True, # convenient for local dev
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
When both `config` and `client` are provided, `connect()` skips client creation (the caller owns the client lifecycle). `config` is required whenever `client` is injected.
|
||||
|
||||
Load a `.env` file explicitly:
|
||||
|
||||
```python
|
||||
from python_repositories import load_dotenv
|
||||
|
||||
load_dotenv() # optional — from_env() also loads .env by default
|
||||
```
|
||||
|
||||
Calling `RedisAdapter()` or `MinioAdapter()` with no arguments still loads configuration from environment variables (and `.env` if present).
|
||||
|
||||
## Quick start
|
||||
|
||||
@@ -69,16 +123,18 @@ with ArtifactObjectRepository() as repo:
|
||||
### Subclassing in your own project
|
||||
|
||||
```python
|
||||
from typing import Any
|
||||
|
||||
from python_repositories import RedisAdapter
|
||||
|
||||
class UserRepository(RedisAdapter):
|
||||
def _key(self, user_id: str) -> str:
|
||||
return f"user:{user_id}"
|
||||
|
||||
def get_user(self, user_id: str) -> dict | None:
|
||||
def get_user(self, user_id: str) -> dict[str, Any] | None:
|
||||
return self.get(self._key(user_id))
|
||||
|
||||
def save_user(self, user_id: str, user: dict) -> None:
|
||||
def save_user(self, user_id: str, user: dict[str, Any]) -> None:
|
||||
self.set(self._key(user_id), user)
|
||||
```
|
||||
|
||||
@@ -89,9 +145,12 @@ from python_repositories import (
|
||||
ConnectionAwareInterface,
|
||||
ContextAwareInterface,
|
||||
JsonRepositoryInterface,
|
||||
MinioAdapter,
|
||||
MinioConfig,
|
||||
ObjectRepositoryInterface,
|
||||
RedisAdapter,
|
||||
MinioAdapter,
|
||||
RedisConfig,
|
||||
load_dotenv,
|
||||
)
|
||||
```
|
||||
|
||||
@@ -100,7 +159,27 @@ from python_repositories import (
|
||||
```bash
|
||||
uv sync --all-extras
|
||||
uv run pre-commit install # once per clone — runs hooks on git commit
|
||||
uv run pytest tests/integration/ -v
|
||||
uv run pytest tests/unit/ -v # fast, no Docker
|
||||
uv run pytest -m "not integration" -v # all non-Docker tests
|
||||
uv run pytest -v # full suite (requires Docker)
|
||||
```
|
||||
|
||||
Integration tests are marked with `@pytest.mark.integration` and require Docker (testcontainers). Run unit tests alone for quick local feedback.
|
||||
|
||||
### CI base image
|
||||
|
||||
Gitea Actions jobs use a pre-built image (`python-repositories-ci`) with Python 3.12,
|
||||
`uv`, and locked dependencies baked in. The image is rebuilt nightly and when
|
||||
`uv.lock` changes; see [`docs/ci-image.md`](docs/ci-image.md) for runner setup and
|
||||
bootstrap order.
|
||||
|
||||
CI runs unit and integration tests in parallel with coverage, then merges `.coverage` artifacts in a follow-up job (via [christopherhx/gitea-\*-artifact@v4](https://github.com/christopherHX/gitea-upload-artifact) for Gitea 1.26 compatibility). Combined coverage must meet the floor in [`fail_under` in `pyproject.toml`](pyproject.toml#L45-L48); enforcement happens after merging unit and integration coverage, not on unit-only runs.
|
||||
|
||||
To check coverage locally (requires Docker for the full suite):
|
||||
|
||||
```bash
|
||||
uv run pytest --cov=python_repositories --cov-report=
|
||||
uv run coverage report
|
||||
```
|
||||
|
||||
`pre-commit` is included in the dev dependency group. `uv sync` installs the CLI, but git does not run hooks until you install them with `pre-commit install` (one time per clone). After that, commits run the checks defined in [`.pre-commit-config.yaml`](.pre-commit-config.yaml) (ruff, mypy, pyupgrade, prettier, and general file hygiene).
|
||||
@@ -111,4 +190,37 @@ To run all hooks manually without committing:
|
||||
uv run pre-commit run --all-files
|
||||
```
|
||||
|
||||
Integration tests require Docker (testcontainers).
|
||||
## Releases
|
||||
|
||||
Releases are automated when a pull request is merged to `main`. CI reads the **merged PR title** to decide whether and how to bump the version.
|
||||
|
||||
### How it works
|
||||
|
||||
1. Open a PR targeting `main` (see [`.gitea/PULL_REQUEST_TEMPLATE.md`](.gitea/PULL_REQUEST_TEMPLATE.md)).
|
||||
2. If the PR changes files under `python_repositories/`, the title **must** start with a version bump prefix (enforced by CI).
|
||||
3. On merge, [`release.yml`](.gitea/workflows/release.yml) bumps [`pyproject.toml`](pyproject.toml) and syncs [`uv.lock`](uv.lock), commits, tags `vX.Y.Z`, creates a Gitea release with auto-generated notes, and pushes the tag.
|
||||
4. [`publish.yml`](.gitea/workflows/publish.yml) builds and publishes the package to the Gitea Package Registry.
|
||||
|
||||
Docs-, CI-, and test-only PRs do not need a prefix and will not trigger a release.
|
||||
|
||||
### PR title prefixes
|
||||
|
||||
| Prefix | Bump | Example |
|
||||
| ------------------------- | ---------- | --------------------- |
|
||||
| `[patch]` or `[fix]` | patch | `1.2.3` → `1.2.4` |
|
||||
| `[minor]` or `[feat]` | minor | `1.2.3` → `1.3.0` |
|
||||
| `[major]` or `[breaking]` | major | `1.2.3` → `2.0.0` |
|
||||
| _(none)_ | no release | docs / CI / deps only |
|
||||
|
||||
Example titles:
|
||||
|
||||
- `[minor] Add public repository interfaces and subclassable adapter CRUD API`
|
||||
- `[patch] Fix mypy context manager typing for adapter subclasses`
|
||||
|
||||
### Release notes
|
||||
|
||||
Release notes are generated from commits since the previous tag (see [`scripts/ci/generate-release-notes.sh`](scripts/ci/generate-release-notes.sh)).
|
||||
|
||||
### Manual release
|
||||
|
||||
You can still push a `v*.*.*` tag manually; `publish.yml` will build and publish. The current released version is in [`pyproject.toml`](pyproject.toml) (and on the latest `v*.*.*` tag).
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# CI job image for Gitea Actions (act_runner requires Node.js in job containers).
|
||||
FROM node:20-bookworm
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
docker.io \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.7.0 /uv /usr/local/bin/uv
|
||||
|
||||
ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python \
|
||||
UV_LINK_MODE=copy
|
||||
|
||||
COPY .python-version /tmp/.python-version
|
||||
RUN uv python install "$(cat /tmp/.python-version)"
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY pyproject.toml uv.lock .python-version ./
|
||||
|
||||
ENV PATH="/app/.venv/bin:${PATH}"
|
||||
|
||||
RUN --mount=type=secret,id=uv_token \
|
||||
UV_INDEX_GITEA_USERNAME=ci-bot \
|
||||
UV_INDEX_GITEA_PASSWORD="$(cat /run/secrets/uv_token)" \
|
||||
uv sync --all-extras --frozen --no-install-project
|
||||
@@ -0,0 +1,169 @@
|
||||
# CI base image
|
||||
|
||||
This repository uses a **per-repo Docker image** for Gitea Actions jobs instead of
|
||||
installing Python and `uv` on every run. Infrastructure images (redis, minio, etc.)
|
||||
are cached cluster-wide via Harbor (see homelab-platform
|
||||
[`docs/harbor-registry-mirror.md`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/docs/harbor-registry-mirror.md)).
|
||||
|
||||
## Image contents
|
||||
|
||||
Built from [`docker/ci/Dockerfile`](../docker/ci/Dockerfile):
|
||||
|
||||
- Node.js 20 (required by act_runner job containers)
|
||||
- Python 3.12 (installed via `uv python install` from [`.python-version`](../.python-version)) and pinned `uv` 0.7.0
|
||||
- Docker CLI (integration tests via testcontainers)
|
||||
- Dev dependencies from `uv.lock` (`uv sync --all-extras --no-install-project`)
|
||||
|
||||
Published to the Gitea container registry:
|
||||
|
||||
- `gitea.lille-vemmelund.dk/brian/python-repositories-ci:latest`
|
||||
- `gitea.lille-vemmelund.dk/brian/python-repositories-ci:YYYYMMDDHHmm` (timestamped rollback tag)
|
||||
|
||||
## Rebuild triggers
|
||||
|
||||
[`ci-image.yml`](../.gitea/workflows/ci-image.yml) runs on:
|
||||
|
||||
- Nightly cron (`0 2 * * *` UTC)
|
||||
- Manual `workflow_dispatch`
|
||||
- Push to `main` when `pyproject.toml`, `uv.lock`, or `docker/ci/**` change
|
||||
|
||||
## Workflow usage
|
||||
|
||||
Python jobs use `runs-on: python-repositories-ci` and a fast incremental sync:
|
||||
|
||||
```yaml
|
||||
runs-on: python-repositories-ci
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Sync dependencies
|
||||
env:
|
||||
UV_LINK_MODE: copy
|
||||
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: uv sync --all-extras --frozen
|
||||
```
|
||||
|
||||
`ci-image.yml` uses `runs-on: ubuntu-latest` so it can bootstrap before the custom
|
||||
image exists.
|
||||
|
||||
## Registry authentication
|
||||
|
||||
act_runner pulls the job image **before any workflow step runs**, so a `docker
|
||||
login` step inside a job cannot authenticate that pull. Authentication must be
|
||||
configured on the runner (or via a job-level `container.credentials` block — not
|
||||
used here).
|
||||
|
||||
### k8s Gitea runners (automatic)
|
||||
|
||||
The homelab-platform runners mount Gitea registry credentials automatically:
|
||||
|
||||
- Secret: `gitea-runners/gitea-registry-dockerconfig` (created by
|
||||
[`scripts/create-gitea-registry-secret.sh`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/scripts/create-gitea-registry-secret.sh))
|
||||
- Mounted at `/root/.docker/config.json` on the **runner** container (not DinD)
|
||||
- Configured in
|
||||
[`platform/gitea-runners/values.yaml`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/platform/gitea-runners/values.yaml)
|
||||
|
||||
No manual `docker login` is required on the in-cluster runners once the secret
|
||||
exists. To create or rotate credentials:
|
||||
|
||||
```bash
|
||||
export KUBECONFIG=/path/to/homelab-cluster/talos/_out/kubeconfig
|
||||
export GITEA_REGISTRY_USERNAME='ci-bot'
|
||||
export GITEA_REGISTRY_PASSWORD='personal-access-token-with-read-package'
|
||||
bash scripts/create-gitea-registry-secret.sh
|
||||
```
|
||||
|
||||
Then restart runner pods so they pick up the updated secret. See homelab-platform
|
||||
[`docs/gitea-actions-runners.md`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/docs/gitea-actions-runners.md)
|
||||
for full runner setup.
|
||||
|
||||
Verify on a running pod:
|
||||
|
||||
```bash
|
||||
kubectl -n gitea-runners exec homelab-cluster-gitea-runner-0 -c runner -- \
|
||||
test -f /root/.docker/config.json && echo "registry auth mounted"
|
||||
```
|
||||
|
||||
### Standalone runners (e.g. Unraid `homelab`)
|
||||
|
||||
The 4th runner is outside the k8s cluster and does **not** get the automatic
|
||||
mount. Configure registry auth manually on that host:
|
||||
|
||||
```bash
|
||||
docker login gitea.lille-vemmelund.dk -u ci-bot -p <token>
|
||||
```
|
||||
|
||||
Also add the `python-repositories-ci` label to that runner's act_runner config.
|
||||
|
||||
### `ci-image.yml` push login
|
||||
|
||||
The build workflow still runs `docker login` before `docker push`. That step
|
||||
authenticates **DinD inside the job** for pushing the image to Gitea — a different
|
||||
code path from act_runner pulling the job container.
|
||||
|
||||
## Runner label
|
||||
|
||||
Jobs use the `python-repositories-ci` act_runner label, configured in
|
||||
homelab-platform
|
||||
[`platform/gitea-runners/values.yaml`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/platform/gitea-runners/values.yaml):
|
||||
|
||||
```yaml
|
||||
python-repositories-ci:docker://gitea.lille-vemmelund.dk/brian/python-repositories-ci:latest
|
||||
```
|
||||
|
||||
After label changes, roll runner pods so they re-register with Gitea.
|
||||
|
||||
## Bootstrap order
|
||||
|
||||
1. Ensure homelab-platform runners have `gitea-registry-dockerconfig` and the
|
||||
`python-repositories-ci` label (see homelab-platform docs).
|
||||
2. Merge `ci-image.yml`, `docker/ci/Dockerfile`, and workflow migrations to `main`.
|
||||
3. Seed the registry with a first image (see below).
|
||||
4. Confirm a test workflow job starts on `python-repositories-ci`.
|
||||
|
||||
Until step 3 completes, jobs targeting `python-repositories-ci` will fail because
|
||||
the image does not exist in the Gitea registry yet.
|
||||
|
||||
### Seed the image
|
||||
|
||||
**After merge to `main`:** Actions → **Build CI Image** → **Run workflow**.
|
||||
|
||||
**Before merge (e.g. PR branch):** the workflow file is not on `main` yet — build
|
||||
and push locally with [`scripts/ci/build-ci-image.sh`](../scripts/ci/build-ci-image.sh):
|
||||
|
||||
```bash
|
||||
export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||
bash scripts/ci/build-ci-image.sh --push
|
||||
```
|
||||
|
||||
Run from the repo root on the branch you want to test. Runners pull
|
||||
`gitea.lille-vemmelund.dk/brian/python-repositories-ci:latest` from the registry;
|
||||
they do not care which git branch built it.
|
||||
|
||||
Override registry settings if needed:
|
||||
|
||||
```bash
|
||||
export REGISTRY=gitea.lille-vemmelund.dk
|
||||
export REGISTRY_USER=ci-bot
|
||||
export CI_RUNNER_TOKEN='...'
|
||||
bash scripts/ci/build-ci-image.sh --push
|
||||
```
|
||||
|
||||
## Local build
|
||||
|
||||
Build only (no registry login or push):
|
||||
|
||||
```bash
|
||||
export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||
bash scripts/ci/build-ci-image.sh --local
|
||||
```
|
||||
|
||||
Or manually:
|
||||
|
||||
```bash
|
||||
echo "$CI_RUNNER_TOKEN" > /tmp/uv_token
|
||||
docker build -f docker/ci/Dockerfile \
|
||||
--secret id=uv_token,src=/tmp/uv_token \
|
||||
-t python-repositories-ci:local .
|
||||
rm -f /tmp/uv_token
|
||||
```
|
||||
+14
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "python-repositories"
|
||||
version = "0.1.0"
|
||||
version = "2.0.2"
|
||||
description = "Various python repository interfaces exposed as a python package."
|
||||
authors = [
|
||||
{ name = "Brian Bjarke Jensen", email = "[email protected]" }
|
||||
@@ -14,6 +14,7 @@ classifiers = [
|
||||
"Operating System :: OS Independent",
|
||||
]
|
||||
dependencies = [
|
||||
"python-dotenv>=1.0.0",
|
||||
"python-utils>=0.1.0",
|
||||
"structlog>=25.4.0",
|
||||
]
|
||||
@@ -33,6 +34,18 @@ build-backend = "hatchling.build"
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
pythonpath = ["."]
|
||||
addopts = "--import-mode=importlib"
|
||||
markers = [
|
||||
"integration: tests requiring Docker containers (deselect with '-m \"not integration\"')",
|
||||
]
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["python_repositories"]
|
||||
|
||||
[tool.coverage.report]
|
||||
fail_under = 100
|
||||
show_missing = true
|
||||
precision = 2
|
||||
|
||||
[tool.uv.sources]
|
||||
python-utils = { index = "gitea" }
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
"""python_repositories: Unified repository interfaces and adapters."""
|
||||
|
||||
from .adapters import MinioAdapter, RedisAdapter
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
# Interfaces are always available; they have no optional backend dependencies.
|
||||
from . import adapters
|
||||
from .config import MinioConfig, RedisConfig, load_dotenv
|
||||
from .interfaces import (
|
||||
ConnectionAwareInterface,
|
||||
ContextAwareInterface,
|
||||
@@ -8,11 +14,30 @@ from .interfaces import (
|
||||
ObjectRepositoryInterface,
|
||||
)
|
||||
|
||||
# Adapters are imported only for static type checkers; runtime loading is delegated below.
|
||||
if TYPE_CHECKING:
|
||||
from .adapters.minio_adapter import MinioAdapter as MinioAdapter
|
||||
from .adapters.redis_adapter import RedisAdapter as RedisAdapter
|
||||
|
||||
__all__ = [
|
||||
"ConnectionAwareInterface",
|
||||
"ContextAwareInterface",
|
||||
"JsonRepositoryInterface",
|
||||
"MinioConfig",
|
||||
"ObjectRepositoryInterface",
|
||||
"RedisAdapter",
|
||||
"MinioAdapter",
|
||||
"RedisConfig",
|
||||
"load_dotenv",
|
||||
*adapters.__all__,
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> object:
|
||||
"""Delegate adapter lookups to adapters; lazy loading is defined there."""
|
||||
if name in adapters.__all__:
|
||||
return getattr(adapters, name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
"""Expose lazy adapter names in tab completion and dir()."""
|
||||
return sorted(__all__)
|
||||
|
||||
@@ -4,10 +4,39 @@ Adapters for various backend repositories (e.g., Redis, Minio).
|
||||
This module exposes concrete implementations for repository interfaces.
|
||||
"""
|
||||
|
||||
from .redis_adapter import RedisAdapter
|
||||
from .minio_adapter import MinioAdapter
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
# Adapters are imported only for static type checkers; runtime loading is deferred below.
|
||||
if TYPE_CHECKING:
|
||||
from .minio_adapter import MinioAdapter
|
||||
from .redis_adapter import RedisAdapter
|
||||
|
||||
# Map public adapter names to their defining module and class.
|
||||
# Each adapter module fails fast with an install hint if its extra is missing.
|
||||
# When adding a new adapter, update this dict and __all__ only.
|
||||
_LAZY_EXPORTS = {
|
||||
"RedisAdapter": (".redis_adapter", "RedisAdapter"),
|
||||
"MinioAdapter": (".minio_adapter", "MinioAdapter"),
|
||||
}
|
||||
|
||||
__all__ = [
|
||||
"RedisAdapter",
|
||||
"MinioAdapter",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> object:
|
||||
"""Load an adapter on first access so the base package installs without backend clients."""
|
||||
if name in _LAZY_EXPORTS:
|
||||
module_path, attr = _LAZY_EXPORTS[name]
|
||||
module = importlib.import_module(module_path, __package__)
|
||||
return getattr(module, attr)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
"""Expose lazy adapter names in tab completion and dir()."""
|
||||
return sorted(__all__)
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Shared connection lifecycle behavior for repository adapters."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from abc import abstractmethod
|
||||
from typing import Self
|
||||
|
||||
import structlog
|
||||
|
||||
from python_repositories.interfaces import (
|
||||
ConnectionAwareInterface,
|
||||
ContextAwareInterface,
|
||||
)
|
||||
|
||||
|
||||
class ConnectionAwareAdapter(ConnectionAwareInterface, ContextAwareInterface):
|
||||
"""Base adapter with context-manager and TTL-cached connection health checks."""
|
||||
|
||||
health_check_ttl_seconds: float = 1.0
|
||||
connection_name: str = ""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.logger = structlog.get_logger(self.__class__.__name__)
|
||||
self._client_injected = False
|
||||
self._health_check_at: float | None = None
|
||||
self._health_check_ok: bool = False
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
"""Enter the context."""
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
||||
) -> None:
|
||||
"""Exit the context."""
|
||||
ctx_info = {"exc_type": exc_type, "exc_val": exc_val, "exc_tb": exc_tb}
|
||||
if any(
|
||||
(
|
||||
exc_type is not None,
|
||||
exc_val is not None,
|
||||
exc_tb is not None,
|
||||
),
|
||||
):
|
||||
self.logger.error("Error while exiting context", **ctx_info)
|
||||
self.disconnect()
|
||||
|
||||
def _invalidate_health_cache(self) -> None:
|
||||
self._health_check_at = None
|
||||
self._health_check_ok = False
|
||||
|
||||
@abstractmethod
|
||||
def _is_client_ready(self) -> bool:
|
||||
"""Return True when internal state is sufficient for a probe."""
|
||||
|
||||
@abstractmethod
|
||||
def _probe_connection(self) -> bool:
|
||||
"""Backend-specific liveness check; called only when client is ready."""
|
||||
|
||||
@abstractmethod
|
||||
def _validate_injected_client(self) -> None:
|
||||
"""Verify an injected client is reachable; raise ConnectionError on failure."""
|
||||
|
||||
@abstractmethod
|
||||
def _establish_connection(self) -> None:
|
||||
"""Create a backend client and set internal connection state."""
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Connect to the backend; idempotent when already connected and healthy."""
|
||||
if self._client_injected:
|
||||
self._validate_injected_client()
|
||||
self._invalidate_health_cache()
|
||||
return
|
||||
if self._is_client_ready() and self.is_connected():
|
||||
self.logger.info(f"Already connected to {self.connection_name}")
|
||||
return
|
||||
self.disconnect()
|
||||
self._establish_connection()
|
||||
self._invalidate_health_cache()
|
||||
|
||||
def is_connected(self) -> bool:
|
||||
"""Check if connected to the backend."""
|
||||
if not self._is_client_ready():
|
||||
return False
|
||||
now = time.monotonic()
|
||||
if self._health_check_at is not None:
|
||||
seconds_since_last_health_check = now - self._health_check_at
|
||||
cache_is_fresh = (
|
||||
seconds_since_last_health_check < self.health_check_ttl_seconds
|
||||
)
|
||||
if cache_is_fresh:
|
||||
return self._health_check_ok
|
||||
result = self._probe_connection()
|
||||
self._health_check_at = now
|
||||
self._health_check_ok = result
|
||||
self.logger.debug("Connection status", connected=result)
|
||||
return result
|
||||
|
||||
def _require_connected(self) -> None:
|
||||
if not self.is_connected():
|
||||
raise ConnectionError(f"Not connected to {self.connection_name}")
|
||||
@@ -1,120 +1,111 @@
|
||||
"""Definition of MinioAdapter class."""
|
||||
|
||||
from __future__ import annotations
|
||||
import os
|
||||
from io import BytesIO
|
||||
from importlib.util import find_spec
|
||||
from typing import Self
|
||||
import structlog
|
||||
from python_utils import check_env
|
||||
|
||||
from python_repositories.interfaces import (
|
||||
ConnectionAwareInterface,
|
||||
ContextAwareInterface,
|
||||
ObjectRepositoryInterface,
|
||||
from python_repositories.adapters.connection_aware_adapter import (
|
||||
ConnectionAwareAdapter,
|
||||
)
|
||||
from python_repositories.config import MinioConfig
|
||||
from python_repositories.interfaces import ObjectRepositoryInterface
|
||||
|
||||
# Handle optional dependencies
|
||||
if find_spec("minio") is not None:
|
||||
try:
|
||||
import minio
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"MinIO support requires the minio extra. "
|
||||
"Install with: pip install python-repositories[minio]"
|
||||
) from exc
|
||||
|
||||
|
||||
class MinioAdapter(
|
||||
ObjectRepositoryInterface,
|
||||
ContextAwareInterface,
|
||||
ConnectionAwareInterface,
|
||||
):
|
||||
class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
||||
"""Minio adapter exposing basic CRUD functionality."""
|
||||
|
||||
endpoint_env_var_name: str = "MINIO_ENDPOINT"
|
||||
access_key_env_var_name: str = "MINIO_ACCESS_KEY"
|
||||
secret_key_env_var_name: str = "MINIO_SECRET_KEY"
|
||||
bucket_env_var_name: str = "MINIO_BUCKET"
|
||||
secure_env_var_name: str = "MINIO_SECURE"
|
||||
create_bucket_if_missing_env_var_name: str = "MINIO_CREATE_BUCKET_IF_MISSING"
|
||||
chunk_size: int = 5 * 2**20 # 5 MiB
|
||||
connection_name: str = "Minio"
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Setup logger
|
||||
self.logger = structlog.get_logger(
|
||||
self.__class__.__name__,
|
||||
)
|
||||
# Check environment variables
|
||||
check_env(
|
||||
{
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
config: MinioConfig | None = None,
|
||||
client: minio.Minio | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if client is not None and config is None:
|
||||
raise ValueError("config is required when client is provided")
|
||||
if config is None:
|
||||
config = MinioConfig.from_env(
|
||||
self.endpoint_env_var_name,
|
||||
self.access_key_env_var_name,
|
||||
self.secret_key_env_var_name,
|
||||
self.bucket_env_var_name,
|
||||
},
|
||||
)
|
||||
# Prepare internal variables
|
||||
self._client: minio.Minio | None = None
|
||||
self._bucket_name: str | None = None
|
||||
self.secure_env_var_name,
|
||||
self.create_bucket_if_missing_env_var_name,
|
||||
)
|
||||
self._config = config
|
||||
self._client_injected = client is not None
|
||||
self._client: minio.Minio | None = client
|
||||
self._bucket_name: str | None = config.bucket if client is not None else None
|
||||
if self._client_injected:
|
||||
self._invalidate_health_cache()
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
"""Enter the context."""
|
||||
self.connect()
|
||||
return self
|
||||
def _is_client_ready(self) -> bool:
|
||||
return self._client is not None and self._bucket_name is not None
|
||||
|
||||
def __exit__(
|
||||
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
||||
) -> None:
|
||||
"""Exit the context."""
|
||||
ctx_info = {"exc_type": exc_type, "exc_val": exc_val, "exc_tb": exc_tb}
|
||||
if any(
|
||||
(
|
||||
exc_type is not None,
|
||||
exc_val is not None,
|
||||
exc_tb is not None,
|
||||
),
|
||||
):
|
||||
self.logger.error("Error while exiting context", **ctx_info)
|
||||
self.disconnect()
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Connect to the Minio server."""
|
||||
# Stop if already connected
|
||||
if self.is_connected:
|
||||
self.logger.info("Already connected to Minio")
|
||||
def _validate_injected_client(self) -> None:
|
||||
if self._client is None:
|
||||
return
|
||||
# Prepare arguments
|
||||
endpoint = str(os.getenv(self.endpoint_env_var_name))
|
||||
access_key = str(os.getenv(self.access_key_env_var_name))
|
||||
secret_key = str(os.getenv(self.secret_key_env_var_name))
|
||||
bucket = str(os.getenv(self.bucket_env_var_name))
|
||||
# Connect client
|
||||
try:
|
||||
_ = self._client.list_buckets()
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
raise ConnectionError(
|
||||
f"Could not connect to Minio at {self._config.endpoint}"
|
||||
) from exc
|
||||
|
||||
def _establish_connection(self) -> None:
|
||||
endpoint = self._config.endpoint
|
||||
access_key = self._config.access_key
|
||||
secret_key = self._config.secret_key
|
||||
bucket = self._config.bucket
|
||||
client = minio.Minio(
|
||||
endpoint=endpoint,
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
secure=False,
|
||||
secure=self._config.secure,
|
||||
)
|
||||
# Test the connection by listing buckets (will raise if connection fails)
|
||||
try:
|
||||
_ = client.list_buckets()
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
raise ConnectionError(f"Could not connect to Minio at {endpoint}") from exc
|
||||
# Ensure bucket exists
|
||||
if not client.bucket_exists(bucket):
|
||||
if not self._config.create_bucket_if_missing:
|
||||
raise ConnectionError(
|
||||
f"Bucket '{bucket}' does not exist on Minio at {endpoint}"
|
||||
)
|
||||
self.logger.info(f"Creating bucket '{bucket}'")
|
||||
client.make_bucket(bucket)
|
||||
# Persist information
|
||||
self._client = client
|
||||
self._bucket_name = bucket
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Disconnect from the Minio server."""
|
||||
# Close connection
|
||||
# N.B. Minio client does not have a close method, but we include this for symmetry with other adapters
|
||||
# Reset client
|
||||
# N.B. Minio client does not have a close method, but we include this for symmetry
|
||||
self._client = None
|
||||
self._bucket_name = None
|
||||
self._invalidate_health_cache()
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Check if connected to Minio server."""
|
||||
res = bool(isinstance(self._client, minio.Minio))
|
||||
self.logger.debug(res)
|
||||
return res
|
||||
def _probe_connection(self) -> bool:
|
||||
assert self._client is not None and self._bucket_name is not None
|
||||
try:
|
||||
return bool(self._client.bucket_exists(self._bucket_name))
|
||||
except Exception: # pylint: disable=broad-except
|
||||
return False
|
||||
|
||||
def put(
|
||||
self,
|
||||
@@ -131,8 +122,8 @@ class MinioAdapter(
|
||||
if not isinstance(content_type, str) or len(content_type) == 0:
|
||||
raise ValueError("content_type must be a non-empty string")
|
||||
# Check connection
|
||||
if self._client is None or not self.is_connected:
|
||||
raise ConnectionError("Not connected to Minio")
|
||||
self._require_connected()
|
||||
assert self._client is not None and self._bucket_name is not None
|
||||
# Prepare buffer for reading
|
||||
num_bytes = data.getbuffer().nbytes
|
||||
data.seek(0)
|
||||
@@ -156,10 +147,11 @@ class MinioAdapter(
|
||||
if not isinstance(object_name, str) or len(object_name) == 0:
|
||||
raise ValueError("object_name must be a non-empty string")
|
||||
# Check connection
|
||||
if self._client is None or not self.is_connected:
|
||||
raise ConnectionError("Not connected to Minio")
|
||||
self._require_connected()
|
||||
assert self._client is not None and self._bucket_name is not None
|
||||
# Get data from bucket
|
||||
# N.B. bucket name is set when connecting
|
||||
response = None
|
||||
try:
|
||||
response = self._client.get_object(
|
||||
bucket_name=self._bucket_name,
|
||||
@@ -179,11 +171,12 @@ class MinioAdapter(
|
||||
self.logger.warning(
|
||||
f"Object '{object_name}' not found in bucket '{self._bucket_name}'"
|
||||
)
|
||||
else:
|
||||
self.logger.error(repr(exc))
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
self.logger.error(repr(exc))
|
||||
return None
|
||||
return None
|
||||
raise
|
||||
finally:
|
||||
if response is not None:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
def delete(self, object_name: str) -> None:
|
||||
"""Delete an object from the Minio bucket."""
|
||||
@@ -191,8 +184,8 @@ class MinioAdapter(
|
||||
if not isinstance(object_name, str) or len(object_name) == 0:
|
||||
raise ValueError("object_name must be a non-empty string")
|
||||
# Check connection
|
||||
if self._client is None or not self.is_connected:
|
||||
raise ConnectionError("Not connected to Minio")
|
||||
self._require_connected()
|
||||
assert self._client is not None and self._bucket_name is not None
|
||||
# Delete object from bucket
|
||||
# N.B. bucket name is set when connecting
|
||||
self._client.remove_object(
|
||||
@@ -210,8 +203,8 @@ class MinioAdapter(
|
||||
raise ValueError("prefix must be a string")
|
||||
# Check connection
|
||||
# N.B. bucket name is set when connecting
|
||||
if self._client is None or not self.is_connected:
|
||||
raise ConnectionError("Not connected to Minio")
|
||||
self._require_connected()
|
||||
assert self._client is not None and self._bucket_name is not None
|
||||
# List objects in bucket
|
||||
objects = self._client.list_objects(
|
||||
bucket_name=self._bucket_name,
|
||||
|
||||
@@ -1,72 +1,69 @@
|
||||
"""Definition of RedisAdapter class."""
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import Self, cast
|
||||
from importlib.util import find_spec
|
||||
import os
|
||||
import structlog
|
||||
from collections.abc import Iterator
|
||||
from typing import Any, cast
|
||||
|
||||
from python_utils import check_env
|
||||
|
||||
from python_repositories.interfaces import (
|
||||
ConnectionAwareInterface,
|
||||
ContextAwareInterface,
|
||||
JsonRepositoryInterface,
|
||||
from python_repositories.adapters.connection_aware_adapter import (
|
||||
ConnectionAwareAdapter,
|
||||
)
|
||||
from python_repositories.config import RedisConfig
|
||||
from python_repositories.interfaces import JsonRepositoryInterface
|
||||
|
||||
# Handle optional dependencies
|
||||
if find_spec("redis") is not None:
|
||||
try:
|
||||
import redis
|
||||
from redis.commands.json.path import Path as RedisPath
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"Redis support requires the redis extra. "
|
||||
"Install with: pip install python-repositories[redis]"
|
||||
) from exc
|
||||
|
||||
|
||||
class RedisAdapter(
|
||||
JsonRepositoryInterface,
|
||||
ContextAwareInterface,
|
||||
ConnectionAwareInterface,
|
||||
):
|
||||
class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
||||
"""Redis adapter exposing basic CRUD functionality."""
|
||||
|
||||
uri_env_var_name: str = "REDIS_URI"
|
||||
path: str = "." # JSON root path, updated in __init__
|
||||
encoding: str = "UTF-8"
|
||||
connection_name: str = "Redis"
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Setup logger
|
||||
self.logger = structlog.get_logger(
|
||||
self.__class__.__name__,
|
||||
)
|
||||
# Check environment variables
|
||||
check_env(self.uri_env_var_name)
|
||||
# Prepare internal variables
|
||||
self._client: redis.Redis | None = None
|
||||
self.path: str = RedisPath.root_path()
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
"""Enter the context."""
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self, exc_type: type | None, exc_val: object | None, exc_tb: object | None
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
config: RedisConfig | None = None,
|
||||
client: redis.Redis | None = None,
|
||||
) -> None:
|
||||
"""Exit the context."""
|
||||
ctx_info = {"exc_type": exc_type, "exc_val": exc_val, "exc_tb": exc_tb}
|
||||
if any(
|
||||
(
|
||||
exc_type is not None,
|
||||
exc_val is not None,
|
||||
exc_tb is not None,
|
||||
),
|
||||
):
|
||||
self.logger.error("Error while exiting context", **ctx_info)
|
||||
self.disconnect()
|
||||
super().__init__()
|
||||
if client is not None and config is None:
|
||||
raise ValueError("config is required when client is provided")
|
||||
if config is None:
|
||||
config = RedisConfig.from_env(self.uri_env_var_name)
|
||||
self._config = config
|
||||
self._client_injected = client is not None
|
||||
self._client: redis.Redis | None = client
|
||||
self.path: str = RedisPath.root_path()
|
||||
if self._client_injected:
|
||||
self._invalidate_health_cache()
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Connect to the Redis server."""
|
||||
# Prepare arguments
|
||||
uri = str(os.getenv(self.uri_env_var_name))
|
||||
# Connect client
|
||||
def _is_client_ready(self) -> bool:
|
||||
return self._client is not None
|
||||
|
||||
def _validate_injected_client(self) -> None:
|
||||
if self._client is None:
|
||||
return
|
||||
try:
|
||||
if not self._client.ping():
|
||||
raise ConnectionError(
|
||||
f"Could not connect to Redis at {self._config.uri}"
|
||||
)
|
||||
except (redis.ConnectionError, redis.TimeoutError) as exc:
|
||||
raise ConnectionError(
|
||||
f"Could not connect to Redis at {self._config.uri}"
|
||||
) from exc
|
||||
|
||||
def _establish_connection(self) -> None:
|
||||
uri = self._config.uri
|
||||
try:
|
||||
client = redis.Redis.from_url(
|
||||
url=uri,
|
||||
@@ -76,25 +73,23 @@ class RedisAdapter(
|
||||
raise ConnectionError(f"Could not connect to Redis at {uri}")
|
||||
except (redis.ConnectionError, redis.TimeoutError) as exc:
|
||||
raise ConnectionError(f"Could not connect to Redis at {uri}") from exc
|
||||
# Persist client
|
||||
self._client = client
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Disconnect from the Redis server."""
|
||||
# Close connection
|
||||
if self._client is not None:
|
||||
if self._client is not None and not self._client_injected:
|
||||
self._client.close()
|
||||
# Reset client
|
||||
self._client = None
|
||||
self._invalidate_health_cache()
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Check if connected to Redis server."""
|
||||
res = bool(isinstance(self._client, redis.Redis))
|
||||
self.logger.debug(res)
|
||||
return res
|
||||
def _probe_connection(self) -> bool:
|
||||
assert self._client is not None
|
||||
try:
|
||||
return bool(self._client.ping())
|
||||
except (redis.ConnectionError, redis.TimeoutError):
|
||||
return False
|
||||
|
||||
def set(self, key: str, data: dict) -> None:
|
||||
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||
"""Set a JSON object in Redis."""
|
||||
# Check input
|
||||
if not isinstance(key, str) or len(key) == 0:
|
||||
@@ -102,23 +97,23 @@ class RedisAdapter(
|
||||
if not isinstance(data, dict) or len(data) == 0:
|
||||
raise ValueError("Data must be a non-empty dictionary")
|
||||
# Check connection
|
||||
if self._client is None or not self.is_connected:
|
||||
raise ConnectionError("Not connected to Redis")
|
||||
self._require_connected()
|
||||
assert self._client is not None
|
||||
# Set data
|
||||
self._client.json().set(key, self.path, data)
|
||||
self.logger.debug(f"Set {key} to {data}")
|
||||
|
||||
def get(self, key: str) -> dict | None:
|
||||
def get(self, key: str) -> dict[str, Any] | None:
|
||||
"""Get a JSON object from Redis."""
|
||||
# Check input
|
||||
if not isinstance(key, str) or len(key) == 0:
|
||||
raise ValueError("Key must be a non-empty string")
|
||||
# Check connection
|
||||
if self._client is None or not self.is_connected:
|
||||
raise ConnectionError("Not connected to Redis")
|
||||
self._require_connected()
|
||||
assert self._client is not None
|
||||
# Get data
|
||||
data = cast(
|
||||
dict | None,
|
||||
dict[str, Any] | None,
|
||||
self._client.json().get(key),
|
||||
)
|
||||
self.logger.debug(f"Got {data} from {key}")
|
||||
@@ -130,20 +125,22 @@ class RedisAdapter(
|
||||
if not isinstance(key, str) or len(key) == 0:
|
||||
raise ValueError("Key must be a non-empty string")
|
||||
# Check connection
|
||||
if self._client is None or not self.is_connected:
|
||||
raise ConnectionError("Not connected to Redis")
|
||||
self._require_connected()
|
||||
assert self._client is not None
|
||||
# Delete data
|
||||
self._client.json().delete(key)
|
||||
self.logger.debug(f"Deleted {key}")
|
||||
|
||||
def list_keys(self, pattern: str) -> list[str]:
|
||||
"""List keys in Redis matching a pattern."""
|
||||
# Check input
|
||||
def _validate_pattern(self, pattern: str) -> None:
|
||||
if not isinstance(pattern, str) or len(pattern) == 0:
|
||||
raise ValueError("Pattern must be a non-empty string")
|
||||
|
||||
def list_keys(self, pattern: str) -> list[str]:
|
||||
"""List keys in Redis using KEYS; may block on large datasets."""
|
||||
self._validate_pattern(pattern)
|
||||
# Check connection
|
||||
if self._client is None or not self.is_connected:
|
||||
raise ConnectionError("Not connected to Redis")
|
||||
self._require_connected()
|
||||
assert self._client is not None
|
||||
# List keys
|
||||
keys_raw = cast(
|
||||
list[bytes],
|
||||
@@ -152,3 +149,29 @@ class RedisAdapter(
|
||||
keys: list[str] = [key.decode(self.encoding) for key in keys_raw]
|
||||
self.logger.debug(f"Got {keys} matching {pattern}")
|
||||
return keys
|
||||
|
||||
def scan_keys(
|
||||
self,
|
||||
pattern: str,
|
||||
*,
|
||||
count: int | None = None,
|
||||
) -> Iterator[str]:
|
||||
"""Yield keys in Redis using SCAN to avoid blocking large datasets."""
|
||||
self._validate_pattern(pattern)
|
||||
self._require_connected()
|
||||
assert self._client is not None
|
||||
client = self._client
|
||||
|
||||
def _decode(key: bytes | str) -> str:
|
||||
return key if isinstance(key, str) else key.decode(self.encoding)
|
||||
|
||||
def _iter() -> Iterator[str]:
|
||||
scan_iter = (
|
||||
client.scan_iter(match=pattern, count=count)
|
||||
if count is not None
|
||||
else client.scan_iter(match=pattern)
|
||||
)
|
||||
for key_raw in scan_iter:
|
||||
yield _decode(key_raw)
|
||||
|
||||
return _iter()
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
from .dotenv_loader import load_dotenv as load_dotenv
|
||||
from .minio_config import MinioConfig as MinioConfig
|
||||
from .redis_config import RedisConfig as RedisConfig
|
||||
|
||||
__all__ = [
|
||||
"MinioConfig",
|
||||
"RedisConfig",
|
||||
"load_dotenv",
|
||||
]
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Load environment variables from a .env file."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv as _load_dotenv
|
||||
|
||||
|
||||
def load_dotenv(path: str | Path | None = None) -> bool:
|
||||
"""Load .env into os.environ. Idempotent; returns True if a file was loaded."""
|
||||
if path is None:
|
||||
return bool(_load_dotenv())
|
||||
return bool(_load_dotenv(path))
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Parse boolean values from environment variables."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
_TRUTHY = frozenset({"1", "true", "yes", "on"})
|
||||
_FALSY = frozenset({"0", "false", "no", "off"})
|
||||
|
||||
|
||||
def env_bool(name: str, default: bool) -> bool:
|
||||
"""Parse an environment variable as a boolean value."""
|
||||
raw = os.getenv(name)
|
||||
if raw is None:
|
||||
return default
|
||||
normalized = raw.strip().lower()
|
||||
if normalized in _TRUTHY:
|
||||
return True
|
||||
if normalized in _FALSY:
|
||||
return False
|
||||
raise ValueError(f"Invalid boolean value for {name}: {raw!r}")
|
||||
@@ -0,0 +1,57 @@
|
||||
"""MinIO connection configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
from python_utils import check_env
|
||||
|
||||
from python_repositories.config.dotenv_loader import load_dotenv
|
||||
from python_repositories.config.env_bool import env_bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MinioConfig:
|
||||
"""Configuration for connecting to MinIO."""
|
||||
|
||||
endpoint: str
|
||||
access_key: str
|
||||
secret_key: str
|
||||
bucket: str
|
||||
secure: bool = True
|
||||
create_bucket_if_missing: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_env(
|
||||
cls,
|
||||
endpoint_env_var_name: str = "MINIO_ENDPOINT",
|
||||
access_key_env_var_name: str = "MINIO_ACCESS_KEY",
|
||||
secret_key_env_var_name: str = "MINIO_SECRET_KEY",
|
||||
bucket_env_var_name: str = "MINIO_BUCKET",
|
||||
secure_env_var_name: str = "MINIO_SECURE",
|
||||
create_bucket_if_missing_env_var_name: str = ("MINIO_CREATE_BUCKET_IF_MISSING"),
|
||||
*,
|
||||
use_dotenv: bool = True,
|
||||
) -> MinioConfig:
|
||||
"""Load configuration from environment variables."""
|
||||
if use_dotenv:
|
||||
load_dotenv()
|
||||
env_var_names = {
|
||||
endpoint_env_var_name,
|
||||
access_key_env_var_name,
|
||||
secret_key_env_var_name,
|
||||
bucket_env_var_name,
|
||||
}
|
||||
check_env(env_var_names)
|
||||
return cls(
|
||||
endpoint=str(os.getenv(endpoint_env_var_name)),
|
||||
access_key=str(os.getenv(access_key_env_var_name)),
|
||||
secret_key=str(os.getenv(secret_key_env_var_name)),
|
||||
bucket=str(os.getenv(bucket_env_var_name)),
|
||||
secure=env_bool(secure_env_var_name, default=True),
|
||||
create_bucket_if_missing=env_bool(
|
||||
create_bucket_if_missing_env_var_name,
|
||||
default=False,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Redis connection configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
from python_utils import check_env
|
||||
|
||||
from python_repositories.config.dotenv_loader import load_dotenv
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RedisConfig:
|
||||
"""Configuration for connecting to Redis."""
|
||||
|
||||
uri: str
|
||||
|
||||
@classmethod
|
||||
def from_env(
|
||||
cls,
|
||||
uri_env_var_name: str = "REDIS_URI",
|
||||
*,
|
||||
use_dotenv: bool = True,
|
||||
) -> RedisConfig:
|
||||
"""Load configuration from environment variables."""
|
||||
if use_dotenv:
|
||||
load_dotenv()
|
||||
check_env(uri_env_var_name)
|
||||
return cls(uri=str(os.getenv(uri_env_var_name)))
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Example domain repository backed by Redis JSON."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||
|
||||
|
||||
@@ -9,10 +11,10 @@ class UserJsonRepository(RedisAdapter):
|
||||
def _key(self, user_id: str) -> str:
|
||||
return f"user:{user_id}"
|
||||
|
||||
def get_user(self, user_id: str) -> dict | None:
|
||||
def get_user(self, user_id: str) -> dict[str, Any] | None:
|
||||
return self.get(self._key(user_id))
|
||||
|
||||
def save_user(self, user_id: str, user: dict) -> None:
|
||||
def save_user(self, user_id: str, user: dict[str, Any]) -> None:
|
||||
self.set(self._key(user_id), user)
|
||||
|
||||
def delete_user(self, user_id: str) -> None:
|
||||
|
||||
@@ -8,7 +8,11 @@ class ConnectionAwareInterface(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def connect(self) -> None:
|
||||
"""Connect to resource."""
|
||||
"""Connect to resource.
|
||||
|
||||
Implementations should be idempotent: calling connect while already
|
||||
connected and healthy is a no-op.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
@@ -16,8 +20,10 @@ class ConnectionAwareInterface(ABC):
|
||||
"""Disconnect from resource."""
|
||||
...
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def is_connected(self) -> bool:
|
||||
"""Check if connected to resource."""
|
||||
"""Return whether the adapter has an active, reachable connection.
|
||||
|
||||
Implementations may perform a cached network probe to verify liveness.
|
||||
"""
|
||||
...
|
||||
|
||||
@@ -1,18 +1,20 @@
|
||||
"""Definition of JsonRepositoryInterface abstract base class."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class JsonRepositoryInterface(ABC):
|
||||
"""Interface that defines JSON document CRUD methods."""
|
||||
|
||||
@abstractmethod
|
||||
def get(self, key: str) -> dict | None:
|
||||
def get(self, key: str) -> dict[str, Any] | None:
|
||||
"""Get a JSON object by key."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def set(self, key: str, data: dict) -> None:
|
||||
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||
"""Set a JSON object by key."""
|
||||
...
|
||||
|
||||
@@ -25,3 +27,13 @@ class JsonRepositoryInterface(ABC):
|
||||
def list_keys(self, pattern: str) -> list[str]:
|
||||
"""List keys matching a glob pattern."""
|
||||
...
|
||||
|
||||
def scan_keys(
|
||||
self,
|
||||
pattern: str,
|
||||
*,
|
||||
count: int | None = None,
|
||||
) -> Iterator[str]:
|
||||
"""Yield keys matching a glob pattern incrementally."""
|
||||
del count
|
||||
yield from self.list_keys(pattern)
|
||||
|
||||
@@ -9,7 +9,11 @@ class ObjectRepositoryInterface(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def get(self, object_name: str) -> BytesIO | None:
|
||||
"""Get an object by name."""
|
||||
"""Get an object by name.
|
||||
|
||||
Returns None when the object does not exist. Raises ConnectionError when
|
||||
not connected. Other backend errors propagate to the caller.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build and optionally push the Gitea Actions CI base image locally.
|
||||
#
|
||||
# Use when ci-image.yml is not yet on main (e.g. bootstrapping a PR branch) or
|
||||
# when you need to rebuild without waiting for the nightly workflow.
|
||||
#
|
||||
# Build only (smoke test):
|
||||
# export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||
# bash scripts/ci/build-ci-image.sh --local
|
||||
#
|
||||
# Build and push to Gitea (unblocks python-repositories-ci jobs):
|
||||
# export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||
# bash scripts/ci/build-ci-image.sh --push
|
||||
#
|
||||
# Optional overrides:
|
||||
# REGISTRY=gitea.lille-vemmelund.dk
|
||||
# REGISTRY_USER=ci-bot
|
||||
# IMAGE_OWNER=brian
|
||||
# IMAGE_NAME=python-repositories-ci
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
|
||||
REGISTRY="${REGISTRY:-gitea.lille-vemmelund.dk}"
|
||||
REGISTRY_USER="${REGISTRY_USER:-ci-bot}"
|
||||
IMAGE_OWNER="${IMAGE_OWNER:-brian}"
|
||||
IMAGE_NAME="${IMAGE_NAME:-python-repositories-ci}"
|
||||
IMAGE="${IMAGE:-${REGISTRY}/${IMAGE_OWNER}/${IMAGE_NAME}}"
|
||||
|
||||
MODE="push"
|
||||
if [[ "${1:-}" == "--local" ]]; then
|
||||
MODE="local"
|
||||
elif [[ "${1:-}" == "--push" || -z "${1:-}" ]]; then
|
||||
MODE="push"
|
||||
elif [[ -n "${1:-}" ]]; then
|
||||
echo "Usage: build-ci-image.sh [--local | --push]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: "${CI_RUNNER_TOKEN:?CI_RUNNER_TOKEN is required (ci-bot token with read/write package access)}"
|
||||
|
||||
TOKEN_FILE="$(mktemp)"
|
||||
cleanup() {
|
||||
rm -f "$TOKEN_FILE"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
printf '%s' "$CI_RUNNER_TOKEN" >"$TOKEN_FILE"
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if [[ "$MODE" == "local" ]]; then
|
||||
echo "=== Building local CI image (no push): python-repositories-ci:local ==="
|
||||
docker build -f docker/ci/Dockerfile \
|
||||
--secret "id=uv_token,src=${TOKEN_FILE}" \
|
||||
-t python-repositories-ci:local \
|
||||
.
|
||||
echo "Built python-repositories-ci:local"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== Logging in to ${REGISTRY} as ${REGISTRY_USER} ==="
|
||||
echo "$CI_RUNNER_TOKEN" | docker login "$REGISTRY" -u "$REGISTRY_USER" --password-stdin
|
||||
|
||||
echo "=== Building ${IMAGE}:latest ==="
|
||||
docker build -f docker/ci/Dockerfile \
|
||||
--secret "id=uv_token,src=${TOKEN_FILE}" \
|
||||
-t "${IMAGE}:latest" \
|
||||
.
|
||||
|
||||
STAMP="$(date -u +%Y%m%d%H%M)"
|
||||
echo "=== Pushing ${IMAGE}:latest and ${IMAGE}:${STAMP} ==="
|
||||
docker tag "${IMAGE}:latest" "${IMAGE}:${STAMP}"
|
||||
docker push "${IMAGE}:latest"
|
||||
docker push "${IMAGE}:${STAMP}"
|
||||
|
||||
echo "Done. Re-run failing CI jobs — runners pull ${IMAGE}:latest"
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=common.sh
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
usage() {
|
||||
echo "Usage: bump-version.sh <major|minor|patch>" >&2
|
||||
echo " bump-version.sh --from-tag <vX.Y.Z>" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [[ "$1" == "--from-tag" ]]; then
|
||||
if [[ $# -ne 2 ]]; then
|
||||
usage
|
||||
fi
|
||||
VERSION="${2#v}"
|
||||
else
|
||||
BUMP_TYPE="$1"
|
||||
CURRENT=$(latest_tag_version)
|
||||
VERSION=$(bump_semver "$CURRENT" "$BUMP_TYPE")
|
||||
fi
|
||||
|
||||
set_pyproject_version "$VERSION"
|
||||
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv lock
|
||||
else
|
||||
echo "uv not found; uv.lock was not updated" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "version=${VERSION}"
|
||||
fi
|
||||
|
||||
echo "Updated pyproject.toml and uv.lock to version ${VERSION}" >&2
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=common.sh
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
PR_TITLE="${1:-${PR_TITLE:-}}"
|
||||
|
||||
if [[ -z "$PR_TITLE" ]]; then
|
||||
echo "Usage: check-pr-title.sh <pr-title> (changed files on stdin)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CHANGED_FILES=$(cat)
|
||||
|
||||
if ! has_source_changes "$CHANGED_FILES"; then
|
||||
echo "No changes under ${SOURCE_DIR}/ — PR title prefix not required."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if title_has_bump_prefix "$PR_TITLE"; then
|
||||
echo "PR title has a valid version bump prefix."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cat >&2 <<EOF
|
||||
PR changes files under ${SOURCE_DIR}/ but the title lacks a version bump prefix.
|
||||
|
||||
Title: ${PR_TITLE}
|
||||
|
||||
When changing package source, start the PR title with one of:
|
||||
[patch] or [fix] — bug fix
|
||||
[minor] or [feat] — new feature
|
||||
[major] or [breaking] — breaking change
|
||||
|
||||
Example: [patch] Fix connection retry in RedisAdapter
|
||||
|
||||
See .gitea/PULL_REQUEST_TEMPLATE.md for details.
|
||||
EOF
|
||||
exit 1
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared helpers for release and PR title CI scripts.
|
||||
|
||||
SOURCE_DIR="python_repositories"
|
||||
|
||||
# Returns bump type (major|minor|patch) or empty if no valid prefix.
|
||||
bump_type_from_title() {
|
||||
local title="$1"
|
||||
local lower
|
||||
lower=$(echo "$title" | tr '[:upper:]' '[:lower:]')
|
||||
|
||||
if echo "$lower" | grep -qE '^\[(major|breaking)\]'; then
|
||||
echo "major"
|
||||
elif echo "$lower" | grep -qE '^\[(minor|feat)\]'; then
|
||||
echo "minor"
|
||||
elif echo "$lower" | grep -qE '^\[(patch|fix)\]'; then
|
||||
echo "patch"
|
||||
fi
|
||||
}
|
||||
|
||||
title_has_bump_prefix() {
|
||||
[[ -n "$(bump_type_from_title "$1")" ]]
|
||||
}
|
||||
|
||||
strip_bump_prefix() {
|
||||
local title="$1"
|
||||
echo "$title" | sed -E 's/^\[(patch|fix|minor|feat|major|breaking)\][[:space:]]*//i'
|
||||
}
|
||||
|
||||
extract_pr_title_from_merge_commit() {
|
||||
local msg="$1"
|
||||
echo "$msg" | sed -n "s/^Merge pull request '\(.*\)' (#.*/\1/p"
|
||||
}
|
||||
|
||||
changed_source_files() {
|
||||
grep -E "^${SOURCE_DIR}/" || true
|
||||
}
|
||||
|
||||
has_source_changes() {
|
||||
local files="$1"
|
||||
echo "$files" | changed_source_files | grep -q .
|
||||
}
|
||||
|
||||
read_pyproject_version() {
|
||||
sed -n 's/^version = "\(.*\)"/\1/p' pyproject.toml | head -1
|
||||
}
|
||||
|
||||
latest_tag_version() {
|
||||
local tag
|
||||
tag=$(git describe --tags --abbrev=0 2>/dev/null || true)
|
||||
if [[ -n "$tag" ]]; then
|
||||
echo "${tag#v}"
|
||||
else
|
||||
read_pyproject_version
|
||||
fi
|
||||
}
|
||||
|
||||
set_pyproject_version() {
|
||||
local version="$1"
|
||||
sed -i.bak -E "s/^version = \".*\"/version = \"${version}\"/" pyproject.toml
|
||||
rm -f pyproject.toml.bak
|
||||
}
|
||||
|
||||
bump_semver() {
|
||||
local current="$1"
|
||||
local bump_type="$2"
|
||||
local major minor patch
|
||||
|
||||
IFS=. read -r major minor patch <<< "$current"
|
||||
case "$bump_type" in
|
||||
major)
|
||||
major=$((major + 1))
|
||||
minor=0
|
||||
patch=0
|
||||
;;
|
||||
minor)
|
||||
minor=$((minor + 1))
|
||||
patch=0
|
||||
;;
|
||||
patch)
|
||||
patch=$((patch + 1))
|
||||
;;
|
||||
*)
|
||||
echo "Unknown bump type: $bump_type" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
echo "${major}.${minor}.${patch}"
|
||||
}
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
: "${API_URL:?API_URL is required}"
|
||||
: "${REPO_OWNER:?REPO_OWNER is required}"
|
||||
: "${REPO_NAME:?REPO_NAME is required}"
|
||||
: "${CI_RUNNER_TOKEN:?CI_RUNNER_TOKEN is required}"
|
||||
: "${GITHUB_SERVER_URL:?GITHUB_SERVER_URL is required}"
|
||||
: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}"
|
||||
|
||||
BRANCH="${BRANCH:-renovate/auto-deps-update}"
|
||||
BASE_BRANCH="${BASE_BRANCH:-main}"
|
||||
LOCKFILE="${LOCKFILE:-uv.lock}"
|
||||
|
||||
if git diff --quiet "$LOCKFILE"; then
|
||||
echo "No dependency updates available."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git config user.name "CI Bot"
|
||||
git config user.email "[email protected]"
|
||||
git add "$LOCKFILE"
|
||||
git commit -m "chore(deps): update dependencies [automated]"
|
||||
|
||||
git remote set-url origin "https://x-access-token:${CI_RUNNER_TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
|
||||
git push origin "HEAD:${BRANCH}"
|
||||
|
||||
if curl -s "${API_URL}/repos/${REPO_OWNER}/${REPO_NAME}/pulls?state=open" \
|
||||
-H "Authorization: token ${CI_RUNNER_TOKEN}" \
|
||||
| BRANCH="$BRANCH" python3 -c '
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
branch = os.environ["BRANCH"]
|
||||
prs = json.load(sys.stdin)
|
||||
sys.exit(0 if any(pr.get("head", {}).get("ref") == branch for pr in prs) else 1)
|
||||
'; then
|
||||
echo "Pull request already exists; branch push updates it."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
PAYLOAD=$(BRANCH="$BRANCH" BASE_BRANCH="$BASE_BRANCH" python3 -c '
|
||||
import json
|
||||
import os
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"title": "chore(deps): update dependencies",
|
||||
"body": "This PR was created automatically to update dependencies.",
|
||||
"head": os.environ["BRANCH"],
|
||||
"base": os.environ["BASE_BRANCH"],
|
||||
}
|
||||
)
|
||||
)
|
||||
')
|
||||
|
||||
curl -sf -X POST "${API_URL}/repos/${REPO_OWNER}/${REPO_NAME}/pulls" \
|
||||
-H "Authorization: token ${CI_RUNNER_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$PAYLOAD"
|
||||
|
||||
echo "Created pull request."
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Fetch the current PR title from the Gitea API.
|
||||
# Requires: API_URL, REPO_OWNER, REPO_NAME, PR_NUMBER, CI_RUNNER_TOKEN
|
||||
|
||||
: "${API_URL:?API_URL is required}"
|
||||
: "${REPO_OWNER:?REPO_OWNER is required}"
|
||||
: "${REPO_NAME:?REPO_NAME is required}"
|
||||
: "${PR_NUMBER:?PR_NUMBER is required}"
|
||||
: "${CI_RUNNER_TOKEN:?CI_RUNNER_TOKEN is required}"
|
||||
|
||||
curl -sf \
|
||||
"${API_URL}/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PR_NUMBER}" \
|
||||
-H "Authorization: token ${CI_RUNNER_TOKEN}" \
|
||||
| python3 -c 'import json, sys; print(json.load(sys.stdin)["title"])'
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=common.sh
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
NEW_TAG="${1:-}"
|
||||
SUMMARY="${2:-}"
|
||||
PREV_TAG="${3:-}"
|
||||
|
||||
if [[ -z "$NEW_TAG" ]]; then
|
||||
echo "Usage: generate-release-notes.sh <new-tag> [summary] [prev-tag]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$PREV_TAG" ]]; then
|
||||
PREV_TAG=$(git describe --tags --abbrev=0 "${NEW_TAG}^" 2>/dev/null || git describe --tags --abbrev=0 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
NOTES="## Summary"
|
||||
if [[ -n "$SUMMARY" ]]; then
|
||||
NOTES="${NOTES}
|
||||
|
||||
$(strip_bump_prefix "$SUMMARY")"
|
||||
else
|
||||
NOTES="${NOTES}
|
||||
|
||||
Automated release ${NEW_TAG}."
|
||||
fi
|
||||
|
||||
if [[ -n "$PREV_TAG" ]]; then
|
||||
NOTES="${NOTES}
|
||||
|
||||
## Changes since ${PREV_TAG}
|
||||
|
||||
$(git log "${PREV_TAG}..HEAD" --pretty=format:'- %h %s' || true)"
|
||||
else
|
||||
NOTES="${NOTES}
|
||||
|
||||
## Changes
|
||||
|
||||
$(git log --pretty=format:'- %h %s' -20 || true)"
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||
{
|
||||
echo 'body<<EOF'
|
||||
echo "$NOTES"
|
||||
echo 'EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "$NOTES"
|
||||
fi
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=common.sh
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
|
||||
COMMIT_MSG="${1:-${COMMIT_MSG:-}}"
|
||||
|
||||
if [[ -z "$COMMIT_MSG" ]]; then
|
||||
echo "Usage: parse-merge-commit.sh <commit-message>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PR_TITLE=$(extract_pr_title_from_merge_commit "$COMMIT_MSG")
|
||||
|
||||
if [[ -z "$PR_TITLE" ]]; then
|
||||
bump="skip"
|
||||
else
|
||||
bump_type=$(bump_type_from_title "$PR_TITLE")
|
||||
if [[ -n "$bump_type" ]]; then
|
||||
bump="$bump_type"
|
||||
else
|
||||
bump="skip"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||
{
|
||||
echo "pr_title=${PR_TITLE}"
|
||||
echo "bump=${bump}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "pr_title=${PR_TITLE}"
|
||||
echo "bump=${bump}"
|
||||
fi
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Shared test configuration constants."""
|
||||
|
||||
from python_repositories.config import MinioConfig, RedisConfig
|
||||
|
||||
TEST_REDIS_CONFIG = RedisConfig(uri="redis://localhost:6379")
|
||||
TEST_MINIO_CONFIG = MinioConfig(
|
||||
endpoint="localhost:9000",
|
||||
access_key="minioadmin",
|
||||
secret_key="minioadmin",
|
||||
bucket="test-bucket",
|
||||
secure=False,
|
||||
)
|
||||
@@ -1,25 +1,56 @@
|
||||
"""Integration tests configuration."""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from collections.abc import Generator
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
import redis
|
||||
import structlog
|
||||
import logging
|
||||
from minio import Minio
|
||||
|
||||
from testcontainers.redis import RedisContainer
|
||||
from testcontainers.core.container import DockerContainer
|
||||
from testcontainers.core.waiting_utils import WaitStrategy, WaitStrategyTarget
|
||||
from testcontainers.minio import MinioContainer
|
||||
|
||||
from python_repositories.config import MinioConfig, RedisConfig
|
||||
|
||||
REDIS_PORT = 6379
|
||||
|
||||
MINIO_ACCESS_KEY = "minioadmin"
|
||||
MINIO_SECRET_KEY = "minioadmin"
|
||||
MINIO_BUCKET = "test-bucket"
|
||||
|
||||
|
||||
class _RedisPingWaitStrategy(WaitStrategy):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.with_transient_exceptions(redis.exceptions.ConnectionError)
|
||||
|
||||
def wait_until_ready(self, container: WaitStrategyTarget) -> None:
|
||||
redis_container = cast("RedisTestContainer", container)
|
||||
if not self._poll(lambda: redis_container.get_client().ping()):
|
||||
raise redis.exceptions.ConnectionError("Could not connect to Redis")
|
||||
|
||||
|
||||
class RedisTestContainer(DockerContainer):
|
||||
"""Redis container using wait strategies instead of the deprecated decorator."""
|
||||
|
||||
def __init__(self, image: str, port: int = REDIS_PORT) -> None:
|
||||
super().__init__(image, _wait_strategy=_RedisPingWaitStrategy())
|
||||
self.port = port
|
||||
self.with_exposed_ports(self.port)
|
||||
|
||||
def get_client(self, **kwargs: Any) -> redis.Redis:
|
||||
return redis.Redis(
|
||||
host=self.get_container_host_ip(),
|
||||
port=self.get_exposed_port(self.port),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def configure_logging() -> None:
|
||||
"""Configure logging for the test session."""
|
||||
# Configure structlog
|
||||
structlog.configure(
|
||||
processors=[
|
||||
structlog.stdlib.filter_by_level,
|
||||
@@ -32,40 +63,34 @@ def configure_logging() -> None:
|
||||
wrapper_class=structlog.stdlib.BoundLogger,
|
||||
cache_logger_on_first_use=True,
|
||||
)
|
||||
# Set up basic logging configuration
|
||||
logging.basicConfig(level=logging.ERROR)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def redis_container() -> Generator[str]:
|
||||
def redis_container() -> Generator[str, None, None]:
|
||||
"""Set up a Redis container for testing and yield the Redis URI."""
|
||||
# Start container
|
||||
container = RedisContainer(
|
||||
container = RedisTestContainer(
|
||||
image="redis/redis-stack:7.2.0-v0",
|
||||
)
|
||||
container.start()
|
||||
# Set environment variable for Redis URI
|
||||
redis_host = container.get_container_host_ip()
|
||||
redis_port = container.get_exposed_port(6379)
|
||||
redis_port = container.get_exposed_port(REDIS_PORT)
|
||||
redis_uri = f"redis://{redis_host}:{redis_port}"
|
||||
|
||||
yield redis_uri
|
||||
|
||||
# Stop container
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def minio_container() -> Generator[dict[str, str]]:
|
||||
"""Set up a Minio container for testing and yield the Minio URI."""
|
||||
# Start container
|
||||
def minio_container() -> Generator[dict[str, str], None, None]:
|
||||
"""Set up a Minio container for testing and yield connection settings."""
|
||||
container = MinioContainer(
|
||||
image="minio/minio:latest",
|
||||
access_key=MINIO_ACCESS_KEY,
|
||||
secret_key=MINIO_SECRET_KEY,
|
||||
)
|
||||
container.start()
|
||||
# Build environment variables dictionary
|
||||
minio_host = container.get_container_host_ip()
|
||||
minio_port = container.get_exposed_port(9000)
|
||||
minio_endpoint = f"{minio_host}:{minio_port}"
|
||||
@@ -74,38 +99,35 @@ def minio_container() -> Generator[dict[str, str]]:
|
||||
"MINIO_ACCESS_KEY": MINIO_ACCESS_KEY,
|
||||
"MINIO_SECRET_KEY": MINIO_SECRET_KEY,
|
||||
"MINIO_BUCKET": MINIO_BUCKET,
|
||||
"MINIO_SECURE": "false",
|
||||
}
|
||||
|
||||
yield env_vars
|
||||
|
||||
# Stop container
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def set_environment_variables(
|
||||
redis_container: str,
|
||||
minio_container: dict[str, str],
|
||||
) -> Generator[dict[str, str]]:
|
||||
"""Set environment variables needed for tests."""
|
||||
# Build environment variables dictionary
|
||||
env_vars = {"REDIS_URI": redis_container}
|
||||
env_vars.update(minio_container)
|
||||
# Set environment variables
|
||||
for key, value in env_vars.items():
|
||||
os.environ[key] = value
|
||||
|
||||
yield env_vars
|
||||
|
||||
# Cleanup
|
||||
for key in env_vars:
|
||||
_ = os.environ.pop(key, default=None)
|
||||
@pytest.fixture(scope="session")
|
||||
def redis_config(redis_container: str) -> RedisConfig:
|
||||
"""Provide RedisConfig built from the test container."""
|
||||
return RedisConfig(uri=redis_container)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def raw_redis_client(redis_container: str) -> Generator[redis.Redis]:
|
||||
def minio_config(minio_container: dict[str, str]) -> MinioConfig:
|
||||
"""Provide MinioConfig built from the test container."""
|
||||
return MinioConfig(
|
||||
endpoint=minio_container["MINIO_ENDPOINT"],
|
||||
access_key=minio_container["MINIO_ACCESS_KEY"],
|
||||
secret_key=minio_container["MINIO_SECRET_KEY"],
|
||||
bucket=minio_container["MINIO_BUCKET"],
|
||||
secure=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def raw_redis_client(redis_container: str) -> Generator[redis.Redis, None, None]:
|
||||
"""Provide a raw Redis client connected to the test Redis container."""
|
||||
# Connect client
|
||||
client = redis.Redis.from_url(
|
||||
url=redis_container,
|
||||
socket_connect_timeout=10,
|
||||
@@ -113,29 +135,25 @@ def raw_redis_client(redis_container: str) -> Generator[redis.Redis]:
|
||||
|
||||
yield client
|
||||
|
||||
# Cleanup
|
||||
client.flushall()
|
||||
client.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio]:
|
||||
def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio, None, None]:
|
||||
"""Provide a raw Minio client connected to the test Minio container."""
|
||||
# Connect client
|
||||
client = Minio(
|
||||
endpoint=minio_container["MINIO_ENDPOINT"],
|
||||
access_key=minio_container["MINIO_ACCESS_KEY"],
|
||||
secret_key=minio_container["MINIO_SECRET_KEY"],
|
||||
secure=False,
|
||||
)
|
||||
# Ensure bucket exists
|
||||
bucket_name = minio_container["MINIO_BUCKET"]
|
||||
if not client.bucket_exists(bucket_name):
|
||||
client.make_bucket(bucket_name)
|
||||
|
||||
yield client
|
||||
|
||||
# Cleanup
|
||||
objects = client.list_objects(bucket_name, recursive=True)
|
||||
for obj in objects:
|
||||
client.remove_object(bucket_name, obj.object_name)
|
||||
|
||||
@@ -2,29 +2,49 @@
|
||||
|
||||
from collections.abc import Generator
|
||||
from io import BytesIO
|
||||
import os
|
||||
import random
|
||||
|
||||
import pytest
|
||||
from minio import Minio
|
||||
|
||||
from python_repositories.examples.artifact_object_repository import (
|
||||
ArtifactObjectRepository,
|
||||
)
|
||||
from python_repositories.examples.user_json_repository import UserJsonRepository
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def set_example_env(
|
||||
redis_container: str,
|
||||
minio_container: dict[str, str],
|
||||
raw_minio_client: Minio,
|
||||
) -> Generator[None, None, None]:
|
||||
"""Set env vars so example repositories can use from_env() defaults."""
|
||||
_ = raw_minio_client
|
||||
env_vars = {"REDIS_URI": redis_container, **minio_container}
|
||||
for key, value in env_vars.items():
|
||||
os.environ[key] = value
|
||||
yield
|
||||
for key in env_vars:
|
||||
_ = os.environ.pop(key, default=None)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def user_data() -> Generator[dict[str, str]]:
|
||||
def user_data() -> Generator[dict[str, str], None, None]:
|
||||
"""Provide sample user data for tests."""
|
||||
yield {"name": "Alice", "email": "[email protected]"}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def artifact_data() -> Generator[BytesIO]:
|
||||
def artifact_data() -> Generator[BytesIO, None, None]:
|
||||
"""Provide sample artifact data for tests."""
|
||||
yield BytesIO(random.randbytes(2**20))
|
||||
|
||||
|
||||
def test_user_json_repository_save_and_get(
|
||||
redis_container: str,
|
||||
user_data: dict[str, str],
|
||||
) -> None:
|
||||
"""Test that UserJsonRepository can save and retrieve a user."""
|
||||
@@ -34,7 +54,6 @@ def test_user_json_repository_save_and_get(
|
||||
|
||||
|
||||
def test_user_json_repository_delete(
|
||||
redis_container: str,
|
||||
user_data: dict[str, str],
|
||||
) -> None:
|
||||
"""Test that UserJsonRepository can delete a user."""
|
||||
@@ -45,7 +64,6 @@ def test_user_json_repository_delete(
|
||||
|
||||
|
||||
def test_artifact_object_repository_store_and_get(
|
||||
minio_container: dict[str, str],
|
||||
artifact_data: BytesIO,
|
||||
) -> None:
|
||||
"""Test that ArtifactObjectRepository can store and retrieve an artifact."""
|
||||
|
||||
@@ -1,16 +1,21 @@
|
||||
"""Integration tests for the MinioAdapter."""
|
||||
|
||||
from collections.abc import Generator
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
from minio import Minio
|
||||
from io import BytesIO
|
||||
import random
|
||||
import os
|
||||
from dataclasses import replace
|
||||
import logging
|
||||
from minio import S3Error
|
||||
import random
|
||||
from io import BytesIO
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from minio import Minio, S3Error
|
||||
from urllib3.response import BaseHTTPResponse
|
||||
|
||||
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||
from python_repositories.interfaces import ObjectRepositoryInterface
|
||||
from python_repositories.config import MinioConfig
|
||||
from tests.conftest import TEST_MINIO_CONFIG
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def same_data(
|
||||
@@ -18,15 +23,10 @@ def same_data(
|
||||
data_b: BytesIO,
|
||||
) -> bool:
|
||||
"""Check if two BytesIO-objects contain the same data."""
|
||||
assert isinstance(data_a, BytesIO)
|
||||
assert isinstance(data_b, BytesIO)
|
||||
# prepare for being read
|
||||
data_a.seek(0)
|
||||
data_b.seek(0)
|
||||
# convert to bytes
|
||||
data_a_bytes = data_a.read()
|
||||
data_b_bytes = data_b.read()
|
||||
# compare size
|
||||
if len(data_a_bytes) != len(data_b_bytes):
|
||||
logging.error(
|
||||
"data has different length: %s and %s",
|
||||
@@ -34,7 +34,6 @@ def same_data(
|
||||
len(data_b_bytes),
|
||||
)
|
||||
return False
|
||||
# compare content
|
||||
if data_a_bytes != data_b_bytes:
|
||||
logging.error("data has different bytes")
|
||||
return False
|
||||
@@ -42,22 +41,21 @@ def same_data(
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def data() -> Generator[BytesIO]:
|
||||
def data() -> Generator[BytesIO, None, None]:
|
||||
"""Provide a sample data bytes for tests."""
|
||||
# Generate random bytes
|
||||
random_bytes = random.randbytes(2**21) # 2 MiB
|
||||
random_bytes = random.randbytes(2**21)
|
||||
yield BytesIO(random_bytes)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def data_in_minio(
|
||||
raw_minio_client: Minio,
|
||||
minio_config: MinioConfig,
|
||||
data: BytesIO,
|
||||
) -> Generator[tuple[str, BytesIO]]:
|
||||
) -> Generator[tuple[str, BytesIO], None, None]:
|
||||
"""Fixture to set up a known value in Minio before each test."""
|
||||
object_name = "test_object"
|
||||
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
||||
# Upload object
|
||||
bucket_name = minio_config.bucket
|
||||
num_bytes = data.getbuffer().nbytes
|
||||
data.seek(0)
|
||||
raw_minio_client.put_object(
|
||||
@@ -67,19 +65,17 @@ def data_in_minio(
|
||||
length=num_bytes,
|
||||
part_size=MinioAdapter.chunk_size,
|
||||
)
|
||||
# Reset data for reading in tests
|
||||
data.seek(0)
|
||||
|
||||
yield object_name, data
|
||||
|
||||
# Cleanup
|
||||
raw_minio_client.remove_object(bucket_name, object_name)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def minio_adapter() -> Generator[MinioAdapter]:
|
||||
def minio_adapter(minio_config: MinioConfig) -> Generator[MinioAdapter, None, None]:
|
||||
"""Fixture to provide a connected MinioAdapter instance."""
|
||||
adapter = MinioAdapter()
|
||||
adapter = MinioAdapter(config=minio_config)
|
||||
adapter.connect()
|
||||
yield adapter
|
||||
adapter.disconnect()
|
||||
@@ -88,10 +84,10 @@ def minio_adapter() -> Generator[MinioAdapter]:
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def clear_minio(
|
||||
raw_minio_client: Minio,
|
||||
minio_config: MinioConfig,
|
||||
) -> None:
|
||||
"""Fixture to clear all Minio objects before each test."""
|
||||
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
||||
# Clear all objects before each test
|
||||
bucket_name = minio_config.bucket
|
||||
objects = raw_minio_client.list_objects(bucket_name, recursive=True)
|
||||
for obj in objects:
|
||||
if not obj.object_name:
|
||||
@@ -99,24 +95,6 @@ def clear_minio(
|
||||
raw_minio_client.remove_object(bucket_name, obj.object_name)
|
||||
|
||||
|
||||
def test_should_adhere_to_interface() -> None:
|
||||
"""Test that the MinioAdapter adheres to the expected interface."""
|
||||
assert issubclass(MinioAdapter, ObjectRepositoryInterface)
|
||||
_ = MinioAdapter()
|
||||
|
||||
|
||||
def test_should_have_logger_when_instantiated() -> None:
|
||||
"""Test that the MinioAdapter has a logger when instantiated."""
|
||||
adapter = MinioAdapter()
|
||||
assert hasattr(adapter, "logger")
|
||||
|
||||
|
||||
def test_should_not_be_connected_when_instantiated() -> None:
|
||||
"""Test that the MinioAdapter is not connected when instantiated."""
|
||||
adapter = MinioAdapter()
|
||||
assert not adapter.is_connected
|
||||
|
||||
|
||||
def test_should_log_info_when_already_connected(
|
||||
minio_adapter: MinioAdapter,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
@@ -127,52 +105,71 @@ def test_should_log_info_when_already_connected(
|
||||
assert "Already connected to Minio" in caplog.text
|
||||
|
||||
|
||||
def test_should_raise_connection_error_when_unable_to_connect(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
def test_should_raise_connection_error_when_unable_to_connect() -> None:
|
||||
"""Test that the MinioAdapter raises a ConnectionError when unable to connect."""
|
||||
# Arrange
|
||||
monkeypatch.setenv("MINIO_ENDPOINT", "invalid_uri")
|
||||
adapter = MinioAdapter()
|
||||
config = MinioConfig(
|
||||
endpoint="invalid_uri",
|
||||
access_key="minioadmin",
|
||||
secret_key="minioadmin",
|
||||
bucket="test-bucket",
|
||||
)
|
||||
adapter = MinioAdapter(config=config)
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter.connect()
|
||||
assert not adapter.is_connected
|
||||
assert not adapter.is_connected()
|
||||
|
||||
|
||||
def test_should_log_info_when_creating_expected_bucket(
|
||||
def test_connect_raises_when_bucket_missing(
|
||||
raw_minio_client: Minio,
|
||||
minio_config: MinioConfig,
|
||||
) -> None:
|
||||
"""Test that connect fails when the configured bucket is missing."""
|
||||
bucket_name = minio_config.bucket
|
||||
raw_minio_client.remove_bucket(bucket_name)
|
||||
adapter = MinioAdapter(config=minio_config)
|
||||
|
||||
try:
|
||||
with pytest.raises(ConnectionError, match="does not exist"):
|
||||
adapter.connect()
|
||||
assert not adapter.is_connected()
|
||||
finally:
|
||||
raw_minio_client.make_bucket(bucket_name)
|
||||
|
||||
|
||||
def test_connect_creates_bucket_when_create_bucket_if_missing_enabled(
|
||||
raw_minio_client: Minio,
|
||||
minio_config: MinioConfig,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter logs info when creating the expected bucket."""
|
||||
# Arrange
|
||||
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
||||
"""Test that connect can create the configured bucket when enabled."""
|
||||
bucket_name = minio_config.bucket
|
||||
raw_minio_client.remove_bucket(bucket_name)
|
||||
adapter = MinioAdapter()
|
||||
# Act
|
||||
config = replace(minio_config, create_bucket_if_missing=True)
|
||||
adapter = MinioAdapter(config=config)
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
adapter.connect()
|
||||
# Assert
|
||||
|
||||
assert f"Creating bucket '{bucket_name}'" in caplog.text
|
||||
|
||||
|
||||
def test_should_log_error_on_exception_during_exit(
|
||||
minio_container: dict[str, str],
|
||||
minio_config: MinioConfig,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter logs an error if an exception occurs during __exit__."""
|
||||
try:
|
||||
with MinioAdapter() as adapter:
|
||||
assert adapter.is_connected
|
||||
with MinioAdapter(config=minio_config) as adapter:
|
||||
assert adapter.is_connected()
|
||||
raise ValueError("Simulated error")
|
||||
except ValueError:
|
||||
pass # Expected
|
||||
# Assert error was logged
|
||||
pass
|
||||
assert "Error while exiting context" in caplog.text
|
||||
|
||||
|
||||
def test_should_have_context_manager() -> None:
|
||||
def test_should_have_context_manager(minio_config: MinioConfig) -> None:
|
||||
"""Test that the MinioAdapter can be used as a context manager."""
|
||||
with MinioAdapter() as adapter:
|
||||
with MinioAdapter(config=minio_config) as adapter:
|
||||
assert adapter._client is not None
|
||||
assert adapter._client is None
|
||||
|
||||
@@ -182,11 +179,8 @@ def test_should_get_data(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter can get data from a bucket."""
|
||||
# Arrange
|
||||
object_name, expected_data = data_in_minio
|
||||
# Act
|
||||
received_data = minio_adapter.get(object_name)
|
||||
# Assert
|
||||
assert received_data is not None
|
||||
assert same_data(expected_data, received_data)
|
||||
|
||||
@@ -195,11 +189,7 @@ def test_should_get_none_for_nonexistent_object(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter returns None for a nonexistent object."""
|
||||
# Arrange
|
||||
object_name = "nonexistent_object"
|
||||
# Act
|
||||
received_data = minio_adapter.get(object_name)
|
||||
# Assert
|
||||
received_data = minio_adapter.get("nonexistent_object")
|
||||
assert received_data is None
|
||||
|
||||
|
||||
@@ -207,21 +197,17 @@ def test_should_raise_value_error_on_invalid_get_object_name(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter raises ValueError when getting with an invalid object name."""
|
||||
# Arrange
|
||||
invalid_object_names = ["", 123, None]
|
||||
# Act & Assert
|
||||
for object_name in invalid_object_names:
|
||||
with pytest.raises(ValueError):
|
||||
minio_adapter.get(object_name) # type: ignore
|
||||
|
||||
|
||||
def test_should_raise_connection_error_on_get_when_not_connected(
|
||||
minio_adapter: MinioAdapter,
|
||||
minio_config: MinioConfig,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter raises ConnectionError when getting while not connected."""
|
||||
# Arrange
|
||||
adapter = MinioAdapter() # not connected
|
||||
# Act & Assert
|
||||
adapter = MinioAdapter(config=minio_config)
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter.get("some_object")
|
||||
|
||||
@@ -230,25 +216,22 @@ def test_should_log_warning_when_getting_nonexistent_object(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter logs a warning when getting a nonexistent object."""
|
||||
# Arrange
|
||||
adapter = MinioAdapter()
|
||||
adapter._client = MagicMock(spec=Minio)
|
||||
adapter._client.get_object.side_effect = S3Error(
|
||||
code="NoSuchKey",
|
||||
message="",
|
||||
resource="",
|
||||
request_id="",
|
||||
host_id="",
|
||||
response="",
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
mock_client.get_object.side_effect = S3Error(
|
||||
MagicMock(spec=BaseHTTPResponse),
|
||||
"NoSuchKey",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
bucket_name="test-bucket",
|
||||
object_name="missing-object",
|
||||
)
|
||||
adapter._bucket_name = "test-bucket"
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
object_name = "missing-object"
|
||||
# Act
|
||||
with caplog.at_level("WARNING"):
|
||||
result = adapter.get(object_name)
|
||||
# Assert
|
||||
assert result is None
|
||||
assert (
|
||||
f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'"
|
||||
@@ -256,71 +239,52 @@ def test_should_log_warning_when_getting_nonexistent_object(
|
||||
)
|
||||
|
||||
|
||||
def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter logs an error when getting a nonexistent object."""
|
||||
# Arrange
|
||||
adapter = MinioAdapter()
|
||||
adapter._client = MagicMock(spec=Minio)
|
||||
def test_should_reraise_s3error_other_than_no_such_key() -> None:
|
||||
"""Test that the MinioAdapter re-raises unhandled S3 errors."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
other_s3error = S3Error(
|
||||
code="UnhandledError",
|
||||
message="",
|
||||
resource="",
|
||||
request_id="",
|
||||
host_id="",
|
||||
response="",
|
||||
MagicMock(spec=BaseHTTPResponse),
|
||||
"UnhandledError",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
bucket_name="test-bucket",
|
||||
object_name="missing-object",
|
||||
)
|
||||
adapter._client.get_object.side_effect = other_s3error
|
||||
adapter._bucket_name = "test-bucket"
|
||||
object_name = "missing-object"
|
||||
# Act
|
||||
with caplog.at_level("ERROR"):
|
||||
result = adapter.get(object_name)
|
||||
# Assert
|
||||
assert result is None
|
||||
assert repr(other_s3error) in caplog.text
|
||||
mock_client.get_object.side_effect = other_s3error
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
with pytest.raises(S3Error) as exc_info:
|
||||
adapter.get("missing-object")
|
||||
assert exc_info.value.code == "UnhandledError"
|
||||
|
||||
|
||||
def test_should_log_error_when_getting_with_general_exception(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter logs an error when getting a nonexistent object."""
|
||||
# Arrange
|
||||
adapter = MinioAdapter()
|
||||
adapter._client = MagicMock(spec=Minio)
|
||||
def test_should_reraise_general_exception() -> None:
|
||||
"""Test that the MinioAdapter re-raises general exceptions during get."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
general_exception = Exception("General failure")
|
||||
adapter._client.get_object.side_effect = general_exception
|
||||
adapter._bucket_name = "test-bucket"
|
||||
object_name = "missing-object"
|
||||
# Act
|
||||
with caplog.at_level("ERROR"):
|
||||
result = adapter.get(object_name)
|
||||
# Assert
|
||||
assert result is None
|
||||
assert repr(general_exception) in caplog.text
|
||||
mock_client.get_object.side_effect = general_exception
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
with pytest.raises(Exception, match="General failure"):
|
||||
adapter.get("missing-object")
|
||||
|
||||
|
||||
def test_should_put_data(
|
||||
data: BytesIO,
|
||||
minio_adapter: MinioAdapter,
|
||||
minio_config: MinioConfig,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter can put data into a bucket."""
|
||||
# Arrange
|
||||
object_name = "new_test_object"
|
||||
received_data = minio_adapter.get(object_name)
|
||||
assert received_data is None # ensure object does not exist yet
|
||||
# Act
|
||||
assert received_data is None
|
||||
minio_adapter.put(object_name, data)
|
||||
# Assert
|
||||
received_data = minio_adapter.get(object_name)
|
||||
assert received_data is not None
|
||||
assert same_data(data, received_data)
|
||||
# Cleanup
|
||||
bucket_name = str(os.getenv("MINIO_BUCKET"))
|
||||
minio_adapter._client.remove_object(bucket_name, object_name) # type: ignore
|
||||
minio_adapter._client.remove_object(minio_config.bucket, object_name) # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_should_update_data(
|
||||
@@ -328,15 +292,12 @@ def test_should_update_data(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter can update data in a bucket."""
|
||||
# Arrange
|
||||
object_name, _ = data_in_minio
|
||||
new_data = BytesIO(random.randbytes(2**21)) # 2 MiB
|
||||
new_data = BytesIO(random.randbytes(2**21))
|
||||
received_data = minio_adapter.get(object_name)
|
||||
assert received_data is not None
|
||||
assert not same_data(received_data, new_data)
|
||||
# Act
|
||||
minio_adapter.put(object_name, new_data)
|
||||
# Assert
|
||||
received_data = minio_adapter.get(object_name)
|
||||
assert received_data is not None
|
||||
assert same_data(new_data, received_data)
|
||||
@@ -347,9 +308,7 @@ def test_should_raise_value_error_on_invalid_put_object_name(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter raises ValueError when putting with an invalid object name."""
|
||||
# Arrange
|
||||
invalid_object_names = ["", 123, None]
|
||||
# Act & Assert
|
||||
for object_name in invalid_object_names:
|
||||
with pytest.raises(ValueError):
|
||||
minio_adapter.put(object_name, data) # type: ignore
|
||||
@@ -359,13 +318,11 @@ def test_should_raise_value_error_on_invalid_put_data(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter raises ValueError when putting with invalid data."""
|
||||
# Arrange
|
||||
object_name = "valid_object_name"
|
||||
invalid_data = ["not_bytesio", 123, None]
|
||||
# Act & Assert
|
||||
for data in invalid_data:
|
||||
for invalid in invalid_data:
|
||||
with pytest.raises(ValueError):
|
||||
minio_adapter.put(object_name, data) # type: ignore
|
||||
minio_adapter.put(object_name, invalid) # type: ignore
|
||||
|
||||
|
||||
def test_should_raise_value_error_on_invalid_put_content_type(
|
||||
@@ -373,23 +330,19 @@ def test_should_raise_value_error_on_invalid_put_content_type(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter raises ValueError when putting with an invalid content type."""
|
||||
# Arrange
|
||||
object_name = "valid_object_name"
|
||||
invalid_content_types = ["", 123, None]
|
||||
# Act & Assert
|
||||
for content_type in invalid_content_types:
|
||||
with pytest.raises(ValueError):
|
||||
minio_adapter.put(object_name, data, content_type) # type: ignore
|
||||
|
||||
|
||||
def test_should_raise_connection_error_on_put_when_not_connected(
|
||||
minio_config: MinioConfig,
|
||||
data: BytesIO,
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter raises ConnectionError when putting while not connected."""
|
||||
# Arrange
|
||||
adapter = MinioAdapter() # not connected
|
||||
# Act & Assert
|
||||
adapter = MinioAdapter(config=minio_config)
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter.put("some_object", data)
|
||||
|
||||
@@ -399,36 +352,28 @@ def test_should_delete_object(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter can delete an object from a bucket."""
|
||||
# Arrange
|
||||
object_name, _ = data_in_minio
|
||||
received_data = minio_adapter.get(object_name)
|
||||
assert received_data is not None # ensure object exists
|
||||
# Act
|
||||
assert received_data is not None
|
||||
minio_adapter.delete(object_name)
|
||||
# Assert
|
||||
received_data = minio_adapter.get(object_name)
|
||||
assert received_data is None
|
||||
assert minio_adapter.get(object_name) is None
|
||||
|
||||
|
||||
def test_should_raise_value_error_on_invalid_delete_object_name(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter raises ValueError when deleting with an invalid object name."""
|
||||
# Arrange
|
||||
invalid_object_names = ["", 123, None]
|
||||
# Act & Assert
|
||||
for object_name in invalid_object_names:
|
||||
with pytest.raises(ValueError):
|
||||
minio_adapter.delete(object_name) # type: ignore
|
||||
|
||||
|
||||
def test_should_raise_connection_error_on_delete_when_not_connected(
|
||||
minio_adapter: MinioAdapter,
|
||||
minio_config: MinioConfig,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter raises ConnectionError when deleting while not connected."""
|
||||
# Arrange
|
||||
adapter = MinioAdapter() # not connected
|
||||
# Act & Assert
|
||||
adapter = MinioAdapter(config=minio_config)
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter.delete("some_object")
|
||||
|
||||
@@ -438,14 +383,11 @@ def test_should_list_objects(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter can list objects in a bucket."""
|
||||
# Arrange
|
||||
object_name, _ = data_in_minio
|
||||
new_data = BytesIO(random.randbytes(2**21)) # 2 MiB
|
||||
new_data = BytesIO(random.randbytes(2**21))
|
||||
new_data_name = "another_test_object"
|
||||
minio_adapter.put(new_data_name, new_data)
|
||||
# Act
|
||||
objects = minio_adapter.list_objects()
|
||||
# Assert
|
||||
assert isinstance(objects, list)
|
||||
assert len(objects) == 2
|
||||
assert object_name in objects
|
||||
@@ -457,15 +399,12 @@ def test_should_list_objects_with_prefix(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter can list objects in a bucket with a prefix."""
|
||||
# Arrange
|
||||
object_name, _ = data_in_minio
|
||||
new_data = BytesIO(random.randbytes(2**21)) # 2 MiB
|
||||
new_data = BytesIO(random.randbytes(2**21))
|
||||
new_data_name = "prefix_test_object"
|
||||
minio_adapter.put(new_data_name, new_data)
|
||||
prefix = "prefix_"
|
||||
# Act
|
||||
objects = minio_adapter.list_objects(prefix)
|
||||
# Assert
|
||||
assert isinstance(objects, list)
|
||||
assert len(objects) == 1
|
||||
assert new_data_name in objects
|
||||
@@ -476,25 +415,20 @@ def test_should_raise_value_error_on_invalid_list_objects_prefix(
|
||||
minio_adapter: MinioAdapter,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter raises ValueError when listing with an invalid prefix."""
|
||||
# Arrange
|
||||
invalid_prefixes = [123, None]
|
||||
# Act & Assert
|
||||
for prefix in invalid_prefixes:
|
||||
with pytest.raises(ValueError):
|
||||
minio_adapter.list_objects(prefix) # type: ignore
|
||||
|
||||
|
||||
def test_should_raise_connection_error_on_list_objects_when_not_connected(
|
||||
minio_adapter: MinioAdapter,
|
||||
minio_config: MinioConfig,
|
||||
) -> None:
|
||||
"""Test that the MinioAdapter raises ConnectionError when listing while not connected."""
|
||||
# Arrange
|
||||
adapter = MinioAdapter() # not connected
|
||||
# Act & Assert
|
||||
adapter = MinioAdapter(config=minio_config)
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter.list_objects()
|
||||
|
||||
|
||||
# allows local debugging by running file as script
|
||||
if __name__ == "__main__":
|
||||
pytest.main(["-s", "-v", __file__])
|
||||
|
||||
@@ -1,15 +1,20 @@
|
||||
"""Integration tests for the RedisAdapter."""
|
||||
|
||||
from collections.abc import Generator
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
import redis
|
||||
from redis.commands.json.path import Path as RedisPath
|
||||
|
||||
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||
from python_repositories.interfaces import JsonRepositoryInterface
|
||||
from python_repositories.config import RedisConfig
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def data() -> Generator[dict[str, str]]:
|
||||
def data() -> Generator[dict[str, str], None, None]:
|
||||
"""Provide a sample data dictionary for tests."""
|
||||
yield {"foo": "bar"}
|
||||
|
||||
@@ -18,7 +23,7 @@ def data() -> Generator[dict[str, str]]:
|
||||
def data_in_redis(
|
||||
raw_redis_client: redis.Redis,
|
||||
data: dict[str, str],
|
||||
) -> Generator[tuple[str, dict[str, str]]]:
|
||||
) -> Generator[tuple[str, dict[str, str]], None, None]:
|
||||
"""Fixture to set up a known value in Redis before each test."""
|
||||
key = "test_key"
|
||||
path = RedisPath.root_path()
|
||||
@@ -26,14 +31,13 @@ def data_in_redis(
|
||||
|
||||
yield key, data
|
||||
|
||||
# Cleanup
|
||||
raw_redis_client.delete(key)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def redis_adapter(redis_container: str) -> Generator[RedisAdapter]:
|
||||
def redis_adapter(redis_config: RedisConfig) -> Generator[RedisAdapter, None, None]:
|
||||
"""Fixture to provide a connected RedisAdapter instance."""
|
||||
adapter = RedisAdapter()
|
||||
adapter = RedisAdapter(config=redis_config)
|
||||
adapter.connect()
|
||||
yield adapter
|
||||
adapter.disconnect()
|
||||
@@ -42,84 +46,63 @@ def redis_adapter(redis_container: str) -> Generator[RedisAdapter]:
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def clear_redis(raw_redis_client: redis.Redis) -> None:
|
||||
"""Fixture to clear all Redis keys before each test."""
|
||||
# Clear all keys before each test
|
||||
raw_redis_client.flushall()
|
||||
|
||||
|
||||
def test_should_adhere_to_interface(redis_container: str) -> None:
|
||||
"""Test that the RedisAdapter adheres to the expected interface."""
|
||||
assert issubclass(RedisAdapter, JsonRepositoryInterface)
|
||||
_ = RedisAdapter()
|
||||
|
||||
|
||||
def test_should_have_logger_when_instantiated(redis_container: str) -> None:
|
||||
"""Test that the RedisAdapter has a logger when instantiated."""
|
||||
adapter = RedisAdapter()
|
||||
assert hasattr(adapter, "logger")
|
||||
assert adapter.logger is not None
|
||||
|
||||
|
||||
def test_should_not_be_connected_when_instantiated(redis_container: str) -> None:
|
||||
"""Test that the RedisAdapter is not connected when instantiated."""
|
||||
adapter = RedisAdapter()
|
||||
assert adapter._client is None
|
||||
assert not adapter.is_connected
|
||||
|
||||
|
||||
def test_should_raise_connection_error_when_unable_to_connect(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
def test_should_log_info_when_already_connected(
|
||||
redis_adapter: RedisAdapter,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter logs info when connect is called while already connected."""
|
||||
with caplog.at_level(logging.INFO):
|
||||
redis_adapter.connect()
|
||||
assert "Already connected to Redis" in caplog.text
|
||||
|
||||
|
||||
def test_should_raise_connection_error_when_unable_to_connect() -> None:
|
||||
"""Test that the RedisAdapter raises ConnectionError when unable to connect."""
|
||||
# Arrange
|
||||
monkeypatch.setenv("REDIS_URI", "redis://invalid:6379")
|
||||
adapter = RedisAdapter()
|
||||
# Act & Assert
|
||||
adapter = RedisAdapter(config=RedisConfig(uri="redis://invalid:6379"))
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter.connect()
|
||||
assert adapter._client is None
|
||||
assert not adapter.is_connected
|
||||
assert not adapter.is_connected()
|
||||
|
||||
|
||||
def test_connect_raises_connection_error_when_unable_to_ping(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ConnectionError when ping fails."""
|
||||
# Set an invalid URI
|
||||
monkeypatch.setenv("REDIS_URI", "redis://invalid:6379")
|
||||
|
||||
# Monkeypatch redis.Redis.from_url to return a mock client
|
||||
class MockRedis:
|
||||
"""A mock Redis client that simulates a failed ping."""
|
||||
|
||||
def ping(self) -> bool:
|
||||
"""Simulate a failed ping."""
|
||||
return False # Simulate failed ping
|
||||
return False
|
||||
|
||||
monkeypatch.setattr("redis.Redis.from_url", lambda *a, **kw: MockRedis())
|
||||
|
||||
adapter = RedisAdapter()
|
||||
adapter = RedisAdapter(config=RedisConfig(uri="redis://invalid:6379"))
|
||||
with pytest.raises(ConnectionError, match="Could not connect to Redis"):
|
||||
adapter.connect()
|
||||
|
||||
|
||||
def test_should_log_error_on_exception_during_exit(
|
||||
redis_container: str,
|
||||
redis_config: RedisConfig,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter logs an error when an exception occurs during context exit."""
|
||||
try:
|
||||
with RedisAdapter() as adapter:
|
||||
assert adapter.is_connected
|
||||
with RedisAdapter(config=redis_config) as adapter:
|
||||
assert adapter.is_connected()
|
||||
raise ValueError("Simulated error")
|
||||
except ValueError:
|
||||
pass # Expected
|
||||
# Assert error was logged
|
||||
pass
|
||||
assert "Error while exiting context" in caplog.text
|
||||
|
||||
|
||||
def test_should_have_context_manager(redis_container: str) -> None:
|
||||
def test_should_have_context_manager(redis_config: RedisConfig) -> None:
|
||||
"""Test that the RedisAdapter can be used as a context manager."""
|
||||
with RedisAdapter() as adapter:
|
||||
with RedisAdapter(config=redis_config) as adapter:
|
||||
assert adapter._client is not None
|
||||
assert adapter._client is None
|
||||
|
||||
@@ -129,11 +112,8 @@ def test_should_get_value(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter can get a value."""
|
||||
# Arrange
|
||||
key, data = data_in_redis
|
||||
# Act
|
||||
value = redis_adapter.get(key)
|
||||
# Assert
|
||||
assert value is not None
|
||||
assert value == data
|
||||
|
||||
@@ -142,9 +122,7 @@ def test_should_get_none_for_missing_key(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that getting a non-existent key returns None."""
|
||||
# Act
|
||||
value = redis_adapter.get("nonexistent_key")
|
||||
# Assert
|
||||
assert value is None
|
||||
|
||||
|
||||
@@ -152,21 +130,17 @@ def test_should_raise_value_error_on_invalid_get_key(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ValueError when getting with an invalid key."""
|
||||
# Arrange
|
||||
invalid_keys = ["", 123, None]
|
||||
# Act & Assert
|
||||
for key in invalid_keys:
|
||||
with pytest.raises(ValueError):
|
||||
redis_adapter.get(key) # type: ignore
|
||||
|
||||
|
||||
def test_should_raise_connection_error_on_get_when_not_connected(
|
||||
redis_adapter: RedisAdapter,
|
||||
redis_config: RedisConfig,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ConnectionError when getting while not connected."""
|
||||
# Arrange
|
||||
adapter = RedisAdapter() # not connected
|
||||
# Act & Assert
|
||||
adapter = RedisAdapter(config=redis_config)
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter.get("some_key")
|
||||
|
||||
@@ -176,13 +150,10 @@ def test_should_set_value(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter can set a value."""
|
||||
# Arrange
|
||||
key = "test_key"
|
||||
received_data = redis_adapter.get(key)
|
||||
assert received_data is None # Ensure key does not exist
|
||||
# Act
|
||||
assert received_data is None
|
||||
redis_adapter.set(key, data)
|
||||
# Assert
|
||||
received_data = redis_adapter.get(key)
|
||||
assert received_data is not None
|
||||
assert received_data == data
|
||||
@@ -193,15 +164,12 @@ def test_should_update_value(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter can update an existing value."""
|
||||
# Arrange
|
||||
key, _ = data_in_redis
|
||||
new_data = {"new_key": "new_value"}
|
||||
received_data = redis_adapter.get(key)
|
||||
assert received_data is not None
|
||||
assert received_data != new_data
|
||||
# Act
|
||||
redis_adapter.set(key, new_data)
|
||||
# Assert
|
||||
assert redis_adapter.get(key) == new_data
|
||||
|
||||
|
||||
@@ -210,9 +178,7 @@ def test_should_raise_value_error_on_invalid_set_key(
|
||||
data: dict[str, str],
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ValueError when setting with an invalid key."""
|
||||
# Arrange
|
||||
invalid_keys = ["", 123, None]
|
||||
# Act & Assert
|
||||
for key in invalid_keys:
|
||||
with pytest.raises(ValueError):
|
||||
redis_adapter.set(key, data) # type: ignore
|
||||
@@ -222,26 +188,21 @@ def test_should_raise_value_error_on_invalid_set_data(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ValueError when setting with invalid data."""
|
||||
# Arrange
|
||||
key = "test_key"
|
||||
invalid_data = ["", 123, None, [], {}]
|
||||
# Act & Assert
|
||||
for data in invalid_data:
|
||||
with pytest.raises(ValueError):
|
||||
redis_adapter.set(key, data) # type: ignore
|
||||
|
||||
|
||||
def test_should_raise_connection_error_on_set_when_not_connected(
|
||||
redis_adapter: RedisAdapter,
|
||||
redis_config: RedisConfig,
|
||||
data: dict[str, str],
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ConnectionError when setting while not connected."""
|
||||
# Arrange
|
||||
adapter = RedisAdapter()
|
||||
key = "test_key"
|
||||
# Act & Assert
|
||||
adapter = RedisAdapter(config=redis_config)
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter.set(key, data)
|
||||
adapter.set("test_key", data)
|
||||
|
||||
|
||||
def test_should_delete_key(
|
||||
@@ -249,13 +210,10 @@ def test_should_delete_key(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that deleting a key removes it from Redis."""
|
||||
# Arrange
|
||||
key, _ = data_in_redis
|
||||
received_data = redis_adapter.get(key)
|
||||
assert received_data is not None # Ensure key exists
|
||||
# Act
|
||||
assert received_data is not None
|
||||
redis_adapter.delete(key)
|
||||
# Assert
|
||||
assert redis_adapter.get(key) is None
|
||||
|
||||
|
||||
@@ -263,21 +221,17 @@ def test_should_raise_value_error_on_invalid_delete_key(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ValueError when deleting with an invalid key."""
|
||||
# Arrange
|
||||
invalid_keys = ["", 123, None]
|
||||
# Act & Assert
|
||||
for key in invalid_keys:
|
||||
with pytest.raises(ValueError):
|
||||
redis_adapter.delete(key) # type: ignore
|
||||
|
||||
|
||||
def test_should_raise_connection_error_on_delete_when_not_connected(
|
||||
redis_adapter: RedisAdapter,
|
||||
redis_config: RedisConfig,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ConnectionError when deleting while not connected."""
|
||||
# Arrange
|
||||
adapter = RedisAdapter()
|
||||
# Act & Assert
|
||||
adapter = RedisAdapter(config=redis_config)
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter.delete("some_key")
|
||||
|
||||
@@ -286,12 +240,9 @@ def test_should_list_keys(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test listing keys matching a pattern returns correct keys."""
|
||||
# Arrange
|
||||
redis_adapter.set("key1", {"a": 1})
|
||||
redis_adapter.set("key2", {"b": 2})
|
||||
# Act
|
||||
keys = redis_adapter.list_keys("key*")
|
||||
# Assert
|
||||
assert set(keys) == {"key1", "key2"}
|
||||
|
||||
|
||||
@@ -299,25 +250,49 @@ def test_should_raise_value_error_on_invalid_list_keys_pattern(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ValueError when listing keys with an invalid pattern."""
|
||||
# Arrange
|
||||
invalid_patterns = ["", 123, None]
|
||||
# Act & Assert
|
||||
for pattern in invalid_patterns:
|
||||
with pytest.raises(ValueError):
|
||||
redis_adapter.list_keys(pattern) # type: ignore
|
||||
|
||||
|
||||
def test_should_raise_connection_error_on_list_keys_when_not_connected(
|
||||
redis_adapter: RedisAdapter,
|
||||
redis_config: RedisConfig,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ConnectionError when listing keys while not connected."""
|
||||
# Arrange
|
||||
adapter = RedisAdapter()
|
||||
# Act & Assert
|
||||
adapter = RedisAdapter(config=redis_config)
|
||||
with pytest.raises(ConnectionError):
|
||||
adapter.list_keys("some_pattern")
|
||||
|
||||
|
||||
# allows local debugging by running file as script
|
||||
def test_should_scan_keys(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test scanning keys matching a pattern returns correct keys."""
|
||||
redis_adapter.set("key1", {"a": 1})
|
||||
redis_adapter.set("key2", {"b": 2})
|
||||
keys = set(redis_adapter.scan_keys("key*"))
|
||||
assert keys == {"key1", "key2"}
|
||||
|
||||
|
||||
def test_should_raise_value_error_on_invalid_scan_keys_pattern(
|
||||
redis_adapter: RedisAdapter,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ValueError when scanning keys with an invalid pattern."""
|
||||
invalid_patterns = ["", 123, None]
|
||||
for pattern in invalid_patterns:
|
||||
with pytest.raises(ValueError):
|
||||
list(redis_adapter.scan_keys(pattern)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_should_raise_connection_error_on_scan_keys_when_not_connected(
|
||||
redis_config: RedisConfig,
|
||||
) -> None:
|
||||
"""Test that the RedisAdapter raises ConnectionError when scanning keys while not connected."""
|
||||
adapter = RedisAdapter(config=redis_config)
|
||||
with pytest.raises(ConnectionError):
|
||||
list(adapter.scan_keys("some_pattern"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main(["-s", "-v", __file__])
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Unit tests for lazy adapter loading in adapters subpackage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import builtins
|
||||
import importlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import python_repositories
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _block_backend_import(blocked_prefix: str) -> Callable[..., ModuleType]:
|
||||
real_import = builtins.__import__
|
||||
|
||||
def fake_import(
|
||||
name: str,
|
||||
globals: Mapping[str, object] | None = None,
|
||||
locals: Mapping[str, object] | None = None,
|
||||
fromlist: Sequence[str] = (),
|
||||
level: int = 0,
|
||||
) -> ModuleType:
|
||||
if name == blocked_prefix or name.startswith(f"{blocked_prefix}."):
|
||||
raise ImportError(f"No module named '{name}'")
|
||||
return real_import(name, globals, locals, fromlist, level)
|
||||
|
||||
return fake_import
|
||||
|
||||
|
||||
def test_base_import_does_not_load_adapters() -> None:
|
||||
"""Base package import does not eagerly load backend adapter modules."""
|
||||
script = """
|
||||
import sys
|
||||
from python_repositories import JsonRepositoryInterface
|
||||
|
||||
assert JsonRepositoryInterface is not None
|
||||
assert "python_repositories.adapters.redis_adapter" not in sys.modules
|
||||
assert "python_repositories.adapters.minio_adapter" not in sys.modules
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=_ROOT,
|
||||
env={**os.environ, "PYTHONPATH": str(_ROOT)},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
|
||||
def test_lazy_adapter_load_succeeds_when_extra_present() -> None:
|
||||
"""Adapters load when their optional dependencies are installed."""
|
||||
from python_repositories import MinioAdapter, RedisAdapter
|
||||
|
||||
assert RedisAdapter.__name__ == "RedisAdapter"
|
||||
assert MinioAdapter.__name__ == "MinioAdapter"
|
||||
|
||||
|
||||
def test_redis_adapter_import_error_without_extra() -> None:
|
||||
"""Missing redis extra raises ImportError with install hint."""
|
||||
import python_repositories.adapters.redis_adapter as redis_adapter_module
|
||||
|
||||
with patch.object(builtins, "__import__", new=_block_backend_import("redis")):
|
||||
with pytest.raises(ImportError, match=r"python-repositories\[redis\]"):
|
||||
importlib.reload(redis_adapter_module)
|
||||
|
||||
importlib.reload(redis_adapter_module)
|
||||
|
||||
|
||||
def test_minio_adapter_import_error_without_extra() -> None:
|
||||
"""Missing minio extra raises ImportError with install hint."""
|
||||
import python_repositories.adapters.minio_adapter as minio_adapter_module
|
||||
|
||||
with patch.object(builtins, "__import__", new=_block_backend_import("minio")):
|
||||
with pytest.raises(ImportError, match=r"python-repositories\[minio\]"):
|
||||
importlib.reload(minio_adapter_module)
|
||||
|
||||
importlib.reload(minio_adapter_module)
|
||||
|
||||
|
||||
def test_top_level_lazy_import_propagates_redis_import_error() -> None:
|
||||
"""Top-level RedisAdapter access surfaces adapter import errors."""
|
||||
with patch(
|
||||
"importlib.import_module",
|
||||
side_effect=ImportError(
|
||||
"Redis support requires the redis extra. "
|
||||
"Install with: pip install python-repositories[redis]"
|
||||
),
|
||||
):
|
||||
with pytest.raises(ImportError, match=r"python-repositories\[redis\]"):
|
||||
_ = python_repositories.RedisAdapter
|
||||
|
||||
|
||||
def test_top_level_lazy_import_propagates_minio_import_error() -> None:
|
||||
"""Top-level MinioAdapter access surfaces adapter import errors."""
|
||||
with patch(
|
||||
"importlib.import_module",
|
||||
side_effect=ImportError(
|
||||
"MinIO support requires the minio extra. "
|
||||
"Install with: pip install python-repositories[minio]"
|
||||
),
|
||||
):
|
||||
with pytest.raises(ImportError, match=r"python-repositories\[minio\]"):
|
||||
_ = python_repositories.MinioAdapter
|
||||
|
||||
|
||||
def test_adapters_subpackage_lazy_import_succeeds() -> None:
|
||||
"""Adapter subpackage imports delegate to the same lazy loader."""
|
||||
from python_repositories.adapters import RedisAdapter
|
||||
|
||||
assert RedisAdapter.__name__ == "RedisAdapter"
|
||||
|
||||
|
||||
def test_adapters_dir_exposes_lazy_exports() -> None:
|
||||
"""dir(adapters) includes lazy adapter names for tab completion."""
|
||||
import python_repositories.adapters as adapters
|
||||
|
||||
assert {"RedisAdapter", "MinioAdapter"}.issubset(set(dir(adapters)))
|
||||
|
||||
|
||||
def test_adapters_getattr_raises_for_unknown() -> None:
|
||||
"""Unknown adapter names raise AttributeError."""
|
||||
import python_repositories.adapters as adapters
|
||||
|
||||
with pytest.raises(AttributeError, match="has no attribute 'NoSuchAdapter'"):
|
||||
_ = adapters.NoSuchAdapter
|
||||
|
||||
|
||||
def test_top_level_dir_exposes_lazy_exports() -> None:
|
||||
"""dir(python_repositories) includes lazy adapter names for tab completion."""
|
||||
assert "RedisAdapter" in dir(python_repositories)
|
||||
assert "MinioAdapter" in dir(python_repositories)
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Unit test fixtures for mocked adapters."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import redis
|
||||
from minio import Minio
|
||||
|
||||
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||
from tests.conftest import TEST_MINIO_CONFIG, TEST_REDIS_CONFIG
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def redis_adapter() -> RedisAdapter:
|
||||
"""Provide a RedisAdapter with an injected mock client."""
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
return RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def minio_adapter() -> MinioAdapter:
|
||||
"""Provide a MinioAdapter with an injected mock client."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
return MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Unit tests for TTL-cached connection health checks on ConnectionAwareAdapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import redis
|
||||
|
||||
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||
from tests.conftest import TEST_MINIO_CONFIG, TEST_REDIS_CONFIG
|
||||
|
||||
|
||||
class TestRedisConnectionHealth:
|
||||
def test_not_connected_when_no_client(self) -> None:
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||
assert not adapter.is_connected()
|
||||
|
||||
def test_connected_when_probe_succeeds(self, redis_adapter: RedisAdapter) -> None:
|
||||
mock_client = cast(MagicMock, redis_adapter._client)
|
||||
mock_client.ping.return_value = True
|
||||
|
||||
assert redis_adapter.is_connected()
|
||||
mock_client.ping.assert_called_once()
|
||||
|
||||
def test_stale_connection_when_probe_fails(
|
||||
self, redis_adapter: RedisAdapter
|
||||
) -> None:
|
||||
cast(MagicMock, redis_adapter._client).ping.side_effect = redis.ConnectionError(
|
||||
"connection lost"
|
||||
)
|
||||
|
||||
assert not redis_adapter.is_connected()
|
||||
|
||||
def test_cache_hit_avoids_second_probe(self, redis_adapter: RedisAdapter) -> None:
|
||||
mock_client = cast(MagicMock, redis_adapter._client)
|
||||
mock_client.ping.return_value = True
|
||||
|
||||
with patch(
|
||||
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||
return_value=100.0,
|
||||
):
|
||||
assert redis_adapter.is_connected()
|
||||
assert redis_adapter.is_connected()
|
||||
|
||||
mock_client.ping.assert_called_once()
|
||||
|
||||
def test_cache_miss_runs_probe_again(self, redis_adapter: RedisAdapter) -> None:
|
||||
mock_client = cast(MagicMock, redis_adapter._client)
|
||||
mock_client.ping.return_value = True
|
||||
|
||||
with patch(
|
||||
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||
side_effect=[100.0, 102.0],
|
||||
):
|
||||
assert redis_adapter.is_connected()
|
||||
assert redis_adapter.is_connected()
|
||||
|
||||
assert mock_client.ping.call_count == 2
|
||||
|
||||
def test_disconnect_clears_cache(self, redis_adapter: RedisAdapter) -> None:
|
||||
mock_client = cast(MagicMock, redis_adapter._client)
|
||||
mock_client.ping.return_value = True
|
||||
|
||||
with patch(
|
||||
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||
return_value=100.0,
|
||||
):
|
||||
assert redis_adapter.is_connected()
|
||||
|
||||
redis_adapter.disconnect()
|
||||
reinjected = RedisAdapter(
|
||||
config=redis_adapter._config,
|
||||
client=mock_client,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||
return_value=100.0,
|
||||
):
|
||||
assert reinjected.is_connected()
|
||||
|
||||
assert mock_client.ping.call_count == 2
|
||||
|
||||
|
||||
class TestMinioConnectionHealth:
|
||||
def test_not_connected_when_no_client(self) -> None:
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||
assert not adapter.is_connected()
|
||||
|
||||
def test_not_connected_when_bucket_name_missing(
|
||||
self, minio_adapter: MinioAdapter
|
||||
) -> None:
|
||||
minio_adapter._bucket_name = None
|
||||
|
||||
assert not minio_adapter.is_connected()
|
||||
|
||||
def test_connected_when_probe_succeeds(self, minio_adapter: MinioAdapter) -> None:
|
||||
mock_client = cast(MagicMock, minio_adapter._client)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
|
||||
assert minio_adapter.is_connected()
|
||||
mock_client.bucket_exists.assert_called_once_with("test-bucket")
|
||||
|
||||
def test_stale_connection_when_probe_fails(
|
||||
self, minio_adapter: MinioAdapter
|
||||
) -> None:
|
||||
cast(MagicMock, minio_adapter._client).bucket_exists.side_effect = Exception(
|
||||
"connection lost"
|
||||
)
|
||||
|
||||
assert not minio_adapter.is_connected()
|
||||
|
||||
def test_cache_hit_avoids_second_probe(self, minio_adapter: MinioAdapter) -> None:
|
||||
mock_client = cast(MagicMock, minio_adapter._client)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
|
||||
with patch(
|
||||
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||
return_value=100.0,
|
||||
):
|
||||
assert minio_adapter.is_connected()
|
||||
assert minio_adapter.is_connected()
|
||||
|
||||
mock_client.bucket_exists.assert_called_once()
|
||||
|
||||
def test_cache_miss_runs_probe_again(self, minio_adapter: MinioAdapter) -> None:
|
||||
mock_client = cast(MagicMock, minio_adapter._client)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
|
||||
with patch(
|
||||
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||
side_effect=[100.0, 102.0],
|
||||
):
|
||||
assert minio_adapter.is_connected()
|
||||
assert minio_adapter.is_connected()
|
||||
|
||||
assert mock_client.bucket_exists.call_count == 2
|
||||
|
||||
def test_disconnect_clears_cache(self, minio_adapter: MinioAdapter) -> None:
|
||||
mock_client = cast(MagicMock, minio_adapter._client)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
|
||||
with patch(
|
||||
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||
return_value=100.0,
|
||||
):
|
||||
assert minio_adapter.is_connected()
|
||||
|
||||
minio_adapter.disconnect()
|
||||
reinjected = MinioAdapter(
|
||||
config=minio_adapter._config,
|
||||
client=mock_client,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"python_repositories.adapters.connection_aware_adapter.time.monotonic",
|
||||
return_value=100.0,
|
||||
):
|
||||
assert reinjected.is_connected()
|
||||
|
||||
assert mock_client.bucket_exists.call_count == 2
|
||||
+1
-3
@@ -1,4 +1,4 @@
|
||||
"""Integration tests for ConnectionAwareInterface."""
|
||||
"""Unit tests for ConnectionAwareInterface."""
|
||||
|
||||
import pytest
|
||||
from python_repositories.interfaces.connection_aware_interface import (
|
||||
@@ -15,7 +15,6 @@ def test_instantiation_fails_when_connect_not_implemented() -> None:
|
||||
def disconnect(self) -> None:
|
||||
pass
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return False
|
||||
|
||||
@@ -32,7 +31,6 @@ def test_instantiation_fails_when_disconnect_not_implemented() -> None:
|
||||
def connect(self) -> None:
|
||||
pass
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return False
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""Integration tests for ContextAwareInterface."""
|
||||
"""Unit tests for ContextAwareInterface."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Unit tests for dotenv_loader."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from python_repositories.config.dotenv_loader import load_dotenv
|
||||
|
||||
|
||||
def test_load_dotenv_loads_file(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
env_file = tmp_path / ".env"
|
||||
env_file.write_text("DOTENV_TEST_VAR=loaded_value\n")
|
||||
monkeypatch.delenv("DOTENV_TEST_VAR", raising=False)
|
||||
loaded = load_dotenv(env_file)
|
||||
assert loaded is True
|
||||
assert os.getenv("DOTENV_TEST_VAR") == "loaded_value"
|
||||
|
||||
|
||||
def test_load_dotenv_returns_false_for_missing_file(tmp_path: Path) -> None:
|
||||
missing = tmp_path / "missing.env"
|
||||
assert load_dotenv(missing) is False
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Unit tests for env_bool."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from python_repositories.config.env_bool import env_bool
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
("true", True),
|
||||
("1", True),
|
||||
("yes", True),
|
||||
("on", True),
|
||||
("false", False),
|
||||
("0", False),
|
||||
("no", False),
|
||||
("off", False),
|
||||
],
|
||||
)
|
||||
def test_env_bool_parses_truthy_and_falsy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
value: str,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
monkeypatch.setenv("TEST_BOOL", value)
|
||||
assert env_bool("TEST_BOOL", default=not expected) is expected
|
||||
|
||||
|
||||
def test_env_bool_returns_default_when_unset(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("TEST_BOOL", raising=False)
|
||||
assert env_bool("TEST_BOOL", default=True) is True
|
||||
assert env_bool("TEST_BOOL", default=False) is False
|
||||
|
||||
|
||||
def test_env_bool_raises_for_invalid_value(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("TEST_BOOL", "not-a-bool")
|
||||
with pytest.raises(ValueError, match="TEST_BOOL"):
|
||||
env_bool("TEST_BOOL", default=False)
|
||||
+30
-7
@@ -1,4 +1,6 @@
|
||||
"""Integration tests for JsonRepositoryInterface."""
|
||||
"""Unit tests for JsonRepositoryInterface."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from python_repositories.interfaces.json_repository_interface import (
|
||||
@@ -12,7 +14,7 @@ def test_instantiation_fails_when_get_not_implemented() -> None:
|
||||
class Incomplete(JsonRepositoryInterface):
|
||||
"""A class that does not implement get."""
|
||||
|
||||
def set(self, key: str, data: dict) -> None:
|
||||
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||
pass
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
@@ -31,7 +33,7 @@ def test_instantiation_fails_when_set_not_implemented() -> None:
|
||||
class Incomplete(JsonRepositoryInterface):
|
||||
"""A class that does not implement set."""
|
||||
|
||||
def get(self, key: str) -> dict | None:
|
||||
def get(self, key: str) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
@@ -50,10 +52,10 @@ def test_instantiation_fails_when_delete_not_implemented() -> None:
|
||||
class Incomplete(JsonRepositoryInterface):
|
||||
"""A class that does not implement delete."""
|
||||
|
||||
def get(self, key: str) -> dict | None:
|
||||
def get(self, key: str) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
def set(self, key: str, data: dict) -> None:
|
||||
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||
pass
|
||||
|
||||
def list_keys(self, pattern: str) -> list[str]:
|
||||
@@ -69,10 +71,10 @@ def test_instantiation_fails_when_list_keys_not_implemented() -> None:
|
||||
class Incomplete(JsonRepositoryInterface):
|
||||
"""A class that does not implement list_keys."""
|
||||
|
||||
def get(self, key: str) -> dict | None:
|
||||
def get(self, key: str) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
def set(self, key: str, data: dict) -> None:
|
||||
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||
pass
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
@@ -80,3 +82,24 @@ def test_instantiation_fails_when_list_keys_not_implemented() -> None:
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
_ = Incomplete() # type: ignore
|
||||
|
||||
|
||||
def test_scan_keys_defaults_to_list_keys() -> None:
|
||||
"""Test that the default scan_keys implementation delegates to list_keys."""
|
||||
|
||||
class Complete(JsonRepositoryInterface):
|
||||
def get(self, key: str) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||
pass
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
pass
|
||||
|
||||
def list_keys(self, pattern: str) -> list[str]:
|
||||
return [f"{pattern}-1", f"{pattern}-2"]
|
||||
|
||||
repository = Complete()
|
||||
|
||||
assert list(repository.scan_keys("user")) == ["user-1", "user-2"]
|
||||
@@ -0,0 +1,252 @@
|
||||
"""Unit tests for MinioAdapter instantiation and injection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from minio import Minio, S3Error
|
||||
from urllib3.response import BaseHTTPResponse
|
||||
|
||||
from python_repositories.adapters.minio_adapter import MinioAdapter
|
||||
from python_repositories.interfaces import ObjectRepositoryInterface
|
||||
from tests.conftest import TEST_MINIO_CONFIG
|
||||
|
||||
|
||||
def test_should_adhere_to_interface() -> None:
|
||||
assert issubclass(MinioAdapter, ObjectRepositoryInterface)
|
||||
_ = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||
|
||||
|
||||
def test_should_have_logger_when_instantiated() -> None:
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||
assert hasattr(adapter, "logger")
|
||||
assert adapter.logger is not None
|
||||
|
||||
|
||||
def test_should_not_be_connected_when_instantiated() -> None:
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||
assert not adapter.is_connected()
|
||||
|
||||
|
||||
def test_constructs_with_injected_config_without_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
for var in (
|
||||
"MINIO_ENDPOINT",
|
||||
"MINIO_ACCESS_KEY",
|
||||
"MINIO_SECRET_KEY",
|
||||
"MINIO_BUCKET",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||
assert adapter._config == TEST_MINIO_CONFIG
|
||||
|
||||
|
||||
def test_injected_client_sets_bucket_name() -> None:
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
assert adapter._bucket_name == "test-bucket"
|
||||
assert adapter._client is mock_client
|
||||
|
||||
|
||||
def test_raises_when_client_provided_without_config() -> None:
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
with pytest.raises(ValueError, match="config is required"):
|
||||
MinioAdapter(client=mock_client)
|
||||
|
||||
|
||||
def test_connect_with_injected_client_succeeds() -> None:
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
adapter.connect()
|
||||
|
||||
mock_client.list_buckets.assert_called_once()
|
||||
|
||||
|
||||
def test_connect_with_injected_client_raises_on_failure() -> None:
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.list_buckets.side_effect = Exception("connection lost")
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
with pytest.raises(ConnectionError, match="Could not connect to Minio"):
|
||||
adapter.connect()
|
||||
|
||||
|
||||
def test_connect_with_injected_client_skips_validation_when_client_cleared() -> None:
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
adapter.disconnect()
|
||||
|
||||
adapter.connect()
|
||||
|
||||
mock_client.list_buckets.assert_not_called()
|
||||
|
||||
|
||||
def test_connect_disconnects_before_reconnect(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
stale_client = MagicMock(spec=Minio)
|
||||
new_client = MagicMock(spec=Minio)
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||
adapter._client = stale_client
|
||||
adapter._bucket_name = None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||
lambda *args, **kwargs: new_client,
|
||||
)
|
||||
|
||||
adapter.connect()
|
||||
|
||||
assert adapter._client is new_client
|
||||
new_client.list_buckets.assert_called_once()
|
||||
|
||||
|
||||
def test_connect_skips_reconnect_when_already_connected(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
stale_client = MagicMock(spec=Minio)
|
||||
stale_client.bucket_exists.return_value = True
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||
adapter._client = stale_client
|
||||
adapter._bucket_name = TEST_MINIO_CONFIG.bucket
|
||||
|
||||
minio_ctor = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||
minio_ctor,
|
||||
)
|
||||
|
||||
adapter.connect()
|
||||
|
||||
minio_ctor.assert_not_called()
|
||||
assert adapter._client is stale_client
|
||||
|
||||
|
||||
def test_connect_raises_when_bucket_missing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = False
|
||||
|
||||
monkeypatch.setattr(
|
||||
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||
lambda *args, **kwargs: mock_client,
|
||||
)
|
||||
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||
|
||||
with pytest.raises(ConnectionError, match="does not exist"):
|
||||
adapter.connect()
|
||||
|
||||
mock_client.make_bucket.assert_not_called()
|
||||
|
||||
|
||||
def test_connect_creates_bucket_when_create_bucket_if_missing_enabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = False
|
||||
config = replace(TEST_MINIO_CONFIG, create_bucket_if_missing=True)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||
lambda *args, **kwargs: mock_client,
|
||||
)
|
||||
|
||||
adapter = MinioAdapter(config=config)
|
||||
adapter.connect()
|
||||
|
||||
mock_client.bucket_exists.assert_called_once_with(config.bucket)
|
||||
mock_client.make_bucket.assert_called_once_with(config.bucket)
|
||||
|
||||
|
||||
def test_get_closes_response_on_success() -> None:
|
||||
"""get() must close and release the get_object HTTP response."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
mock_response = MagicMock()
|
||||
mock_response.read.side_effect = [b"data", b""]
|
||||
mock_client.get_object.return_value = mock_response
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
result = adapter.get("some-object")
|
||||
|
||||
assert result is not None
|
||||
assert result.read() == b"data"
|
||||
mock_response.close.assert_called_once()
|
||||
mock_response.release_conn.assert_called_once()
|
||||
|
||||
|
||||
def test_get_closes_response_when_read_fails() -> None:
|
||||
"""get() must close and release the response even if read() raises."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
mock_response = MagicMock()
|
||||
mock_response.read.side_effect = OSError("connection reset")
|
||||
mock_client.get_object.return_value = mock_response
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
with pytest.raises(OSError, match="connection reset"):
|
||||
adapter.get("some-object")
|
||||
|
||||
mock_response.close.assert_called_once()
|
||||
mock_response.release_conn.assert_called_once()
|
||||
|
||||
|
||||
def test_get_returns_none_for_no_such_key() -> None:
|
||||
"""get() returns None when the object does not exist."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
mock_client.get_object.side_effect = S3Error(
|
||||
MagicMock(spec=BaseHTTPResponse),
|
||||
"NoSuchKey",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
bucket_name="test-bucket",
|
||||
object_name="missing-object",
|
||||
)
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
result = adapter.get("missing-object")
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_get_reraises_other_s3_errors() -> None:
|
||||
"""get() re-raises S3 errors other than NoSuchKey."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
other_s3error = S3Error(
|
||||
MagicMock(spec=BaseHTTPResponse),
|
||||
"AccessDenied",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
bucket_name="test-bucket",
|
||||
object_name="some-object",
|
||||
)
|
||||
mock_client.get_object.side_effect = other_s3error
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
with pytest.raises(S3Error) as exc_info:
|
||||
adapter.get("some-object")
|
||||
|
||||
assert exc_info.value.code == "AccessDenied"
|
||||
|
||||
|
||||
def test_get_reraises_general_exception_from_get_object() -> None:
|
||||
"""get() re-raises unexpected exceptions from get_object."""
|
||||
mock_client = MagicMock(spec=Minio)
|
||||
mock_client.bucket_exists.return_value = True
|
||||
mock_client.get_object.side_effect = Exception("General failure")
|
||||
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||
|
||||
with pytest.raises(Exception, match="General failure"):
|
||||
adapter.get("some-object")
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Unit tests for MinioConfig."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from python_repositories.config import MinioConfig
|
||||
|
||||
|
||||
def _set_required_minio_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("MINIO_ENDPOINT", "localhost:9000")
|
||||
monkeypatch.setenv("MINIO_ACCESS_KEY", "access")
|
||||
monkeypatch.setenv("MINIO_SECRET_KEY", "secret")
|
||||
monkeypatch.setenv("MINIO_BUCKET", "my-bucket")
|
||||
|
||||
|
||||
def test_from_env_loads_all_fields(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_set_required_minio_env(monkeypatch)
|
||||
config = MinioConfig.from_env(use_dotenv=False)
|
||||
assert config.endpoint == "localhost:9000"
|
||||
assert config.access_key == "access"
|
||||
assert config.secret_key == "secret"
|
||||
assert config.bucket == "my-bucket"
|
||||
assert config.secure is True
|
||||
assert config.create_bucket_if_missing is False
|
||||
|
||||
|
||||
def test_from_env_defaults_secure_to_true_when_unset(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_set_required_minio_env(monkeypatch)
|
||||
monkeypatch.delenv("MINIO_SECURE", raising=False)
|
||||
config = MinioConfig.from_env(use_dotenv=False)
|
||||
assert config.secure is True
|
||||
|
||||
|
||||
def test_from_env_reads_secure_from_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_set_required_minio_env(monkeypatch)
|
||||
monkeypatch.setenv("MINIO_SECURE", "false")
|
||||
assert MinioConfig.from_env(use_dotenv=False).secure is False
|
||||
|
||||
|
||||
def test_from_env_reads_create_bucket_if_missing_from_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_set_required_minio_env(monkeypatch)
|
||||
monkeypatch.setenv("MINIO_CREATE_BUCKET_IF_MISSING", "true")
|
||||
assert MinioConfig.from_env(use_dotenv=False).create_bucket_if_missing is True
|
||||
|
||||
|
||||
def test_from_env_defaults_create_bucket_if_missing_to_false_when_unset(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_set_required_minio_env(monkeypatch)
|
||||
monkeypatch.delenv("MINIO_CREATE_BUCKET_IF_MISSING", raising=False)
|
||||
config = MinioConfig.from_env(use_dotenv=False)
|
||||
assert config.create_bucket_if_missing is False
|
||||
|
||||
|
||||
def test_from_env_raises_when_endpoint_missing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("MINIO_ENDPOINT", raising=False)
|
||||
monkeypatch.setenv("MINIO_ACCESS_KEY", "access")
|
||||
monkeypatch.setenv("MINIO_SECRET_KEY", "secret")
|
||||
monkeypatch.setenv("MINIO_BUCKET", "my-bucket")
|
||||
with pytest.raises(Exception):
|
||||
MinioConfig.from_env(use_dotenv=False)
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""Integration tests for ObjectRepositoryInterface."""
|
||||
"""Unit tests for ObjectRepositoryInterface."""
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Unit tests for RedisAdapter instantiation and injection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import redis
|
||||
|
||||
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||
from python_repositories.interfaces import JsonRepositoryInterface
|
||||
from tests.conftest import TEST_REDIS_CONFIG
|
||||
|
||||
|
||||
def test_should_adhere_to_interface() -> None:
|
||||
assert issubclass(RedisAdapter, JsonRepositoryInterface)
|
||||
_ = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||
|
||||
|
||||
def test_should_have_logger_when_instantiated() -> None:
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||
assert hasattr(adapter, "logger")
|
||||
assert adapter.logger is not None
|
||||
|
||||
|
||||
def test_should_not_be_connected_when_instantiated() -> None:
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||
assert adapter._client is None
|
||||
assert not adapter.is_connected()
|
||||
|
||||
|
||||
def test_constructs_with_injected_config_without_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("REDIS_URI", raising=False)
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||
assert adapter._config == TEST_REDIS_CONFIG
|
||||
|
||||
|
||||
def test_constructs_with_injected_client_without_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("REDIS_URI", raising=False)
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
assert adapter._client is mock_client
|
||||
assert adapter._client_injected is True
|
||||
|
||||
|
||||
def test_raises_when_client_provided_without_config() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
with pytest.raises(ValueError, match="config is required"):
|
||||
RedisAdapter(client=mock_client)
|
||||
|
||||
|
||||
def test_disconnect_does_not_close_injected_client() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
adapter.disconnect()
|
||||
mock_client.close.assert_not_called()
|
||||
assert adapter._client is None
|
||||
|
||||
|
||||
class CustomEnvRedisAdapter(RedisAdapter):
|
||||
uri_env_var_name = "CUSTOM_REDIS_URI"
|
||||
|
||||
|
||||
def test_subclass_custom_env_var_name(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("CUSTOM_REDIS_URI", "redis://custom:6379")
|
||||
adapter = CustomEnvRedisAdapter()
|
||||
assert adapter._config.uri == "redis://custom:6379"
|
||||
|
||||
|
||||
def test_connect_with_injected_client_succeeds_when_ping_ok() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
mock_client.ping.return_value = True
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
adapter.connect()
|
||||
|
||||
mock_client.ping.assert_called_once()
|
||||
|
||||
|
||||
def test_connect_with_injected_client_raises_when_ping_false() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
mock_client.ping.return_value = False
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
with pytest.raises(ConnectionError, match="Could not connect to Redis"):
|
||||
adapter.connect()
|
||||
|
||||
|
||||
def test_connect_with_injected_client_raises_on_redis_error() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
mock_client.ping.side_effect = redis.ConnectionError("connection lost")
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
with pytest.raises(ConnectionError, match="Could not connect to Redis"):
|
||||
adapter.connect()
|
||||
|
||||
|
||||
def test_connect_with_injected_client_skips_validation_when_client_cleared() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
adapter.disconnect()
|
||||
|
||||
adapter.connect()
|
||||
|
||||
mock_client.ping.assert_not_called()
|
||||
|
||||
|
||||
def test_connect_reconnects_when_existing_client_unhealthy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
stale_client = MagicMock(spec=redis.Redis)
|
||||
stale_client.ping.side_effect = redis.ConnectionError("connection lost")
|
||||
new_client = MagicMock(spec=redis.Redis)
|
||||
new_client.ping.return_value = True
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||
adapter._client = stale_client
|
||||
|
||||
monkeypatch.setattr("redis.Redis.from_url", lambda *args, **kwargs: new_client)
|
||||
|
||||
adapter.connect()
|
||||
|
||||
stale_client.close.assert_called_once()
|
||||
assert adapter._client is new_client
|
||||
|
||||
|
||||
def test_connect_skips_reconnect_when_already_connected(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
stale_client = MagicMock(spec=redis.Redis)
|
||||
stale_client.ping.return_value = True
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||
adapter._client = stale_client
|
||||
|
||||
from_url = MagicMock()
|
||||
monkeypatch.setattr("redis.Redis.from_url", from_url)
|
||||
|
||||
adapter.connect()
|
||||
|
||||
stale_client.close.assert_not_called()
|
||||
from_url.assert_not_called()
|
||||
assert adapter._client is stale_client
|
||||
|
||||
|
||||
def test_scan_keys_yields_decoded_keys() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
mock_client.scan_iter.return_value = iter([b"key1", b"key2"])
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
keys = list(adapter.scan_keys("key*"))
|
||||
|
||||
assert keys == ["key1", "key2"]
|
||||
mock_client.scan_iter.assert_called_once_with(match="key*")
|
||||
mock_client.keys.assert_not_called()
|
||||
|
||||
|
||||
def test_scan_keys_forwards_count() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
mock_client.scan_iter.return_value = iter([b"key1"])
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
keys = list(adapter.scan_keys("key*", count=50))
|
||||
|
||||
assert keys == ["key1"]
|
||||
mock_client.scan_iter.assert_called_once_with(match="key*", count=50)
|
||||
|
||||
|
||||
def test_list_keys_raises_value_error_on_invalid_pattern() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
invalid_patterns = ["", 123, None]
|
||||
for pattern in invalid_patterns:
|
||||
with pytest.raises(ValueError, match="Pattern must be a non-empty string"):
|
||||
adapter.list_keys(pattern) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_scan_keys_raises_value_error_on_invalid_pattern() -> None:
|
||||
mock_client = MagicMock(spec=redis.Redis)
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||
|
||||
invalid_patterns = ["", 123, None]
|
||||
for pattern in invalid_patterns:
|
||||
with pytest.raises(ValueError, match="Pattern must be a non-empty string"):
|
||||
list(adapter.scan_keys(pattern)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_scan_keys_raises_connection_error_when_not_connected() -> None:
|
||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||
|
||||
with pytest.raises(ConnectionError):
|
||||
list(adapter.scan_keys("key*"))
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Unit tests for RedisConfig."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from python_repositories.config import RedisConfig
|
||||
|
||||
|
||||
def test_from_env_loads_uri(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("REDIS_URI", "redis://example:6379")
|
||||
config = RedisConfig.from_env(use_dotenv=False)
|
||||
assert config.uri == "redis://example:6379"
|
||||
|
||||
|
||||
def test_from_env_raises_when_uri_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("REDIS_URI", raising=False)
|
||||
with pytest.raises(Exception):
|
||||
RedisConfig.from_env(use_dotenv=False)
|
||||
|
||||
|
||||
def test_from_env_respects_custom_env_var_name(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("CUSTOM_REDIS_URI", "redis://custom:6379")
|
||||
config = RedisConfig.from_env("CUSTOM_REDIS_URI", use_dotenv=False)
|
||||
assert config.uri == "redis://custom:6379"
|
||||
Reference in New Issue
Block a user