Merge pull request 'tests_for_datastore' (#63) from tests_for_datastore into main
Reviewed-on: #63
This commit was merged in pull request #63.
This commit is contained in:
Brian Bjarke Jensen
@@ -17,8 +17,8 @@ jobs:
|
|||||||
architecture: "x64"
|
architecture: "x64"
|
||||||
- name: Install Packages
|
- name: Install Packages
|
||||||
env:
|
env:
|
||||||
PIP_INDEX_URL: http://192.168.1.2:5001/index/
|
PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
|
||||||
PIP_TRUSTED_HOST: 192.168.1.2
|
PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
|
||||||
run: |
|
run: |
|
||||||
pip install poetry
|
pip install poetry
|
||||||
poetry install
|
poetry install
|
||||||
@@ -28,6 +28,13 @@ jobs:
|
|||||||
- name: Type Check
|
- name: Type Check
|
||||||
run: |
|
run: |
|
||||||
poetry run mypy .
|
poetry run mypy .
|
||||||
- name: Pytest
|
- name: Pytest & Calculate Coverage
|
||||||
|
env:
|
||||||
|
MINIO_ENDPOINT: ${{ vars.MINIO_ENDPOINT }}
|
||||||
|
MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
|
||||||
|
MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }}
|
||||||
run: |
|
run: |
|
||||||
poetry run pytest .
|
poetry run coverage run -m pytest .
|
||||||
|
- name: Coverage Report
|
||||||
|
run: |
|
||||||
|
poetry run coverage report -m
|
||||||
|
|||||||
@@ -58,4 +58,4 @@ repos:
|
|||||||
rev: 'v2.6'
|
rev: 'v2.6'
|
||||||
hooks:
|
hooks:
|
||||||
- id: vulture
|
- id: vulture
|
||||||
entry: vulture . --min-confidence 90 --exclude */.venv/*.py
|
entry: vulture . --min-confidence 90 --exclude */.venv/*.py,*/tests/*.py
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ if __name__ == '__main__':
|
|||||||
# get image from minio
|
# get image from minio
|
||||||
buffer = get(
|
buffer = get(
|
||||||
client=minio_client,
|
client=minio_client,
|
||||||
|
bucket_name=BUCKET_NAME,
|
||||||
object_name=obj.object_name,
|
object_name=obj.object_name,
|
||||||
)
|
)
|
||||||
# convert data to image
|
# convert data to image
|
||||||
|
|||||||
@@ -3,16 +3,15 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from io import BytesIO
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from bson import ObjectId
|
from bson import ObjectId
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from pymongo.collection import Collection
|
from pymongo.collection import Collection
|
||||||
|
|
||||||
from shared.datastore import connect_minio, put
|
from shared.datastore import connect_minio, put_image
|
||||||
from shared.mongodb import connect_mongodb
|
from shared.mongodb import connect_mongodb
|
||||||
from shared.mongodb.classes import VisualCommunication
|
from shared.mongodb.src.classes import VisualCommunication
|
||||||
from shared.utils import check_env, setup_logging
|
from shared.utils import check_env, setup_logging
|
||||||
from web_ui.src.main import NECESSARY_ENV_VAR_LIST
|
from web_ui.src.main import NECESSARY_ENV_VAR_LIST
|
||||||
|
|
||||||
@@ -96,13 +95,14 @@ if __name__ == '__main__':
|
|||||||
logging.error('failed getting image from document: %s', doc_id)
|
logging.error('failed getting image from document: %s', doc_id)
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
# save image to buffer
|
# get image
|
||||||
buffer = BytesIO()
|
image = vis_com.get_image(
|
||||||
vis_com.image.save(buffer, 'png') # type: ignore
|
minio_client=minio_client,
|
||||||
|
)
|
||||||
# put buffer in minio
|
# put buffer in minio
|
||||||
object_name = put(
|
object_name = put_image(
|
||||||
client=minio_client,
|
client=minio_client,
|
||||||
buffer=buffer,
|
image=image,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logging.debug(exc)
|
logging.debug(exc)
|
||||||
|
|||||||
Generated
+303
-1
@@ -363,6 +363,85 @@ type = "legacy"
|
|||||||
url = "http://192.168.1.2:5001/index"
|
url = "http://192.168.1.2:5001/index"
|
||||||
reference = "threadripper"
|
reference = "threadripper"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "coverage"
|
||||||
|
version = "7.6.4"
|
||||||
|
description = "Code coverage measurement for Python"
|
||||||
|
optional = false
|
||||||
|
python-versions = ">=3.9"
|
||||||
|
files = [
|
||||||
|
{file = "coverage-7.6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5f8ae553cba74085db385d489c7a792ad66f7f9ba2ee85bfa508aeb84cf0ba07"},
|
||||||
|
{file = "coverage-7.6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8165b796df0bd42e10527a3f493c592ba494f16ef3c8b531288e3d0d72c1f6f0"},
|
||||||
|
{file = "coverage-7.6.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7c8b95bf47db6d19096a5e052ffca0a05f335bc63cef281a6e8fe864d450a72"},
|
||||||
|
{file = "coverage-7.6.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ed9281d1b52628e81393f5eaee24a45cbd64965f41857559c2b7ff19385df51"},
|
||||||
|
{file = "coverage-7.6.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0809082ee480bb8f7416507538243c8863ac74fd8a5d2485c46f0f7499f2b491"},
|
||||||
|
{file = "coverage-7.6.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d541423cdd416b78626b55f123412fcf979d22a2c39fce251b350de38c15c15b"},
|
||||||
|
{file = "coverage-7.6.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:58809e238a8a12a625c70450b48e8767cff9eb67c62e6154a642b21ddf79baea"},
|
||||||
|
{file = "coverage-7.6.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c9b8e184898ed014884ca84c70562b4a82cbc63b044d366fedc68bc2b2f3394a"},
|
||||||
|
{file = "coverage-7.6.4-cp310-cp310-win32.whl", hash = "sha256:6bd818b7ea14bc6e1f06e241e8234508b21edf1b242d49831831a9450e2f35fa"},
|
||||||
|
{file = "coverage-7.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:06babbb8f4e74b063dbaeb74ad68dfce9186c595a15f11f5d5683f748fa1d172"},
|
||||||
|
{file = "coverage-7.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:73d2b73584446e66ee633eaad1a56aad577c077f46c35ca3283cd687b7715b0b"},
|
||||||
|
{file = "coverage-7.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:51b44306032045b383a7a8a2c13878de375117946d68dcb54308111f39775a25"},
|
||||||
|
{file = "coverage-7.6.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b3fb02fe73bed561fa12d279a417b432e5b50fe03e8d663d61b3d5990f29546"},
|
||||||
|
{file = "coverage-7.6.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed8fe9189d2beb6edc14d3ad19800626e1d9f2d975e436f84e19efb7fa19469b"},
|
||||||
|
{file = "coverage-7.6.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b369ead6527d025a0fe7bd3864e46dbee3aa8f652d48df6174f8d0bac9e26e0e"},
|
||||||
|
{file = "coverage-7.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ade3ca1e5f0ff46b678b66201f7ff477e8fa11fb537f3b55c3f0568fbfe6e718"},
|
||||||
|
{file = "coverage-7.6.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:27fb4a050aaf18772db513091c9c13f6cb94ed40eacdef8dad8411d92d9992db"},
|
||||||
|
{file = "coverage-7.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4f704f0998911abf728a7783799444fcbbe8261c4a6c166f667937ae6a8aa522"},
|
||||||
|
{file = "coverage-7.6.4-cp311-cp311-win32.whl", hash = "sha256:29155cd511ee058e260db648b6182c419422a0d2e9a4fa44501898cf918866cf"},
|
||||||
|
{file = "coverage-7.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:8902dd6a30173d4ef09954bfcb24b5d7b5190cf14a43170e386979651e09ba19"},
|
||||||
|
{file = "coverage-7.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12394842a3a8affa3ba62b0d4ab7e9e210c5e366fbac3e8b2a68636fb19892c2"},
|
||||||
|
{file = "coverage-7.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b6b4c83d8e8ea79f27ab80778c19bc037759aea298da4b56621f4474ffeb117"},
|
||||||
|
{file = "coverage-7.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d5b8007f81b88696d06f7df0cb9af0d3b835fe0c8dbf489bad70b45f0e45613"},
|
||||||
|
{file = "coverage-7.6.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b57b768feb866f44eeed9f46975f3d6406380275c5ddfe22f531a2bf187eda27"},
|
||||||
|
{file = "coverage-7.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5915fcdec0e54ee229926868e9b08586376cae1f5faa9bbaf8faf3561b393d52"},
|
||||||
|
{file = "coverage-7.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b58c672d14f16ed92a48db984612f5ce3836ae7d72cdd161001cc54512571f2"},
|
||||||
|
{file = "coverage-7.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2fdef0d83a2d08d69b1f2210a93c416d54e14d9eb398f6ab2f0a209433db19e1"},
|
||||||
|
{file = "coverage-7.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8cf717ee42012be8c0cb205dbbf18ffa9003c4cbf4ad078db47b95e10748eec5"},
|
||||||
|
{file = "coverage-7.6.4-cp312-cp312-win32.whl", hash = "sha256:7bb92c539a624cf86296dd0c68cd5cc286c9eef2d0c3b8b192b604ce9de20a17"},
|
||||||
|
{file = "coverage-7.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:1032e178b76a4e2b5b32e19d0fd0abbce4b58e77a1ca695820d10e491fa32b08"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:023bf8ee3ec6d35af9c1c6ccc1d18fa69afa1cb29eaac57cb064dbb262a517f9"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b0ac3d42cb51c4b12df9c5f0dd2f13a4f24f01943627120ec4d293c9181219ba"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8fe4984b431f8621ca53d9380901f62bfb54ff759a1348cd140490ada7b693c"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5fbd612f8a091954a0c8dd4c0b571b973487277d26476f8480bfa4b2a65b5d06"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dacbc52de979f2823a819571f2e3a350a7e36b8cb7484cdb1e289bceaf35305f"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dab4d16dfef34b185032580e2f2f89253d302facba093d5fa9dbe04f569c4f4b"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:862264b12ebb65ad8d863d51f17758b1684560b66ab02770d4f0baf2ff75da21"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5beb1ee382ad32afe424097de57134175fea3faf847b9af002cc7895be4e2a5a"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313-win32.whl", hash = "sha256:bf20494da9653f6410213424f5f8ad0ed885e01f7e8e59811f572bdb20b8972e"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:182e6cd5c040cec0a1c8d415a87b67ed01193ed9ad458ee427741c7d8513d963"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a181e99301a0ae128493a24cfe5cfb5b488c4e0bf2f8702091473d033494d04f"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:df57bdbeffe694e7842092c5e2e0bc80fff7f43379d465f932ef36f027179806"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bcd1069e710600e8e4cf27f65c90c7843fa8edfb4520fb0ccb88894cad08b11"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99b41d18e6b2a48ba949418db48159d7a2e81c5cc290fc934b7d2380515bd0e3"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1e54712ba3474f34b7ef7a41e65bd9037ad47916ccb1cc78769bae324c01a"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:53d202fd109416ce011578f321460795abfe10bb901b883cafd9b3ef851bacfc"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:c48167910a8f644671de9f2083a23630fbf7a1cb70ce939440cd3328e0919f70"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cc8ff50b50ce532de2fa7a7daae9dd12f0a699bfcd47f20945364e5c31799fef"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313t-win32.whl", hash = "sha256:b8d3a03d9bfcaf5b0141d07a88456bb6a4c3ce55c080712fec8418ef3610230e"},
|
||||||
|
{file = "coverage-7.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:f3ddf056d3ebcf6ce47bdaf56142af51bb7fad09e4af310241e9db7a3a8022e1"},
|
||||||
|
{file = "coverage-7.6.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9cb7fa111d21a6b55cbf633039f7bc2749e74932e3aa7cb7333f675a58a58bf3"},
|
||||||
|
{file = "coverage-7.6.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:11a223a14e91a4693d2d0755c7a043db43d96a7450b4f356d506c2562c48642c"},
|
||||||
|
{file = "coverage-7.6.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a413a096c4cbac202433c850ee43fa326d2e871b24554da8327b01632673a076"},
|
||||||
|
{file = "coverage-7.6.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00a1d69c112ff5149cabe60d2e2ee948752c975d95f1e1096742e6077affd376"},
|
||||||
|
{file = "coverage-7.6.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f76846299ba5c54d12c91d776d9605ae33f8ae2b9d1d3c3703cf2db1a67f2c0"},
|
||||||
|
{file = "coverage-7.6.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fe439416eb6380de434886b00c859304338f8b19f6f54811984f3420a2e03858"},
|
||||||
|
{file = "coverage-7.6.4-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:0294ca37f1ba500667b1aef631e48d875ced93ad5e06fa665a3295bdd1d95111"},
|
||||||
|
{file = "coverage-7.6.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6f01ba56b1c0e9d149f9ac85a2f999724895229eb36bd997b61e62999e9b0901"},
|
||||||
|
{file = "coverage-7.6.4-cp39-cp39-win32.whl", hash = "sha256:bc66f0bf1d7730a17430a50163bb264ba9ded56739112368ba985ddaa9c3bd09"},
|
||||||
|
{file = "coverage-7.6.4-cp39-cp39-win_amd64.whl", hash = "sha256:c481b47f6b5845064c65a7bc78bc0860e635a9b055af0df46fdf1c58cebf8e8f"},
|
||||||
|
{file = "coverage-7.6.4-pp39.pp310-none-any.whl", hash = "sha256:3c65d37f3a9ebb703e710befdc489a38683a5b152242664b973a7b7b22348a4e"},
|
||||||
|
{file = "coverage-7.6.4.tar.gz", hash = "sha256:29fc0f17b1d3fea332f8001d4558f8214af7f1d87a345f3a133c901d60347c73"},
|
||||||
|
]
|
||||||
|
|
||||||
|
[package.extras]
|
||||||
|
toml = ["tomli"]
|
||||||
|
|
||||||
|
[package.source]
|
||||||
|
type = "legacy"
|
||||||
|
url = "http://192.168.1.2:5001/index"
|
||||||
|
reference = "threadripper"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dash"
|
name = "dash"
|
||||||
version = "2.17.0"
|
version = "2.17.0"
|
||||||
@@ -552,6 +631,33 @@ type = "legacy"
|
|||||||
url = "http://192.168.1.2:5001/index"
|
url = "http://192.168.1.2:5001/index"
|
||||||
reference = "threadripper"
|
reference = "threadripper"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "docker"
|
||||||
|
version = "7.1.0"
|
||||||
|
description = "A Python library for the Docker Engine API."
|
||||||
|
optional = false
|
||||||
|
python-versions = ">=3.8"
|
||||||
|
files = [
|
||||||
|
{file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"},
|
||||||
|
{file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"},
|
||||||
|
]
|
||||||
|
|
||||||
|
[package.dependencies]
|
||||||
|
pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""}
|
||||||
|
requests = ">=2.26.0"
|
||||||
|
urllib3 = ">=1.26.0"
|
||||||
|
|
||||||
|
[package.extras]
|
||||||
|
dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"]
|
||||||
|
docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"]
|
||||||
|
ssh = ["paramiko (>=2.4.3)"]
|
||||||
|
websockets = ["websocket-client (>=1.3.0)"]
|
||||||
|
|
||||||
|
[package.source]
|
||||||
|
type = "legacy"
|
||||||
|
url = "http://192.168.1.2:5001/index"
|
||||||
|
reference = "threadripper"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "filelock"
|
name = "filelock"
|
||||||
version = "3.14.0"
|
version = "3.14.0"
|
||||||
@@ -2040,6 +2146,29 @@ type = "legacy"
|
|||||||
url = "http://192.168.1.2:5001/index"
|
url = "http://192.168.1.2:5001/index"
|
||||||
reference = "threadripper"
|
reference = "threadripper"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pytest-cov"
|
||||||
|
version = "6.0.0"
|
||||||
|
description = "Pytest plugin for measuring coverage."
|
||||||
|
optional = false
|
||||||
|
python-versions = ">=3.9"
|
||||||
|
files = [
|
||||||
|
{file = "pytest-cov-6.0.0.tar.gz", hash = "sha256:fde0b595ca248bb8e2d76f020b465f3b107c9632e6a1d1705f17834c89dcadc0"},
|
||||||
|
{file = "pytest_cov-6.0.0-py3-none-any.whl", hash = "sha256:eee6f1b9e61008bd34975a4d5bab25801eb31898b032dd55addc93e96fcaaa35"},
|
||||||
|
]
|
||||||
|
|
||||||
|
[package.dependencies]
|
||||||
|
coverage = {version = ">=7.5", extras = ["toml"]}
|
||||||
|
pytest = ">=4.6"
|
||||||
|
|
||||||
|
[package.extras]
|
||||||
|
testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"]
|
||||||
|
|
||||||
|
[package.source]
|
||||||
|
type = "legacy"
|
||||||
|
url = "http://192.168.1.2:5001/index"
|
||||||
|
reference = "threadripper"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "python-dateutil"
|
name = "python-dateutil"
|
||||||
version = "2.9.0.post0"
|
version = "2.9.0.post0"
|
||||||
@@ -2114,6 +2243,38 @@ type = "legacy"
|
|||||||
url = "http://192.168.1.2:5001/index"
|
url = "http://192.168.1.2:5001/index"
|
||||||
reference = "threadripper"
|
reference = "threadripper"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pywin32"
|
||||||
|
version = "308"
|
||||||
|
description = "Python for Window Extensions"
|
||||||
|
optional = false
|
||||||
|
python-versions = "*"
|
||||||
|
files = [
|
||||||
|
{file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"},
|
||||||
|
{file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"},
|
||||||
|
{file = "pywin32-308-cp310-cp310-win_arm64.whl", hash = "sha256:a5ab5381813b40f264fa3495b98af850098f814a25a63589a8e9eb12560f450c"},
|
||||||
|
{file = "pywin32-308-cp311-cp311-win32.whl", hash = "sha256:5d8c8015b24a7d6855b1550d8e660d8daa09983c80e5daf89a273e5c6fb5095a"},
|
||||||
|
{file = "pywin32-308-cp311-cp311-win_amd64.whl", hash = "sha256:575621b90f0dc2695fec346b2d6302faebd4f0f45c05ea29404cefe35d89442b"},
|
||||||
|
{file = "pywin32-308-cp311-cp311-win_arm64.whl", hash = "sha256:100a5442b7332070983c4cd03f2e906a5648a5104b8a7f50175f7906efd16bb6"},
|
||||||
|
{file = "pywin32-308-cp312-cp312-win32.whl", hash = "sha256:587f3e19696f4bf96fde9d8a57cec74a57021ad5f204c9e627e15c33ff568897"},
|
||||||
|
{file = "pywin32-308-cp312-cp312-win_amd64.whl", hash = "sha256:00b3e11ef09ede56c6a43c71f2d31857cf7c54b0ab6e78ac659497abd2834f47"},
|
||||||
|
{file = "pywin32-308-cp312-cp312-win_arm64.whl", hash = "sha256:9b4de86c8d909aed15b7011182c8cab38c8850de36e6afb1f0db22b8959e3091"},
|
||||||
|
{file = "pywin32-308-cp313-cp313-win32.whl", hash = "sha256:1c44539a37a5b7b21d02ab34e6a4d314e0788f1690d65b48e9b0b89f31abbbed"},
|
||||||
|
{file = "pywin32-308-cp313-cp313-win_amd64.whl", hash = "sha256:fd380990e792eaf6827fcb7e187b2b4b1cede0585e3d0c9e84201ec27b9905e4"},
|
||||||
|
{file = "pywin32-308-cp313-cp313-win_arm64.whl", hash = "sha256:ef313c46d4c18dfb82a2431e3051ac8f112ccee1a34f29c263c583c568db63cd"},
|
||||||
|
{file = "pywin32-308-cp37-cp37m-win32.whl", hash = "sha256:1f696ab352a2ddd63bd07430080dd598e6369152ea13a25ebcdd2f503a38f1ff"},
|
||||||
|
{file = "pywin32-308-cp37-cp37m-win_amd64.whl", hash = "sha256:13dcb914ed4347019fbec6697a01a0aec61019c1046c2b905410d197856326a6"},
|
||||||
|
{file = "pywin32-308-cp38-cp38-win32.whl", hash = "sha256:5794e764ebcabf4ff08c555b31bd348c9025929371763b2183172ff4708152f0"},
|
||||||
|
{file = "pywin32-308-cp38-cp38-win_amd64.whl", hash = "sha256:3b92622e29d651c6b783e368ba7d6722b1634b8e70bd376fd7610fe1992e19de"},
|
||||||
|
{file = "pywin32-308-cp39-cp39-win32.whl", hash = "sha256:7873ca4dc60ab3287919881a7d4f88baee4a6e639aa6962de25a98ba6b193341"},
|
||||||
|
{file = "pywin32-308-cp39-cp39-win_amd64.whl", hash = "sha256:71b3322d949b4cc20776436a9c9ba0eeedcbc9c650daa536df63f0ff111bb920"},
|
||||||
|
]
|
||||||
|
|
||||||
|
[package.source]
|
||||||
|
type = "legacy"
|
||||||
|
url = "http://192.168.1.2:5001/index"
|
||||||
|
reference = "threadripper"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "requests"
|
name = "requests"
|
||||||
version = "2.32.3"
|
version = "2.32.3"
|
||||||
@@ -2353,6 +2514,63 @@ type = "legacy"
|
|||||||
url = "http://192.168.1.2:5001/index"
|
url = "http://192.168.1.2:5001/index"
|
||||||
reference = "threadripper"
|
reference = "threadripper"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "testcontainers"
|
||||||
|
version = "4.8.2"
|
||||||
|
description = "Python library for throwaway instances of anything that can run in a Docker container"
|
||||||
|
optional = false
|
||||||
|
python-versions = ">=3.9,<4.0"
|
||||||
|
files = [
|
||||||
|
{file = "testcontainers-4.8.2-py3-none-any.whl", hash = "sha256:9e19af077cd96e1957c13ee466f1f32905bc6c5bc1bc98643eb18be1a989bfb0"},
|
||||||
|
{file = "testcontainers-4.8.2.tar.gz", hash = "sha256:dd4a6a2ea09e3c3ecd39e180b6548105929d0bb78d665ce9919cb3f8c98f9853"},
|
||||||
|
]
|
||||||
|
|
||||||
|
[package.dependencies]
|
||||||
|
docker = "*"
|
||||||
|
typing-extensions = "*"
|
||||||
|
urllib3 = "*"
|
||||||
|
wrapt = "*"
|
||||||
|
|
||||||
|
[package.extras]
|
||||||
|
arangodb = ["python-arango (>=7.8,<8.0)"]
|
||||||
|
aws = ["boto3", "httpx"]
|
||||||
|
azurite = ["azure-storage-blob (>=12.19,<13.0)"]
|
||||||
|
chroma = ["chromadb-client"]
|
||||||
|
clickhouse = ["clickhouse-driver"]
|
||||||
|
cosmosdb = ["azure-cosmos"]
|
||||||
|
db2 = ["ibm_db_sa", "sqlalchemy"]
|
||||||
|
generic = ["httpx", "redis"]
|
||||||
|
google = ["google-cloud-datastore (>=2)", "google-cloud-pubsub (>=2)"]
|
||||||
|
influxdb = ["influxdb", "influxdb-client"]
|
||||||
|
k3s = ["kubernetes", "pyyaml"]
|
||||||
|
keycloak = ["python-keycloak"]
|
||||||
|
localstack = ["boto3"]
|
||||||
|
mailpit = ["cryptography"]
|
||||||
|
minio = ["minio"]
|
||||||
|
mongodb = ["pymongo"]
|
||||||
|
mssql = ["pymssql", "sqlalchemy"]
|
||||||
|
mysql = ["pymysql[rsa]", "sqlalchemy"]
|
||||||
|
nats = ["nats-py"]
|
||||||
|
neo4j = ["neo4j"]
|
||||||
|
opensearch = ["opensearch-py"]
|
||||||
|
oracle = ["oracledb", "sqlalchemy"]
|
||||||
|
oracle-free = ["oracledb", "sqlalchemy"]
|
||||||
|
qdrant = ["qdrant-client"]
|
||||||
|
rabbitmq = ["pika"]
|
||||||
|
redis = ["redis"]
|
||||||
|
registry = ["bcrypt"]
|
||||||
|
scylla = ["cassandra-driver (==3.29.1)"]
|
||||||
|
selenium = ["selenium"]
|
||||||
|
sftp = ["cryptography"]
|
||||||
|
test-module-import = ["httpx"]
|
||||||
|
trino = ["trino"]
|
||||||
|
weaviate = ["weaviate-client (>=4.5.4,<5.0.0)"]
|
||||||
|
|
||||||
|
[package.source]
|
||||||
|
type = "legacy"
|
||||||
|
url = "http://192.168.1.2:5001/index"
|
||||||
|
reference = "threadripper"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "torch"
|
name = "torch"
|
||||||
version = "2.2.2"
|
version = "2.2.2"
|
||||||
@@ -2730,6 +2948,90 @@ type = "legacy"
|
|||||||
url = "http://192.168.1.2:5001/index"
|
url = "http://192.168.1.2:5001/index"
|
||||||
reference = "threadripper"
|
reference = "threadripper"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wrapt"
|
||||||
|
version = "1.16.0"
|
||||||
|
description = "Module for decorators, wrappers and monkey patching."
|
||||||
|
optional = false
|
||||||
|
python-versions = ">=3.6"
|
||||||
|
files = [
|
||||||
|
{file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"},
|
||||||
|
{file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"},
|
||||||
|
{file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"},
|
||||||
|
{file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"},
|
||||||
|
{file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"},
|
||||||
|
{file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"},
|
||||||
|
{file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"},
|
||||||
|
{file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"},
|
||||||
|
{file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"},
|
||||||
|
{file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"},
|
||||||
|
{file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"},
|
||||||
|
{file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"},
|
||||||
|
{file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"},
|
||||||
|
{file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"},
|
||||||
|
{file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"},
|
||||||
|
{file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"},
|
||||||
|
{file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"},
|
||||||
|
{file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"},
|
||||||
|
{file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"},
|
||||||
|
{file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"},
|
||||||
|
{file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"},
|
||||||
|
{file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"},
|
||||||
|
{file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"},
|
||||||
|
{file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"},
|
||||||
|
{file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"},
|
||||||
|
{file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"},
|
||||||
|
{file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"},
|
||||||
|
{file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"},
|
||||||
|
{file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"},
|
||||||
|
{file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"},
|
||||||
|
{file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"},
|
||||||
|
{file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"},
|
||||||
|
{file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"},
|
||||||
|
{file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"},
|
||||||
|
{file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"},
|
||||||
|
{file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"},
|
||||||
|
{file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"},
|
||||||
|
{file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"},
|
||||||
|
{file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"},
|
||||||
|
{file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"},
|
||||||
|
{file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"},
|
||||||
|
{file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"},
|
||||||
|
{file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"},
|
||||||
|
{file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"},
|
||||||
|
{file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"},
|
||||||
|
{file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"},
|
||||||
|
{file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"},
|
||||||
|
{file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"},
|
||||||
|
{file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"},
|
||||||
|
{file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"},
|
||||||
|
{file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"},
|
||||||
|
{file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"},
|
||||||
|
{file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"},
|
||||||
|
{file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"},
|
||||||
|
{file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"},
|
||||||
|
{file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"},
|
||||||
|
{file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"},
|
||||||
|
{file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"},
|
||||||
|
{file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"},
|
||||||
|
{file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"},
|
||||||
|
{file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"},
|
||||||
|
{file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"},
|
||||||
|
{file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"},
|
||||||
|
{file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"},
|
||||||
|
{file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"},
|
||||||
|
{file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"},
|
||||||
|
{file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"},
|
||||||
|
{file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"},
|
||||||
|
{file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"},
|
||||||
|
{file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"},
|
||||||
|
]
|
||||||
|
|
||||||
|
[package.source]
|
||||||
|
type = "legacy"
|
||||||
|
url = "http://192.168.1.2:5001/index"
|
||||||
|
reference = "threadripper"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wsproto"
|
name = "wsproto"
|
||||||
version = "1.2.0"
|
version = "1.2.0"
|
||||||
@@ -2772,4 +3074,4 @@ reference = "threadripper"
|
|||||||
[metadata]
|
[metadata]
|
||||||
lock-version = "2.0"
|
lock-version = "2.0"
|
||||||
python-versions = "^3.12"
|
python-versions = "^3.12"
|
||||||
content-hash = "33dbc73047d591ae63dd2d934519882654074a1f3c9ca757d0571d869b8541c2"
|
content-hash = "539b697eb7c6ea07594c923ca2aaf26ea841be843c18e696882ccd5dc83339f7"
|
||||||
|
|||||||
@@ -22,6 +22,9 @@ flake8-pyproject = "^1.2.3"
|
|||||||
pandas-stubs = "^2.2.2.240603"
|
pandas-stubs = "^2.2.2.240603"
|
||||||
types-tqdm = "^4.66.0.20240417"
|
types-tqdm = "^4.66.0.20240417"
|
||||||
pytest = "^8.3.3"
|
pytest = "^8.3.3"
|
||||||
|
testcontainers = "^4.8.2"
|
||||||
|
coverage = "^7.6.4"
|
||||||
|
pytest-cov = "^6.0.0"
|
||||||
|
|
||||||
|
|
||||||
[tool.poetry.group.dev.dependencies]
|
[tool.poetry.group.dev.dependencies]
|
||||||
@@ -118,3 +121,7 @@ ignore_missing_imports = true
|
|||||||
[[tool.mypy.overrides]]
|
[[tool.mypy.overrides]]
|
||||||
module = "shared.mongodb.*"
|
module = "shared.mongodb.*"
|
||||||
ignore_missing_imports = true
|
ignore_missing_imports = true
|
||||||
|
|
||||||
|
[[tool.mypy.overrides]]
|
||||||
|
module = "testcontainers.*"
|
||||||
|
ignore_missing_imports = true
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
from .connect_minio import connect_minio
|
from .src import (
|
||||||
from .delete import delete
|
connect_minio,
|
||||||
from .get import get
|
delete,
|
||||||
from .get_image import get_image
|
get,
|
||||||
from .get_model import get_model
|
get_image,
|
||||||
from .put import put
|
get_model,
|
||||||
from .put_image import put_image
|
put,
|
||||||
from .put_model import put_model
|
put_image,
|
||||||
|
put_model,
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,41 +0,0 @@
|
|||||||
"""Definition of get_image function."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
from io import BytesIO
|
|
||||||
|
|
||||||
from minio import Minio
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
|
|
||||||
def get_image(
|
|
||||||
client: Minio,
|
|
||||||
object_name: str,
|
|
||||||
) -> Image.Image:
|
|
||||||
"""Get image from image subfolder in bucket in Minio."""
|
|
||||||
assert isinstance(client, Minio)
|
|
||||||
assert isinstance(object_name, str)
|
|
||||||
assert len(object_name) > 0
|
|
||||||
# prepare arguments
|
|
||||||
assert 'MINIO_BUCKET_NAME' in os.environ
|
|
||||||
bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
|
|
||||||
subfolder = 'images'
|
|
||||||
# get object from bucket
|
|
||||||
object_name = f'{subfolder}/{object_name}'
|
|
||||||
try:
|
|
||||||
response = client.get_object(
|
|
||||||
bucket_name=bucket_name,
|
|
||||||
object_name=object_name,
|
|
||||||
)
|
|
||||||
buffer = BytesIO(response.data)
|
|
||||||
except Exception as exc:
|
|
||||||
logging.error('failed getting data from MinIO')
|
|
||||||
raise exc
|
|
||||||
finally:
|
|
||||||
response.close()
|
|
||||||
response.release_conn()
|
|
||||||
buffer.seek(0)
|
|
||||||
# convert data to image
|
|
||||||
image = Image.open(buffer)
|
|
||||||
logging.debug('got data from %s', object_name)
|
|
||||||
return image
|
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
from .connect_minio import connect_minio
|
||||||
|
from .delete import delete
|
||||||
|
from .get import get
|
||||||
|
from .get_image import get_image
|
||||||
|
from .get_model import get_model
|
||||||
|
from .put import put
|
||||||
|
from .put_image import put_image
|
||||||
|
from .put_model import put_model
|
||||||
@@ -5,28 +5,24 @@ import os
|
|||||||
|
|
||||||
from minio import Minio
|
from minio import Minio
|
||||||
|
|
||||||
|
from shared.utils import check_env
|
||||||
|
|
||||||
|
|
||||||
def connect_minio() -> Minio:
|
def connect_minio() -> Minio:
|
||||||
"""Connect to MinIO server."""
|
"""Connect to MinIO server."""
|
||||||
# ensure necessary env vars available
|
# ensure necessary env vars available
|
||||||
env_var_list = [
|
var_list = {
|
||||||
'MINIO_ENDPOINT',
|
'MINIO_ENDPOINT',
|
||||||
'MINIO_ACCESS_KEY',
|
'MINIO_ACCESS_KEY',
|
||||||
'MINIO_SECRET_KEY',
|
'MINIO_SECRET_KEY',
|
||||||
'MINIO_BUCKET_NAME',
|
'MINIO_BUCKET_NAME',
|
||||||
]
|
}
|
||||||
for env_var in env_var_list:
|
check_env(var_list)
|
||||||
# ensure env var set
|
|
||||||
assert (
|
|
||||||
env_var in os.environ
|
|
||||||
), f"environment variable not set: {
|
|
||||||
env_var
|
|
||||||
}"
|
|
||||||
# prepare arguments
|
# prepare arguments
|
||||||
minio_endpoint = os.getenv('MINIO_ENDPOINT', default='')
|
minio_endpoint = str(os.getenv('MINIO_ENDPOINT'))
|
||||||
minio_access_key = os.getenv('MINIO_ACCESS_KEY', default='')
|
minio_access_key = str(os.getenv('MINIO_ACCESS_KEY'))
|
||||||
minio_secret_key = os.getenv('MINIO_SECRET_KEY', default='')
|
minio_secret_key = str(os.getenv('MINIO_SECRET_KEY'))
|
||||||
minio_bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
|
minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||||
# connect client
|
# connect client
|
||||||
client = Minio(
|
client = Minio(
|
||||||
endpoint=minio_endpoint,
|
endpoint=minio_endpoint,
|
||||||
@@ -1,22 +1,22 @@
|
|||||||
"""Definition of delete function."""
|
"""Definition of delete function."""
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
from traceback import print_exc
|
||||||
|
|
||||||
from minio import Minio
|
from minio import Minio
|
||||||
|
|
||||||
|
|
||||||
def delete(
|
def delete(
|
||||||
client: Minio,
|
client: Minio,
|
||||||
|
bucket_name: str,
|
||||||
object_name: str,
|
object_name: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Delete object from MinIO."""
|
"""Delete object from MinIO."""
|
||||||
assert isinstance(client, Minio)
|
assert isinstance(client, Minio)
|
||||||
assert isinstance(object_name, str)
|
assert isinstance(bucket_name, str)
|
||||||
bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
|
|
||||||
assert len(bucket_name) > 0
|
assert len(bucket_name) > 0
|
||||||
|
assert isinstance(object_name, str)
|
||||||
|
assert len(object_name) > 0
|
||||||
# remove object
|
# remove object
|
||||||
try:
|
try:
|
||||||
client.remove_object(
|
client.remove_object(
|
||||||
@@ -24,7 +24,8 @@ def delete(
|
|||||||
object_name=object_name,
|
object_name=object_name,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logging.debug(exc)
|
|
||||||
logging.error('failed deleting %s', object_name)
|
logging.error('failed deleting %s', object_name)
|
||||||
|
print_exc()
|
||||||
|
raise exc
|
||||||
else:
|
else:
|
||||||
logging.debug('deleted %s', object_name)
|
logging.debug('deleted %s', object_name)
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
"""Definition of get function."""
|
"""Definition of get function."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from traceback import print_exc
|
from traceback import print_exc
|
||||||
|
|
||||||
@@ -10,19 +9,23 @@ from minio import Minio
|
|||||||
|
|
||||||
def get(
|
def get(
|
||||||
client: Minio,
|
client: Minio,
|
||||||
|
bucket_name: str,
|
||||||
object_name: str,
|
object_name: str,
|
||||||
) -> BytesIO:
|
) -> BytesIO:
|
||||||
"""Get buffer from bucket in MinIO."""
|
"""Get buffer from bucket in MinIO."""
|
||||||
assert isinstance(client, Minio)
|
assert isinstance(client, Minio)
|
||||||
|
assert isinstance(bucket_name, str)
|
||||||
|
assert len(bucket_name) > 0
|
||||||
assert isinstance(object_name, str)
|
assert isinstance(object_name, str)
|
||||||
bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
|
assert len(object_name) > 0
|
||||||
# get buffer
|
|
||||||
try:
|
try:
|
||||||
|
# make request
|
||||||
response = client.get_object(
|
response = client.get_object(
|
||||||
bucket_name=bucket_name,
|
bucket_name=bucket_name,
|
||||||
object_name=object_name,
|
object_name=object_name,
|
||||||
)
|
)
|
||||||
assert response.status == 200
|
assert response.status == 200
|
||||||
|
# get buffer
|
||||||
buffer = BytesIO()
|
buffer = BytesIO()
|
||||||
chunk_size = 2**14
|
chunk_size = 2**14
|
||||||
while chunk := response.read(chunk_size):
|
while chunk := response.read(chunk_size):
|
||||||
@@ -35,5 +38,7 @@ def get(
|
|||||||
print_exc()
|
print_exc()
|
||||||
raise exc
|
raise exc
|
||||||
finally:
|
finally:
|
||||||
response.close()
|
# close connection if established
|
||||||
response.release_conn()
|
if 'response' in locals():
|
||||||
|
response.close()
|
||||||
|
response.release_conn()
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
"""Definition of get_image function."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from shared.utils import check_env
|
||||||
|
|
||||||
|
from .get import get
|
||||||
|
|
||||||
|
|
||||||
|
def get_image(
|
||||||
|
client: Minio,
|
||||||
|
object_name: str,
|
||||||
|
) -> Image.Image:
|
||||||
|
"""Get image from image subfolder in bucket in Minio."""
|
||||||
|
assert isinstance(client, Minio)
|
||||||
|
assert isinstance(object_name, str)
|
||||||
|
assert len(object_name) > 0
|
||||||
|
# prepare arguments
|
||||||
|
check_env({'MINIO_BUCKET_NAME'})
|
||||||
|
bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||||
|
object_name = f'images/{object_name}'
|
||||||
|
# get object from bucket
|
||||||
|
buffer = get(
|
||||||
|
client=client,
|
||||||
|
bucket_name=bucket_name,
|
||||||
|
object_name=object_name,
|
||||||
|
)
|
||||||
|
# convert data to image
|
||||||
|
image = Image.open(buffer)
|
||||||
|
logging.debug('got data from %s', object_name)
|
||||||
|
return image
|
||||||
@@ -1,11 +1,14 @@
|
|||||||
"""Definition of get_model function."""
|
"""Definition of get_model function."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from minio import Minio
|
from minio import Minio
|
||||||
|
|
||||||
|
from shared.utils import check_env
|
||||||
|
|
||||||
from .get import get
|
from .get import get
|
||||||
|
|
||||||
|
|
||||||
@@ -16,15 +19,18 @@ def get_model(
|
|||||||
"""Get model from model subfolder in bucket in Minio."""
|
"""Get model from model subfolder in bucket in Minio."""
|
||||||
assert isinstance(client, Minio)
|
assert isinstance(client, Minio)
|
||||||
assert isinstance(object_name, str)
|
assert isinstance(object_name, str)
|
||||||
subfolder = 'models'
|
assert len(object_name) > 0
|
||||||
object_name = f'{subfolder}/{object_name}'
|
# prepare arguments
|
||||||
# get buffer
|
check_env({'MINIO_BUCKET_NAME'})
|
||||||
|
bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||||
|
object_name = f'models/{object_name}'
|
||||||
|
# get object from bucket
|
||||||
buffer = get(
|
buffer = get(
|
||||||
client=client,
|
client=client,
|
||||||
|
bucket_name=bucket_name,
|
||||||
object_name=object_name,
|
object_name=object_name,
|
||||||
)
|
)
|
||||||
# convert data to model checkpoint
|
# convert data to model checkpoint
|
||||||
buffer.seek(0)
|
|
||||||
model_content = torch.load(buffer)
|
model_content = torch.load(buffer)
|
||||||
logging.debug('finished')
|
logging.debug('finished')
|
||||||
return model_content
|
return model_content
|
||||||
@@ -1,10 +1,6 @@
|
|||||||
"""Definition of put function."""
|
"""Definition of put function."""
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
from hashlib import md5
|
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
|
||||||
from minio import Minio
|
from minio import Minio
|
||||||
@@ -13,27 +9,28 @@ from minio import Minio
|
|||||||
def put(
|
def put(
|
||||||
client: Minio,
|
client: Minio,
|
||||||
buffer: BytesIO,
|
buffer: BytesIO,
|
||||||
) -> str:
|
bucket_name: str,
|
||||||
|
object_name: str,
|
||||||
|
) -> None:
|
||||||
"""Put buffer in bucket in MinIO and return MD5 checksum as object name."""
|
"""Put buffer in bucket in MinIO and return MD5 checksum as object name."""
|
||||||
assert isinstance(client, Minio)
|
assert isinstance(client, Minio)
|
||||||
assert isinstance(buffer, BytesIO)
|
assert isinstance(buffer, BytesIO)
|
||||||
bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
|
assert isinstance(bucket_name, str)
|
||||||
assert len(bucket_name) > 0
|
assert len(bucket_name) > 0
|
||||||
# get md5 of image
|
assert isinstance(object_name, str)
|
||||||
checksum = md5(buffer.getbuffer()).hexdigest()
|
assert len(object_name) > 0
|
||||||
# prepare for saving
|
# prepare for saving
|
||||||
num_bytes = buffer.tell()
|
num_bytes = len(buffer.getvalue())
|
||||||
buffer.seek(0)
|
buffer.seek(0)
|
||||||
# send data to bucket
|
# send data to bucket
|
||||||
try:
|
try:
|
||||||
client.put_object(
|
client.put_object(
|
||||||
bucket_name=bucket_name,
|
bucket_name=bucket_name,
|
||||||
object_name=checksum,
|
object_name=object_name,
|
||||||
length=num_bytes,
|
length=num_bytes,
|
||||||
data=buffer,
|
data=buffer,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logging.error('failed saving data to MinIO')
|
logging.error('failed saving data to MinIO')
|
||||||
raise exc
|
raise exc
|
||||||
logging.debug('saved data to %s', checksum)
|
logging.debug('saved data to %s', object_name)
|
||||||
return checksum
|
|
||||||
@@ -8,6 +8,8 @@ from io import BytesIO
|
|||||||
from minio import Minio
|
from minio import Minio
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
|
from .put import put
|
||||||
|
|
||||||
|
|
||||||
def put_image(
|
def put_image(
|
||||||
client: Minio,
|
client: Minio,
|
||||||
@@ -17,28 +19,22 @@ def put_image(
|
|||||||
used as object name."""
|
used as object name."""
|
||||||
assert isinstance(client, Minio)
|
assert isinstance(client, Minio)
|
||||||
assert isinstance(image, Image.Image)
|
assert isinstance(image, Image.Image)
|
||||||
|
# get bucket name from env
|
||||||
bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
|
bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
|
||||||
assert len(bucket_name) > 0
|
assert len(bucket_name) > 0
|
||||||
subfolder = 'images'
|
# save data to buffer
|
||||||
# save image to buffer
|
|
||||||
buffer = BytesIO()
|
buffer = BytesIO()
|
||||||
image.save(buffer, 'png')
|
image.save(buffer, 'png')
|
||||||
# get md5 of image
|
# get md5 of buffer
|
||||||
checksum = md5(buffer.getbuffer()).hexdigest()
|
checksum = md5(buffer.getbuffer()).hexdigest()
|
||||||
# prepare for saving
|
# set object name
|
||||||
num_bytes = buffer.tell()
|
object_name = f'images/{checksum}'
|
||||||
buffer.seek(0)
|
|
||||||
# send data to bucket
|
# send data to bucket
|
||||||
object_name = f'{subfolder}/{checksum}'
|
put(
|
||||||
try:
|
client=client,
|
||||||
client.put_object(
|
buffer=buffer,
|
||||||
bucket_name=bucket_name,
|
bucket_name=bucket_name,
|
||||||
object_name=object_name,
|
object_name=object_name,
|
||||||
length=num_bytes,
|
)
|
||||||
data=buffer,
|
logging.debug('finished')
|
||||||
)
|
|
||||||
except Exception as exc:
|
|
||||||
logging.error('failed saving data to MinIO')
|
|
||||||
raise exc
|
|
||||||
logging.debug('saved data to %s', object_name)
|
|
||||||
return checksum
|
return checksum
|
||||||
@@ -9,6 +9,10 @@ import torch
|
|||||||
from minio import Minio
|
from minio import Minio
|
||||||
from torch.nn import Module
|
from torch.nn import Module
|
||||||
|
|
||||||
|
from shared.utils import check_env
|
||||||
|
|
||||||
|
from .put import put
|
||||||
|
|
||||||
|
|
||||||
def put_model(
|
def put_model(
|
||||||
client: Minio,
|
client: Minio,
|
||||||
@@ -18,28 +22,22 @@ def put_model(
|
|||||||
used as object name."""
|
used as object name."""
|
||||||
assert isinstance(client, Minio)
|
assert isinstance(client, Minio)
|
||||||
assert isinstance(model, Module)
|
assert isinstance(model, Module)
|
||||||
bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
|
# get bucket name from env
|
||||||
assert len(bucket_name) > 0
|
check_env({'MINIO_BUCKET_NAME'})
|
||||||
subfolder = 'models'
|
bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||||
# save image to buffer
|
# save data to buffer
|
||||||
buffer = BytesIO()
|
buffer = BytesIO()
|
||||||
torch.save(model.state_dict(), buffer)
|
torch.save(model.state_dict(), buffer)
|
||||||
# get md5 of image
|
# get md5 of image
|
||||||
checksum = md5(buffer.getbuffer()).hexdigest()
|
checksum = md5(buffer.getbuffer()).hexdigest()
|
||||||
# prepare for saving
|
# set object name
|
||||||
num_bytes = buffer.tell()
|
object_name = f'models/{checksum}'
|
||||||
buffer.seek(0)
|
|
||||||
# send data to bucket
|
# send data to bucket
|
||||||
object_name = f'{subfolder}/{checksum}'
|
put(
|
||||||
try:
|
client=client,
|
||||||
client.put_object(
|
buffer=buffer,
|
||||||
bucket_name=bucket_name,
|
bucket_name=bucket_name,
|
||||||
object_name=object_name,
|
object_name=object_name,
|
||||||
length=num_bytes,
|
)
|
||||||
data=buffer,
|
logging.debug('finished')
|
||||||
)
|
|
||||||
except Exception as exc:
|
|
||||||
logging.error('failed saving data to MinIO')
|
|
||||||
raise exc
|
|
||||||
logging.debug('saved data to %s', object_name)
|
|
||||||
return checksum
|
return checksum
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
"""Integration tests related to base CRUD functions."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
import minio
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from shared.datastore import delete, get, put
|
||||||
|
|
||||||
|
|
||||||
|
def same_data(
|
||||||
|
data_a: BytesIO,
|
||||||
|
data_b: BytesIO,
|
||||||
|
) -> bool:
|
||||||
|
"""Check if two BytesIO-objects contain the same data."""
|
||||||
|
assert isinstance(data_a, BytesIO)
|
||||||
|
assert isinstance(data_b, BytesIO)
|
||||||
|
# prepare for being read
|
||||||
|
data_a.seek(0)
|
||||||
|
data_b.seek(0)
|
||||||
|
# convert to bytes
|
||||||
|
data_a_bytes = data_a.read()
|
||||||
|
data_b_bytes = data_b.read()
|
||||||
|
# compare size
|
||||||
|
if len(data_a_bytes) != len(data_b_bytes):
|
||||||
|
logging.error(
|
||||||
|
'data has different length: %s and %s',
|
||||||
|
len(data_a_bytes),
|
||||||
|
len(data_b_bytes),
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
# compare content
|
||||||
|
if data_a_bytes != data_b_bytes:
|
||||||
|
logging.error('data has different bytes')
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_get_data(
|
||||||
|
minio_client,
|
||||||
|
data_in_minio,
|
||||||
|
):
|
||||||
|
data, bucket_name, object_name = data_in_minio
|
||||||
|
received_data = get(
|
||||||
|
client=minio_client,
|
||||||
|
bucket_name=bucket_name,
|
||||||
|
object_name=object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(data, BytesIO)
|
||||||
|
assert same_data(data, received_data)
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_delete_data(
|
||||||
|
minio_client,
|
||||||
|
data_in_minio,
|
||||||
|
):
|
||||||
|
_, bucket_name, object_name = data_in_minio
|
||||||
|
delete(
|
||||||
|
client=minio_client,
|
||||||
|
bucket_name=bucket_name,
|
||||||
|
object_name=object_name,
|
||||||
|
)
|
||||||
|
with pytest.raises(minio.error.S3Error):
|
||||||
|
_ = get(
|
||||||
|
client=minio_client,
|
||||||
|
bucket_name=bucket_name,
|
||||||
|
object_name=object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_put_data(
|
||||||
|
minio_client,
|
||||||
|
data,
|
||||||
|
):
|
||||||
|
# prepare variables
|
||||||
|
minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||||
|
minio_object_name = str(os.getenv('MINIO_OBJECT_NAME'))
|
||||||
|
buffer = BytesIO(data)
|
||||||
|
put(
|
||||||
|
client=minio_client,
|
||||||
|
buffer=buffer,
|
||||||
|
bucket_name=minio_bucket_name,
|
||||||
|
object_name=minio_object_name,
|
||||||
|
)
|
||||||
|
received_data = get(
|
||||||
|
client=minio_client,
|
||||||
|
bucket_name=minio_bucket_name,
|
||||||
|
object_name=minio_object_name,
|
||||||
|
)
|
||||||
|
assert isinstance(received_data, BytesIO)
|
||||||
|
assert same_data(received_data, buffer)
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_update_data(
|
||||||
|
minio_client,
|
||||||
|
data_in_minio,
|
||||||
|
):
|
||||||
|
buffer, bucket_name, object_name = data_in_minio
|
||||||
|
put(
|
||||||
|
client=minio_client,
|
||||||
|
buffer=buffer,
|
||||||
|
bucket_name=bucket_name,
|
||||||
|
object_name=object_name,
|
||||||
|
)
|
||||||
|
received_data = get(
|
||||||
|
client=minio_client,
|
||||||
|
bucket_name=bucket_name,
|
||||||
|
object_name=object_name,
|
||||||
|
)
|
||||||
|
assert same_data(received_data, buffer)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
pytest.main()
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
"""Integration test configurations."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from io import BytesIO
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from minio import Minio
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
env_var_map = {
|
||||||
|
'MINIO_BUCKET_NAME': 'test-bucket',
|
||||||
|
'MINIO_OBJECT_NAME': '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='session', autouse=True)
|
||||||
|
def populate_env(
|
||||||
|
request: pytest.FixtureRequest,
|
||||||
|
) -> None:
|
||||||
|
"""Populate environment with variables used for testing."""
|
||||||
|
# read env-file for local testing
|
||||||
|
load_dotenv(
|
||||||
|
dotenv_path=Path(__file__).parent.parent.parent.parent.parent / 'server.env',
|
||||||
|
)
|
||||||
|
# update env
|
||||||
|
for key, val in env_var_map.items():
|
||||||
|
os.environ[key] = val
|
||||||
|
|
||||||
|
# ensure cleanup
|
||||||
|
def cleanup_env():
|
||||||
|
for key in env_var_map:
|
||||||
|
_ = os.environ.pop(key, default=None)
|
||||||
|
|
||||||
|
request.addfinalizer(cleanup_env)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='session')
|
||||||
|
def minio_client(
|
||||||
|
populate_env,
|
||||||
|
) -> Iterator[Minio]:
|
||||||
|
# prepare arguments
|
||||||
|
minio_endpoint = str(os.getenv('MINIO_ENDPOINT'))
|
||||||
|
minio_access_key = str(os.getenv('MINIO_ACCESS_KEY'))
|
||||||
|
minio_secret_key = str(os.getenv('MINIO_SECRET_KEY'))
|
||||||
|
minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||||
|
# connect to minio
|
||||||
|
client = Minio(
|
||||||
|
endpoint=minio_endpoint,
|
||||||
|
access_key=minio_access_key,
|
||||||
|
secret_key=minio_secret_key,
|
||||||
|
secure=False,
|
||||||
|
)
|
||||||
|
# ensure bucket exists
|
||||||
|
if not client.bucket_exists(bucket_name=minio_bucket_name):
|
||||||
|
client.make_bucket(bucket_name=minio_bucket_name)
|
||||||
|
# expose client
|
||||||
|
yield client
|
||||||
|
# remove objects left behind by tests
|
||||||
|
for obj in client.list_objects(bucket_name=minio_bucket_name, recursive=True):
|
||||||
|
client.remove_object(
|
||||||
|
bucket_name=obj.bucket_name,
|
||||||
|
object_name=obj.object_name,
|
||||||
|
)
|
||||||
|
# remove bucket
|
||||||
|
client.remove_bucket(bucket_name=minio_bucket_name)
|
||||||
|
assert not client.bucket_exists(bucket_name=minio_bucket_name)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def data() -> Iterator[bytes]:
|
||||||
|
# generate random data
|
||||||
|
num_bytes = 2**21 # 2 MB
|
||||||
|
data = random.randbytes(n=num_bytes)
|
||||||
|
# expose data
|
||||||
|
yield data
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def data_in_minio(
|
||||||
|
minio_client,
|
||||||
|
data,
|
||||||
|
) -> Iterator[tuple[BytesIO, str, str]]:
|
||||||
|
# prepare arguments
|
||||||
|
minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
|
||||||
|
minio_object_name = str(os.getenv('MINIO_OBJECT_NAME'))
|
||||||
|
# convert data
|
||||||
|
buffer = BytesIO(data)
|
||||||
|
# prepare for saving
|
||||||
|
num_bytes = len(buffer.getvalue())
|
||||||
|
buffer.seek(0)
|
||||||
|
# send data to bucket
|
||||||
|
minio_client.put_object(
|
||||||
|
bucket_name=minio_bucket_name,
|
||||||
|
object_name=minio_object_name,
|
||||||
|
length=num_bytes,
|
||||||
|
data=buffer,
|
||||||
|
)
|
||||||
|
# expose data
|
||||||
|
yield buffer, minio_bucket_name, minio_object_name
|
||||||
|
# clean up
|
||||||
|
minio_client.remove_object(
|
||||||
|
bucket_name=minio_bucket_name,
|
||||||
|
object_name=minio_object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def image() -> Iterator[Image.Image]:
|
||||||
|
# generate image
|
||||||
|
image = Image.new(mode='RGB', size=(480, 480))
|
||||||
|
# expose image
|
||||||
|
yield image
|
||||||
|
|
||||||
|
|
||||||
|
# @pytest.fixture
|
||||||
|
# def image_in_minio(
|
||||||
|
# image: Image.Image,
|
||||||
|
# ) -> tuple(Image.Image, str):
|
||||||
|
# #
|
||||||
|
|
||||||
|
|
||||||
|
# # expose image and object name
|
||||||
|
# yield image, object_name
|
||||||
|
# # cleanup
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
"""Integration test for connect_minio function."""
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
|
||||||
|
from shared.datastore import connect_minio
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_return_correct_type():
|
||||||
|
client = connect_minio()
|
||||||
|
assert isinstance(client, Minio)
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
"""Integration tests related to image CRUD."""
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
# from shared.datastore import connect_minio, get_image, put_image
|
||||||
|
|
||||||
|
|
||||||
|
def same_image(
|
||||||
|
img_a: Image.Image,
|
||||||
|
img_b: Image.Image,
|
||||||
|
) -> bool:
|
||||||
|
"""Check if two images contain the same data."""
|
||||||
|
assert isinstance(img_a, Image.Image)
|
||||||
|
assert isinstance(img_b, Image.Image)
|
||||||
|
# check if images have a comparable number of channels
|
||||||
|
if img_a.getbands() != img_b.getbands():
|
||||||
|
return False
|
||||||
|
# calculate pixel difference between images
|
||||||
|
img_a_arr = np.asarray(img_a)
|
||||||
|
img_b_arr = np.asarray(img_b)
|
||||||
|
diff = np.subtract(img_a_arr, img_b_arr)
|
||||||
|
if np.sum(diff) != 0:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
# def test_should_get_image(
|
||||||
|
# image_in_minio,
|
||||||
|
# ):
|
||||||
|
# image, object_name = image_in_minio
|
||||||
|
# client = connect_minio()
|
||||||
|
# received_image = get_image(
|
||||||
|
# client=client,
|
||||||
|
# object_name=object_name,
|
||||||
|
# )
|
||||||
|
# assert isinstance(image, Image.Image)
|
||||||
|
# assert received_image == image
|
||||||
|
|
||||||
|
|
||||||
|
# def test_should_put_image(
|
||||||
|
# image,
|
||||||
|
# ):
|
||||||
|
# client = connect_minio()
|
||||||
|
# object_name = put_image(
|
||||||
|
# client=client,
|
||||||
|
# image=image,
|
||||||
|
# )
|
||||||
|
# assert isinstance(object_name, str)
|
||||||
|
# assert len(object_name) > 0
|
||||||
|
# received_image = get_image(
|
||||||
|
# client=client,
|
||||||
|
# object_name=object_name,
|
||||||
|
# )
|
||||||
|
# assert same_image(received_image, image)
|
||||||
|
|
||||||
|
|
||||||
|
# def test_should_update_image(
|
||||||
|
# image_in_minio,
|
||||||
|
# ):
|
||||||
|
# image, object_name = image_in_minio
|
||||||
|
# client = connect_minio()
|
||||||
|
# object_name = put_image(
|
||||||
|
# client=client,
|
||||||
|
# image=image,
|
||||||
|
# )
|
||||||
|
# assert isinstance(object_name, str)
|
||||||
|
# assert len(object_name) > 0
|
||||||
|
# received_image = get_image(
|
||||||
|
# client=client,
|
||||||
|
# object_name=object_name,
|
||||||
|
# )
|
||||||
|
# assert received_image == image
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
"""Integration tests related to model CRUD."""
|
||||||
|
|
||||||
|
# from torch.nn import Module
|
||||||
|
|
||||||
|
# from shared.datastore import connect_minio, get_model, put_model
|
||||||
|
|
||||||
|
# def test_should_get_model(
|
||||||
|
# model_in_minio,
|
||||||
|
# ):
|
||||||
|
# model, object_name = model_in_minio
|
||||||
|
# client = connect_minio()
|
||||||
|
# received_model = get_model(
|
||||||
|
# client=client,
|
||||||
|
# object_name=object_name,
|
||||||
|
# )
|
||||||
|
# assert isinstance(model, Module)
|
||||||
|
# assert received_model == model
|
||||||
|
|
||||||
|
|
||||||
|
# def test_should_put_model(
|
||||||
|
# model,
|
||||||
|
# ):
|
||||||
|
# client = connect_minio()
|
||||||
|
# object_name = put_model(
|
||||||
|
# client=client,
|
||||||
|
# model=model,
|
||||||
|
# )
|
||||||
|
# assert isinstance(object_name, str)
|
||||||
|
# assert len(object_name) > 0
|
||||||
|
# received_model = get_model(
|
||||||
|
# client=client,
|
||||||
|
# object_name=object_name,
|
||||||
|
# )
|
||||||
|
# assert received_model == model
|
||||||
|
|
||||||
|
|
||||||
|
# def test_should_update_model(
|
||||||
|
# model_in_minio,
|
||||||
|
# ):
|
||||||
|
# model, object_name = model_in_minio
|
||||||
|
# client = connect_minio()
|
||||||
|
# object_name = put_model(
|
||||||
|
# client=client,
|
||||||
|
# model=model,
|
||||||
|
# )
|
||||||
|
# assert isinstance(object_name, str)
|
||||||
|
# assert len(object_name) > 0
|
||||||
|
# received_model = get_model(
|
||||||
|
# client=client,
|
||||||
|
# object_name=object_name,
|
||||||
|
# )
|
||||||
|
# assert received_model == model
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
"""Definition of unittests for connect_minio function."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from shared.datastore import connect_minio
|
||||||
|
|
||||||
|
|
||||||
|
class TestConnectMinio(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
# define relevant env vars
|
||||||
|
self.env_var_map = {
|
||||||
|
'MINIO_ENDPOINT': '192.168.1.2',
|
||||||
|
'MINIO_ACCESS_KEY': 'randomAccess_key',
|
||||||
|
'MINIO_SECRET_KEY': 'randomSecret_key',
|
||||||
|
'MINIO_BUCKET_NAME': 'test-bucket-name',
|
||||||
|
}
|
||||||
|
# set env vars
|
||||||
|
for key, val in self.env_var_map.items():
|
||||||
|
os.environ[key] = val
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
# clear env vars
|
||||||
|
for key in self.env_var_map:
|
||||||
|
_ = os.environ.pop(key, default=None)
|
||||||
|
|
||||||
|
def test_should_fail_when_env_not_set(self):
|
||||||
|
# ensure env not set
|
||||||
|
self.tearDown()
|
||||||
|
# run test
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
_ = connect_minio()
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
"""Definition of unittests for delete function."""
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
|
||||||
|
from shared.datastore import delete
|
||||||
|
|
||||||
|
|
||||||
|
class TestDelete(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
# set relevant variables
|
||||||
|
self.client = Mock(spec=Minio)
|
||||||
|
self.bucket_name = 'test-bucket'
|
||||||
|
self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX'
|
||||||
|
# set bad arguments
|
||||||
|
self.bad_client = 'not-minio-type'
|
||||||
|
self.bad_string = float(0.0)
|
||||||
|
self.len_0_string = ''
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_client(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
delete(
|
||||||
|
client=self.bad_client,
|
||||||
|
bucket_name=self.bucket_name,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_bucket_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
delete(
|
||||||
|
client=self.client,
|
||||||
|
bucket_name=self.bad_string,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_length_bucket_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
delete(
|
||||||
|
client=self.client,
|
||||||
|
bucket_name=self.len_0_string,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_object_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
delete(
|
||||||
|
client=self.client,
|
||||||
|
bucket_name=self.bucket_name,
|
||||||
|
object_name=self.bad_string,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_length_object_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
delete(
|
||||||
|
client=self.client,
|
||||||
|
bucket_name=self.bucket_name,
|
||||||
|
object_name=self.len_0_string,
|
||||||
|
)
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
"""Definition of unittests for get_image function."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
|
||||||
|
from shared.datastore import get_image
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetImage(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
# set relevant variables
|
||||||
|
self.client = MagicMock(spec=Minio)
|
||||||
|
self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX'
|
||||||
|
self.env_var_map = {
|
||||||
|
'MINIO_BUCKET_NAME': 'test-bucket',
|
||||||
|
}
|
||||||
|
# set bad arguments
|
||||||
|
self.bad_client = 'not-minio-type'
|
||||||
|
self.bad_string = float(0.0)
|
||||||
|
self.len_0_string = ''
|
||||||
|
# populate env
|
||||||
|
for key, val in self.env_var_map.items():
|
||||||
|
os.environ[key] = val
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
# clean env
|
||||||
|
for key in self.env_var_map:
|
||||||
|
_ = os.environ.pop(key, default=None)
|
||||||
|
|
||||||
|
def test_should_fail_when_env_not_set(self):
|
||||||
|
# ensure env not set
|
||||||
|
self.tearDown()
|
||||||
|
# run test
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get_image(
|
||||||
|
client=self.client,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_client(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get_image(
|
||||||
|
client=self.bad_client,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_object_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get_image(
|
||||||
|
client=self.client,
|
||||||
|
object_name=self.bad_string,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_length_object_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get_image(
|
||||||
|
client=self.client,
|
||||||
|
object_name=self.len_0_string,
|
||||||
|
)
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
"""Definition of unittest for get_model function."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
|
||||||
|
from shared.datastore import get_model
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetModel(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
# set relevant variables
|
||||||
|
self.client = MagicMock(spec=Minio)
|
||||||
|
self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX'
|
||||||
|
self.env_var_map = {
|
||||||
|
'MINIO_BUCKET_NAME': 'test-bucket',
|
||||||
|
}
|
||||||
|
# set bad arguments
|
||||||
|
self.bad_client = 'not-minio-type'
|
||||||
|
self.bad_string = float(0.0)
|
||||||
|
self.len_0_string = ''
|
||||||
|
# populate env
|
||||||
|
for key, val in self.env_var_map.items():
|
||||||
|
os.environ[key] = val
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
# clean env
|
||||||
|
for key in self.env_var_map:
|
||||||
|
_ = os.environ.pop(key, default=None)
|
||||||
|
|
||||||
|
def test_should_fail_when_env_not_set(self):
|
||||||
|
# ensure env not set
|
||||||
|
self.tearDown()
|
||||||
|
# run test
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get_model(
|
||||||
|
client=self.client,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_client(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get_model(
|
||||||
|
client=self.bad_client,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_object_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get_model(
|
||||||
|
client=self.client,
|
||||||
|
object_name=self.bad_string,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_length_object_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get_model(
|
||||||
|
client=self.client,
|
||||||
|
object_name=self.len_0_string,
|
||||||
|
)
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
"""Definition of unittests for get function."""
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
|
||||||
|
from shared.datastore import get
|
||||||
|
|
||||||
|
|
||||||
|
class TestGet(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
# set relevant variables
|
||||||
|
self.client = MagicMock(spec=Minio)
|
||||||
|
self.bucket_name = 'test-bucket'
|
||||||
|
self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX'
|
||||||
|
# set bad arguments
|
||||||
|
self.bad_client = 'not-minio-type'
|
||||||
|
self.bad_string = float(0.0)
|
||||||
|
self.len_0_string = ''
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_client(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get(
|
||||||
|
client=self.bad_client,
|
||||||
|
bucket_name=self.bucket_name,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_bucket_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get(
|
||||||
|
client=self.client,
|
||||||
|
bucket_name=self.bad_string,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_length_bucket_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get(
|
||||||
|
client=self.client,
|
||||||
|
bucket_name=self.len_0_string,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_object_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get(
|
||||||
|
client=self.client,
|
||||||
|
bucket_name=self.bucket_name,
|
||||||
|
object_name=self.bad_string,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_length_object_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
get(
|
||||||
|
client=self.client,
|
||||||
|
bucket_name=self.bucket_name,
|
||||||
|
object_name=self.len_0_string,
|
||||||
|
)
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
"""Definition of unittests for put_image function."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from shared.datastore import put_image
|
||||||
|
|
||||||
|
|
||||||
|
class TestPutImage(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
# set relevant variables
|
||||||
|
self.client = MagicMock(spec=Minio)
|
||||||
|
self.image = Image.new(mode='RGB', size=(480, 480))
|
||||||
|
self.env_var_map = {
|
||||||
|
'MINIO_BUCKET_NAME': 'test-bucket',
|
||||||
|
}
|
||||||
|
# set bad arguments
|
||||||
|
self.bad_client = 'not-minio-type'
|
||||||
|
self.bad_image = 'not-image-type'
|
||||||
|
# populate env
|
||||||
|
for key, val in self.env_var_map.items():
|
||||||
|
os.environ[key] = val
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_client(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
put_image(
|
||||||
|
client=self.bad_client,
|
||||||
|
image=self.image,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_image(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
put_image(
|
||||||
|
client=self.client,
|
||||||
|
image=self.bad_image,
|
||||||
|
)
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""Definition of unittests for put_model function."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
from torch.nn import Module
|
||||||
|
|
||||||
|
from shared.datastore import put_model
|
||||||
|
|
||||||
|
|
||||||
|
class TestPutModel(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
# set relevant variables
|
||||||
|
self.client = MagicMock(spec=Minio)
|
||||||
|
self.model = MagicMock(spec=Module)
|
||||||
|
self.env_var_map = {
|
||||||
|
'MINIO_BUCKET_NAME': 'test-bucket',
|
||||||
|
}
|
||||||
|
# set bad arguments
|
||||||
|
self.bad_client = 'not-minio-type'
|
||||||
|
self.bad_model = 'not-image-type'
|
||||||
|
# populate env
|
||||||
|
for key, val in self.env_var_map.items():
|
||||||
|
os.environ[key] = val
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_client(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
put_model(
|
||||||
|
client=self.bad_client,
|
||||||
|
model=self.model,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_model(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
put_model(
|
||||||
|
client=self.client,
|
||||||
|
model=self.bad_model,
|
||||||
|
)
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
"""Definition of unittests for put function."""
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from io import BytesIO
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from shared.datastore import put
|
||||||
|
|
||||||
|
|
||||||
|
class TestPut(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
# set relevant variables
|
||||||
|
self.client = MagicMock(spec=Minio)
|
||||||
|
self.bucket_name = 'test-bucket'
|
||||||
|
self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX'
|
||||||
|
self.image = Image.new(mode='RGB', size=(480, 480))
|
||||||
|
self.buffer = BytesIO()
|
||||||
|
self.image.save(self.buffer, 'png')
|
||||||
|
# set bad arguments
|
||||||
|
self.bad_client = 'not-minio-type'
|
||||||
|
self.bad_string = float(0.0)
|
||||||
|
self.bad_buffer = 'not-buffer-type'
|
||||||
|
self.len_0_string = ''
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_client(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
put(
|
||||||
|
client=self.bad_client,
|
||||||
|
buffer=self.buffer,
|
||||||
|
bucket_name=self.bucket_name,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_buffer(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
put(
|
||||||
|
client=self.client,
|
||||||
|
buffer=self.bad_buffer,
|
||||||
|
bucket_name=self.bucket_name,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_bucket_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
put(
|
||||||
|
client=self.client,
|
||||||
|
buffer=self.buffer,
|
||||||
|
bucket_name=self.bad_string,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_length_bucket_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
put(
|
||||||
|
client=self.client,
|
||||||
|
buffer=self.buffer,
|
||||||
|
bucket_name=self.len_0_string,
|
||||||
|
object_name=self.object_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_type_object_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
put(
|
||||||
|
client=self.client,
|
||||||
|
buffer=self.buffer,
|
||||||
|
bucket_name=self.bucket_name,
|
||||||
|
object_name=self.bad_string,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_should_fail_on_wrong_input_length_object_name(self):
|
||||||
|
with self.assertRaises(AssertionError):
|
||||||
|
put(
|
||||||
|
client=self.client,
|
||||||
|
buffer=self.buffer,
|
||||||
|
bucket_name=self.bucket_name,
|
||||||
|
object_name=self.len_0_string,
|
||||||
|
)
|
||||||
@@ -12,8 +12,8 @@ from PIL import Image
|
|||||||
from pydantic import BaseModel, ConfigDict
|
from pydantic import BaseModel, ConfigDict
|
||||||
from pymongo.collection import Collection
|
from pymongo.collection import Collection
|
||||||
|
|
||||||
from shared.datastore import get, put
|
from shared.datastore import get_image, put_image
|
||||||
from shared.mongodb.classes import ModelData
|
from shared.mongodb.src.classes import ModelData
|
||||||
|
|
||||||
|
|
||||||
class VisualCommunication(BaseModel):
|
class VisualCommunication(BaseModel):
|
||||||
@@ -39,11 +39,9 @@ class VisualCommunication(BaseModel):
|
|||||||
"""Upload image to MinIO and return MD5 checksum of hashed image."""
|
"""Upload image to MinIO and return MD5 checksum of hashed image."""
|
||||||
assert isinstance(image, Image.Image)
|
assert isinstance(image, Image.Image)
|
||||||
assert isinstance(minio_client, Minio)
|
assert isinstance(minio_client, Minio)
|
||||||
buffer = BytesIO()
|
object_name = put_image(
|
||||||
image.save(buffer, 'png')
|
|
||||||
object_name = put(
|
|
||||||
client=minio_client,
|
client=minio_client,
|
||||||
buffer=buffer,
|
image=image,
|
||||||
)
|
)
|
||||||
return object_name
|
return object_name
|
||||||
|
|
||||||
@@ -92,14 +90,12 @@ class VisualCommunication(BaseModel):
|
|||||||
def get_image(self, minio_client: Minio) -> Image.Image:
|
def get_image(self, minio_client: Minio) -> Image.Image:
|
||||||
"""Load image data from minio."""
|
"""Load image data from minio."""
|
||||||
assert isinstance(minio_client, Minio)
|
assert isinstance(minio_client, Minio)
|
||||||
# get buffer from minio
|
# get image from minio
|
||||||
buffer = get(
|
image = get_image(
|
||||||
client=minio_client,
|
client=minio_client,
|
||||||
object_name=self.object_name,
|
object_name=self.object_name,
|
||||||
)
|
)
|
||||||
# convert data to image
|
return image
|
||||||
im = Image.open(buffer)
|
|
||||||
return im
|
|
||||||
|
|
||||||
def save_to_mongo(self, collection: Collection) -> None:
|
def save_to_mongo(self, collection: Collection) -> None:
|
||||||
"""Save self as document in MongoDB."""
|
"""Save self as document in MongoDB."""
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ from pymongo.collection import Collection
|
|||||||
|
|
||||||
from shared.datastore import delete as delete_from_minio
|
from shared.datastore import delete as delete_from_minio
|
||||||
from shared.mongodb import count_documents, get_visual_communication, upsert_annotation
|
from shared.mongodb import count_documents, get_visual_communication, upsert_annotation
|
||||||
from shared.mongodb.classes import ModelData, VisualCommunication
|
from shared.mongodb.src.classes import ModelData, VisualCommunication
|
||||||
from shared.mongodb.exceptions import NoDocumentFoundException
|
from shared.mongodb.src.exceptions import NoDocumentFoundException
|
||||||
|
|
||||||
from .layout import app_layout
|
from .layout import app_layout
|
||||||
|
|
||||||
@@ -146,8 +146,10 @@ def init_app(
|
|||||||
logging.debug(exc)
|
logging.debug(exc)
|
||||||
failed_filename_list.append(filename)
|
failed_filename_list.append(filename)
|
||||||
# remove document from minio
|
# remove document from minio
|
||||||
|
bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
|
||||||
delete_from_minio(
|
delete_from_minio(
|
||||||
client=minio_client,
|
client=minio_client,
|
||||||
|
bucket_name=bucket_name,
|
||||||
object_name=vis_com.object_name,
|
object_name=vis_com.object_name,
|
||||||
)
|
)
|
||||||
assert (
|
assert (
|
||||||
|
|||||||
Reference in New Issue
Block a user