From d8e3d077e3af00f598d1879047e37163360d8fa5 Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 24 Oct 2024 16:40:26 +0000 Subject: [PATCH 01/45] moved files into src folder --- shared/datastore/__init__.py | 18 ++++++++++-------- shared/datastore/src/__init__.py | 8 ++++++++ shared/datastore/{ => src}/connect_minio.py | 0 shared/datastore/{ => src}/delete.py | 0 shared/datastore/{ => src}/get.py | 0 shared/datastore/{ => src}/get_image.py | 0 shared/datastore/{ => src}/get_model.py | 0 shared/datastore/{ => src}/put.py | 0 shared/datastore/{ => src}/put_image.py | 0 shared/datastore/{ => src}/put_model.py | 0 10 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 shared/datastore/src/__init__.py rename shared/datastore/{ => src}/connect_minio.py (100%) rename shared/datastore/{ => src}/delete.py (100%) rename shared/datastore/{ => src}/get.py (100%) rename shared/datastore/{ => src}/get_image.py (100%) rename shared/datastore/{ => src}/get_model.py (100%) rename shared/datastore/{ => src}/put.py (100%) rename shared/datastore/{ => src}/put_image.py (100%) rename shared/datastore/{ => src}/put_model.py (100%) diff --git a/shared/datastore/__init__.py b/shared/datastore/__init__.py index f4e0653..63e7a47 100644 --- a/shared/datastore/__init__.py +++ b/shared/datastore/__init__.py @@ -1,8 +1,10 @@ -from .connect_minio import connect_minio -from .delete import delete -from .get import get -from .get_image import get_image -from .get_model import get_model -from .put import put -from .put_image import put_image -from .put_model import put_model +from .src import ( + connect_minio, + delete, + get, + get_image, + get_model, + put, + put_image, + put_model, +) diff --git a/shared/datastore/src/__init__.py b/shared/datastore/src/__init__.py new file mode 100644 index 0000000..f4e0653 --- /dev/null +++ b/shared/datastore/src/__init__.py @@ -0,0 +1,8 @@ +from .connect_minio import connect_minio +from .delete import delete +from .get import get +from .get_image import get_image +from .get_model import get_model +from .put import put +from .put_image import put_image +from .put_model import put_model diff --git a/shared/datastore/connect_minio.py b/shared/datastore/src/connect_minio.py similarity index 100% rename from shared/datastore/connect_minio.py rename to shared/datastore/src/connect_minio.py diff --git a/shared/datastore/delete.py b/shared/datastore/src/delete.py similarity index 100% rename from shared/datastore/delete.py rename to shared/datastore/src/delete.py diff --git a/shared/datastore/get.py b/shared/datastore/src/get.py similarity index 100% rename from shared/datastore/get.py rename to shared/datastore/src/get.py diff --git a/shared/datastore/get_image.py b/shared/datastore/src/get_image.py similarity index 100% rename from shared/datastore/get_image.py rename to shared/datastore/src/get_image.py diff --git a/shared/datastore/get_model.py b/shared/datastore/src/get_model.py similarity index 100% rename from shared/datastore/get_model.py rename to shared/datastore/src/get_model.py diff --git a/shared/datastore/put.py b/shared/datastore/src/put.py similarity index 100% rename from shared/datastore/put.py rename to shared/datastore/src/put.py diff --git a/shared/datastore/put_image.py b/shared/datastore/src/put_image.py similarity index 100% rename from shared/datastore/put_image.py rename to shared/datastore/src/put_image.py diff --git a/shared/datastore/put_model.py b/shared/datastore/src/put_model.py similarity index 100% rename from shared/datastore/put_model.py rename to shared/datastore/src/put_model.py -- 2.54.0 From ae18041d6ea2e1525deb05f421ac67a975e24525 Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 24 Oct 2024 16:46:35 +0000 Subject: [PATCH 02/45] simplified code --- shared/datastore/src/connect_minio.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/shared/datastore/src/connect_minio.py b/shared/datastore/src/connect_minio.py index c0bd3f5..aeef685 100644 --- a/shared/datastore/src/connect_minio.py +++ b/shared/datastore/src/connect_minio.py @@ -5,23 +5,19 @@ import os from minio import Minio +from shared.utils import check_env + def connect_minio() -> Minio: """Connect to MinIO server.""" # ensure necessary env vars available - env_var_list = [ + var_list = { 'MINIO_ENDPOINT', 'MINIO_ACCESS_KEY', 'MINIO_SECRET_KEY', 'MINIO_BUCKET_NAME', - ] - for env_var in env_var_list: - # ensure env var set - assert ( - env_var in os.environ - ), f"environment variable not set: { - env_var - }" + } + check_env(var_list) # prepare arguments minio_endpoint = os.getenv('MINIO_ENDPOINT', default='') minio_access_key = os.getenv('MINIO_ACCESS_KEY', default='') -- 2.54.0 From b755a81ef9e70f3082dabb8bc4a7c15cad2f33e5 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 25 Oct 2024 09:25:52 +0000 Subject: [PATCH 03/45] simplified code --- shared/datastore/src/connect_minio.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shared/datastore/src/connect_minio.py b/shared/datastore/src/connect_minio.py index aeef685..7dadb4e 100644 --- a/shared/datastore/src/connect_minio.py +++ b/shared/datastore/src/connect_minio.py @@ -19,10 +19,10 @@ def connect_minio() -> Minio: } check_env(var_list) # prepare arguments - minio_endpoint = os.getenv('MINIO_ENDPOINT', default='') - minio_access_key = os.getenv('MINIO_ACCESS_KEY', default='') - minio_secret_key = os.getenv('MINIO_SECRET_KEY', default='') - minio_bucket_name = os.getenv('MINIO_BUCKET_NAME', default='') + minio_endpoint = str(os.getenv('MINIO_ENDPOINT')) + minio_access_key = str(os.getenv('MINIO_ACCESS_KEY')) + minio_secret_key = str(os.getenv('MINIO_SECRET_KEY')) + minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) # connect client client = Minio( endpoint=minio_endpoint, -- 2.54.0 From b722ddcb83ea8fd8588fe5fca317a9c1f89aae00 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 25 Oct 2024 09:26:10 +0000 Subject: [PATCH 04/45] installed new packages for testing --- poetry.lock | 202 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 202 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 83e3e02..b646a8c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -552,6 +552,33 @@ type = "legacy" url = "http://192.168.1.2:5001/index" reference = "threadripper" +[[package]] +name = "docker" +version = "7.1.0" +description = "A Python library for the Docker Engine API." +optional = false +python-versions = ">=3.8" +files = [ + {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"}, + {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"}, +] + +[package.dependencies] +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" + +[package.extras] +dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"] +docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"] +ssh = ["paramiko (>=2.4.3)"] +websockets = ["websocket-client (>=1.3.0)"] + +[package.source] +type = "legacy" +url = "http://192.168.1.2:5001/index" +reference = "threadripper" + [[package]] name = "filelock" version = "3.14.0" @@ -2114,6 +2141,38 @@ type = "legacy" url = "http://192.168.1.2:5001/index" reference = "threadripper" +[[package]] +name = "pywin32" +version = "308" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"}, + {file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"}, + {file = "pywin32-308-cp310-cp310-win_arm64.whl", hash = "sha256:a5ab5381813b40f264fa3495b98af850098f814a25a63589a8e9eb12560f450c"}, + {file = "pywin32-308-cp311-cp311-win32.whl", hash = "sha256:5d8c8015b24a7d6855b1550d8e660d8daa09983c80e5daf89a273e5c6fb5095a"}, + {file = "pywin32-308-cp311-cp311-win_amd64.whl", hash = "sha256:575621b90f0dc2695fec346b2d6302faebd4f0f45c05ea29404cefe35d89442b"}, + {file = "pywin32-308-cp311-cp311-win_arm64.whl", hash = "sha256:100a5442b7332070983c4cd03f2e906a5648a5104b8a7f50175f7906efd16bb6"}, + {file = "pywin32-308-cp312-cp312-win32.whl", hash = "sha256:587f3e19696f4bf96fde9d8a57cec74a57021ad5f204c9e627e15c33ff568897"}, + {file = "pywin32-308-cp312-cp312-win_amd64.whl", hash = "sha256:00b3e11ef09ede56c6a43c71f2d31857cf7c54b0ab6e78ac659497abd2834f47"}, + {file = "pywin32-308-cp312-cp312-win_arm64.whl", hash = "sha256:9b4de86c8d909aed15b7011182c8cab38c8850de36e6afb1f0db22b8959e3091"}, + {file = "pywin32-308-cp313-cp313-win32.whl", hash = "sha256:1c44539a37a5b7b21d02ab34e6a4d314e0788f1690d65b48e9b0b89f31abbbed"}, + {file = "pywin32-308-cp313-cp313-win_amd64.whl", hash = "sha256:fd380990e792eaf6827fcb7e187b2b4b1cede0585e3d0c9e84201ec27b9905e4"}, + {file = "pywin32-308-cp313-cp313-win_arm64.whl", hash = "sha256:ef313c46d4c18dfb82a2431e3051ac8f112ccee1a34f29c263c583c568db63cd"}, + {file = "pywin32-308-cp37-cp37m-win32.whl", hash = "sha256:1f696ab352a2ddd63bd07430080dd598e6369152ea13a25ebcdd2f503a38f1ff"}, + {file = "pywin32-308-cp37-cp37m-win_amd64.whl", hash = "sha256:13dcb914ed4347019fbec6697a01a0aec61019c1046c2b905410d197856326a6"}, + {file = "pywin32-308-cp38-cp38-win32.whl", hash = "sha256:5794e764ebcabf4ff08c555b31bd348c9025929371763b2183172ff4708152f0"}, + {file = "pywin32-308-cp38-cp38-win_amd64.whl", hash = "sha256:3b92622e29d651c6b783e368ba7d6722b1634b8e70bd376fd7610fe1992e19de"}, + {file = "pywin32-308-cp39-cp39-win32.whl", hash = "sha256:7873ca4dc60ab3287919881a7d4f88baee4a6e639aa6962de25a98ba6b193341"}, + {file = "pywin32-308-cp39-cp39-win_amd64.whl", hash = "sha256:71b3322d949b4cc20776436a9c9ba0eeedcbc9c650daa536df63f0ff111bb920"}, +] + +[package.source] +type = "legacy" +url = "http://192.168.1.2:5001/index" +reference = "threadripper" + [[package]] name = "requests" version = "2.32.3" @@ -2353,6 +2412,63 @@ type = "legacy" url = "http://192.168.1.2:5001/index" reference = "threadripper" +[[package]] +name = "testcontainers" +version = "4.8.2" +description = "Python library for throwaway instances of anything that can run in a Docker container" +optional = false +python-versions = ">=3.9,<4.0" +files = [ + {file = "testcontainers-4.8.2-py3-none-any.whl", hash = "sha256:9e19af077cd96e1957c13ee466f1f32905bc6c5bc1bc98643eb18be1a989bfb0"}, + {file = "testcontainers-4.8.2.tar.gz", hash = "sha256:dd4a6a2ea09e3c3ecd39e180b6548105929d0bb78d665ce9919cb3f8c98f9853"}, +] + +[package.dependencies] +docker = "*" +typing-extensions = "*" +urllib3 = "*" +wrapt = "*" + +[package.extras] +arangodb = ["python-arango (>=7.8,<8.0)"] +aws = ["boto3", "httpx"] +azurite = ["azure-storage-blob (>=12.19,<13.0)"] +chroma = ["chromadb-client"] +clickhouse = ["clickhouse-driver"] +cosmosdb = ["azure-cosmos"] +db2 = ["ibm_db_sa", "sqlalchemy"] +generic = ["httpx", "redis"] +google = ["google-cloud-datastore (>=2)", "google-cloud-pubsub (>=2)"] +influxdb = ["influxdb", "influxdb-client"] +k3s = ["kubernetes", "pyyaml"] +keycloak = ["python-keycloak"] +localstack = ["boto3"] +mailpit = ["cryptography"] +minio = ["minio"] +mongodb = ["pymongo"] +mssql = ["pymssql", "sqlalchemy"] +mysql = ["pymysql[rsa]", "sqlalchemy"] +nats = ["nats-py"] +neo4j = ["neo4j"] +opensearch = ["opensearch-py"] +oracle = ["oracledb", "sqlalchemy"] +oracle-free = ["oracledb", "sqlalchemy"] +qdrant = ["qdrant-client"] +rabbitmq = ["pika"] +redis = ["redis"] +registry = ["bcrypt"] +scylla = ["cassandra-driver (==3.29.1)"] +selenium = ["selenium"] +sftp = ["cryptography"] +test-module-import = ["httpx"] +trino = ["trino"] +weaviate = ["weaviate-client (>=4.5.4,<5.0.0)"] + +[package.source] +type = "legacy" +url = "http://192.168.1.2:5001/index" +reference = "threadripper" + [[package]] name = "torch" version = "2.2.2" @@ -2730,6 +2846,90 @@ type = "legacy" url = "http://192.168.1.2:5001/index" reference = "threadripper" +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[package.source] +type = "legacy" +url = "http://192.168.1.2:5001/index" +reference = "threadripper" + [[package]] name = "wsproto" version = "1.2.0" @@ -2772,4 +2972,4 @@ reference = "threadripper" [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "33dbc73047d591ae63dd2d934519882654074a1f3c9ca757d0571d869b8541c2" +content-hash = "c5e7d72d26658899174ec00fe2cb435b0e571efe4efe17988b0c94a80c893597" diff --git a/pyproject.toml b/pyproject.toml index 24c6fb4..00dab39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ flake8-pyproject = "^1.2.3" pandas-stubs = "^2.2.2.240603" types-tqdm = "^4.66.0.20240417" pytest = "^8.3.3" +testcontainers = "^4.8.2" [tool.poetry.group.dev.dependencies] -- 2.54.0 From dc535167fa477ccca797da9266eb79c4f4135a99 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 25 Oct 2024 09:27:01 +0000 Subject: [PATCH 05/45] started making tests --- shared/datastore/tests/integration_test.py | 62 ++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 shared/datastore/tests/integration_test.py diff --git a/shared/datastore/tests/integration_test.py b/shared/datastore/tests/integration_test.py new file mode 100644 index 0000000..6763761 --- /dev/null +++ b/shared/datastore/tests/integration_test.py @@ -0,0 +1,62 @@ +"""Integration tests for functions exposed by the datastore module.""" + +import os + +import pytest +from minio import Minio +from testcontainers.minio import MinioContainer + +from shared.datastore import connect_minio + +minio = MinioContainer() +MINIO_BUCKET_NAME = 'visual-critical-discourse-analysis' + + +@pytest.fixture(scope='module', autouse=True) +def setup(request): + """Setup function for datastore module integration tests.""" + # start minio + minio.start() + + # ensure minio is stopped after testing + def remove_container(): + minio.stop() + + request.addfinalizer(remove_container) + # extract information + minio_ip = minio.get_container_host_ip() + minio_port = minio.get_exposed_port(port=9000) + os.environ['MINIO_ENDPOINT'] = f'{minio_ip}:{minio_port}' + os.environ['MINIO_ACCESS_KEY'] = minio.access_key + os.environ['MINIO_SECRET_KEY'] = minio.secret_key + os.environ['MINIO_BUCKET_NAME'] = MINIO_BUCKET_NAME + # save random image to Minio + pass + # save random model to Minio + pass + + +def test_connect_minio(): + """Test function connect_minio.""" + minio_client = connect_minio() + assert isinstance(minio_client, Minio) + + +def test_get_image(): + """Test function get_image.""" + raise NotImplementedError() + + +def test_put_image(): + """Test function put_image.""" + raise NotImplementedError() + + +def test_get_model(): + """Test function get_model.""" + raise NotImplementedError() + + +def test_put_model(): + """Test function put_model.""" + raise NotImplementedError() -- 2.54.0 From e189b7c1f7509115e45b604bb21707c0d676dc3f Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 25 Oct 2024 17:09:43 +0000 Subject: [PATCH 06/45] updated for more general use --- shared/datastore/src/put.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/shared/datastore/src/put.py b/shared/datastore/src/put.py index fdd64c3..3e6c99e 100644 --- a/shared/datastore/src/put.py +++ b/shared/datastore/src/put.py @@ -1,10 +1,6 @@ """Definition of put function.""" -from __future__ import annotations - import logging -import os -from hashlib import md5 from io import BytesIO from minio import Minio @@ -13,14 +9,16 @@ from minio import Minio def put( client: Minio, buffer: BytesIO, -) -> str: + bucket_name: str, + object_name: str, +) -> None: """Put buffer in bucket in MinIO and return MD5 checksum as object name.""" assert isinstance(client, Minio) assert isinstance(buffer, BytesIO) - bucket_name = os.getenv('MINIO_BUCKET_NAME', default='') + assert isinstance(bucket_name, str) assert len(bucket_name) > 0 - # get md5 of image - checksum = md5(buffer.getbuffer()).hexdigest() + assert isinstance(object_name, str) + assert len(object_name) > 0 # prepare for saving num_bytes = buffer.tell() buffer.seek(0) @@ -28,12 +26,11 @@ def put( try: client.put_object( bucket_name=bucket_name, - object_name=checksum, + object_name=object_name, length=num_bytes, data=buffer, ) except Exception as exc: logging.error('failed saving data to MinIO') raise exc - logging.debug('saved data to %s', checksum) - return checksum + logging.debug('saved data to %s', object_name) -- 2.54.0 From 0676011f987ff476921734a12e0807d485867c13 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 25 Oct 2024 17:10:03 +0000 Subject: [PATCH 07/45] implemented new more general function --- shared/datastore/src/put_image.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/shared/datastore/src/put_image.py b/shared/datastore/src/put_image.py index 0865fa7..b5de63c 100644 --- a/shared/datastore/src/put_image.py +++ b/shared/datastore/src/put_image.py @@ -8,6 +8,8 @@ from io import BytesIO from minio import Minio from PIL import Image +from .put import put + def put_image( client: Minio, @@ -17,28 +19,23 @@ def put_image( used as object name.""" assert isinstance(client, Minio) assert isinstance(image, Image.Image) + # get bucket name from env bucket_name = os.getenv('MINIO_BUCKET_NAME', default='') assert len(bucket_name) > 0 - subfolder = 'images' # save image to buffer buffer = BytesIO() image.save(buffer, 'png') - # get md5 of image + # get md5 of buffer checksum = md5(buffer.getbuffer()).hexdigest() - # prepare for saving - num_bytes = buffer.tell() - buffer.seek(0) - # send data to bucket + # determine object name + subfolder = 'images' object_name = f'{subfolder}/{checksum}' - try: - client.put_object( - bucket_name=bucket_name, - object_name=object_name, - length=num_bytes, - data=buffer, - ) - except Exception as exc: - logging.error('failed saving data to MinIO') - raise exc - logging.debug('saved data to %s', object_name) + # send data to bucket + put( + client=client, + buffer=buffer, + bucket_name=bucket_name, + object_name=object_name, + ) + logging.debug('finished') return checksum -- 2.54.0 From 42400d6f32934ed7729d6c27e2da3fc6b85088aa Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 25 Oct 2024 17:18:53 +0000 Subject: [PATCH 08/45] updated for readability --- shared/datastore/src/get_image.py | 9 +++++---- shared/datastore/src/get_model.py | 29 +++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/shared/datastore/src/get_image.py b/shared/datastore/src/get_image.py index 8f92f7d..ef81762 100644 --- a/shared/datastore/src/get_image.py +++ b/shared/datastore/src/get_image.py @@ -7,6 +7,8 @@ from io import BytesIO from minio import Minio from PIL import Image +from shared.utils import check_env + def get_image( client: Minio, @@ -17,11 +19,10 @@ def get_image( assert isinstance(object_name, str) assert len(object_name) > 0 # prepare arguments - assert 'MINIO_BUCKET_NAME' in os.environ - bucket_name = os.getenv('MINIO_BUCKET_NAME', default='') - subfolder = 'images' + check_env({'MINIO_BUCKET_NAME'}) + bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) + object_name = f'images/{object_name}' # get object from bucket - object_name = f'{subfolder}/{object_name}' try: response = client.get_object( bucket_name=bucket_name, diff --git a/shared/datastore/src/get_model.py b/shared/datastore/src/get_model.py index dd6f38d..2cc40e7 100644 --- a/shared/datastore/src/get_model.py +++ b/shared/datastore/src/get_model.py @@ -1,12 +1,14 @@ """Definition of get_model function.""" import logging +import os from collections import OrderedDict +from io import BytesIO import torch from minio import Minio -from .get import get +from shared.utils import check_env def get_model( @@ -16,13 +18,24 @@ def get_model( """Get model from model subfolder in bucket in Minio.""" assert isinstance(client, Minio) assert isinstance(object_name, str) - subfolder = 'models' - object_name = f'{subfolder}/{object_name}' - # get buffer - buffer = get( - client=client, - object_name=object_name, - ) + assert len(object_name) > 0 + # prepare arguments + check_env({'MINIO_BUCKET_NAME_MODELS'}) + bucket_name = str(os.getenv('MINIO_BUCKET_NAME_MODELS')) + object_name = f'models/{object_name}' + # get object from bucket + try: + response = client.get_object( + bucket_name=bucket_name, + object_name=object_name, + ) + buffer = BytesIO(response.data) + except Exception as exc: + logging.error('failed getting data from MinIO') + raise exc + finally: + response.close() + response.release_conn() # convert data to model checkpoint buffer.seek(0) model_content = torch.load(buffer) -- 2.54.0 From bdf3e73ba9a6b17cad7d528b99f74fb06009db08 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 25 Oct 2024 17:19:15 +0000 Subject: [PATCH 09/45] updated for more general use --- shared/datastore/src/get.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/shared/datastore/src/get.py b/shared/datastore/src/get.py index b41ef63..dfd9206 100644 --- a/shared/datastore/src/get.py +++ b/shared/datastore/src/get.py @@ -1,7 +1,6 @@ """Definition of get function.""" import logging -import os from io import BytesIO from traceback import print_exc @@ -10,19 +9,22 @@ from minio import Minio def get( client: Minio, + bucket_name: str, object_name: str, ) -> BytesIO: """Get buffer from bucket in MinIO.""" assert isinstance(client, Minio) assert isinstance(object_name, str) - bucket_name = os.getenv('MINIO_BUCKET_NAME', default='') - # get buffer + assert isinstance(bucket_name, str) + assert len(bucket_name) > 0 try: + # make request response = client.get_object( bucket_name=bucket_name, object_name=object_name, ) assert response.status == 200 + # get buffer buffer = BytesIO() chunk_size = 2**14 while chunk := response.read(chunk_size): -- 2.54.0 From 10bfac73bd07a723686a3fc6cc582f7371d5d982 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 25 Oct 2024 17:19:47 +0000 Subject: [PATCH 10/45] extended tests --- shared/datastore/tests/integration_test.py | 58 ++++++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/shared/datastore/tests/integration_test.py b/shared/datastore/tests/integration_test.py index 6763761..b8277c9 100644 --- a/shared/datastore/tests/integration_test.py +++ b/shared/datastore/tests/integration_test.py @@ -1,12 +1,15 @@ """Integration tests for functions exposed by the datastore module.""" import os +from io import BytesIO +from random import randbytes import pytest from minio import Minio from testcontainers.minio import MinioContainer -from shared.datastore import connect_minio +from shared.datastore import connect_minio, get, put +from shared.utils import setup_logging minio = MinioContainer() MINIO_BUCKET_NAME = 'visual-critical-discourse-analysis' @@ -30,10 +33,6 @@ def setup(request): os.environ['MINIO_ACCESS_KEY'] = minio.access_key os.environ['MINIO_SECRET_KEY'] = minio.secret_key os.environ['MINIO_BUCKET_NAME'] = MINIO_BUCKET_NAME - # save random image to Minio - pass - # save random model to Minio - pass def test_connect_minio(): @@ -42,6 +41,49 @@ def test_connect_minio(): assert isinstance(minio_client, Minio) +def test_put(): + """Test function put.""" + # connect to minio + minio_client = connect_minio() + # generate random bytes + buffer = BytesIO(randbytes(2**20)) + # put data in Minio + put( + client=minio_client, + buffer=buffer, + bucket_name=MINIO_BUCKET_NAME, + object_name='test_put_data', + ) + + +def test_get(): + """Test function get.""" + # connect to minio + minio_client = connect_minio() + # generate random bytes + buffer = BytesIO(randbytes(2**20)) + # put data in minio + put( + client=minio_client, + buffer=buffer, + bucket_name=MINIO_BUCKET_NAME, + object_name='test_get_data', + ) + # get data back from minio + buffer_rec = get( + client=minio_client, + bucket_name=MINIO_BUCKET_NAME, + object_name='test_get_data', + ) + # check that data are the same + assert buffer.read() == buffer_rec.read() + + +def test_delete(): + """Test function delete.""" + raise NotImplementedError() + + def test_get_image(): """Test function get_image.""" raise NotImplementedError() @@ -60,3 +102,9 @@ def test_get_model(): def test_put_model(): """Test function put_model.""" raise NotImplementedError() + + +if __name__ == '__main__': + os.environ['LOG_LEVEL'] = 'debug' + setup_logging() + pytest.main() -- 2.54.0 From 0c9537975cf72fd5f36afc7616793fdf459c0bd8 Mon Sep 17 00:00:00 2001 From: brian Date: Mon, 28 Oct 2024 19:43:37 +0000 Subject: [PATCH 11/45] updated bucket name --- shared/datastore/tests/integration_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/datastore/tests/integration_test.py b/shared/datastore/tests/integration_test.py index b8277c9..959d8b1 100644 --- a/shared/datastore/tests/integration_test.py +++ b/shared/datastore/tests/integration_test.py @@ -11,12 +11,12 @@ from testcontainers.minio import MinioContainer from shared.datastore import connect_minio, get, put from shared.utils import setup_logging +MINIO_BUCKET_NAME = 'test-bucket' minio = MinioContainer() -MINIO_BUCKET_NAME = 'visual-critical-discourse-analysis' @pytest.fixture(scope='module', autouse=True) -def setup(request): +def setup(request: pytest.FixtureRequest): """Setup function for datastore module integration tests.""" # start minio minio.start() -- 2.54.0 From 462f7c79b48985daf40d66c97a5ceb12d7d02698 Mon Sep 17 00:00:00 2001 From: brian Date: Mon, 28 Oct 2024 19:43:56 +0000 Subject: [PATCH 12/45] added step to print coverage report --- .gitea/workflows/pull.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitea/workflows/pull.yaml b/.gitea/workflows/pull.yaml index 422132f..f0c93c6 100644 --- a/.gitea/workflows/pull.yaml +++ b/.gitea/workflows/pull.yaml @@ -31,3 +31,6 @@ jobs: - name: Pytest run: | poetry run pytest . + - name: Coverage Report + run: | + poetry run coverage report -m -- 2.54.0 From 2854f74db4f73ffae4c0c98a96764c67a42cc72e Mon Sep 17 00:00:00 2001 From: brian Date: Mon, 28 Oct 2024 19:44:16 +0000 Subject: [PATCH 13/45] installed package for generating coverage report --- poetry.lock | 81 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index b646a8c..6a68bfd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -363,6 +363,85 @@ type = "legacy" url = "http://192.168.1.2:5001/index" reference = "threadripper" +[[package]] +name = "coverage" +version = "7.6.4" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "coverage-7.6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5f8ae553cba74085db385d489c7a792ad66f7f9ba2ee85bfa508aeb84cf0ba07"}, + {file = "coverage-7.6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8165b796df0bd42e10527a3f493c592ba494f16ef3c8b531288e3d0d72c1f6f0"}, + {file = "coverage-7.6.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7c8b95bf47db6d19096a5e052ffca0a05f335bc63cef281a6e8fe864d450a72"}, + {file = "coverage-7.6.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ed9281d1b52628e81393f5eaee24a45cbd64965f41857559c2b7ff19385df51"}, + {file = "coverage-7.6.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0809082ee480bb8f7416507538243c8863ac74fd8a5d2485c46f0f7499f2b491"}, + {file = "coverage-7.6.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d541423cdd416b78626b55f123412fcf979d22a2c39fce251b350de38c15c15b"}, + {file = "coverage-7.6.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:58809e238a8a12a625c70450b48e8767cff9eb67c62e6154a642b21ddf79baea"}, + {file = "coverage-7.6.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c9b8e184898ed014884ca84c70562b4a82cbc63b044d366fedc68bc2b2f3394a"}, + {file = "coverage-7.6.4-cp310-cp310-win32.whl", hash = "sha256:6bd818b7ea14bc6e1f06e241e8234508b21edf1b242d49831831a9450e2f35fa"}, + {file = "coverage-7.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:06babbb8f4e74b063dbaeb74ad68dfce9186c595a15f11f5d5683f748fa1d172"}, + {file = "coverage-7.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:73d2b73584446e66ee633eaad1a56aad577c077f46c35ca3283cd687b7715b0b"}, + {file = "coverage-7.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:51b44306032045b383a7a8a2c13878de375117946d68dcb54308111f39775a25"}, + {file = "coverage-7.6.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b3fb02fe73bed561fa12d279a417b432e5b50fe03e8d663d61b3d5990f29546"}, + {file = "coverage-7.6.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed8fe9189d2beb6edc14d3ad19800626e1d9f2d975e436f84e19efb7fa19469b"}, + {file = "coverage-7.6.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b369ead6527d025a0fe7bd3864e46dbee3aa8f652d48df6174f8d0bac9e26e0e"}, + {file = "coverage-7.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ade3ca1e5f0ff46b678b66201f7ff477e8fa11fb537f3b55c3f0568fbfe6e718"}, + {file = "coverage-7.6.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:27fb4a050aaf18772db513091c9c13f6cb94ed40eacdef8dad8411d92d9992db"}, + {file = "coverage-7.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4f704f0998911abf728a7783799444fcbbe8261c4a6c166f667937ae6a8aa522"}, + {file = "coverage-7.6.4-cp311-cp311-win32.whl", hash = "sha256:29155cd511ee058e260db648b6182c419422a0d2e9a4fa44501898cf918866cf"}, + {file = "coverage-7.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:8902dd6a30173d4ef09954bfcb24b5d7b5190cf14a43170e386979651e09ba19"}, + {file = "coverage-7.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12394842a3a8affa3ba62b0d4ab7e9e210c5e366fbac3e8b2a68636fb19892c2"}, + {file = "coverage-7.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b6b4c83d8e8ea79f27ab80778c19bc037759aea298da4b56621f4474ffeb117"}, + {file = "coverage-7.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d5b8007f81b88696d06f7df0cb9af0d3b835fe0c8dbf489bad70b45f0e45613"}, + {file = "coverage-7.6.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b57b768feb866f44eeed9f46975f3d6406380275c5ddfe22f531a2bf187eda27"}, + {file = "coverage-7.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5915fcdec0e54ee229926868e9b08586376cae1f5faa9bbaf8faf3561b393d52"}, + {file = "coverage-7.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b58c672d14f16ed92a48db984612f5ce3836ae7d72cdd161001cc54512571f2"}, + {file = "coverage-7.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2fdef0d83a2d08d69b1f2210a93c416d54e14d9eb398f6ab2f0a209433db19e1"}, + {file = "coverage-7.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8cf717ee42012be8c0cb205dbbf18ffa9003c4cbf4ad078db47b95e10748eec5"}, + {file = "coverage-7.6.4-cp312-cp312-win32.whl", hash = "sha256:7bb92c539a624cf86296dd0c68cd5cc286c9eef2d0c3b8b192b604ce9de20a17"}, + {file = "coverage-7.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:1032e178b76a4e2b5b32e19d0fd0abbce4b58e77a1ca695820d10e491fa32b08"}, + {file = "coverage-7.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:023bf8ee3ec6d35af9c1c6ccc1d18fa69afa1cb29eaac57cb064dbb262a517f9"}, + {file = "coverage-7.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b0ac3d42cb51c4b12df9c5f0dd2f13a4f24f01943627120ec4d293c9181219ba"}, + {file = "coverage-7.6.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8fe4984b431f8621ca53d9380901f62bfb54ff759a1348cd140490ada7b693c"}, + {file = "coverage-7.6.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5fbd612f8a091954a0c8dd4c0b571b973487277d26476f8480bfa4b2a65b5d06"}, + {file = "coverage-7.6.4-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dacbc52de979f2823a819571f2e3a350a7e36b8cb7484cdb1e289bceaf35305f"}, + {file = "coverage-7.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dab4d16dfef34b185032580e2f2f89253d302facba093d5fa9dbe04f569c4f4b"}, + {file = "coverage-7.6.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:862264b12ebb65ad8d863d51f17758b1684560b66ab02770d4f0baf2ff75da21"}, + {file = "coverage-7.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5beb1ee382ad32afe424097de57134175fea3faf847b9af002cc7895be4e2a5a"}, + {file = "coverage-7.6.4-cp313-cp313-win32.whl", hash = "sha256:bf20494da9653f6410213424f5f8ad0ed885e01f7e8e59811f572bdb20b8972e"}, + {file = "coverage-7.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:182e6cd5c040cec0a1c8d415a87b67ed01193ed9ad458ee427741c7d8513d963"}, + {file = "coverage-7.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a181e99301a0ae128493a24cfe5cfb5b488c4e0bf2f8702091473d033494d04f"}, + {file = "coverage-7.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:df57bdbeffe694e7842092c5e2e0bc80fff7f43379d465f932ef36f027179806"}, + {file = "coverage-7.6.4-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bcd1069e710600e8e4cf27f65c90c7843fa8edfb4520fb0ccb88894cad08b11"}, + {file = "coverage-7.6.4-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99b41d18e6b2a48ba949418db48159d7a2e81c5cc290fc934b7d2380515bd0e3"}, + {file = "coverage-7.6.4-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1e54712ba3474f34b7ef7a41e65bd9037ad47916ccb1cc78769bae324c01a"}, + {file = "coverage-7.6.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:53d202fd109416ce011578f321460795abfe10bb901b883cafd9b3ef851bacfc"}, + {file = "coverage-7.6.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:c48167910a8f644671de9f2083a23630fbf7a1cb70ce939440cd3328e0919f70"}, + {file = "coverage-7.6.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cc8ff50b50ce532de2fa7a7daae9dd12f0a699bfcd47f20945364e5c31799fef"}, + {file = "coverage-7.6.4-cp313-cp313t-win32.whl", hash = "sha256:b8d3a03d9bfcaf5b0141d07a88456bb6a4c3ce55c080712fec8418ef3610230e"}, + {file = "coverage-7.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:f3ddf056d3ebcf6ce47bdaf56142af51bb7fad09e4af310241e9db7a3a8022e1"}, + {file = "coverage-7.6.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9cb7fa111d21a6b55cbf633039f7bc2749e74932e3aa7cb7333f675a58a58bf3"}, + {file = "coverage-7.6.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:11a223a14e91a4693d2d0755c7a043db43d96a7450b4f356d506c2562c48642c"}, + {file = "coverage-7.6.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a413a096c4cbac202433c850ee43fa326d2e871b24554da8327b01632673a076"}, + {file = "coverage-7.6.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00a1d69c112ff5149cabe60d2e2ee948752c975d95f1e1096742e6077affd376"}, + {file = "coverage-7.6.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f76846299ba5c54d12c91d776d9605ae33f8ae2b9d1d3c3703cf2db1a67f2c0"}, + {file = "coverage-7.6.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fe439416eb6380de434886b00c859304338f8b19f6f54811984f3420a2e03858"}, + {file = "coverage-7.6.4-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:0294ca37f1ba500667b1aef631e48d875ced93ad5e06fa665a3295bdd1d95111"}, + {file = "coverage-7.6.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6f01ba56b1c0e9d149f9ac85a2f999724895229eb36bd997b61e62999e9b0901"}, + {file = "coverage-7.6.4-cp39-cp39-win32.whl", hash = "sha256:bc66f0bf1d7730a17430a50163bb264ba9ded56739112368ba985ddaa9c3bd09"}, + {file = "coverage-7.6.4-cp39-cp39-win_amd64.whl", hash = "sha256:c481b47f6b5845064c65a7bc78bc0860e635a9b055af0df46fdf1c58cebf8e8f"}, + {file = "coverage-7.6.4-pp39.pp310-none-any.whl", hash = "sha256:3c65d37f3a9ebb703e710befdc489a38683a5b152242664b973a7b7b22348a4e"}, + {file = "coverage-7.6.4.tar.gz", hash = "sha256:29fc0f17b1d3fea332f8001d4558f8214af7f1d87a345f3a133c901d60347c73"}, +] + +[package.extras] +toml = ["tomli"] + +[package.source] +type = "legacy" +url = "http://192.168.1.2:5001/index" +reference = "threadripper" + [[package]] name = "dash" version = "2.17.0" @@ -2972,4 +3051,4 @@ reference = "threadripper" [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "c5e7d72d26658899174ec00fe2cb435b0e571efe4efe17988b0c94a80c893597" +content-hash = "8ac4e962d93b5a2f23e263d60116c5bb2ab27956ff6eacb5dbc9a6434ac2669c" diff --git a/pyproject.toml b/pyproject.toml index 00dab39..1bee0e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ pandas-stubs = "^2.2.2.240603" types-tqdm = "^4.66.0.20240417" pytest = "^8.3.3" testcontainers = "^4.8.2" +coverage = "^7.6.4" [tool.poetry.group.dev.dependencies] -- 2.54.0 From 6be837025b7603bcbcfb6e20c4d42b484b76028f Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 8 Nov 2024 21:03:53 +0000 Subject: [PATCH 14/45] moved files --- shared/utils/tests/{ => unit}/check_env_test.py | 0 shared/utils/tests/{ => unit}/setup_logging_test.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename shared/utils/tests/{ => unit}/check_env_test.py (100%) rename shared/utils/tests/{ => unit}/setup_logging_test.py (100%) diff --git a/shared/utils/tests/check_env_test.py b/shared/utils/tests/unit/check_env_test.py similarity index 100% rename from shared/utils/tests/check_env_test.py rename to shared/utils/tests/unit/check_env_test.py diff --git a/shared/utils/tests/setup_logging_test.py b/shared/utils/tests/unit/setup_logging_test.py similarity index 100% rename from shared/utils/tests/setup_logging_test.py rename to shared/utils/tests/unit/setup_logging_test.py -- 2.54.0 From 53d179663b8975783627e1df5070a59f0f3a7572 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 8 Nov 2024 21:05:20 +0000 Subject: [PATCH 15/45] added unittest --- .../tests/unit/connect_minio_test.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 shared/datastore/tests/unit/connect_minio_test.py diff --git a/shared/datastore/tests/unit/connect_minio_test.py b/shared/datastore/tests/unit/connect_minio_test.py new file mode 100644 index 0000000..c8b381f --- /dev/null +++ b/shared/datastore/tests/unit/connect_minio_test.py @@ -0,0 +1,33 @@ +"""Definition of unittests for connect_minio function.""" + +import os +import unittest + +from shared.datastore import connect_minio + + +class TestConnectMinio(unittest.TestCase): + + def setUp(self): + # define relevant env vars + self.env_var_map = { + 'MINIO_ENDPOINT': '192.168.1.2', + 'MINIO_ACCESS_KEY': 'randomAccess_key', + 'MINIO_SECRET_KEY': 'randomSecret_key', + 'MINIO_BUCKET_NAME': 'test-bucket-name', + } + # set env vars + for key, val in self.env_var_map.items(): + os.environ[key] = val + + def tearDown(self): + # clear env vars + for key in self.env_var_map: + _ = os.environ.pop(key, default=None) + + def test_should_fail_when_env_not_set(self): + # ensure env not set + self.tearDown() + # run test + with self.assertRaises(AssertionError): + connect_minio() -- 2.54.0 From cddfced1779b2a54c6e56f983b150eefe0d83bbc Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 8 Nov 2024 22:27:58 +0000 Subject: [PATCH 16/45] added unittest --- shared/datastore/src/delete.py | 13 ++-- shared/datastore/tests/unit/delete_test.py | 77 ++++++++++++++++++++++ 2 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 shared/datastore/tests/unit/delete_test.py diff --git a/shared/datastore/src/delete.py b/shared/datastore/src/delete.py index f3cd179..6bd926a 100644 --- a/shared/datastore/src/delete.py +++ b/shared/datastore/src/delete.py @@ -1,22 +1,22 @@ """Definition of delete function.""" -from __future__ import annotations - import logging -import os +from traceback import print_exc from minio import Minio def delete( client: Minio, + bucket_name: str, object_name: str, ) -> None: """Delete object from MinIO.""" assert isinstance(client, Minio) - assert isinstance(object_name, str) - bucket_name = os.getenv('MINIO_BUCKET_NAME', default='') + assert isinstance(bucket_name, str) assert len(bucket_name) > 0 + assert isinstance(object_name, str) + assert len(object_name) > 0 # remove object try: client.remove_object( @@ -24,7 +24,8 @@ def delete( object_name=object_name, ) except Exception as exc: - logging.debug(exc) logging.error('failed deleting %s', object_name) + print_exc() + raise exc else: logging.debug('deleted %s', object_name) diff --git a/shared/datastore/tests/unit/delete_test.py b/shared/datastore/tests/unit/delete_test.py new file mode 100644 index 0000000..8471d10 --- /dev/null +++ b/shared/datastore/tests/unit/delete_test.py @@ -0,0 +1,77 @@ +"""Definition of unittests for delete function.""" + +import random +import string +import unittest +from unittest.mock import Mock + +from minio import Minio + +from shared.datastore import delete + + +class TestDelete(unittest.TestCase): + + def setUp(self): + # mock minio client + self.client = Mock(spec=Minio) + # set bucket name + self.bucket_name = 'test-bucket' + # set object name + self.object_name = ''.join( + random.choices( + string.ascii_uppercase + string.digits, + k=24, + ), + ) + + def test_should_fail_on_wrong_input_type_client(self): + with self.assertRaises(AssertionError): + delete( + client='not-minio-type', + bucket_name=self.bucket_name, + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_type_bucket_name(self): + with self.assertRaises(AssertionError): + delete( + client=self.client, + bucket_name=0.0, + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_length_bucket_name(self): + with self.assertRaises(AssertionError): + delete( + client=self.client, + bucket_name='', + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_type_object_name(self): + with self.assertRaises(AssertionError): + delete( + client=self.client, + bucket_name=self.bucket_name, + object_name=0.0, + ) + + def test_should_fail_on_wrong_input_length_object_name(self): + with self.assertRaises(AssertionError): + delete( + client=self.client, + bucket_name=self.bucket_name, + object_name='', + ) + + def test_should_call_client__remove_object(self): + delete( + client=self.client, + bucket_name=self.bucket_name, + object_name=self.object_name, + ) + self.client.remove_object.assert_called_with( + bucket_name=self.bucket_name, + object_name=self.object_name, + ) -- 2.54.0 From 0b86acd1a9e3b1b3e654f864f55987be229dbfb6 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 8 Nov 2024 22:29:03 +0000 Subject: [PATCH 17/45] began adding tests --- shared/datastore/tests/unit/get_image_test.py | 62 +++++++++++++++++++ shared/datastore/tests/unit/get_model_test.py | 62 +++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 shared/datastore/tests/unit/get_image_test.py create mode 100644 shared/datastore/tests/unit/get_model_test.py diff --git a/shared/datastore/tests/unit/get_image_test.py b/shared/datastore/tests/unit/get_image_test.py new file mode 100644 index 0000000..4322838 --- /dev/null +++ b/shared/datastore/tests/unit/get_image_test.py @@ -0,0 +1,62 @@ +# """Definition of unittests for get_image function.""" + +# import os +# import unittest +# from unittest.mock import Mock +# from minio import Minio +# import random +# import string +# from PIL import Image +# from io import BytesIO + +# from shared.datastore import get_image + + +# class TestGetImage(unittest.TestCase): + +# def setUp(self): +# # mock minio client +# self.client = Mock(spec=Minio) +# # mock response object +# image = Image.new(mode='RGB', size=(480,480)) +# buffer = BytesIO() +# image.save(buffer, 'png') +# self.client.get_object.return_value.data = buffer.getvalue() +# # set object name +# self.object_name = ''.join( +# random.choices( +# string.ascii_uppercase + string.digits, +# k=24 +# ) +# ) +# # populate env +# self.env_var_map = { +# 'MINIO_BUCKET_NAME': 'test-bucket' +# } +# for key, val in self.env_var_map.items(): +# os.environ[key] = val + +# def tearDown(self): +# # clean env +# for key in self.env_var_map: +# _ = os.environ.pop(key, default=None) + +# def test_should_fail_when_env_not_set(self): +# # ensure env not set +# self.tearDown() +# # run test +# with self.assertRaises(AssertionError): +# get_image( +# client=self.client, +# object_name=self.object_name +# ) + +# def test_should_call_client__get_object(self): +# get_image( +# client=self.client, +# object_name=self.object_name, +# ) +# self.client.get_object.assert_called_with( +# bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], +# object_name=f'images/{self.object_name}', +# ) diff --git a/shared/datastore/tests/unit/get_model_test.py b/shared/datastore/tests/unit/get_model_test.py new file mode 100644 index 0000000..c2c132f --- /dev/null +++ b/shared/datastore/tests/unit/get_model_test.py @@ -0,0 +1,62 @@ +# """Definition of unittest for get_model function.""" + +# import os +# import unittest +# from unittest.mock import Mock +# from minio import Minio +# import random +# import string + +# from io import BytesIO + +# from shared.datastore import get_model + + +# class TestGetModel(unittest.TestCase): + +# def setUp(self): +# # mock minio client +# self.client = Mock(spec=Minio) +# # mock response object +# image = Image.new(mode='RGB', size=(480,480)) +# buffer = BytesIO() +# image.save(buffer, 'png') +# self.client.get_object.return_value.data = buffer.getvalue() +# # set object name +# self.object_name = ''.join( +# random.choices( +# string.ascii_uppercase + string.digits, +# k=24 +# ) +# ) +# # populate env +# self.env_var_map = { +# 'MINIO_BUCKET_NAME': 'test-bucket' +# } +# for key, val in self.env_var_map.items(): +# os.environ[key] = val + +# def tearDown(self): +# # clean env +# for key in self.env_var_map: +# _ = os.environ.pop(key, default=None) + +# def test_should_fail_when_env_not_set(self): +# # ensure env not set +# self.tearDown() +# # run test +# with self.assertRaises(AssertionError): +# get_image( +# client=self.client, +# object_name=self.object_name +# ) + +# def test_should_call_client__get_object(self): +# get_image( +# client=self.client, +# object_name=self.object_name, +# ) +# self.client.get_object.assert_called_with( +# bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], +# object_name=f'images/{self.object_name}', +# ) -- 2.54.0 From 3c46275015e5c0f15de750a8fd1c7b9d6daf20f1 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 8 Nov 2024 22:29:33 +0000 Subject: [PATCH 18/45] updated input check --- shared/datastore/src/get.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shared/datastore/src/get.py b/shared/datastore/src/get.py index dfd9206..6c4cc69 100644 --- a/shared/datastore/src/get.py +++ b/shared/datastore/src/get.py @@ -14,9 +14,10 @@ def get( ) -> BytesIO: """Get buffer from bucket in MinIO.""" assert isinstance(client, Minio) - assert isinstance(object_name, str) assert isinstance(bucket_name, str) assert len(bucket_name) > 0 + assert isinstance(object_name, str) + assert len(object_name) > 0 try: # make request response = client.get_object( -- 2.54.0 From 169e0531d94bc35f9f3d811a8f9229ecf5290388 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 8 Nov 2024 22:32:27 +0000 Subject: [PATCH 19/45] started making unittest --- shared/datastore/tests/unit/get_test.py | 83 +++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 shared/datastore/tests/unit/get_test.py diff --git a/shared/datastore/tests/unit/get_test.py b/shared/datastore/tests/unit/get_test.py new file mode 100644 index 0000000..017132d --- /dev/null +++ b/shared/datastore/tests/unit/get_test.py @@ -0,0 +1,83 @@ +"""Definition of unittests for get function.""" + +import random +import string +import unittest +from io import BytesIO +from unittest.mock import Mock + +from minio import Minio + +from shared.datastore import get + + +class TestGet(unittest.TestCase): + + def setUp(self): + # mock minio client + self.client = Mock(spec=Minio) + # set bucket name + self.bucket_name = 'test-bucket' + # set object name + self.object_name = ''.join( + random.choices( + string.ascii_uppercase + string.digits, + k=24, + ), + ) + # mock response object + self.client.get_object.return_value.status = 200 + buffer = BytesIO(random.randbytes(2**20)) + self.client.return_value.get_object = buffer + self.client.get_object.return_value.read = buffer.getvalue() + + def test_should_fail_on_wrong_input_type_client(self): + with self.assertRaises(AssertionError): + get( + client='not-minio-type', + bucket_name=self.bucket_name, + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_type_bucket_name(self): + with self.assertRaises(AssertionError): + get( + client=self.client, + bucket_name=0.0, + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_length_bucket_name(self): + with self.assertRaises(AssertionError): + get( + client=self.client, + bucket_name='', + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_type_object_name(self): + with self.assertRaises(AssertionError): + get( + client=self.client, + bucket_name=self.bucket_name, + object_name=0.0, + ) + + def test_should_fail_on_wrong_input_length_object_name(self): + with self.assertRaises(AssertionError): + get( + client=self.client, + bucket_name=self.bucket_name, + object_name='', + ) + + def test_should_call_client__get_object(self): + get( + client=self.client, + bucket_name=self.bucket_name, + object_name=self.object_name, + ) + self.client.get_object.assert_called_with( + bucket_name=self.bucket_name, + object_name=self.object_name, + ) -- 2.54.0 From 01ee4664ffe23730958bb775bfda1f6ab5759b7a Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 16 Nov 2024 18:18:13 +0000 Subject: [PATCH 20/45] implemented base functions --- shared/datastore/src/get_image.py | 21 ++++++------------ shared/datastore/src/get_model.py | 25 ++++++++------------- shared/datastore/src/put_image.py | 7 +++--- shared/datastore/src/put_model.py | 36 +++++++++++++++---------------- 4 files changed, 36 insertions(+), 53 deletions(-) diff --git a/shared/datastore/src/get_image.py b/shared/datastore/src/get_image.py index ef81762..596226b 100644 --- a/shared/datastore/src/get_image.py +++ b/shared/datastore/src/get_image.py @@ -2,13 +2,14 @@ import logging import os -from io import BytesIO from minio import Minio from PIL import Image from shared.utils import check_env +from .get import get + def get_image( client: Minio, @@ -23,19 +24,11 @@ def get_image( bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) object_name = f'images/{object_name}' # get object from bucket - try: - response = client.get_object( - bucket_name=bucket_name, - object_name=object_name, - ) - buffer = BytesIO(response.data) - except Exception as exc: - logging.error('failed getting data from MinIO') - raise exc - finally: - response.close() - response.release_conn() - buffer.seek(0) + buffer = get( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) # convert data to image image = Image.open(buffer) logging.debug('got data from %s', object_name) diff --git a/shared/datastore/src/get_model.py b/shared/datastore/src/get_model.py index 2cc40e7..59c349a 100644 --- a/shared/datastore/src/get_model.py +++ b/shared/datastore/src/get_model.py @@ -3,13 +3,14 @@ import logging import os from collections import OrderedDict -from io import BytesIO import torch from minio import Minio from shared.utils import check_env +from .get import get + def get_model( client: Minio, @@ -20,24 +21,16 @@ def get_model( assert isinstance(object_name, str) assert len(object_name) > 0 # prepare arguments - check_env({'MINIO_BUCKET_NAME_MODELS'}) - bucket_name = str(os.getenv('MINIO_BUCKET_NAME_MODELS')) + check_env({'MINIO_BUCKET_NAME'}) + bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) object_name = f'models/{object_name}' # get object from bucket - try: - response = client.get_object( - bucket_name=bucket_name, - object_name=object_name, - ) - buffer = BytesIO(response.data) - except Exception as exc: - logging.error('failed getting data from MinIO') - raise exc - finally: - response.close() - response.release_conn() + buffer = get( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) # convert data to model checkpoint - buffer.seek(0) model_content = torch.load(buffer) logging.debug('finished') return model_content diff --git a/shared/datastore/src/put_image.py b/shared/datastore/src/put_image.py index b5de63c..bea9dd6 100644 --- a/shared/datastore/src/put_image.py +++ b/shared/datastore/src/put_image.py @@ -22,14 +22,13 @@ def put_image( # get bucket name from env bucket_name = os.getenv('MINIO_BUCKET_NAME', default='') assert len(bucket_name) > 0 - # save image to buffer + # save data to buffer buffer = BytesIO() image.save(buffer, 'png') # get md5 of buffer checksum = md5(buffer.getbuffer()).hexdigest() - # determine object name - subfolder = 'images' - object_name = f'{subfolder}/{checksum}' + # set object name + object_name = f'images/{checksum}' # send data to bucket put( client=client, diff --git a/shared/datastore/src/put_model.py b/shared/datastore/src/put_model.py index 164b499..69436a9 100644 --- a/shared/datastore/src/put_model.py +++ b/shared/datastore/src/put_model.py @@ -9,6 +9,10 @@ import torch from minio import Minio from torch.nn import Module +from shared.utils import check_env + +from .put import put + def put_model( client: Minio, @@ -18,28 +22,22 @@ def put_model( used as object name.""" assert isinstance(client, Minio) assert isinstance(model, Module) - bucket_name = os.getenv('MINIO_BUCKET_NAME', default='') - assert len(bucket_name) > 0 - subfolder = 'models' - # save image to buffer + # get bucket name from env + check_env({'MINIO_BUCKET_NAME'}) + bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) + # save data to buffer buffer = BytesIO() torch.save(model.state_dict(), buffer) # get md5 of image checksum = md5(buffer.getbuffer()).hexdigest() - # prepare for saving - num_bytes = buffer.tell() - buffer.seek(0) + # set object name + object_name = f'models/{checksum}' # send data to bucket - object_name = f'{subfolder}/{checksum}' - try: - client.put_object( - bucket_name=bucket_name, - object_name=object_name, - length=num_bytes, - data=buffer, - ) - except Exception as exc: - logging.error('failed saving data to MinIO') - raise exc - logging.debug('saved data to %s', object_name) + put( + client=client, + buffer=buffer, + bucket_name=bucket_name, + object_name=object_name, + ) + logging.debug('finished') return checksum -- 2.54.0 From 556e26f21d5ef10f4659ae7a05fe4e64a52fd8f3 Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 16 Nov 2024 18:27:59 +0000 Subject: [PATCH 21/45] added unittests for base functions --- .../tests/unit/connect_minio_test.py | 2 +- shared/datastore/tests/unit/delete_test.py | 38 +++------ shared/datastore/tests/unit/get_test.py | 49 +++--------- shared/datastore/tests/unit/put_test.py | 80 +++++++++++++++++++ 4 files changed, 105 insertions(+), 64 deletions(-) create mode 100644 shared/datastore/tests/unit/put_test.py diff --git a/shared/datastore/tests/unit/connect_minio_test.py b/shared/datastore/tests/unit/connect_minio_test.py index c8b381f..508ac60 100644 --- a/shared/datastore/tests/unit/connect_minio_test.py +++ b/shared/datastore/tests/unit/connect_minio_test.py @@ -30,4 +30,4 @@ class TestConnectMinio(unittest.TestCase): self.tearDown() # run test with self.assertRaises(AssertionError): - connect_minio() + _ = connect_minio() diff --git a/shared/datastore/tests/unit/delete_test.py b/shared/datastore/tests/unit/delete_test.py index 8471d10..45da173 100644 --- a/shared/datastore/tests/unit/delete_test.py +++ b/shared/datastore/tests/unit/delete_test.py @@ -1,7 +1,5 @@ """Definition of unittests for delete function.""" -import random -import string import unittest from unittest.mock import Mock @@ -13,22 +11,19 @@ from shared.datastore import delete class TestDelete(unittest.TestCase): def setUp(self): - # mock minio client + # set relevant variables self.client = Mock(spec=Minio) - # set bucket name self.bucket_name = 'test-bucket' - # set object name - self.object_name = ''.join( - random.choices( - string.ascii_uppercase + string.digits, - k=24, - ), - ) + self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX' + # set bad arguments + self.bad_minio_client = 'not-minio-type' + self.bad_string = float(0.0) + self.len_0_string = '' def test_should_fail_on_wrong_input_type_client(self): with self.assertRaises(AssertionError): delete( - client='not-minio-type', + client=self.bad_minio_client, bucket_name=self.bucket_name, object_name=self.object_name, ) @@ -37,7 +32,7 @@ class TestDelete(unittest.TestCase): with self.assertRaises(AssertionError): delete( client=self.client, - bucket_name=0.0, + bucket_name=self.bad_string, object_name=self.object_name, ) @@ -45,7 +40,7 @@ class TestDelete(unittest.TestCase): with self.assertRaises(AssertionError): delete( client=self.client, - bucket_name='', + bucket_name=self.len_0_string, object_name=self.object_name, ) @@ -54,7 +49,7 @@ class TestDelete(unittest.TestCase): delete( client=self.client, bucket_name=self.bucket_name, - object_name=0.0, + object_name=self.bad_string, ) def test_should_fail_on_wrong_input_length_object_name(self): @@ -62,16 +57,5 @@ class TestDelete(unittest.TestCase): delete( client=self.client, bucket_name=self.bucket_name, - object_name='', + object_name=self.len_0_string, ) - - def test_should_call_client__remove_object(self): - delete( - client=self.client, - bucket_name=self.bucket_name, - object_name=self.object_name, - ) - self.client.remove_object.assert_called_with( - bucket_name=self.bucket_name, - object_name=self.object_name, - ) diff --git a/shared/datastore/tests/unit/get_test.py b/shared/datastore/tests/unit/get_test.py index 017132d..1280b55 100644 --- a/shared/datastore/tests/unit/get_test.py +++ b/shared/datastore/tests/unit/get_test.py @@ -1,10 +1,7 @@ """Definition of unittests for get function.""" -import random -import string import unittest -from io import BytesIO -from unittest.mock import Mock +from unittest.mock import MagicMock from minio import Minio @@ -12,29 +9,20 @@ from shared.datastore import get class TestGet(unittest.TestCase): - def setUp(self): - # mock minio client - self.client = Mock(spec=Minio) - # set bucket name + # set relevant variables + self.client = MagicMock(spec=Minio) self.bucket_name = 'test-bucket' - # set object name - self.object_name = ''.join( - random.choices( - string.ascii_uppercase + string.digits, - k=24, - ), - ) - # mock response object - self.client.get_object.return_value.status = 200 - buffer = BytesIO(random.randbytes(2**20)) - self.client.return_value.get_object = buffer - self.client.get_object.return_value.read = buffer.getvalue() + self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX' + # set bad arguments + self.bad_minio_client = 'not-minio-type' + self.bad_string = float(0.0) + self.len_0_string = '' def test_should_fail_on_wrong_input_type_client(self): with self.assertRaises(AssertionError): get( - client='not-minio-type', + client=self.bad_minio_client, bucket_name=self.bucket_name, object_name=self.object_name, ) @@ -43,7 +31,7 @@ class TestGet(unittest.TestCase): with self.assertRaises(AssertionError): get( client=self.client, - bucket_name=0.0, + bucket_name=self.bad_string, object_name=self.object_name, ) @@ -51,7 +39,7 @@ class TestGet(unittest.TestCase): with self.assertRaises(AssertionError): get( client=self.client, - bucket_name='', + bucket_name=self.len_0_string, object_name=self.object_name, ) @@ -60,7 +48,7 @@ class TestGet(unittest.TestCase): get( client=self.client, bucket_name=self.bucket_name, - object_name=0.0, + object_name=self.bad_string, ) def test_should_fail_on_wrong_input_length_object_name(self): @@ -68,16 +56,5 @@ class TestGet(unittest.TestCase): get( client=self.client, bucket_name=self.bucket_name, - object_name='', + object_name=self.len_0_string, ) - - def test_should_call_client__get_object(self): - get( - client=self.client, - bucket_name=self.bucket_name, - object_name=self.object_name, - ) - self.client.get_object.assert_called_with( - bucket_name=self.bucket_name, - object_name=self.object_name, - ) diff --git a/shared/datastore/tests/unit/put_test.py b/shared/datastore/tests/unit/put_test.py new file mode 100644 index 0000000..8bfb82f --- /dev/null +++ b/shared/datastore/tests/unit/put_test.py @@ -0,0 +1,80 @@ +"""Definition of unittests for put function.""" + +import unittest +from io import BytesIO +from unittest.mock import MagicMock + +from minio import Minio +from PIL import Image + +from shared.datastore import put + + +class TestPut(unittest.TestCase): + def setUp(self): + # set relevant variables + self.client = MagicMock(spec=Minio) + self.bucket_name = 'test-bucket' + self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX' + self.image = Image.new(mode='RGB', size=(480, 480)) + self.buffer = BytesIO() + self.image.save(self.buffer, 'png') + # set bad arguments + self.bad_minio_client = 'not-minio-type' + self.bad_string = float(0.0) + self.bad_buffer = 'not-buffer-type' + self.len_0_string = '' + + def test_should_fail_on_wrong_input_type_client(self): + with self.assertRaises(AssertionError): + put( + client=self.bad_minio_client, + buffer=self.buffer, + bucket_name=self.bucket_name, + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_type_buffer(self): + with self.assertRaises(AssertionError): + put( + client=self.bad_minio_client, + buffer=self.bad_buffer, + bucket_name=self.bucket_name, + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_type_bucket_name(self): + with self.assertRaises(AssertionError): + put( + client=self.client, + buffer=self.buffer, + bucket_name=self.bad_string, + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_length_bucket_name(self): + with self.assertRaises(AssertionError): + put( + client=self.client, + buffer=self.buffer, + bucket_name=self.len_0_string, + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_type_object_name(self): + with self.assertRaises(AssertionError): + put( + client=self.client, + buffer=self.buffer, + bucket_name=self.bucket_name, + object_name=self.bad_string, + ) + + def test_should_fail_on_wrong_input_length_object_name(self): + with self.assertRaises(AssertionError): + put( + client=self.client, + buffer=self.buffer, + bucket_name=self.bucket_name, + object_name=self.len_0_string, + ) -- 2.54.0 From 3fa79faa9e33dadf102230744a41e84aceb4fe9c Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 16 Nov 2024 18:39:25 +0000 Subject: [PATCH 22/45] shortened variable name --- shared/datastore/tests/unit/delete_test.py | 4 ++-- shared/datastore/tests/unit/get_test.py | 4 ++-- shared/datastore/tests/unit/put_test.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/shared/datastore/tests/unit/delete_test.py b/shared/datastore/tests/unit/delete_test.py index 45da173..06c902d 100644 --- a/shared/datastore/tests/unit/delete_test.py +++ b/shared/datastore/tests/unit/delete_test.py @@ -16,14 +16,14 @@ class TestDelete(unittest.TestCase): self.bucket_name = 'test-bucket' self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX' # set bad arguments - self.bad_minio_client = 'not-minio-type' + self.bad_client = 'not-minio-type' self.bad_string = float(0.0) self.len_0_string = '' def test_should_fail_on_wrong_input_type_client(self): with self.assertRaises(AssertionError): delete( - client=self.bad_minio_client, + client=self.bad_client, bucket_name=self.bucket_name, object_name=self.object_name, ) diff --git a/shared/datastore/tests/unit/get_test.py b/shared/datastore/tests/unit/get_test.py index 1280b55..056f2da 100644 --- a/shared/datastore/tests/unit/get_test.py +++ b/shared/datastore/tests/unit/get_test.py @@ -15,14 +15,14 @@ class TestGet(unittest.TestCase): self.bucket_name = 'test-bucket' self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX' # set bad arguments - self.bad_minio_client = 'not-minio-type' + self.bad_client = 'not-minio-type' self.bad_string = float(0.0) self.len_0_string = '' def test_should_fail_on_wrong_input_type_client(self): with self.assertRaises(AssertionError): get( - client=self.bad_minio_client, + client=self.bad_client, bucket_name=self.bucket_name, object_name=self.object_name, ) diff --git a/shared/datastore/tests/unit/put_test.py b/shared/datastore/tests/unit/put_test.py index 8bfb82f..b0c8501 100644 --- a/shared/datastore/tests/unit/put_test.py +++ b/shared/datastore/tests/unit/put_test.py @@ -20,7 +20,7 @@ class TestPut(unittest.TestCase): self.buffer = BytesIO() self.image.save(self.buffer, 'png') # set bad arguments - self.bad_minio_client = 'not-minio-type' + self.bad_client = 'not-minio-type' self.bad_string = float(0.0) self.bad_buffer = 'not-buffer-type' self.len_0_string = '' @@ -28,7 +28,7 @@ class TestPut(unittest.TestCase): def test_should_fail_on_wrong_input_type_client(self): with self.assertRaises(AssertionError): put( - client=self.bad_minio_client, + client=self.bad_client, buffer=self.buffer, bucket_name=self.bucket_name, object_name=self.object_name, @@ -37,7 +37,7 @@ class TestPut(unittest.TestCase): def test_should_fail_on_wrong_input_type_buffer(self): with self.assertRaises(AssertionError): put( - client=self.bad_minio_client, + client=self.client, buffer=self.bad_buffer, bucket_name=self.bucket_name, object_name=self.object_name, -- 2.54.0 From f9d23c5bd43612621764e1a1682e5ed874f9c77f Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 16 Nov 2024 18:50:28 +0000 Subject: [PATCH 23/45] added type-specific unittests --- shared/datastore/tests/unit/get_image_test.py | 109 +++++++++--------- shared/datastore/tests/unit/get_model_test.py | 107 ++++++++--------- shared/datastore/tests/unit/put_image_test.py | 41 +++++++ shared/datastore/tests/unit/put_model_test.py | 40 +++++++ 4 files changed, 190 insertions(+), 107 deletions(-) create mode 100644 shared/datastore/tests/unit/put_image_test.py create mode 100644 shared/datastore/tests/unit/put_model_test.py diff --git a/shared/datastore/tests/unit/get_image_test.py b/shared/datastore/tests/unit/get_image_test.py index 4322838..ab72324 100644 --- a/shared/datastore/tests/unit/get_image_test.py +++ b/shared/datastore/tests/unit/get_image_test.py @@ -1,62 +1,63 @@ -# """Definition of unittests for get_image function.""" +"""Definition of unittests for get_image function.""" -# import os -# import unittest -# from unittest.mock import Mock -# from minio import Minio -# import random -# import string -# from PIL import Image -# from io import BytesIO +import os +import unittest +from unittest.mock import MagicMock -# from shared.datastore import get_image +from minio import Minio + +from shared.datastore import get_image -# class TestGetImage(unittest.TestCase): +class TestGetImage(unittest.TestCase): -# def setUp(self): -# # mock minio client -# self.client = Mock(spec=Minio) -# # mock response object -# image = Image.new(mode='RGB', size=(480,480)) -# buffer = BytesIO() -# image.save(buffer, 'png') -# self.client.get_object.return_value.data = buffer.getvalue() -# # set object name -# self.object_name = ''.join( -# random.choices( -# string.ascii_uppercase + string.digits, -# k=24 -# ) -# ) -# # populate env -# self.env_var_map = { -# 'MINIO_BUCKET_NAME': 'test-bucket' -# } -# for key, val in self.env_var_map.items(): -# os.environ[key] = val + def setUp(self): + # set relevant variables + self.client = MagicMock(spec=Minio) + self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX' + self.env_var_map = { + 'MINIO_BUCKET_NAME': 'test-bucket', + } + # set bad arguments + self.bad_client = 'not-minio-type' + self.bad_string = float(0.0) + self.len_0_string = '' + # populate env + for key, val in self.env_var_map.items(): + os.environ[key] = val -# def tearDown(self): -# # clean env -# for key in self.env_var_map: -# _ = os.environ.pop(key, default=None) + def tearDown(self): + # clean env + for key in self.env_var_map: + _ = os.environ.pop(key, default=None) -# def test_should_fail_when_env_not_set(self): -# # ensure env not set -# self.tearDown() -# # run test -# with self.assertRaises(AssertionError): -# get_image( -# client=self.client, -# object_name=self.object_name -# ) + def test_should_fail_when_env_not_set(self): + # ensure env not set + self.tearDown() + # run test + with self.assertRaises(AssertionError): + get_image( + client=self.client, + object_name=self.object_name, + ) -# def test_should_call_client__get_object(self): -# get_image( -# client=self.client, -# object_name=self.object_name, -# ) -# self.client.get_object.assert_called_with( -# bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], -# object_name=f'images/{self.object_name}', -# ) + def test_should_fail_on_wrong_input_type_client(self): + with self.assertRaises(AssertionError): + get_image( + client=self.bad_client, + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_type_object_name(self): + with self.assertRaises(AssertionError): + get_image( + client=self.client, + object_name=self.bad_string, + ) + + def test_should_fail_on_wrong_input_length_object_name(self): + with self.assertRaises(AssertionError): + get_image( + client=self.client, + object_name=self.len_0_string, + ) diff --git a/shared/datastore/tests/unit/get_model_test.py b/shared/datastore/tests/unit/get_model_test.py index c2c132f..0ba1f76 100644 --- a/shared/datastore/tests/unit/get_model_test.py +++ b/shared/datastore/tests/unit/get_model_test.py @@ -1,62 +1,63 @@ -# """Definition of unittest for get_model function.""" +"""Definition of unittest for get_model function.""" -# import os -# import unittest -# from unittest.mock import Mock -# from minio import Minio -# import random -# import string +import os +import unittest +from unittest.mock import MagicMock -# from io import BytesIO +from minio import Minio -# from shared.datastore import get_model +from shared.datastore import get_model -# class TestGetModel(unittest.TestCase): +class TestGetModel(unittest.TestCase): -# def setUp(self): -# # mock minio client -# self.client = Mock(spec=Minio) -# # mock response object -# image = Image.new(mode='RGB', size=(480,480)) -# buffer = BytesIO() -# image.save(buffer, 'png') -# self.client.get_object.return_value.data = buffer.getvalue() -# # set object name -# self.object_name = ''.join( -# random.choices( -# string.ascii_uppercase + string.digits, -# k=24 -# ) -# ) -# # populate env -# self.env_var_map = { -# 'MINIO_BUCKET_NAME': 'test-bucket' -# } -# for key, val in self.env_var_map.items(): -# os.environ[key] = val + def setUp(self): + # set relevant variables + self.client = MagicMock(spec=Minio) + self.object_name = '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX' + self.env_var_map = { + 'MINIO_BUCKET_NAME': 'test-bucket', + } + # set bad arguments + self.bad_client = 'not-minio-type' + self.bad_string = float(0.0) + self.len_0_string = '' + # populate env + for key, val in self.env_var_map.items(): + os.environ[key] = val -# def tearDown(self): -# # clean env -# for key in self.env_var_map: -# _ = os.environ.pop(key, default=None) + def tearDown(self): + # clean env + for key in self.env_var_map: + _ = os.environ.pop(key, default=None) -# def test_should_fail_when_env_not_set(self): -# # ensure env not set -# self.tearDown() -# # run test -# with self.assertRaises(AssertionError): -# get_image( -# client=self.client, -# object_name=self.object_name -# ) + def test_should_fail_when_env_not_set(self): + # ensure env not set + self.tearDown() + # run test + with self.assertRaises(AssertionError): + get_model( + client=self.client, + object_name=self.object_name, + ) -# def test_should_call_client__get_object(self): -# get_image( -# client=self.client, -# object_name=self.object_name, -# ) -# self.client.get_object.assert_called_with( -# bucket_name=self.env_var_map['MINIO_BUCKET_NAME'], -# object_name=f'images/{self.object_name}', -# ) + def test_should_fail_on_wrong_input_type_client(self): + with self.assertRaises(AssertionError): + get_model( + client=self.bad_client, + object_name=self.object_name, + ) + + def test_should_fail_on_wrong_input_type_object_name(self): + with self.assertRaises(AssertionError): + get_model( + client=self.client, + object_name=self.bad_string, + ) + + def test_should_fail_on_wrong_input_length_object_name(self): + with self.assertRaises(AssertionError): + get_model( + client=self.client, + object_name=self.len_0_string, + ) diff --git a/shared/datastore/tests/unit/put_image_test.py b/shared/datastore/tests/unit/put_image_test.py new file mode 100644 index 0000000..e971478 --- /dev/null +++ b/shared/datastore/tests/unit/put_image_test.py @@ -0,0 +1,41 @@ +"""Definition of unittests for put_image function.""" + +import os +import unittest +from unittest.mock import MagicMock + +from minio import Minio +from PIL import Image + +from shared.datastore import put_image + + +class TestPutImage(unittest.TestCase): + + def setUp(self): + # set relevant variables + self.client = MagicMock(spec=Minio) + self.image = Image.new(mode='RGB', size=(480, 480)) + self.env_var_map = { + 'MINIO_BUCKET_NAME': 'test-bucket', + } + # set bad arguments + self.bad_client = 'not-minio-type' + self.bad_image = 'not-image-type' + # populate env + for key, val in self.env_var_map.items(): + os.environ[key] = val + + def test_should_fail_on_wrong_input_type_client(self): + with self.assertRaises(AssertionError): + put_image( + client=self.bad_client, + image=self.image, + ) + + def test_should_fail_on_wrong_input_type_image(self): + with self.assertRaises(AssertionError): + put_image( + client=self.client, + image=self.bad_image, + ) diff --git a/shared/datastore/tests/unit/put_model_test.py b/shared/datastore/tests/unit/put_model_test.py new file mode 100644 index 0000000..790e839 --- /dev/null +++ b/shared/datastore/tests/unit/put_model_test.py @@ -0,0 +1,40 @@ +"""Definition of unittests for put_model function.""" + +import os +import unittest +from unittest.mock import MagicMock + +from minio import Minio +from torch.nn import Module + +from shared.datastore import put_model + + +class TestPutModel(unittest.TestCase): + def setUp(self): + # set relevant variables + self.client = MagicMock(spec=Minio) + self.model = MagicMock(spec=Module) + self.env_var_map = { + 'MINIO_BUCKET_NAME': 'test-bucket', + } + # set bad arguments + self.bad_client = 'not-minio-type' + self.bad_model = 'not-image-type' + # populate env + for key, val in self.env_var_map.items(): + os.environ[key] = val + + def test_should_fail_on_wrong_input_type_client(self): + with self.assertRaises(AssertionError): + put_model( + client=self.bad_client, + model=self.model, + ) + + def test_should_fail_on_wrong_input_type_model(self): + with self.assertRaises(AssertionError): + put_model( + client=self.client, + model=self.bad_model, + ) -- 2.54.0 From c630498168dbfd639bcf8578a4328e2afae31293 Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 16 Nov 2024 19:46:18 +0000 Subject: [PATCH 24/45] vulture ignore test folders --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0da6fde..12c3384 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,4 +58,4 @@ repos: rev: 'v2.6' hooks: - id: vulture - entry: vulture . --min-confidence 90 --exclude */.venv/*.py + entry: vulture . --min-confidence 90 --exclude */.venv/*.py,*/tests/*.py -- 2.54.0 From cbbb0441778a81e0e70d0e20ce61fd2e5cc8c62f Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 16 Nov 2024 19:46:37 +0000 Subject: [PATCH 25/45] updated integration tests --- .../tests/integration/base_crud_test.py | 77 ++++++++++++ .../datastore/tests/integration/conftest.py | 78 +++++++++++++ .../tests/integration/connect_test.py | 10 ++ .../tests/integration/image_crud_test.py | 53 +++++++++ .../tests/integration/model_crud_test.py | 53 +++++++++ shared/datastore/tests/integration_test.py | 110 ------------------ 6 files changed, 271 insertions(+), 110 deletions(-) create mode 100644 shared/datastore/tests/integration/base_crud_test.py create mode 100644 shared/datastore/tests/integration/conftest.py create mode 100644 shared/datastore/tests/integration/connect_test.py create mode 100644 shared/datastore/tests/integration/image_crud_test.py create mode 100644 shared/datastore/tests/integration/model_crud_test.py delete mode 100644 shared/datastore/tests/integration_test.py diff --git a/shared/datastore/tests/integration/base_crud_test.py b/shared/datastore/tests/integration/base_crud_test.py new file mode 100644 index 0000000..93b4c61 --- /dev/null +++ b/shared/datastore/tests/integration/base_crud_test.py @@ -0,0 +1,77 @@ +"""Integration tests related to base CRUD functions.""" + +from io import BytesIO + +import pytest + +from shared.datastore import connect_minio, delete, get, put + + +def test_should_get_data( + data_in_minio, +): + data, bucket_name, object_name = data_in_minio + client = connect_minio() + received_data = get( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) + assert isinstance(data, BytesIO) + assert received_data == data + + +def test_should_delete_data( + data_in_minio, +): + _, bucket_name, object_name = data_in_minio + client = connect_minio() + delete( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) + with pytest.raises(ValueError): + _ = get( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) + + +def test_should_put_data( + data, +): + buffer, bucket_name, object_name = data + client = connect_minio() + put( + client=client, + buffer=buffer, + bucket_name=bucket_name, + object_name=object_name, + ) + received_data = get( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) + assert received_data == data + + +def test_should_update_data( + data_in_minio, +): + buffer, bucket_name, object_name = data_in_minio + client = connect_minio() + put( + client=client, + buffer=buffer, + bucket_name=bucket_name, + object_name=object_name, + ) + received_data = get( + client=client, + bucket_name=bucket_name, + object_name=object_name, + ) + assert received_data == buffer diff --git a/shared/datastore/tests/integration/conftest.py b/shared/datastore/tests/integration/conftest.py new file mode 100644 index 0000000..98a22ac --- /dev/null +++ b/shared/datastore/tests/integration/conftest.py @@ -0,0 +1,78 @@ +"""Integration test configurations.""" + +import os + +import pytest +from PIL import Image +from testcontainers.minio import MinioContainer + +env_var_map = { + 'MINIO_ENDPOINT': 'localhost:9000', + 'MINIO_ACCESS_KEY': 'test-access-key', + 'MINIO_SECRET_KEY': 'test-secret-key', + 'MINIO_BUCKET_NAME': 'test-bucket', +} +container_map = { + 'minio': MinioContainer( + port=env_var_map['MINIO_ENDPOINT'].rsplit(':', maxsplit=1)[-1], + access_key=env_var_map['MINIO_ACCESS_KEY'], + secret_key=env_var_map['MINIO_SECRET_KEY'], + ), +} + + +@pytest.fixture(scope='session', autouse=True) +def setup_infrastructure(request: pytest.FixtureRequest) -> None: + """Prepare infrastructure for integration test.""" + # prepare infrastructure + for container in container_map.values(): + container.start() + # update env var map + minio_host = container_map['minio'].get_container_host_ip() + minio_port = container_map['minio'].get_exposed_port(9000) + env_var_map['MINIO_ENDPOINT'] = f'{minio_host}:{minio_port}' + + # ensure cleanup + def cleanup_infrastructure(): + for container in container_map.values(): + container.stop() + + request.addfinalizer(cleanup_infrastructure) + + +@pytest.fixture(scope='session', autouse=True) +def populate_env( + request: pytest.FixtureRequest, + setup_infrastructure, +) -> None: + """Populate environment with variables used for testing.""" + # update env + for key, val in env_var_map.items(): + os.environ[key] = val + + # ensure cleanup + def cleanup_env(): + for key in env_var_map: + _ = os.environ.pop(key, default=None) + + request.addfinalizer(cleanup_env) + + +@pytest.fixture +def image() -> Image.Image: + # generate image + image = Image.new(mode='RGB', size=(480, 480)) + # expose image + yield image + + +# @pytest.fixture +# def image_in_minio( +# image: Image.Image, +# ) -> tuple(Image.Image, str): +# # + + +# # expose image and object name +# yield image, object_name +# # cleanup diff --git a/shared/datastore/tests/integration/connect_test.py b/shared/datastore/tests/integration/connect_test.py new file mode 100644 index 0000000..25b5aaf --- /dev/null +++ b/shared/datastore/tests/integration/connect_test.py @@ -0,0 +1,10 @@ +"""Integration test for connect_minio function.""" + +from minio import Minio + +from shared.datastore import connect_minio + + +def test_should_return_correct_type(): + client = connect_minio() + assert isinstance(client, Minio) diff --git a/shared/datastore/tests/integration/image_crud_test.py b/shared/datastore/tests/integration/image_crud_test.py new file mode 100644 index 0000000..bb1589e --- /dev/null +++ b/shared/datastore/tests/integration/image_crud_test.py @@ -0,0 +1,53 @@ +"""Integration tests related to image CRUD.""" + +from PIL import Image + +from shared.datastore import connect_minio, get_image, put_image + + +def test_should_get_image( + image_in_minio, +): + image, object_name = image_in_minio + client = connect_minio() + received_image = get_image( + client=client, + object_name=object_name, + ) + assert isinstance(image, Image.Image) + assert received_image == image + + +def test_should_put_image( + image, +): + client = connect_minio() + object_name = put_image( + client=client, + image=image, + ) + assert isinstance(object_name, str) + assert len(object_name) > 0 + received_image = get_image( + client=client, + object_name=object_name, + ) + assert received_image == image + + +def test_should_update_image( + image_in_minio, +): + image, object_name = image_in_minio + client = connect_minio() + object_name = put_image( + client=client, + image=image, + ) + assert isinstance(object_name, str) + assert len(object_name) > 0 + received_image = get_image( + client=client, + object_name=object_name, + ) + assert received_image == image diff --git a/shared/datastore/tests/integration/model_crud_test.py b/shared/datastore/tests/integration/model_crud_test.py new file mode 100644 index 0000000..dd9d859 --- /dev/null +++ b/shared/datastore/tests/integration/model_crud_test.py @@ -0,0 +1,53 @@ +"""Integration tests related to model CRUD.""" + +from torch.nn import Module + +from shared.datastore import connect_minio, get_model, put_model + + +def test_should_get_model( + model_in_minio, +): + model, object_name = model_in_minio + client = connect_minio() + received_model = get_model( + client=client, + object_name=object_name, + ) + assert isinstance(model, Module) + assert received_model == model + + +def test_should_put_model( + model, +): + client = connect_minio() + object_name = put_model( + client=client, + model=model, + ) + assert isinstance(object_name, str) + assert len(object_name) > 0 + received_model = get_model( + client=client, + object_name=object_name, + ) + assert received_model == model + + +def test_should_update_model( + model_in_minio, +): + model, object_name = model_in_minio + client = connect_minio() + object_name = put_model( + client=client, + model=model, + ) + assert isinstance(object_name, str) + assert len(object_name) > 0 + received_model = get_model( + client=client, + object_name=object_name, + ) + assert received_model == model diff --git a/shared/datastore/tests/integration_test.py b/shared/datastore/tests/integration_test.py deleted file mode 100644 index 959d8b1..0000000 --- a/shared/datastore/tests/integration_test.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Integration tests for functions exposed by the datastore module.""" - -import os -from io import BytesIO -from random import randbytes - -import pytest -from minio import Minio -from testcontainers.minio import MinioContainer - -from shared.datastore import connect_minio, get, put -from shared.utils import setup_logging - -MINIO_BUCKET_NAME = 'test-bucket' -minio = MinioContainer() - - -@pytest.fixture(scope='module', autouse=True) -def setup(request: pytest.FixtureRequest): - """Setup function for datastore module integration tests.""" - # start minio - minio.start() - - # ensure minio is stopped after testing - def remove_container(): - minio.stop() - - request.addfinalizer(remove_container) - # extract information - minio_ip = minio.get_container_host_ip() - minio_port = minio.get_exposed_port(port=9000) - os.environ['MINIO_ENDPOINT'] = f'{minio_ip}:{minio_port}' - os.environ['MINIO_ACCESS_KEY'] = minio.access_key - os.environ['MINIO_SECRET_KEY'] = minio.secret_key - os.environ['MINIO_BUCKET_NAME'] = MINIO_BUCKET_NAME - - -def test_connect_minio(): - """Test function connect_minio.""" - minio_client = connect_minio() - assert isinstance(minio_client, Minio) - - -def test_put(): - """Test function put.""" - # connect to minio - minio_client = connect_minio() - # generate random bytes - buffer = BytesIO(randbytes(2**20)) - # put data in Minio - put( - client=minio_client, - buffer=buffer, - bucket_name=MINIO_BUCKET_NAME, - object_name='test_put_data', - ) - - -def test_get(): - """Test function get.""" - # connect to minio - minio_client = connect_minio() - # generate random bytes - buffer = BytesIO(randbytes(2**20)) - # put data in minio - put( - client=minio_client, - buffer=buffer, - bucket_name=MINIO_BUCKET_NAME, - object_name='test_get_data', - ) - # get data back from minio - buffer_rec = get( - client=minio_client, - bucket_name=MINIO_BUCKET_NAME, - object_name='test_get_data', - ) - # check that data are the same - assert buffer.read() == buffer_rec.read() - - -def test_delete(): - """Test function delete.""" - raise NotImplementedError() - - -def test_get_image(): - """Test function get_image.""" - raise NotImplementedError() - - -def test_put_image(): - """Test function put_image.""" - raise NotImplementedError() - - -def test_get_model(): - """Test function get_model.""" - raise NotImplementedError() - - -def test_put_model(): - """Test function put_model.""" - raise NotImplementedError() - - -if __name__ == '__main__': - os.environ['LOG_LEVEL'] = 'debug' - setup_logging() - pytest.main() -- 2.54.0 From ce08ee4ccd183ab95c07126116551871307e3f65 Mon Sep 17 00:00:00 2001 From: brian Date: Mon, 25 Nov 2024 16:37:52 +0000 Subject: [PATCH 26/45] added function to compare images --- .../tests/integration/image_crud_test.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/shared/datastore/tests/integration/image_crud_test.py b/shared/datastore/tests/integration/image_crud_test.py index bb1589e..c9e7d40 100644 --- a/shared/datastore/tests/integration/image_crud_test.py +++ b/shared/datastore/tests/integration/image_crud_test.py @@ -1,10 +1,30 @@ """Integration tests related to image CRUD.""" +import numpy as np from PIL import Image from shared.datastore import connect_minio, get_image, put_image +def same_image( + img_a: Image.Image, + img_b: Image.Image, +) -> bool: + """Check if two images contain the same data.""" + assert isinstance(img_a, Image.Image) + assert isinstance(img_b, Image.Image) + # check if images have a comparable number of channels + if img_a.getbands() != img_b.getbands(): + return False + # calculate pixel difference between images + img_a_arr = np.asarray(img_a) + img_b_arr = np.asarray(img_b) + diff = np.subtract(img_a_arr, img_b_arr) + if np.sum(diff) != 0: + return False + return True + + def test_should_get_image( image_in_minio, ): @@ -32,7 +52,7 @@ def test_should_put_image( client=client, object_name=object_name, ) - assert received_image == image + assert same_image(received_image, image) def test_should_update_image( -- 2.54.0 From 499996329f08fbaee626ea9c2641fba24213bdbf Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 19 Dec 2024 15:33:25 +0000 Subject: [PATCH 27/45] added fixtures --- .../datastore/tests/integration/conftest.py | 69 ++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/shared/datastore/tests/integration/conftest.py b/shared/datastore/tests/integration/conftest.py index 98a22ac..0ad380a 100644 --- a/shared/datastore/tests/integration/conftest.py +++ b/shared/datastore/tests/integration/conftest.py @@ -1,8 +1,12 @@ """Integration test configurations.""" import os +import random +from collections.abc import Iterator +from io import BytesIO import pytest +from minio import Minio from PIL import Image from testcontainers.minio import MinioContainer @@ -11,6 +15,7 @@ env_var_map = { 'MINIO_ACCESS_KEY': 'test-access-key', 'MINIO_SECRET_KEY': 'test-secret-key', 'MINIO_BUCKET_NAME': 'test-bucket', + 'MINIO_OBJECT_NAME': '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX', } container_map = { 'minio': MinioContainer( @@ -58,8 +63,70 @@ def populate_env( request.addfinalizer(cleanup_env) +@pytest.fixture(scope='session') +def minio_client( + setup_infrastructure, + populate_env, +) -> Iterator[Minio]: + # prepare arguments + minio_endpoint = str(os.getenv('MINIO_ENDPOINT')) + minio_access_key = str(os.getenv('MINIO_ACCESS_KEY')) + minio_secret_key = str(os.getenv('MINIO_SECRET_KEY')) + minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) + # connect to minio + client = Minio( + endpoint=minio_endpoint, + access_key=minio_access_key, + secret_key=minio_secret_key, + secure=False, + ) + # ensure bucket exists + if not client.bucket_exists(bucket_name=minio_bucket_name): + client.make_bucket(bucket_name=minio_bucket_name) + # expose client + yield client + + @pytest.fixture -def image() -> Image.Image: +def data() -> Iterator[bytes]: + # generate random data + num_bytes = 2**21 # 2 MB + data = random.randbytes(n=num_bytes) + # expose data + yield data + + +@pytest.fixture +def data_in_minio( + minio_client, + data, +) -> Iterator[tuple[BytesIO, str, str]]: + # prepare arguments + minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) + minio_object_name = str(os.getenv('MINIO_OBJECT_NAME')) + # convert data + buffer = BytesIO(data) + # prepare for saving + num_bytes = buffer.tell() + buffer.seek(0) + # send data to bucket + minio_client.put_object( + bucket_name=minio_bucket_name, + object_name=minio_object_name, + length=num_bytes, + data=buffer, + ) + # expose data + yield buffer, minio_bucket_name, minio_object_name + # clean up + minio_client.remove_object( + bucket_name=minio_bucket_name, + object_name=minio_object_name, + ) + + +@pytest.fixture +def image() -> Iterator[Image.Image]: # generate image image = Image.new(mode='RGB', size=(480, 480)) # expose image -- 2.54.0 From 084a13426ad975b75ab88abb36b39a9e43ddc014 Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 19 Dec 2024 15:34:11 +0000 Subject: [PATCH 28/45] commented out unused tests temporarily --- .../tests/integration/base_crud_test.py | 153 +++++++++++------- .../tests/integration/image_crud_test.py | 84 +++++----- .../tests/integration/model_crud_test.py | 85 +++++----- 3 files changed, 175 insertions(+), 147 deletions(-) diff --git a/shared/datastore/tests/integration/base_crud_test.py b/shared/datastore/tests/integration/base_crud_test.py index 93b4c61..411bfe7 100644 --- a/shared/datastore/tests/integration/base_crud_test.py +++ b/shared/datastore/tests/integration/base_crud_test.py @@ -1,5 +1,6 @@ """Integration tests related to base CRUD functions.""" +import logging from io import BytesIO import pytest @@ -7,71 +8,99 @@ import pytest from shared.datastore import connect_minio, delete, get, put -def test_should_get_data( - data_in_minio, -): - data, bucket_name, object_name = data_in_minio - client = connect_minio() - received_data = get( - client=client, - bucket_name=bucket_name, - object_name=object_name, - ) - assert isinstance(data, BytesIO) - assert received_data == data +def same_data( + data_a: BytesIO, + data_b: BytesIO, +) -> bool: + """Check if two BytesIO-objects contain the same data.""" + assert isinstance(data_a, BytesIO) + assert isinstance(data_b, BytesIO) + # prepare for being read + data_a.seek(0) + data_b.seek(0) + # convert to bytes + data_a_bytes = data_a.read() + data_b_bytes = data_b.read() + # compare size + logging.error(len(data_a_bytes)) + logging.error(len(data_b_bytes)) + if len(data_a_bytes) != len(data_b_bytes): + return False + # compare content + if data_a_bytes != data_b_bytes: + return False + return True -def test_should_delete_data( - data_in_minio, -): - _, bucket_name, object_name = data_in_minio - client = connect_minio() - delete( - client=client, - bucket_name=bucket_name, - object_name=object_name, - ) - with pytest.raises(ValueError): - _ = get( - client=client, - bucket_name=bucket_name, - object_name=object_name, - ) +# def test_should_get_data( +# minio_client, +# data_in_minio, +# ): +# data, bucket_name, object_name = data_in_minio +# received_data = get( +# client=minio_client, +# bucket_name=bucket_name, +# object_name=object_name, +# ) + +# assert isinstance(data, BytesIO) +# assert same_data(data, received_data) -def test_should_put_data( - data, -): - buffer, bucket_name, object_name = data - client = connect_minio() - put( - client=client, - buffer=buffer, - bucket_name=bucket_name, - object_name=object_name, - ) - received_data = get( - client=client, - bucket_name=bucket_name, - object_name=object_name, - ) - assert received_data == data +# def test_should_delete_data( +# data_in_minio, +# ): +# _, bucket_name, object_name = data_in_minio +# client = connect_minio() +# delete( +# client=client, +# bucket_name=bucket_name, +# object_name=object_name, +# ) +# with pytest.raises(ValueError): +# _ = get( +# client=client, +# bucket_name=bucket_name, +# object_name=object_name, +# ) -def test_should_update_data( - data_in_minio, -): - buffer, bucket_name, object_name = data_in_minio - client = connect_minio() - put( - client=client, - buffer=buffer, - bucket_name=bucket_name, - object_name=object_name, - ) - received_data = get( - client=client, - bucket_name=bucket_name, - object_name=object_name, - ) - assert received_data == buffer +# def test_should_put_data( +# data, +# ): +# buffer, bucket_name, object_name = data +# client = connect_minio() +# put( +# client=client, +# buffer=buffer, +# bucket_name=bucket_name, +# object_name=object_name, +# ) +# received_data = get( +# client=client, +# bucket_name=bucket_name, +# object_name=object_name, +# ) +# assert received_data == data + + +# def test_should_update_data( +# data_in_minio, +# ): +# buffer, bucket_name, object_name = data_in_minio +# client = connect_minio() +# put( +# client=client, +# buffer=buffer, +# bucket_name=bucket_name, +# object_name=object_name, +# ) +# received_data = get( +# client=client, +# bucket_name=bucket_name, +# object_name=object_name, +# ) +# assert received_data == buffer + +if __name__ == '__main__': + pytest.main() diff --git a/shared/datastore/tests/integration/image_crud_test.py b/shared/datastore/tests/integration/image_crud_test.py index c9e7d40..41f03a6 100644 --- a/shared/datastore/tests/integration/image_crud_test.py +++ b/shared/datastore/tests/integration/image_crud_test.py @@ -25,49 +25,49 @@ def same_image( return True -def test_should_get_image( - image_in_minio, -): - image, object_name = image_in_minio - client = connect_minio() - received_image = get_image( - client=client, - object_name=object_name, - ) - assert isinstance(image, Image.Image) - assert received_image == image +# def test_should_get_image( +# image_in_minio, +# ): +# image, object_name = image_in_minio +# client = connect_minio() +# received_image = get_image( +# client=client, +# object_name=object_name, +# ) +# assert isinstance(image, Image.Image) +# assert received_image == image -def test_should_put_image( - image, -): - client = connect_minio() - object_name = put_image( - client=client, - image=image, - ) - assert isinstance(object_name, str) - assert len(object_name) > 0 - received_image = get_image( - client=client, - object_name=object_name, - ) - assert same_image(received_image, image) +# def test_should_put_image( +# image, +# ): +# client = connect_minio() +# object_name = put_image( +# client=client, +# image=image, +# ) +# assert isinstance(object_name, str) +# assert len(object_name) > 0 +# received_image = get_image( +# client=client, +# object_name=object_name, +# ) +# assert same_image(received_image, image) -def test_should_update_image( - image_in_minio, -): - image, object_name = image_in_minio - client = connect_minio() - object_name = put_image( - client=client, - image=image, - ) - assert isinstance(object_name, str) - assert len(object_name) > 0 - received_image = get_image( - client=client, - object_name=object_name, - ) - assert received_image == image +# def test_should_update_image( +# image_in_minio, +# ): +# image, object_name = image_in_minio +# client = connect_minio() +# object_name = put_image( +# client=client, +# image=image, +# ) +# assert isinstance(object_name, str) +# assert len(object_name) > 0 +# received_image = get_image( +# client=client, +# object_name=object_name, +# ) +# assert received_image == image diff --git a/shared/datastore/tests/integration/model_crud_test.py b/shared/datastore/tests/integration/model_crud_test.py index dd9d859..be331b7 100644 --- a/shared/datastore/tests/integration/model_crud_test.py +++ b/shared/datastore/tests/integration/model_crud_test.py @@ -4,50 +4,49 @@ from torch.nn import Module from shared.datastore import connect_minio, get_model, put_model - -def test_should_get_model( - model_in_minio, -): - model, object_name = model_in_minio - client = connect_minio() - received_model = get_model( - client=client, - object_name=object_name, - ) - assert isinstance(model, Module) - assert received_model == model +# def test_should_get_model( +# model_in_minio, +# ): +# model, object_name = model_in_minio +# client = connect_minio() +# received_model = get_model( +# client=client, +# object_name=object_name, +# ) +# assert isinstance(model, Module) +# assert received_model == model -def test_should_put_model( - model, -): - client = connect_minio() - object_name = put_model( - client=client, - model=model, - ) - assert isinstance(object_name, str) - assert len(object_name) > 0 - received_model = get_model( - client=client, - object_name=object_name, - ) - assert received_model == model +# def test_should_put_model( +# model, +# ): +# client = connect_minio() +# object_name = put_model( +# client=client, +# model=model, +# ) +# assert isinstance(object_name, str) +# assert len(object_name) > 0 +# received_model = get_model( +# client=client, +# object_name=object_name, +# ) +# assert received_model == model -def test_should_update_model( - model_in_minio, -): - model, object_name = model_in_minio - client = connect_minio() - object_name = put_model( - client=client, - model=model, - ) - assert isinstance(object_name, str) - assert len(object_name) > 0 - received_model = get_model( - client=client, - object_name=object_name, - ) - assert received_model == model +# def test_should_update_model( +# model_in_minio, +# ): +# model, object_name = model_in_minio +# client = connect_minio() +# object_name = put_model( +# client=client, +# model=model, +# ) +# assert isinstance(object_name, str) +# assert len(object_name) > 0 +# received_model = get_model( +# client=client, +# object_name=object_name, +# ) +# assert received_model == model -- 2.54.0 From b2773ecf0943dbf3519cb068c25a7dab49d92598 Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 19 Dec 2024 15:38:49 +0000 Subject: [PATCH 29/45] commented out unused imports --- shared/datastore/tests/integration/base_crud_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/datastore/tests/integration/base_crud_test.py b/shared/datastore/tests/integration/base_crud_test.py index 411bfe7..063c7cc 100644 --- a/shared/datastore/tests/integration/base_crud_test.py +++ b/shared/datastore/tests/integration/base_crud_test.py @@ -5,7 +5,7 @@ from io import BytesIO import pytest -from shared.datastore import connect_minio, delete, get, put +# from shared.datastore import connect_minio, delete, get, put def same_data( -- 2.54.0 From c9da7378a188caf7a52b45aa7f9664b539eb49b0 Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 19 Dec 2024 15:41:28 +0000 Subject: [PATCH 30/45] commented out more unused imports --- shared/datastore/tests/integration/image_crud_test.py | 2 +- shared/datastore/tests/integration/model_crud_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shared/datastore/tests/integration/image_crud_test.py b/shared/datastore/tests/integration/image_crud_test.py index 41f03a6..e91c9a0 100644 --- a/shared/datastore/tests/integration/image_crud_test.py +++ b/shared/datastore/tests/integration/image_crud_test.py @@ -3,7 +3,7 @@ import numpy as np from PIL import Image -from shared.datastore import connect_minio, get_image, put_image +# from shared.datastore import connect_minio, get_image, put_image def same_image( diff --git a/shared/datastore/tests/integration/model_crud_test.py b/shared/datastore/tests/integration/model_crud_test.py index be331b7..a5e126a 100644 --- a/shared/datastore/tests/integration/model_crud_test.py +++ b/shared/datastore/tests/integration/model_crud_test.py @@ -1,8 +1,8 @@ """Integration tests related to model CRUD.""" -from torch.nn import Module +# from torch.nn import Module -from shared.datastore import connect_minio, get_model, put_model +# from shared.datastore import connect_minio, get_model, put_model # def test_should_get_model( # model_in_minio, -- 2.54.0 From 99d5c88c8dc662180d9e15a0b1c687c95d836e41 Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 19 Dec 2024 16:01:29 +0000 Subject: [PATCH 31/45] fixed argument types --- other/transfer_images_minio_subfolder.py | 1 + other/transfer_images_mongo_minio.py | 16 ++++++++-------- .../src/classes/visual_communication.py | 18 +++++++----------- web_ui/src/app/init_app.py | 6 ++++-- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/other/transfer_images_minio_subfolder.py b/other/transfer_images_minio_subfolder.py index c531892..cb5305e 100644 --- a/other/transfer_images_minio_subfolder.py +++ b/other/transfer_images_minio_subfolder.py @@ -27,6 +27,7 @@ if __name__ == '__main__': # get image from minio buffer = get( client=minio_client, + bucket_name=BUCKET_NAME, object_name=obj.object_name, ) # convert data to image diff --git a/other/transfer_images_mongo_minio.py b/other/transfer_images_mongo_minio.py index b8d38c9..7e8f5cc 100644 --- a/other/transfer_images_mongo_minio.py +++ b/other/transfer_images_mongo_minio.py @@ -3,16 +3,15 @@ from __future__ import annotations import logging -from io import BytesIO from pathlib import Path from bson import ObjectId from dotenv import load_dotenv from pymongo.collection import Collection -from shared.datastore import connect_minio, put +from shared.datastore import connect_minio, put_image from shared.mongodb import connect_mongodb -from shared.mongodb.classes import VisualCommunication +from shared.mongodb.src.classes import VisualCommunication from shared.utils import check_env, setup_logging from web_ui.src.main import NECESSARY_ENV_VAR_LIST @@ -96,13 +95,14 @@ if __name__ == '__main__': logging.error('failed getting image from document: %s', doc_id) continue try: - # save image to buffer - buffer = BytesIO() - vis_com.image.save(buffer, 'png') # type: ignore + # get image + image = vis_com.get_image( + minio_client=minio_client, + ) # put buffer in minio - object_name = put( + object_name = put_image( client=minio_client, - buffer=buffer, + image=image, ) except Exception as exc: logging.debug(exc) diff --git a/shared/mongodb/src/classes/visual_communication.py b/shared/mongodb/src/classes/visual_communication.py index 9c0383f..a21d0b4 100755 --- a/shared/mongodb/src/classes/visual_communication.py +++ b/shared/mongodb/src/classes/visual_communication.py @@ -12,8 +12,8 @@ from PIL import Image from pydantic import BaseModel, ConfigDict from pymongo.collection import Collection -from shared.datastore import get, put -from shared.mongodb.classes import ModelData +from shared.datastore import get_image, put_image +from shared.mongodb.src.classes import ModelData class VisualCommunication(BaseModel): @@ -39,11 +39,9 @@ class VisualCommunication(BaseModel): """Upload image to MinIO and return MD5 checksum of hashed image.""" assert isinstance(image, Image.Image) assert isinstance(minio_client, Minio) - buffer = BytesIO() - image.save(buffer, 'png') - object_name = put( + object_name = put_image( client=minio_client, - buffer=buffer, + image=image, ) return object_name @@ -92,14 +90,12 @@ class VisualCommunication(BaseModel): def get_image(self, minio_client: Minio) -> Image.Image: """Load image data from minio.""" assert isinstance(minio_client, Minio) - # get buffer from minio - buffer = get( + # get image from minio + image = get_image( client=minio_client, object_name=self.object_name, ) - # convert data to image - im = Image.open(buffer) - return im + return image def save_to_mongo(self, collection: Collection) -> None: """Save self as document in MongoDB.""" diff --git a/web_ui/src/app/init_app.py b/web_ui/src/app/init_app.py index 1187e73..897aa69 100644 --- a/web_ui/src/app/init_app.py +++ b/web_ui/src/app/init_app.py @@ -14,8 +14,8 @@ from pymongo.collection import Collection from shared.datastore import delete as delete_from_minio from shared.mongodb import count_documents, get_visual_communication, upsert_annotation -from shared.mongodb.classes import ModelData, VisualCommunication -from shared.mongodb.exceptions import NoDocumentFoundException +from shared.mongodb.src.classes import ModelData, VisualCommunication +from shared.mongodb.src.exceptions import NoDocumentFoundException from .layout import app_layout @@ -146,8 +146,10 @@ def init_app( logging.debug(exc) failed_filename_list.append(filename) # remove document from minio + bucket_name = os.getenv('MINIO_BUCKET_NAME', default='') delete_from_minio( client=minio_client, + bucket_name=bucket_name, object_name=vis_com.object_name, ) assert ( -- 2.54.0 From 8a602264e6b502d12a5e736d105d7eee18a7e889 Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 19 Dec 2024 16:01:52 +0000 Subject: [PATCH 32/45] mypy ignore testcontainers missing imports --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1bee0e4..f3f6ddc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,3 +120,7 @@ ignore_missing_imports = true [[tool.mypy.overrides]] module = "shared.mongodb.*" ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "testcontainers.*" +ignore_missing_imports = true -- 2.54.0 From 8e366f26bcc7380c3c231f8b7a31956bd13a6af1 Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 19 Dec 2024 21:58:15 +0000 Subject: [PATCH 33/45] updated port mapping --- shared/datastore/tests/integration/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shared/datastore/tests/integration/conftest.py b/shared/datastore/tests/integration/conftest.py index 0ad380a..5c3812c 100644 --- a/shared/datastore/tests/integration/conftest.py +++ b/shared/datastore/tests/integration/conftest.py @@ -34,7 +34,9 @@ def setup_infrastructure(request: pytest.FixtureRequest) -> None: container.start() # update env var map minio_host = container_map['minio'].get_container_host_ip() - minio_port = container_map['minio'].get_exposed_port(9000) + minio_port = container_map['minio'].get_exposed_port( + port=env_var_map['MINIO_ENDPOINT'].rsplit(':', maxsplit=1)[-1], + ) env_var_map['MINIO_ENDPOINT'] = f'{minio_host}:{minio_port}' # ensure cleanup -- 2.54.0 From 1a3ee96932d360e5b4d24b4f816ab12f06c6c2cf Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 19 Dec 2024 22:11:07 +0000 Subject: [PATCH 34/45] added fix to allow running Minio container in CI --- shared/datastore/tests/integration/conftest.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/shared/datastore/tests/integration/conftest.py b/shared/datastore/tests/integration/conftest.py index 5c3812c..98ab924 100644 --- a/shared/datastore/tests/integration/conftest.py +++ b/shared/datastore/tests/integration/conftest.py @@ -4,12 +4,24 @@ import os import random from collections.abc import Iterator from io import BytesIO +from pathlib import Path import pytest from minio import Minio from PIL import Image +from testcontainers.core.container import inside_container from testcontainers.minio import MinioContainer + +class FixedMinioContainer(MinioContainer): + """Fixed Minio Container to allow running inside CI pipeline.""" + + def get_container_host_ip(self) -> str: + if inside_container() and Path('/var/run/docker.sock').exists(): + return self.get_docker_client().gateway_ip(self._container.id) + return super().get_container_host_ip() + + env_var_map = { 'MINIO_ENDPOINT': 'localhost:9000', 'MINIO_ACCESS_KEY': 'test-access-key', @@ -18,7 +30,7 @@ env_var_map = { 'MINIO_OBJECT_NAME': '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX', } container_map = { - 'minio': MinioContainer( + 'minio': FixedMinioContainer( port=env_var_map['MINIO_ENDPOINT'].rsplit(':', maxsplit=1)[-1], access_key=env_var_map['MINIO_ACCESS_KEY'], secret_key=env_var_map['MINIO_SECRET_KEY'], -- 2.54.0 From d860b71b1a1d8a62622068f5e7b73a3247420d4a Mon Sep 17 00:00:00 2001 From: brian Date: Thu, 19 Dec 2024 22:18:58 +0000 Subject: [PATCH 35/45] attempt at showing the determined minio endpoint --- shared/datastore/tests/integration/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shared/datastore/tests/integration/conftest.py b/shared/datastore/tests/integration/conftest.py index 98ab924..0afb893 100644 --- a/shared/datastore/tests/integration/conftest.py +++ b/shared/datastore/tests/integration/conftest.py @@ -1,5 +1,6 @@ """Integration test configurations.""" +import logging import os import random from collections.abc import Iterator @@ -50,6 +51,7 @@ def setup_infrastructure(request: pytest.FixtureRequest) -> None: port=env_var_map['MINIO_ENDPOINT'].rsplit(':', maxsplit=1)[-1], ) env_var_map['MINIO_ENDPOINT'] = f'{minio_host}:{minio_port}' + logging.error('MINIO_ENDPOINT: %s', env_var_map['MINIO_ENDPOINT']) # ensure cleanup def cleanup_infrastructure(): -- 2.54.0 From 150a213ae2050549e3d6c31e46062f3dffc2c697 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 20 Dec 2024 22:39:53 +0000 Subject: [PATCH 36/45] fixed bug when putting data --- shared/datastore/src/put.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/datastore/src/put.py b/shared/datastore/src/put.py index 3e6c99e..372a1d7 100644 --- a/shared/datastore/src/put.py +++ b/shared/datastore/src/put.py @@ -20,7 +20,7 @@ def put( assert isinstance(object_name, str) assert len(object_name) > 0 # prepare for saving - num_bytes = buffer.tell() + num_bytes = len(buffer.getvalue()) buffer.seek(0) # send data to bucket try: -- 2.54.0 From aa6a097e4fd382fee8eef530f7e0ac6665832a4b Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 20 Dec 2024 22:40:53 +0000 Subject: [PATCH 37/45] updated to use server db for testing --- .../datastore/tests/integration/conftest.py | 63 +++++-------------- 1 file changed, 15 insertions(+), 48 deletions(-) diff --git a/shared/datastore/tests/integration/conftest.py b/shared/datastore/tests/integration/conftest.py index 0afb893..e4fce47 100644 --- a/shared/datastore/tests/integration/conftest.py +++ b/shared/datastore/tests/integration/conftest.py @@ -1,6 +1,5 @@ """Integration test configurations.""" -import logging import os import random from collections.abc import Iterator @@ -8,65 +7,25 @@ from io import BytesIO from pathlib import Path import pytest +from dotenv import load_dotenv from minio import Minio from PIL import Image -from testcontainers.core.container import inside_container -from testcontainers.minio import MinioContainer - - -class FixedMinioContainer(MinioContainer): - """Fixed Minio Container to allow running inside CI pipeline.""" - - def get_container_host_ip(self) -> str: - if inside_container() and Path('/var/run/docker.sock').exists(): - return self.get_docker_client().gateway_ip(self._container.id) - return super().get_container_host_ip() - env_var_map = { - 'MINIO_ENDPOINT': 'localhost:9000', - 'MINIO_ACCESS_KEY': 'test-access-key', - 'MINIO_SECRET_KEY': 'test-secret-key', 'MINIO_BUCKET_NAME': 'test-bucket', 'MINIO_OBJECT_NAME': '46KXJMFIAPVLM0TKRFZR5YPPTVJ6PJNX', } -container_map = { - 'minio': FixedMinioContainer( - port=env_var_map['MINIO_ENDPOINT'].rsplit(':', maxsplit=1)[-1], - access_key=env_var_map['MINIO_ACCESS_KEY'], - secret_key=env_var_map['MINIO_SECRET_KEY'], - ), -} - - -@pytest.fixture(scope='session', autouse=True) -def setup_infrastructure(request: pytest.FixtureRequest) -> None: - """Prepare infrastructure for integration test.""" - # prepare infrastructure - for container in container_map.values(): - container.start() - # update env var map - minio_host = container_map['minio'].get_container_host_ip() - minio_port = container_map['minio'].get_exposed_port( - port=env_var_map['MINIO_ENDPOINT'].rsplit(':', maxsplit=1)[-1], - ) - env_var_map['MINIO_ENDPOINT'] = f'{minio_host}:{minio_port}' - logging.error('MINIO_ENDPOINT: %s', env_var_map['MINIO_ENDPOINT']) - - # ensure cleanup - def cleanup_infrastructure(): - for container in container_map.values(): - container.stop() - - request.addfinalizer(cleanup_infrastructure) @pytest.fixture(scope='session', autouse=True) def populate_env( request: pytest.FixtureRequest, - setup_infrastructure, ) -> None: """Populate environment with variables used for testing.""" + # read env-file for local testing + load_dotenv( + dotenv_path=Path(__file__).parent.parent.parent.parent.parent / 'server.env', + ) # update env for key, val in env_var_map.items(): os.environ[key] = val @@ -81,7 +40,6 @@ def populate_env( @pytest.fixture(scope='session') def minio_client( - setup_infrastructure, populate_env, ) -> Iterator[Minio]: # prepare arguments @@ -101,6 +59,15 @@ def minio_client( client.make_bucket(bucket_name=minio_bucket_name) # expose client yield client + # remove objects left behind by tests + for obj_name in client.list_objects(bucket_name=minio_bucket_name, recursive=True): + client.remove_object( + bucket_name=minio_bucket_name, + object_name=obj_name, + ) + # remove bucket + client.remove_bucket(bucket_name=minio_bucket_name) + assert not client.bucket_exists(bucket_name=minio_bucket_name) @pytest.fixture @@ -123,7 +90,7 @@ def data_in_minio( # convert data buffer = BytesIO(data) # prepare for saving - num_bytes = buffer.tell() + num_bytes = len(buffer.getvalue()) buffer.seek(0) # send data to bucket minio_client.put_object( -- 2.54.0 From db4ce6b42533ab60e1f77b3af08a0d84c1542574 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 20 Dec 2024 23:15:55 +0000 Subject: [PATCH 38/45] fixed bug referencing unset variable on exception --- shared/datastore/src/get.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shared/datastore/src/get.py b/shared/datastore/src/get.py index 6c4cc69..09ce2c6 100644 --- a/shared/datastore/src/get.py +++ b/shared/datastore/src/get.py @@ -38,5 +38,7 @@ def get( print_exc() raise exc finally: - response.close() - response.release_conn() + # close connection if established + if 'response' in locals(): + response.close() + response.release_conn() -- 2.54.0 From b17c1745aba2423bdcb6353bfb797004ea3bdff7 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 20 Dec 2024 23:31:33 +0000 Subject: [PATCH 39/45] fixed bug referencing object in minio incorrectly --- shared/datastore/tests/integration/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shared/datastore/tests/integration/conftest.py b/shared/datastore/tests/integration/conftest.py index e4fce47..342ebae 100644 --- a/shared/datastore/tests/integration/conftest.py +++ b/shared/datastore/tests/integration/conftest.py @@ -60,10 +60,10 @@ def minio_client( # expose client yield client # remove objects left behind by tests - for obj_name in client.list_objects(bucket_name=minio_bucket_name, recursive=True): + for obj in client.list_objects(bucket_name=minio_bucket_name, recursive=True): client.remove_object( - bucket_name=minio_bucket_name, - object_name=obj_name, + bucket_name=obj.bucket_name, + object_name=obj.object_name, ) # remove bucket client.remove_bucket(bucket_name=minio_bucket_name) -- 2.54.0 From 7b3374d451c049df03ef78e89661da73686879f1 Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 20 Dec 2024 23:33:43 +0000 Subject: [PATCH 40/45] fixed tests running against server db --- .../tests/integration/base_crud_test.py | 141 ++++++++++-------- 1 file changed, 76 insertions(+), 65 deletions(-) diff --git a/shared/datastore/tests/integration/base_crud_test.py b/shared/datastore/tests/integration/base_crud_test.py index 063c7cc..dda9ee2 100644 --- a/shared/datastore/tests/integration/base_crud_test.py +++ b/shared/datastore/tests/integration/base_crud_test.py @@ -1,11 +1,13 @@ """Integration tests related to base CRUD functions.""" import logging +import os from io import BytesIO +import minio import pytest -# from shared.datastore import connect_minio, delete, get, put +from shared.datastore import delete, get, put def same_data( @@ -22,85 +24,94 @@ def same_data( data_a_bytes = data_a.read() data_b_bytes = data_b.read() # compare size - logging.error(len(data_a_bytes)) - logging.error(len(data_b_bytes)) if len(data_a_bytes) != len(data_b_bytes): + logging.error( + 'data has different length: %s and %s', + len(data_a_bytes), + len(data_b_bytes), + ) return False # compare content if data_a_bytes != data_b_bytes: + logging.error('data has different bytes') return False return True -# def test_should_get_data( -# minio_client, -# data_in_minio, -# ): -# data, bucket_name, object_name = data_in_minio -# received_data = get( -# client=minio_client, -# bucket_name=bucket_name, -# object_name=object_name, -# ) +def test_should_get_data( + minio_client, + data_in_minio, +): + data, bucket_name, object_name = data_in_minio + received_data = get( + client=minio_client, + bucket_name=bucket_name, + object_name=object_name, + ) -# assert isinstance(data, BytesIO) -# assert same_data(data, received_data) + assert isinstance(data, BytesIO) + assert same_data(data, received_data) -# def test_should_delete_data( -# data_in_minio, -# ): -# _, bucket_name, object_name = data_in_minio -# client = connect_minio() -# delete( -# client=client, -# bucket_name=bucket_name, -# object_name=object_name, -# ) -# with pytest.raises(ValueError): -# _ = get( -# client=client, -# bucket_name=bucket_name, -# object_name=object_name, -# ) +def test_should_delete_data( + minio_client, + data_in_minio, +): + _, bucket_name, object_name = data_in_minio + delete( + client=minio_client, + bucket_name=bucket_name, + object_name=object_name, + ) + with pytest.raises(minio.error.S3Error): + _ = get( + client=minio_client, + bucket_name=bucket_name, + object_name=object_name, + ) -# def test_should_put_data( -# data, -# ): -# buffer, bucket_name, object_name = data -# client = connect_minio() -# put( -# client=client, -# buffer=buffer, -# bucket_name=bucket_name, -# object_name=object_name, -# ) -# received_data = get( -# client=client, -# bucket_name=bucket_name, -# object_name=object_name, -# ) -# assert received_data == data +def test_should_put_data( + minio_client, + data, +): + # prepare variables + minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME')) + minio_object_name = str(os.getenv('MINIO_OBJECT_NAME')) + buffer = BytesIO(data) + put( + client=minio_client, + buffer=buffer, + bucket_name=minio_bucket_name, + object_name=minio_object_name, + ) + received_data = get( + client=minio_client, + bucket_name=minio_bucket_name, + object_name=minio_object_name, + ) + assert isinstance(received_data, BytesIO) + assert same_data(received_data, buffer) -# def test_should_update_data( -# data_in_minio, -# ): -# buffer, bucket_name, object_name = data_in_minio -# client = connect_minio() -# put( -# client=client, -# buffer=buffer, -# bucket_name=bucket_name, -# object_name=object_name, -# ) -# received_data = get( -# client=client, -# bucket_name=bucket_name, -# object_name=object_name, -# ) -# assert received_data == buffer +def test_should_update_data( + minio_client, + data_in_minio, +): + buffer, bucket_name, object_name = data_in_minio + put( + client=minio_client, + buffer=buffer, + bucket_name=bucket_name, + object_name=object_name, + ) + received_data = get( + client=minio_client, + bucket_name=bucket_name, + object_name=object_name, + ) + assert same_data(received_data, buffer) + if __name__ == '__main__': pytest.main() -- 2.54.0 From 7da34823664370f5ecd2aae02e2e3581b974d61f Mon Sep 17 00:00:00 2001 From: brian Date: Fri, 20 Dec 2024 23:46:23 +0000 Subject: [PATCH 41/45] updated to use env var from CI --- .gitea/workflows/pull.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/pull.yaml b/.gitea/workflows/pull.yaml index f0c93c6..ad6d35e 100644 --- a/.gitea/workflows/pull.yaml +++ b/.gitea/workflows/pull.yaml @@ -17,7 +17,7 @@ jobs: architecture: "x64" - name: Install Packages env: - PIP_INDEX_URL: http://192.168.1.2:5001/index/ + PIP_INDEX_URL: http://${{ env.DOCKER_REPO_URL }}/index/ PIP_TRUSTED_HOST: 192.168.1.2 run: | pip install poetry -- 2.54.0 From 4e6499cee39172ee85661d6ec47da766bde408eb Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 21 Dec 2024 15:55:31 +0000 Subject: [PATCH 42/45] next attempt at using repo vars --- .gitea/workflows/pull.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/pull.yaml b/.gitea/workflows/pull.yaml index ad6d35e..ee6efce 100644 --- a/.gitea/workflows/pull.yaml +++ b/.gitea/workflows/pull.yaml @@ -17,8 +17,8 @@ jobs: architecture: "x64" - name: Install Packages env: - PIP_INDEX_URL: http://${{ env.DOCKER_REPO_URL }}/index/ - PIP_TRUSTED_HOST: 192.168.1.2 + PIP_INDEX_URL: http://${{ vars.DOCKER_REPO_URL }}/index/ + PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }} run: | pip install poetry poetry install -- 2.54.0 From 06d9753c46067dd6227b55189224c10bd85e94c6 Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 21 Dec 2024 16:03:04 +0000 Subject: [PATCH 43/45] added CI vars for tests --- .gitea/workflows/pull.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitea/workflows/pull.yaml b/.gitea/workflows/pull.yaml index ee6efce..d7258d7 100644 --- a/.gitea/workflows/pull.yaml +++ b/.gitea/workflows/pull.yaml @@ -29,6 +29,10 @@ jobs: run: | poetry run mypy . - name: Pytest + env: + MINIO_ENDPOINT: ${{ vars.MINIO_ENDPOINT }} + MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} + MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }} run: | poetry run pytest . - name: Coverage Report -- 2.54.0 From a430366bbef2349672b6f0ab92fc9b22c0b0a242 Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 21 Dec 2024 16:09:23 +0000 Subject: [PATCH 44/45] updated variable name for clarity --- .gitea/workflows/pull.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/pull.yaml b/.gitea/workflows/pull.yaml index d7258d7..127136e 100644 --- a/.gitea/workflows/pull.yaml +++ b/.gitea/workflows/pull.yaml @@ -17,7 +17,7 @@ jobs: architecture: "x64" - name: Install Packages env: - PIP_INDEX_URL: http://${{ vars.DOCKER_REPO_URL }}/index/ + PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }} PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }} run: | pip install poetry -- 2.54.0 From f61eb59afb313344e8b42e344c5e5c56727a0fbd Mon Sep 17 00:00:00 2001 From: brian Date: Sat, 21 Dec 2024 16:27:48 +0000 Subject: [PATCH 45/45] installed coverage package and included coverage report in CI --- .gitea/workflows/pull.yaml | 4 ++-- poetry.lock | 25 ++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/pull.yaml b/.gitea/workflows/pull.yaml index 127136e..452ad07 100644 --- a/.gitea/workflows/pull.yaml +++ b/.gitea/workflows/pull.yaml @@ -28,13 +28,13 @@ jobs: - name: Type Check run: | poetry run mypy . - - name: Pytest + - name: Pytest & Calculate Coverage env: MINIO_ENDPOINT: ${{ vars.MINIO_ENDPOINT }} MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }} run: | - poetry run pytest . + poetry run coverage run -m pytest . - name: Coverage Report run: | poetry run coverage report -m diff --git a/poetry.lock b/poetry.lock index 6a68bfd..971ea1f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2146,6 +2146,29 @@ type = "legacy" url = "http://192.168.1.2:5001/index" reference = "threadripper" +[[package]] +name = "pytest-cov" +version = "6.0.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.9" +files = [ + {file = "pytest-cov-6.0.0.tar.gz", hash = "sha256:fde0b595ca248bb8e2d76f020b465f3b107c9632e6a1d1705f17834c89dcadc0"}, + {file = "pytest_cov-6.0.0-py3-none-any.whl", hash = "sha256:eee6f1b9e61008bd34975a4d5bab25801eb31898b032dd55addc93e96fcaaa35"}, +] + +[package.dependencies] +coverage = {version = ">=7.5", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] + +[package.source] +type = "legacy" +url = "http://192.168.1.2:5001/index" +reference = "threadripper" + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -3051,4 +3074,4 @@ reference = "threadripper" [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "8ac4e962d93b5a2f23e263d60116c5bb2ab27956ff6eacb5dbc9a6434ac2669c" +content-hash = "539b697eb7c6ea07594c923ca2aaf26ea841be843c18e696882ccd5dc83339f7" diff --git a/pyproject.toml b/pyproject.toml index f3f6ddc..314593d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ types-tqdm = "^4.66.0.20240417" pytest = "^8.3.3" testcontainers = "^4.8.2" coverage = "^7.6.4" +pytest-cov = "^6.0.0" [tool.poetry.group.dev.dependencies] -- 2.54.0