Merge pull request 'add_check_env_function' (#54) from add_check_env_function into main

Reviewed-on: #54

.gitea/workflows/pull.yaml

@@ -28,3 +28,6 @@ jobs:
       - name: Type Check
         run: |
           poetry run mypy .
+      - name: Pytest
+        run: |
+          poetry run pytest .

misc/generate_random_prediction.py

@@ -8,6 +8,7 @@ from shared.data_store import connect_minio
 from shared.database import connect_mongodb
 from shared.database.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
+from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 
 if __name__ == '__main__':
     # load in env file
@@ -15,7 +16,7 @@ if __name__ == '__main__':
     assert env_path.exists()
     load_dotenv(env_path)
     # ensure env vars set
-    check_env()
+    check_env(NECESSARY_ENV_VAR_LIST)
     # setup logging
     setup_logging()
     # connect to minIO

misc/get_visual_communication.py

@@ -7,6 +7,7 @@ from dotenv import load_dotenv
 from shared.data_store import connect_minio
 from shared.database import connect_mongodb, get_visual_communication
 from shared.utils import check_env, setup_logging
+from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 
 if __name__ == '__main__':
     # load in env file
@@ -14,7 +15,7 @@ if __name__ == '__main__':
     assert env_path.exists()
     load_dotenv(env_path)
     # ensure env vars set
-    check_env()
+    check_env(NECESSARY_ENV_VAR_LIST)
     # setup logging
     setup_logging()
     # connect to minIO

misc/image_download.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.database import connect
+from shared.database import connect_mongodb
 from shared.database.classes import VisualCommunication
 
 if __name__ == '__main__':
@@ -15,7 +15,7 @@ if __name__ == '__main__':
     load_dotenv(env_path)
     os.environ['MONGO_HOST'] = 'localhost'
     # connect to database
-    collection, db, client = connect()
+    collection, db, client = connect_mongodb()
     print(client.server_info())
     # download images
     data = None

misc/image_upload.py

@@ -9,6 +9,7 @@ from shared.data_store import connect_minio
 from shared.database import connect_mongodb
 from shared.database.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
+from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 
 if __name__ == '__main__':
     # load in env file
@@ -16,7 +17,7 @@ if __name__ == '__main__':
     assert env_path.exists()
     load_dotenv(env_path)
     # ensure env vars set
-    check_env()
+    check_env(NECESSARY_ENV_VAR_LIST)
     # setup logging
     setup_logging()
     # connect to minIO

misc/image_upload_to_server.py

@@ -8,6 +8,7 @@ from pymongo.errors import DuplicateKeyError
 from shared.data_store import connect_minio
 from shared.database import VisualCommunication, connect_mongodb
 from shared.utils import check_env, setup_logging
+from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 
 if __name__ == '__main__':
     # load in env file
@@ -15,7 +16,7 @@ if __name__ == '__main__':
     assert env_path.exists()
     load_dotenv(env_path)
     # ensure env vars set
-    check_env()
+    check_env(NECESSARY_ENV_VAR_LIST)
     # setup logging
     setup_logging()
     # connect to minIO

misc/model_outputs_from_annotation.py

@@ -2,18 +2,10 @@ from __future__ import annotations
 
 from shared.dto import ModelData
 
-
 if __name__ == '__main__':
     # instantiate data object
-    vis_com_list = [
+    vis_com_list = [ModelData.from_random() for i in range(3)]
-        ModelData.from_random()
-        for i
-        in range(3)
-    ]
     # generate random predictions
-    [
+    [vis_com.from_random() for vis_com in vis_com_list]
-        vis_com.from_random()
-        for vis_com in vis_com_list
-    ]
     for vis_com in vis_com_list:
         print(vis_com)

misc/prediction_upload.py

@@ -8,6 +8,7 @@ from shared.data_store import connect_minio
 from shared.database import connect_mongodb, upsert_prediction
 from shared.database.classes import VisualCommunication
 from shared.utils import check_env, setup_logging
+from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 
 if __name__ == '__main__':
     # load in env file
@@ -15,7 +16,7 @@ if __name__ == '__main__':
     assert env_path.exists()
     load_dotenv(env_path)
     # ensure env vars set
-    check_env()
+    check_env(NECESSARY_ENV_VAR_LIST)
     # setup logging
     setup_logging()
     # connect to minIO

other/transfer_images_mongo_minio.py

@@ -13,6 +13,7 @@ from pymongo.collection import Collection
 from shared.data_store import connect_minio, put
 from shared.database import VisualCommunication, connect_mongodb
 from shared.utils import check_env, setup_logging
+from web_ui.src.main import NECESSARY_ENV_VAR_LIST
 
 
 def list_mongo_document_ids(
@@ -76,7 +77,7 @@ if __name__ == '__main__':
     assert env_path.exists()
     load_dotenv(env_path)
     # ensure env vars set
-    check_env()
+    check_env(NECESSARY_ENV_VAR_LIST)
     # setup logging
     setup_logging()
     # connect to minIO

poetry.lock

@@ -830,6 +830,22 @@ type = "legacy"
 url = "http://192.168.1.2:5001/index"
 reference = "threadripper"
 
+[[package]]
+name = "iniconfig"
+version = "2.0.0"
+description = "brain-dead simple config-ini parsing"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
+    {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
+]
+
+[package.source]
+type = "legacy"
+url = "http://192.168.1.2:5001/index"
+reference = "threadripper"
+
 [[package]]
 name = "itsdangerous"
 version = "2.2.0"
@@ -1621,6 +1637,26 @@ type = "legacy"
 url = "http://192.168.1.2:5001/index"
 reference = "threadripper"
 
+[[package]]
+name = "pluggy"
+version = "1.5.0"
+description = "plugin and hook calling mechanisms for python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
+    {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
+]
+
+[package.extras]
+dev = ["pre-commit", "tox"]
+testing = ["pytest", "pytest-benchmark"]
+
+[package.source]
+type = "legacy"
+url = "http://192.168.1.2:5001/index"
+reference = "threadripper"
+
 [[package]]
 name = "protobuf"
 version = "5.28.0"
@@ -1979,6 +2015,31 @@ type = "legacy"
 url = "http://192.168.1.2:5001/index"
 reference = "threadripper"
 
+[[package]]
+name = "pytest"
+version = "8.3.3"
+description = "pytest: simple powerful testing with Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"},
+    {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+iniconfig = "*"
+packaging = "*"
+pluggy = ">=1.5,<2"
+
+[package.extras]
+dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+
+[package.source]
+type = "legacy"
+url = "http://192.168.1.2:5001/index"
+reference = "threadripper"
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"
@@ -2711,4 +2772,4 @@ reference = "threadripper"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "e311eaec3058b444c5bb980a3700ca9de273d7948ace5d5b94363f4f967b622a"
+content-hash = "33dbc73047d591ae63dd2d934519882654074a1f3c9ca757d0571d869b8541c2"

pyproject.toml

@@ -21,6 +21,7 @@ types-retry = "^0.9.9.4"
 flake8-pyproject = "^1.2.3"
 pandas-stubs = "^2.2.2.240603"
 types-tqdm = "^4.66.0.20240417"
+pytest = "^8.3.3"
 
 
 [tool.poetry.group.dev.dependencies]

shared/database/classes/visual_communication.py

@@ -1,19 +1,18 @@
 """Definition of VisualCommunication model."""
+
 from __future__ import annotations
 
 import logging
-from base64 import b64decode
+from base64 import b64decode, b64encode
-from base64 import b64encode
 from io import BytesIO
 from pathlib import Path
 
 from minio import Minio
 from PIL import Image
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 from pymongo.collection import Collection
 
-from shared.data_store import get
+from shared.data_store import get, put
-from shared.data_store import put
 from shared.dto import ModelData
 
 
@@ -24,10 +23,7 @@ class VisualCommunication(BaseModel):
     object_name: str
     annotation: ModelData | None = None
     prediction: ModelData | None = None
-
+    model_config = ConfigDict(arbitrary_types_allowed=True)
-    class Config:
-        """BaseModel configuration."""
-        arbitrary_types_allowed = True
 
     @classmethod
     def classname(cls) -> str:
@@ -58,10 +54,8 @@ class VisualCommunication(BaseModel):
         image: Image.Image,
         minio_client: Minio,
     ) -> VisualCommunication:
-        """
+        """Instantiate from filename and image that is automatically uploaded
-        Instantiate from filename and image
+        to MinIO."""
-        that is automatically uploaded to MinIO.
-        """
         assert isinstance(name, str)
         assert isinstance(image, Image.Image)
         assert isinstance(minio_client, Minio)

shared/utils/check_env.py

@@ -1,26 +1,14 @@
 """Definition of check_env function."""
 
-from __future__ import annotations
-
 import os
 
 
-def check_env() -> None:
+def check_env(
+    var_list: set[str],
+) -> None:
     """Check necessary environment variables are set."""
-    necesasary_var_list = {
+    assert isinstance(var_list, set)
-        'MONGO_HOST',
+    assert all(isinstance(elem, str) for elem in var_list)
-        'MONGO_DB',
+    # check that env vars are set
-        'MONGO_COLLECTION',
+    for env_var in var_list:
-        'MONGO_USER',
-        'MONGO_PASSWORD',
-        'DASH_AUTH_USERNAME',
-        'DASH_AUTH_PASSWORD',
-        'MINIO_ENDPOINT',
-        'MINIO_ACCESS_KEY',
-        'MINIO_SECRET_KEY',
-        'MINIO_BUCKET_NAME',
-        'MINIO_BUCKET_NAME_MODELS',
-    }
-    for env_var in necesasary_var_list:
-        # ensure env var set
         assert env_var in os.environ, f"environment variable not set: {env_var}"

shared/utils/tests/check_env_test.py

@@ -0,0 +1,53 @@
+"""Definition of tests for check_env function."""
+
+import os
+import unittest
+
+from shared.utils import check_env
+
+
+class TestFunctionCheckEnv(unittest.TestCase):
+    """Testing function check_env."""
+
+    def setUp(self):
+        """Prepare for tests."""
+        # define variables for test
+        self.not_set_env_var = 'NOT_SET_ENV_VAR'
+        self.set_env_var_map = {
+            'ENV_VAR_UPPER': 'ENV_VAR_UPPER',
+            'ENV_VAR_LOWER': 'env_var_lower',
+        }
+        # set env vars
+        for key, val in self.set_env_var_map.items():
+            os.environ[key] = val
+
+    def tearDown(self):
+        """Clean up after tests."""
+        # remove env vars
+        for key in self.set_env_var_map:
+            _ = os.environ.pop(key, default=None)
+
+    def test_input_type(self):
+        """Test that function fails on wrong input types."""
+        with self.assertRaises(AssertionError):
+            check_env('not_a_list')
+        with self.assertRaises(AssertionError):
+            check_env({1, 2, 3})
+
+    def test_env_var_not_set(self):
+        """Test that function raises AssertionError when checking environment
+        variable that is not set."""
+        var_list = {self.not_set_env_var}
+        msg = f'environment variable not set: {self.not_set_env_var}'
+        with self.assertRaises(AssertionError, msg=msg):
+            check_env(var_list)
+
+    def test_env_vars_set(self):
+        """Test that function does not raise any errors when environment
+        variables are set."""
+        var_list = set(self.set_env_var_map.keys())
+        check_env(var_list)
+
+
+if __name__ == '__main__':
+    unittest.main()

web_ui/src/main.py

@@ -11,7 +11,21 @@ from shared.utils import check_env, setup_logging
 from .app import init_app
 
 # ensure env vars set
-check_env()
+NECESSARY_ENV_VAR_LIST = {
+    'MONGO_HOST',
+    'MONGO_DB',
+    'MONGO_COLLECTION',
+    'MONGO_USER',
+    'MONGO_PASSWORD',
+    'DASH_AUTH_USERNAME',
+    'DASH_AUTH_PASSWORD',
+    'MINIO_ENDPOINT',
+    'MINIO_ACCESS_KEY',
+    'MINIO_SECRET_KEY',
+    'MINIO_BUCKET_NAME',
+    'MINIO_BUCKET_NAME_MODELS',
+}
+check_env(NECESSARY_ENV_VAR_LIST)
 
 # setup logging stream handler
 setup_logging()