Merge pull request 'add_model_saving' (#37) from add_model_saving into main

Reviewed-on: http://192.168.1.2:3000/brian/visual_critical_discourse_analysis/pulls/37

.flake8

@@ -1,2 +0,0 @@
-[flake8]
-per-file-ignores = __init__.py:F401

.gitea/workflows/default.yaml

@@ -22,4 +22,4 @@ jobs:
           poetry run flake8 . --benchmark
       - name: Type Check
         run: |
-          poetry run mypy . --disable-error-code=import-untyped
+          poetry run mypy .

.gitea/workflows/publish.yaml

@@ -25,7 +25,7 @@ jobs:
           poetry run flake8 . --benchmark
       - name: Type Check
         run: |
-          poetry run mypy . --disable-error-code=import-untyped
+          poetry run mypy .
   publish:
     name: Build and Publish
     runs-on: ubuntu-latest
@@ -37,9 +37,9 @@ jobs:
           -
             dockerfile: ./Dockerfile.web_ui
             image: ${{ vars.docker_repo_url }}/${{ gitea.repository }}/web_ui
-          # -
-          #   dockerfile: ./model/Dockerfile
-          #   image: ${{ vars.docker_repo_url }}/${{ gitea.repository }}/model
+          -
+            dockerfile: ./Dockerfile.model
+            image: ${{ vars.docker_repo_url }}/${{ gitea.repository }}/model
     steps:
       -
         name: Checkout Code

.pre-commit-config.yaml

@@ -36,6 +36,9 @@ repos:
     rev: 7.0.0
     hooks:
     -   id: flake8
+        entry: pflake8
+        additional_dependencies:
+          - "pyproject-flake8"
 -   repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.8.0
     hooks:

Dockerfile.model

@@ -0,0 +1,53 @@
+# build stage
+FROM python:3.12-slim-bookworm as BUILDER
+
+# set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=ON \
+    PIP_DEFAULT_TIMEOUT=100 \
+    DEBIAN_FRONTEND=noninteractive \
+    POETRY_HOME=/etc/poetry \
+    POETRY_VERSION=1.7.1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache \
+    APP_HOME=/home/app
+
+# update system
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    && apt-get clean
+
+# install poetry
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="${POETRY_HOME}/bin:$PATH"
+
+# install runtime dependencies
+WORKDIR ${APP_HOME}
+COPY ./poetry.lock ./pyproject.toml ./
+RUN --mount=type=cache,target=${POETRY_CACHE_DIR} poetry install \
+    --with shared,model
+
+# final stage
+FROM python:3.12-slim-bookworm
+
+# copy virtualenv made by poetry
+ENV APP_HOME=/home/app \
+    VIRTUAL_ENV=/home/app/.venv
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+
+# create home directory and app user
+RUN mkdir -p /home/app
+
+# add code while changing ownership
+WORKDIR $APP_HOME
+COPY ./shared ./shared
+COPY ./model/src ./
+
+ENTRYPOINT [ "python", "main.py" ]

model/docker-compose.server.yml

@@ -0,0 +1,11 @@
+services:
+  vcda_model:
+    image: vcda_model:test
+    container_name: vcda_model_alone
+    build:
+      context: ../
+      dockerfile: ./Dockerfile.model
+    env_file:
+      - ../server.env
+    environment:
+      - ENV=TEST

model/src/data_store/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .connect import connect
+from .get import get
+from .put import put

model/src/data_store/connect.py

@@ -0,0 +1,30 @@
+"""Definition of connect function."""
+from __future__ import annotations
+
+import os
+
+from minio import Minio
+
+
+def connect(
+) -> Minio:
+    """Connect to MinIO server"""
+    minio_endpoint = os.getenv('MINIO_ENDPOINT', default=None)
+    assert isinstance(minio_endpoint, str)
+    minio_access_key = os.getenv('MINIO_ACCESS_KEY', default=None)
+    assert isinstance(minio_access_key, str)
+    minio_secret_key = os.getenv('MINIO_SECRET_KEY', default=None)
+    assert isinstance(minio_secret_key, str)
+    minio_bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None)
+    assert isinstance(minio_bucket_name, str)
+    # connect client
+    client = Minio(
+        endpoint=minio_endpoint,
+        access_key=minio_access_key,
+        secret_key=minio_secret_key,
+        secure=False,
+    )
+    # ensure bucket exists
+    if not client.bucket_exists(bucket_name=minio_bucket_name):
+        client.make_bucket(bucket_name=minio_bucket_name)
+    return client

model/src/data_store/get.py

@@ -0,0 +1,27 @@
+"""Definition of get function."""
+from __future__ import annotations
+
+import os
+from io import BytesIO
+
+from minio import Minio
+
+
+def get(
+    client: Minio,
+    object_name: str,
+) -> BytesIO:
+    """Get buffer from bucket in MinIO."""
+    assert isinstance(client, Minio)
+    assert isinstance(object_name, str)
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None)
+    assert isinstance(bucket_name, str)
+    # get buffer
+    try:
+        response = client.get_object(bucket_name, object_name)
+        buffer = BytesIO(response.data)
+    finally:
+        response.close()
+        response.release_conn()
+    buffer.seek(0)
+    return buffer

model/src/data_store/put.py

@@ -0,0 +1,30 @@
+"""Definition of put function."""
+from __future__ import annotations
+
+import os
+from io import BytesIO
+
+from minio import Minio
+
+
+def put(
+    client: Minio,
+    buffer: BytesIO,
+    object_name: str,
+) -> None:
+    """Put buffer in bucket in MinIO."""
+    assert isinstance(client, Minio)
+    assert isinstance(buffer, BytesIO)
+    assert isinstance(object_name, str)
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None)
+    assert isinstance(bucket_name, str)
+    # prepare for saving
+    num_bytes = buffer.tell()
+    buffer.seek(0)
+    # send data to bucket
+    client.put_object(
+        bucket_name=bucket_name,
+        object_name=object_name,
+        length=num_bytes,
+        data=buffer,
+    )

model/src/main.py

@@ -0,0 +1,45 @@
+"""Main script to be run by service."""
+from __future__ import annotations
+
+from hashlib import md5
+from io import BytesIO
+
+import torch
+from data_store import connect
+from data_store import put
+from models import VisualCommunicationModel
+from torchinfo import summary
+from utils import check_env
+
+from shared.utils import setup_logging
+
+
+DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+if __name__ == '__main__':
+    # check that environment variables are set
+    check_env()
+    # setup logging
+    setup_logging()
+    # instantiate model
+    model = VisualCommunicationModel().to(DEVICE)
+    # show model weights
+    summary(model)
+    # for param in model.parameters():
+    #     logging.info(param.data)
+    # save model to buffer
+    buffer = BytesIO()
+    torch.save(model.state_dict(), buffer)
+    print(f"buffer size: {len(buffer.getvalue())}")
+    # calculate buffer hash
+    hash_str = md5(buffer.getbuffer()).hexdigest()
+    print(f"hash string: {hash_str}")
+    # connect to minio
+    client = connect()
+    # put buffer in minio bucket
+    put(
+        client=client,
+        buffer=buffer,
+        object_name=hash_str,
+    )
+    print('saved to Minio')

model/src/models/visual_communication.py

@@ -1,6 +1,7 @@
+"""Definition of VisualCommunicationModel class."""
 from __future__ import annotations
 
-import torch.nn as nn
+from torch import nn
 
 from .angle import AngleTail
 from .contact import ContactTail
@@ -28,6 +29,8 @@ from shared.dto import VisualSyntaxData
 
 
 class VisualCommunicationModel(nn.Module):
+    """Visual communication model."""
+
     def __init__(self):
         super().__init__()
         # store other models
@@ -45,6 +48,7 @@ class VisualCommunicationModel(nn.Module):
         self.salience_tail = SalienceTail()
 
     def forward(self, x):
+        """Calculate model output on data."""
         # generate visual representation
         vis_rep = self.resnet_head(x)
         # prepare result map

model/src/utils/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import annotations
+
+from .check_env import check_env

model/src/utils/check_env.py

@@ -0,0 +1,25 @@
+"""Definition of check_env function."""
+from __future__ import annotations
+
+import os
+
+
+def check_env() -> None:
+    """Check necessary environment variables are set."""
+    necesasary_var_list = {
+        'MONGO_HOST',
+        'MONGO_DB',
+        'MONGO_COLLECTION',
+        'MONGO_USER',
+        'MONGO_PASSWORD',
+        'MINIO_ENDPOINT',
+        'MINIO_ACCESS_KEY',
+        'MINIO_SECRET_KEY',
+    }
+    for env_var in necesasary_var_list:
+        # ensure env var set
+        assert (
+            env_var in os.environ
+        ), (
+            f"environment variable not set: {env_var}"
+        )

poetry.lock

@@ -11,6 +11,63 @@ files = [
     {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"},
 ]
 
+[[package]]
+name = "argon2-cffi"
+version = "23.1.0"
+description = "Argon2 for Python"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "argon2_cffi-23.1.0-py3-none-any.whl", hash = "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"},
+    {file = "argon2_cffi-23.1.0.tar.gz", hash = "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08"},
+]
+
+[package.dependencies]
+argon2-cffi-bindings = "*"
+
+[package.extras]
+dev = ["argon2-cffi[tests,typing]", "tox (>4)"]
+docs = ["furo", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-notfound-page"]
+tests = ["hypothesis", "pytest"]
+typing = ["mypy"]
+
+[[package]]
+name = "argon2-cffi-bindings"
+version = "21.2.0"
+description = "Low-level CFFI bindings for Argon2"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3"},
+    {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367"},
+    {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d"},
+    {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae"},
+    {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c"},
+    {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86"},
+    {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_i686.whl", hash = "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f"},
+    {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e"},
+    {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win32.whl", hash = "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082"},
+    {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f"},
+    {file = "argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93"},
+    {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3e385d1c39c520c08b53d63300c3ecc28622f076f4c2b0e6d7e796e9f6502194"},
+    {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3e3cc67fdb7d82c4718f19b4e7a87123caf8a93fde7e23cf66ac0337d3cb3f"},
+    {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a22ad9800121b71099d0fb0a65323810a15f2e292f2ba450810a7316e128ee5"},
+    {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351"},
+    {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:93f9bf70084f97245ba10ee36575f0c3f1e7d7724d67d8e5b08e61787c320ed7"},
+    {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3b9ef65804859d335dc6b31582cad2c5166f0c3e7975f324d9ffaa34ee7e6583"},
+    {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4966ef5848d820776f5f562a7d45fdd70c2f330c961d0d745b784034bd9f48d"},
+    {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ef543a89dee4db46a1a6e206cd015360e5a75822f76df533845c3cbaf72670"},
+    {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed2937d286e2ad0cc79a7087d3c272832865f779430e0cc2b4f3718d3159b0cb"},
+    {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5e00316dabdaea0b2dd82d141cc66889ced0cdcbfa599e8b471cf22c620c329a"},
+]
+
+[package.dependencies]
+cffi = ">=1.0.1"
+
+[package.extras]
+dev = ["cogapp", "pre-commit", "pytest", "wheel"]
+tests = ["pytest"]
+
 [[package]]
 name = "attrs"
 version = "23.2.0"
@@ -416,6 +473,22 @@ mccabe = ">=0.7.0,<0.8.0"
 pycodestyle = ">=2.11.0,<2.12.0"
 pyflakes = ">=3.2.0,<3.3.0"
 
+[[package]]
+name = "flake8-pyproject"
+version = "1.2.3"
+description = "Flake8 plug-in loading the configuration from pyproject.toml"
+optional = false
+python-versions = ">= 3.6"
+files = [
+    {file = "flake8_pyproject-1.2.3-py3-none-any.whl", hash = "sha256:6249fe53545205af5e76837644dc80b4c10037e73a0e5db87ff562d75fb5bd4a"},
+]
+
+[package.dependencies]
+Flake8 = ">=5"
+
+[package.extras]
+dev = ["pyTest", "pyTest-cov"]
+
 [[package]]
 name = "flask"
 version = "3.0.3"
@@ -645,6 +718,24 @@ files = [
     {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"},
 ]
 
+[[package]]
+name = "minio"
+version = "7.2.7"
+description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage"
+optional = false
+python-versions = "*"
+files = [
+    {file = "minio-7.2.7-py3-none-any.whl", hash = "sha256:59d1f255d852fe7104018db75b3bebbd987e538690e680f7c5de835e422de837"},
+    {file = "minio-7.2.7.tar.gz", hash = "sha256:473d5d53d79f340f3cd632054d0c82d2f93177ce1af2eac34a235bea55708d98"},
+]
+
+[package.dependencies]
+argon2-cffi = "*"
+certifi = "*"
+pycryptodome = "*"
+typing-extensions = "*"
+urllib3 = "*"
+
 [[package]]
 name = "mpmath"
 version = "1.3.0"
@@ -1028,6 +1119,21 @@ sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-d
 test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"]
 xml = ["lxml (>=4.9.2)"]
 
+[[package]]
+name = "pandas-stubs"
+version = "2.2.2.240603"
+description = "Type annotations for pandas"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "pandas_stubs-2.2.2.240603-py3-none-any.whl", hash = "sha256:e08ce7f602a4da2bff5a67475ba881c39f2a4d4f7fccc1cba57c6f35a379c6c0"},
+    {file = "pandas_stubs-2.2.2.240603.tar.gz", hash = "sha256:2dcc86e8fa6ea41535a4561c1f08b3942ba5267b464eff2e99caeee66f9e4cd1"},
+]
+
+[package.dependencies]
+numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\" and python_version < \"3.13\""}
+types-pytz = ">=2022.1.1"
+
 [[package]]
 name = "pillow"
 version = "10.3.0"
@@ -1162,6 +1268,47 @@ files = [
     {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
 ]
 
+[[package]]
+name = "pycryptodome"
+version = "3.20.0"
+description = "Cryptographic library for Python"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+files = [
+    {file = "pycryptodome-3.20.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a"},
+    {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f"},
+    {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d"},
+    {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd"},
+    {file = "pycryptodome-3.20.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33"},
+    {file = "pycryptodome-3.20.0-cp27-cp27m-win32.whl", hash = "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690"},
+    {file = "pycryptodome-3.20.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f"},
+    {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091"},
+    {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4"},
+    {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc"},
+    {file = "pycryptodome-3.20.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818"},
+    {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"},
+    {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"},
+    {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"},
+    {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"},
+    {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"},
+    {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"},
+    {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"},
+    {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"},
+    {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"},
+    {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"},
+    {file = "pycryptodome-3.20.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a"},
+    {file = "pycryptodome-3.20.0-pp27-pypy_73-win32.whl", hash = "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e"},
+    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04"},
+    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3"},
+    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea"},
+    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b"},
+    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6"},
+    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab"},
+    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5"},
+    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e"},
+    {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"},
+]
+
 [[package]]
 name = "pydantic"
 version = "2.7.1"
@@ -1618,6 +1765,17 @@ typing-extensions = ">=4.8.0"
 opt-einsum = ["opt-einsum (>=3.3)"]
 optree = ["optree (>=0.9.1)"]
 
+[[package]]
+name = "torchinfo"
+version = "1.8.0"
+description = "Model summary in PyTorch, based off of the original torchsummary."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "torchinfo-1.8.0-py3-none-any.whl", hash = "sha256:2e911c2918603f945c26ff21a3a838d12709223dc4ccf243407bce8b6e897b46"},
+    {file = "torchinfo-1.8.0.tar.gz", hash = "sha256:72e94b0e9a3e64dc583a8e5b7940b8938a1ac0f033f795457f27e6f4e7afa2e9"},
+]
+
 [[package]]
 name = "torchvision"
 version = "0.17.2"
@@ -1705,6 +1863,42 @@ files = [
     {file = "types_Pillow-10.2.0.20240520-py3-none-any.whl", hash = "sha256:33c36494b380e2a269bb742181bea5d9b00820367822dbd3760f07210a1da23d"},
 ]
 
+[[package]]
+name = "types-pytz"
+version = "2024.1.0.20240417"
+description = "Typing stubs for pytz"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "types-pytz-2024.1.0.20240417.tar.gz", hash = "sha256:6810c8a1f68f21fdf0f4f374a432487c77645a0ac0b31de4bf4690cf21ad3981"},
+    {file = "types_pytz-2024.1.0.20240417-py3-none-any.whl", hash = "sha256:8335d443310e2db7b74e007414e74c4f53b67452c0cb0d228ca359ccfba59659"},
+]
+
+[[package]]
+name = "types-requests"
+version = "2.32.0.20240602"
+description = "Typing stubs for requests"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "types-requests-2.32.0.20240602.tar.gz", hash = "sha256:3f98d7bbd0dd94ebd10ff43a7fbe20c3b8528acace6d8efafef0b6a184793f06"},
+    {file = "types_requests-2.32.0.20240602-py3-none-any.whl", hash = "sha256:ed3946063ea9fbc6b5fc0c44fa279188bae42d582cb63760be6cb4b9d06c3de8"},
+]
+
+[package.dependencies]
+urllib3 = ">=2"
+
+[[package]]
+name = "types-retry"
+version = "0.9.9.4"
+description = "Typing stubs for retry"
+optional = false
+python-versions = "*"
+files = [
+    {file = "types-retry-0.9.9.4.tar.gz", hash = "sha256:e4731dc684b56b875d9746459ad665d3bc281a56b530acdf1c97730167799941"},
+    {file = "types_retry-0.9.9.4-py3-none-any.whl", hash = "sha256:f29760a9fe8b1fefe253e5fe6be7e4c0eba243932c600e0eccffb42a21d17765"},
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.11.0"
@@ -1812,4 +2006,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "2db221a28d46b210a8fa0b78ad2fa3ac606a52052ad5e02b50d5b07db1e1c5b7"
+content-hash = "0928d867877ddfb70a5d2e6442e4f45454f1acd40a1931e7d20264cc506f4f31"

pyproject.toml

@@ -16,6 +16,10 @@ python = "^3.12"
 flake8 = "^7.0.0"
 mypy = "^1.8.0"
 types-pillow = "^10.2.0.20240213"
+types-requests = "^2.32.0.20240602"
+types-retry = "^0.9.9.4"
+flake8-pyproject = "^1.2.3"
+pandas-stubs = "^2.2.2.240603"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -28,6 +32,8 @@ retry = "^0.9.2"
 [tool.poetry.group.model.dependencies]
 torch = "^2.2.1"
 torchvision = "^0.17.1"
+torchinfo = "^1.8.0"
+minio = "^7.2.7"
 
 
 [tool.poetry.group.shared.dependencies]
@@ -46,3 +52,49 @@ dash-auth = "^2.2.0"
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
+
+[tool.flake8]
+per-file-ignores = "__init__.py:F401"
+
+[tool.mypy]
+exclude = "image_download"
+
+[[tool.mypy.overrides]]
+module = "dash.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "dash_auth.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "dash_mantine_components.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "dash_bootstrap_components.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "torchvision.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "image_download.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "dataloader.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "utils.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "data_store.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "models.*"
+ignore_missing_imports = true