Merge pull request 'upload-images-to-minio' (#2) from upload-images-to-minio into main

Reviewed-on: #2

.gitea/workflows/sync-annotations.yml

@@ -0,0 +1,28 @@
+name: Sync Label Studio
+
+on:
+  schedule:
+    # This job will run every day at 03:00 AM
+    - cron: "0 3 * * *"
+
+jobs:
+  sync_storage:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Sync Label Studio Target Storage
+        env:
+          LABEL_STUDIO_API_TOKEN: ${{ secrets.LABEL_STUDIO_API_TOKEN }}
+        run: |
+          curl -X POST \
+          -H "Authorization: Token $LABEL_STUDIO_API_TOKEN" \
+          "https://label-studio.gt-proj.com/api/storages/export/s3/1/sync"
+
+      - name: Fix Label Studio output content type in MinIO
+        env:
+          MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
+          MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }}
+        run: |
+          mc alias set myminio http://10.0.0.2:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
+          mc mirror --recursive --overwrite --attr 'Content-Type=application/json' \
+          myminio/visual-semiotic-ai-analysis/label-studio-output \
+          myminio/visual-semiotic-ai-analysis/label-studio-output

data_store/repositories/annotation_repository.py

@@ -23,6 +23,7 @@ class AnnotationRepository(MinioAdapter, AnnotationRepositoryInterface):
 
     encoding: str = "utf-8"
     data_format: str = "json"
+    content_type: str = "application/json"
     folder_name: str = "annotations"
 
     @classmethod
@@ -81,7 +82,7 @@ class AnnotationRepository(MinioAdapter, AnnotationRepositoryInterface):
         buffer = BytesIO(data_bytes)
         # Put object
         object_name = self._object_name(job_id)
-        self._put(object_name, buffer)
+        self._put(object_name, buffer, self.content_type)
 
     def delete(self, job_id: UUID) -> None:
         """Delete an Annotation by its job ID."""

data_store/repositories/image_repository.py

@@ -15,6 +15,7 @@ class ImageRepository(MinioAdapter, ImageRepositoryInterface):
     """ImageRepository implementation using MinIO as the backend."""
 
     image_format: str = "PNG"
+    content_type: str = "image/png"
     folder_name: str = "images"
 
     @classmethod
@@ -57,7 +58,7 @@ class ImageRepository(MinioAdapter, ImageRepositoryInterface):
         image.save(buffer, self.image_format)
         # Put object
         object_name = self._object_name(image_id)
-        self._put(object_name, buffer)
+        self._put(object_name, buffer, self.content_type)
 
     def delete(self, image_id: UUID) -> None:
         """Delete an Image by its ID."""

pyproject.toml

@@ -77,6 +77,7 @@ dev = [
     "pre-commit>=4.3.0",
     "pytest>=8.4.2",
     "pytest-cov>=7.0.0",
+    "python-dotenv>=1.1.1",
     "pyupgrade>=3.20.0",
     "ruff>=0.13.0",
     "safety>=3.2.11",

scripts/upload_images_to_minio.py

@@ -0,0 +1,67 @@
+"""Script to upload images to MinIO server."""
+
+from pathlib import Path
+from uuid import uuid4
+import pandas as pd
+from PIL import Image
+from dotenv import load_dotenv
+
+from data_store.repositories import ImageRepository
+
+UPLOAD_DIR = Path("/Volumes/BW-PSSD/maersk_data/images")
+
+
+def upload_images_to_minio(
+    image_repo: ImageRepository,
+    upload_dir: Path,
+) -> None:
+    """Upload images from the specified directory to MinIO server."""
+    # Check input
+    if not isinstance(image_repo, ImageRepository):
+        raise ValueError("image_repo must be an instance of ImageRepository.")
+    if not image_repo.is_connected:
+        raise ConnectionError("image_repo must be connected to MinIO server.")
+    if not isinstance(upload_dir, Path):
+        raise ValueError("upload_dir must be a pathlib.Path instance.")
+    if not upload_dir.is_dir():
+        raise NotADirectoryError(f"upload_dir must be a valid directory: {upload_dir}")
+    if not upload_dir.exists():
+        raise FileNotFoundError(f"upload_dir does not exist: {upload_dir}")
+    # Scan directory for images
+    images = list(upload_dir.glob("*.png"))
+    if not images:
+        raise FileNotFoundError(f"No PNG images found in directory: {upload_dir}")
+    print(f"Found {len(images)} images to upload.")
+    # Initialize list to store image name and id mapping
+    image_name_id_map: dict[str, str] = {}
+    # Begin uploading images
+    for image_path in images:
+        try:
+            # Load image
+            img = Image.open(image_path)
+            # Upload image to MinIO
+            img_id = uuid4()
+            image_repo.put(img, img_id)
+            print(f"Uploaded {image_path.name} with ID {img_id}.")
+        except Exception as e:
+            print(f"Failed to upload {image_path.name}: {e}")
+            continue
+        else:
+            # Store mapping
+            image_name_id_map[image_path.name] = str(img_id)
+    # Save mapping to CSV
+    df = pd.DataFrame(
+        list(image_name_id_map.items()), columns=["image_name", "image_id"]
+    )
+    csv_path = Path(__file__).parent / "image_name_id_map.csv"
+    df.to_csv(csv_path, index=False)
+
+
+if __name__ == "__main__":
+    # Load environment variables from .env file
+    load_dotenv()
+    # Instantiate ImageRepository
+    with ImageRepository() as image_repo:
+        # Upload images from the specified directory
+        upload_images_to_minio(image_repo, UPLOAD_DIR)
+    print("Image upload process completed.")

uv.lock

@@ -60,14 +60,14 @@ wheels = [
 
 [[package]]
 name = "authlib"
-version = "1.6.3"
+version = "1.6.4"
 source = { registry = "http://10.0.0.2:5001/index/" }
 dependencies = [
     { name = "cryptography" },
 ]
-sdist = { url = "http://10.0.0.2:5001/index/authlib/authlib-1.6.3.tar.gz", hash = "sha256:9f7a982cc395de719e4c2215c5707e7ea690ecf84f1ab126f28c053f4219e610" }
+sdist = { url = "http://10.0.0.2:5001/index/authlib/authlib-1.6.4.tar.gz", hash = "sha256:104b0442a43061dc8bc23b133d1d06a2b0a9c2e3e33f34c4338929e816287649" }
 wheels = [
-    { url = "http://10.0.0.2:5001/index/authlib/authlib-1.6.3-py2.py3-none-any.whl", hash = "sha256:7ea0f082edd95a03b7b72edac65ec7f8f68d703017d7e37573aee4fc603f2a48" },
+    { url = "http://10.0.0.2:5001/index/authlib/authlib-1.6.4-py2.py3-none-any.whl", hash = "sha256:39313d2a2caac3ecf6d8f95fbebdfd30ae6ea6ae6a6db794d976405fdd9aa796" },
 ]
 
 [[package]]
@@ -946,15 +946,15 @@ wheels = [
 
 [[package]]
 name = "python-repositories"
-version = "0.2.2"
+version = "0.2.3"
 source = { registry = "https://gitea.gt-proj.com/api/packages/brian/pypi/simple/" }
 dependencies = [
     { name = "python-utils" },
     { name = "structlog" },
 ]
-sdist = { url = "https://gitea.gt-proj.com/api/packages/brian/pypi/files/python-repositories/0.2.2/python_repositories-0.2.2.tar.gz", hash = "sha256:81d690ca632366f24c2cdf7d550d087810ebc6d305be5bc3e0fad6dacfd9acfe" }
+sdist = { url = "https://gitea.gt-proj.com/api/packages/brian/pypi/files/python-repositories/0.2.3/python_repositories-0.2.3.tar.gz", hash = "sha256:216d1f7cd14ff690c5e6af502a05c7a5de8bfb3202eabf25aa9dbce83b0f384d" }
 wheels = [
-    { url = "https://gitea.gt-proj.com/api/packages/brian/pypi/files/python-repositories/0.2.2/python_repositories-0.2.2-py3-none-any.whl", hash = "sha256:3d521b31a9ba21e2113f8ab4b24449ba6767a60c9c22cf75c29a9eb198fc250a" },
+    { url = "https://gitea.gt-proj.com/api/packages/brian/pypi/files/python-repositories/0.2.3/python_repositories-0.2.3-py3-none-any.whl", hash = "sha256:aa2ef4702f341b89cd973e6e7cfc8a288a5340c67d09ae281285f0a30edec96e" },
 ]
 
 [package.optional-dependencies]
@@ -1327,6 +1327,7 @@ dev = [
     { name = "pre-commit" },
     { name = "pytest" },
     { name = "pytest-cov" },
+    { name = "python-dotenv" },
     { name = "pyupgrade" },
     { name = "ruff" },
     { name = "safety" },
@@ -1349,6 +1350,7 @@ dev = [
     { name = "pre-commit", specifier = ">=4.3.0" },
     { name = "pytest", specifier = ">=8.4.2" },
     { name = "pytest-cov", specifier = ">=7.0.0" },
+    { name = "python-dotenv", specifier = ">=1.1.1" },
     { name = "pyupgrade", specifier = ">=3.20.0" },
     { name = "ruff", specifier = ">=0.13.0" },
     { name = "safety", specifier = ">=3.2.11" },