Merge pull request 'add-script-to-convert-label-studio-output-to-annotation' (#9) from add-script-to-convert-label-studio-output-to-annotation into main

Reviewed-on: #9

.gitea/workflows/code-quality.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ inputs.python-version }}
+          python-version: ${{ vars.PYTHON_VERSION }}
 
       - name: Install uv
         run: pip install uv

.gitea/workflows/sync-annotations.yml

@@ -1,4 +1,4 @@
-name: Sync Label Studio
+name: Sync Label Studio Annotations
 
 on:
   schedule:
@@ -9,39 +9,27 @@ jobs:
   sync_storage:
     runs-on: ubuntu-latest
     steps:
-      - name: Sync Label Studio Target Storage
-        env:
-          LABEL_STUDIO_API_TOKEN: ${{ secrets.LABEL_STUDIO_API_TOKEN }}
-        run: |
-          curl -X POST \
-          -H "Authorization: Token $LABEL_STUDIO_API_TOKEN" \
-          "https://label-studio.gt-proj.com/api/storages/export/s3/1/sync"
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-      - name: Install MinIO Client
-        run: |
-          wget https://dl.min.io/client/mc/release/linux-amd64/mc
-          chmod +x mc
-          sudo mv mc /usr/local/bin/
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ vars.PYTHON_VERSION }}
 
-      - name: Fix Label Studio output content type in MinIO
+      - name: Install uv
+        run: pip install uv
+
+      - name: Install dependencies
         env:
+          UV_LINK_MODE: copy
+        run: uv sync --all-extras
+
+      - name: Sync annotations
+        env:
+          MINIO_ENDPOINT: ${{ vars.MINIO_ENDPOINT }}
           MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
           MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }}
+          MINIO_BUCKET: ${{ vars.MINIO_BUCKET }}
         run: |
-          mc alias set myminio http://10.0.0.2:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
-          mc cp --recursive --attr 'Content-Type=application/json' \
-          myminio/visual-semiotic-ai-analysis/label-studio-output/ \
-          myminio/visual-semiotic-ai-analysis/label-studio-output/
-
-      - name: Ensure .json extension on files
-        env:
-          MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
-          MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }}
-        run: |
-          mc alias set myminio http://10.0.0.2:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
-          for file in $(mc ls --recursive myminio/visual-semiotic-ai-analysis/label-studio-output/ | awk '{print $NF}'); do
-            if [[ "$file" != *.json ]]; then
-              mc mv "myminio/visual-semiotic-ai-analysis/label-studio-output/$file" \
-                    "myminio/visual-semiotic-ai-analysis/label-studio-output/$file.json"
-            fi
-          done
+          uv run python scripts/sync_label_studio_annotations.py

.gitea/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ inputs.python-version }}
+          python-version: ${{ vars.PYTHON_VERSION }}
 
       - name: Install uv
         run: pip install uv

data_store/dto/annotation.py

@@ -13,17 +13,17 @@ from .point_of_view_enum import PointOfViewEnum
 class Annotation(TypeCheckingBaseModel):
     """Data Transfer Object representing an Annotation."""
 
-    angle: AngleEnum
+    angle: AngleEnum | None
     angle_uncertainty: bool
-    colour: ColourEnum
+    colour: ColourEnum | None
     colour_uncertainty: bool
-    contact: ContactEnum
+    contact: ContactEnum | None
     contact_uncertainty: bool
-    depth: DepthEnum
+    depth: DepthEnum | None
     depth_uncertainty: bool
-    distance: DistanceEnum
+    distance: DistanceEnum | None
     distance_uncertainty: bool
-    lighting: LightingEnum
+    lighting: LightingEnum | None
     lighting_uncertainty: bool
-    point_of_view: PointOfViewEnum
+    point_of_view: PointOfViewEnum | None
     point_of_view_uncertainty: bool

scripts/sync_label_studio_annotations.py

@@ -0,0 +1,129 @@
+"""Script to convert Label Studio output JSON files to an Annotation."""
+
+import json
+from uuid import UUID
+from dotenv import load_dotenv
+from python_repositories.adapters import MinioAdapter
+from data_store.dto import Annotation
+from data_store.repositories import AnnotationRepository
+
+
+def list_label_studio_output_files(minio_adapter: MinioAdapter) -> list[str]:
+    """List the Label Studio output files in MinIO."""
+    # Check input
+    if not isinstance(minio_adapter, MinioAdapter):
+        raise ValueError("minio_adapter must be an instance of MinioAdapter")
+    if not minio_adapter.is_connected:
+        raise ConnectionError("minio_adapter must be connected to MinIO")
+    # List objects with the Label Studio output prefix
+    prefix = "label-studio-output/"
+    return minio_adapter._list_objects(prefix)
+
+
+def get_label_studio_output(minio_adapter: MinioAdapter, object_name: str) -> dict:
+    """Get a Label Studio output file from MinIO."""
+    # Check input
+    if not isinstance(minio_adapter, MinioAdapter):
+        raise ValueError("minio_adapter must be an instance of MinioAdapter")
+    if not minio_adapter.is_connected:
+        raise ConnectionError("minio_adapter must be connected to MinIO")
+    if not isinstance(object_name, str) or len(object_name) == 0:
+        raise ValueError("object_name must be a non-empty string")
+    # Get object from MinIO
+    buffer = minio_adapter._get(object_name)
+    if buffer is None:
+        raise FileNotFoundError(f"File not found in MinIO: {object_name}")
+    # Convert buffer to JSON
+    data: dict = json.load(buffer)
+    return data
+
+
+def convert_label_studio_output(label_studio_output: dict) -> tuple[Annotation, UUID]:
+    """Convert Label Studio output to Annotation and corresponding job ID."""
+    # Check input
+    if not isinstance(label_studio_output, dict):
+        raise ValueError("label_studio_output must be a dictionary")
+    # Extract job id
+    image_path = label_studio_output["task"]["data"]["image"]
+    job_id_str = image_path.rsplit("/")[-1].split(".")[0]
+    job_id = UUID(job_id_str)
+    # Extract relevant information from the Label Studio output
+    annotation_data: dict[str, str | None | bool] = {}
+    for elem in label_studio_output["result"]:
+        # Extract key and value
+        key = elem["from_name"]
+        choices = elem["value"]["choices"]
+        # Check for presence of uncertainty flag
+        uncertainty_key = f"{key}_uncertainty"
+        uncertainty_value = bool("uncertain" in choices)
+        if uncertainty_value:
+            choices.remove("uncertain")
+        # If no choices left, set to None
+        value: str | None = choices[0] if choices else None
+        # Store in annotation data
+        annotation_data[key] = value
+        annotation_data[uncertainty_key] = uncertainty_value
+    # Create the Annotation object
+    annotation = Annotation(**annotation_data)  # type: ignore[arg-type]
+    return annotation, job_id
+
+
+def sync_label_studio_annotations(
+    minio_adapter: MinioAdapter,
+    annotation_repository: AnnotationRepository,
+) -> None:
+    """Sync Label Studio output files to AnnotationRepository."""
+    # Check inputs
+    if not isinstance(minio_adapter, MinioAdapter):
+        raise ValueError("minio_adapter must be an instance of MinioAdapter")
+    if not minio_adapter.is_connected:
+        raise ConnectionError("minio_adapter must be connected to MinIO")
+    if not isinstance(annotation_repository, AnnotationRepository):
+        raise ValueError(
+            "annotation_repository must be an instance of AnnotationRepository"
+        )
+    if not annotation_repository.is_connected:
+        raise ConnectionError("annotation_repository must be connected to MinIO")
+    # List annotations in repository
+    annotation_ids = annotation_repository.list_all()
+    print(f"Found {len(annotation_ids)} annotations in repository.")
+    # Count Label Studio output files
+    label_studio_files = list_label_studio_output_files(minio_adapter)
+    print(f"Found {len(label_studio_files)} Label Studio output files.")
+    # Stop early if equal number of annotations and Label Studio files
+    if len(annotation_ids) == len(label_studio_files):
+        print("No new Label Studio output files to process. Exiting.")
+        return
+    # Process each Label Studio output file
+    for file_name in label_studio_files:
+        print(f"Processing file: {file_name}")
+        # Get Label Studio output
+        try:
+            label_studio_output = get_label_studio_output(
+                minio_adapter,
+                file_name,
+            )
+        except Exception as exc:
+            print(f"Failed to get Label Studio output for {file_name}: {exc}")
+            continue
+        # Convert to Annotation
+        try:
+            annotation, job_id = convert_label_studio_output(label_studio_output)
+        except Exception as exc:
+            print(f"Failed to convert Label Studio output for {file_name}: {exc}")
+            continue
+        # Store Annotation in repository
+        try:
+            annotation_repository.put(annotation, job_id)
+        except Exception as exc:
+            print(f"Failed to store annotation for {file_name}: {exc}")
+            continue
+        print(f"Successfully processed and stored annotation for {file_name}.")
+
+
+if __name__ == "__main__":
+    # Load environment variables from .env file
+    load_dotenv()
+    # Connect to data sources
+    with MinioAdapter() as minio_adapter, AnnotationRepository() as annotation_repo:
+        sync_label_studio_annotations(minio_adapter, annotation_repo)