From a124a72c96c1db863d76125b82f5026d6bdbe4b0 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Thu, 18 Sep 2025 23:47:56 +0200 Subject: [PATCH 1/5] allowed annotation to have None when no choice made --- data_store/dto/annotation.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/data_store/dto/annotation.py b/data_store/dto/annotation.py index afeafcd..67c9ddf 100644 --- a/data_store/dto/annotation.py +++ b/data_store/dto/annotation.py @@ -13,17 +13,17 @@ from .point_of_view_enum import PointOfViewEnum class Annotation(TypeCheckingBaseModel): """Data Transfer Object representing an Annotation.""" - angle: AngleEnum + angle: AngleEnum | None angle_uncertainty: bool - colour: ColourEnum + colour: ColourEnum | None colour_uncertainty: bool - contact: ContactEnum + contact: ContactEnum | None contact_uncertainty: bool - depth: DepthEnum + depth: DepthEnum | None depth_uncertainty: bool - distance: DistanceEnum + distance: DistanceEnum | None distance_uncertainty: bool - lighting: LightingEnum + lighting: LightingEnum | None lighting_uncertainty: bool - point_of_view: PointOfViewEnum + point_of_view: PointOfViewEnum | None point_of_view_uncertainty: bool -- 2.54.0 From f0a3bf89b0181f591feb7f6a401899ff4250b178 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Thu, 18 Sep 2025 23:48:07 +0200 Subject: [PATCH 2/5] added script to sync annotations from label studio --- scripts/sync_label_studio_annotations.py | 127 +++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 scripts/sync_label_studio_annotations.py diff --git a/scripts/sync_label_studio_annotations.py b/scripts/sync_label_studio_annotations.py new file mode 100644 index 0000000..74e2dd6 --- /dev/null +++ b/scripts/sync_label_studio_annotations.py @@ -0,0 +1,127 @@ +"""Script to convert Label Studio output JSON files to an Annotation.""" + +import json +from uuid import UUID +from dotenv import load_dotenv +from python_repositories.adapters import MinioAdapter +from data_store.dto import Annotation +from data_store.repositories import AnnotationRepository + + +def list_label_studio_output_files(minio_adapter: MinioAdapter) -> list[str]: + """List the Label Studio output files in MinIO.""" + # Check input + if not isinstance(minio_adapter, MinioAdapter): + raise ValueError("minio_adapter must be an instance of MinioAdapter") + if not minio_adapter.is_connected: + raise ConnectionError("minio_adapter must be connected to MinIO") + # List objects with the Label Studio output prefix + prefix = "label-studio-output/" + return minio_adapter._list_objects(prefix) + + +def get_label_studio_output(minio_adapter: MinioAdapter, object_name: str) -> dict: + """Get a Label Studio output file from MinIO.""" + # Check input + if not isinstance(minio_adapter, MinioAdapter): + raise ValueError("minio_adapter must be an instance of MinioAdapter") + if not minio_adapter.is_connected: + raise ConnectionError("minio_adapter must be connected to MinIO") + if not isinstance(object_name, str) or len(object_name) == 0: + raise ValueError("object_name must be a non-empty string") + # Get object from MinIO + buffer = minio_adapter._get(object_name) + if buffer is None: + raise FileNotFoundError(f"File not found in MinIO: {object_name}") + # Convert buffer to JSON + data: dict = json.load(buffer) + return data + + +def convert_label_studio_output(label_studio_output: dict) -> tuple[Annotation, UUID]: + """Convert Label Studio output to Annotation and corresponding job ID.""" + # Check input + if not isinstance(label_studio_output, dict): + raise ValueError("label_studio_output must be a dictionary") + # Extract job id + image_path = label_studio_output["task"]["data"]["image"] + job_id_str = image_path.rsplit("/")[-1].split(".")[0] + job_id = UUID(job_id_str) + # Extract relevant information from the Label Studio output + annotation_data: dict[str, str | None | bool] = {} + for elem in label_studio_output["result"]: + # Extract key and value + key = elem["from_name"] + choices = elem["value"]["choices"] + # Check for presence of uncertainty flag + uncertainty_key = f"{key}_uncertainty" + uncertainty_value = bool("uncertain" in choices) + if uncertainty_value: + choices.remove("uncertain") + # If no choices left, set to None + value: str | None = choices[0] if choices else None + # Store in annotation data + annotation_data[key] = value + annotation_data[uncertainty_key] = uncertainty_value + # Create the Annotation object + annotation = Annotation(**annotation_data) + return annotation, job_id + + +def sync_label_studio_annotations( + minio_adapter: MinioAdapter, + annotation_repository: AnnotationRepository, +) -> None: + """Sync Label Studio output files to AnnotationRepository.""" + # Check inputs + if not isinstance(minio_adapter, MinioAdapter): + raise ValueError("minio_adapter must be an instance of MinioAdapter") + if not minio_adapter.is_connected: + raise ConnectionError("minio_adapter must be connected to MinIO") + if not isinstance(annotation_repository, AnnotationRepository): + raise ValueError("annotation_repository must be an instance of AnnotationRepository") + if not annotation_repository.is_connected: + raise ConnectionError("annotation_repository must be connected to MinIO") + # List annotations in repository + annotation_ids = annotation_repository.list_all() + print(f"Found {len(annotation_ids)} annotations in repository.") + # Count Label Studio output files + label_studio_files = list_label_studio_output_files(minio_adapter) + print(f"Found {len(label_studio_files)} Label Studio output files.") + # Stop early if equal number of annotations and Label Studio files + if len(annotation_ids) == len(label_studio_files): + print("No new Label Studio output files to process. Exiting.") + return + # Process each Label Studio output file + for file_name in label_studio_files: + print(f"Processing file: {file_name}") + # Get Label Studio output + try: + label_studio_output = get_label_studio_output( + minio_adapter, + file_name, + ) + except Exception as exc: + print(f"Failed to get Label Studio output for {file_name}: {exc}") + continue + # Convert to Annotation + try: + annotation, job_id = convert_label_studio_output(label_studio_output) + except Exception as exc: + print(f"Failed to convert Label Studio output for {file_name}: {exc}") + continue + # Store Annotation in repository + try: + annotation_repository.put(annotation, job_id) + except Exception as exc: + print(f"Failed to store annotation for {file_name}: {exc}") + continue + print(f"Successfully processed and stored annotation for {file_name}.") + + +if __name__ == "__main__": + # Load environment variables from .env file + load_dotenv() + # Connect to data sources + with MinioAdapter() as minio_adapter, AnnotationRepository() as annotation_repo: + sync_label_studio_annotations(minio_adapter, annotation_repo) -- 2.54.0 From 829cfd2e003719fae56b6f208a124f7e41e19a0a Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Thu, 18 Sep 2025 23:48:22 +0200 Subject: [PATCH 3/5] updated CI to use correct variable --- .gitea/workflows/code-quality.yml | 2 +- .gitea/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/code-quality.yml b/.gitea/workflows/code-quality.yml index 273d281..50878bb 100644 --- a/.gitea/workflows/code-quality.yml +++ b/.gitea/workflows/code-quality.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ inputs.python-version }} + python-version: ${{ vars.PYTHON_VERSION }} - name: Install uv run: pip install uv diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 0edfdf9..a4d6b14 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ inputs.python-version }} + python-version: ${{ vars.PYTHON_VERSION }} - name: Install uv run: pip install uv -- 2.54.0 From 28abd22571ebea67d1371f4846935b05abd87897 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Thu, 18 Sep 2025 23:48:49 +0200 Subject: [PATCH 4/5] updated to run python script to synchronize annotations from label studio to annotation repository --- .gitea/workflows/sync-annotations.yml | 50 ++++++++++----------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/.gitea/workflows/sync-annotations.yml b/.gitea/workflows/sync-annotations.yml index 9417a74..6638a57 100644 --- a/.gitea/workflows/sync-annotations.yml +++ b/.gitea/workflows/sync-annotations.yml @@ -1,4 +1,4 @@ -name: Sync Label Studio +name: Sync Label Studio Annotations on: schedule: @@ -9,39 +9,27 @@ jobs: sync_storage: runs-on: ubuntu-latest steps: - - name: Sync Label Studio Target Storage - env: - LABEL_STUDIO_API_TOKEN: ${{ secrets.LABEL_STUDIO_API_TOKEN }} - run: | - curl -X POST \ - -H "Authorization: Token $LABEL_STUDIO_API_TOKEN" \ - "https://label-studio.gt-proj.com/api/storages/export/s3/1/sync" + - name: Checkout code + uses: actions/checkout@v4 - - name: Install MinIO Client - run: | - wget https://dl.min.io/client/mc/release/linux-amd64/mc - chmod +x mc - sudo mv mc /usr/local/bin/ + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ vars.PYTHON_VERSION }} - - name: Fix Label Studio output content type in MinIO + - name: Install uv + run: pip install uv + + - name: Install dependencies env: + UV_LINK_MODE: copy + run: uv sync --all-extras + + - name: Sync annotations + env: + MINIO_ENDPOINT: ${{ vars.MINIO_ENDPOINT }} MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }} + MINIO_BUCKET: ${{ vars.MINIO_BUCKET }} run: | - mc alias set myminio http://10.0.0.2:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY - mc cp --recursive --attr 'Content-Type=application/json' \ - myminio/visual-semiotic-ai-analysis/label-studio-output/ \ - myminio/visual-semiotic-ai-analysis/label-studio-output/ - - - name: Ensure .json extension on files - env: - MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} - MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }} - run: | - mc alias set myminio http://10.0.0.2:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY - for file in $(mc ls --recursive myminio/visual-semiotic-ai-analysis/label-studio-output/ | awk '{print $NF}'); do - if [[ "$file" != *.json ]]; then - mc mv "myminio/visual-semiotic-ai-analysis/label-studio-output/$file" \ - "myminio/visual-semiotic-ai-analysis/label-studio-output/$file.json" - fi - done + uv run python scripts/sync_label_studio_annotations.py -- 2.54.0 From b6b35448b002407af7f0507c6bde155af331d2e0 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Thu, 18 Sep 2025 23:50:36 +0200 Subject: [PATCH 5/5] code quality fixes --- .gitea/workflows/sync-annotations.yml | 2 +- scripts/sync_label_studio_annotations.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/sync-annotations.yml b/.gitea/workflows/sync-annotations.yml index 6638a57..2ca408f 100644 --- a/.gitea/workflows/sync-annotations.yml +++ b/.gitea/workflows/sync-annotations.yml @@ -26,7 +26,7 @@ jobs: run: uv sync --all-extras - name: Sync annotations - env: + env: MINIO_ENDPOINT: ${{ vars.MINIO_ENDPOINT }} MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }} diff --git a/scripts/sync_label_studio_annotations.py b/scripts/sync_label_studio_annotations.py index 74e2dd6..0ab0f3a 100644 --- a/scripts/sync_label_studio_annotations.py +++ b/scripts/sync_label_studio_annotations.py @@ -64,7 +64,7 @@ def convert_label_studio_output(label_studio_output: dict) -> tuple[Annotation, annotation_data[key] = value annotation_data[uncertainty_key] = uncertainty_value # Create the Annotation object - annotation = Annotation(**annotation_data) + annotation = Annotation(**annotation_data) # type: ignore[arg-type] return annotation, job_id @@ -79,7 +79,9 @@ def sync_label_studio_annotations( if not minio_adapter.is_connected: raise ConnectionError("minio_adapter must be connected to MinIO") if not isinstance(annotation_repository, AnnotationRepository): - raise ValueError("annotation_repository must be an instance of AnnotationRepository") + raise ValueError( + "annotation_repository must be an instance of AnnotationRepository" + ) if not annotation_repository.is_connected: raise ConnectionError("annotation_repository must be connected to MinIO") # List annotations in repository -- 2.54.0