diff --git a/.gitea/workflows/code-quality.yml b/.gitea/workflows/code-quality.yml index 273d281..50878bb 100644 --- a/.gitea/workflows/code-quality.yml +++ b/.gitea/workflows/code-quality.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ inputs.python-version }} + python-version: ${{ vars.PYTHON_VERSION }} - name: Install uv run: pip install uv diff --git a/.gitea/workflows/sync-annotations.yml b/.gitea/workflows/sync-annotations.yml index 9417a74..2ca408f 100644 --- a/.gitea/workflows/sync-annotations.yml +++ b/.gitea/workflows/sync-annotations.yml @@ -1,4 +1,4 @@ -name: Sync Label Studio +name: Sync Label Studio Annotations on: schedule: @@ -9,39 +9,27 @@ jobs: sync_storage: runs-on: ubuntu-latest steps: - - name: Sync Label Studio Target Storage - env: - LABEL_STUDIO_API_TOKEN: ${{ secrets.LABEL_STUDIO_API_TOKEN }} - run: | - curl -X POST \ - -H "Authorization: Token $LABEL_STUDIO_API_TOKEN" \ - "https://label-studio.gt-proj.com/api/storages/export/s3/1/sync" + - name: Checkout code + uses: actions/checkout@v4 - - name: Install MinIO Client - run: | - wget https://dl.min.io/client/mc/release/linux-amd64/mc - chmod +x mc - sudo mv mc /usr/local/bin/ + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ vars.PYTHON_VERSION }} - - name: Fix Label Studio output content type in MinIO + - name: Install uv + run: pip install uv + + - name: Install dependencies env: + UV_LINK_MODE: copy + run: uv sync --all-extras + + - name: Sync annotations + env: + MINIO_ENDPOINT: ${{ vars.MINIO_ENDPOINT }} MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }} + MINIO_BUCKET: ${{ vars.MINIO_BUCKET }} run: | - mc alias set myminio http://10.0.0.2:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY - mc cp --recursive --attr 'Content-Type=application/json' \ - myminio/visual-semiotic-ai-analysis/label-studio-output/ \ - myminio/visual-semiotic-ai-analysis/label-studio-output/ - - - name: Ensure .json extension on files - env: - MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} - MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }} - run: | - mc alias set myminio http://10.0.0.2:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY - for file in $(mc ls --recursive myminio/visual-semiotic-ai-analysis/label-studio-output/ | awk '{print $NF}'); do - if [[ "$file" != *.json ]]; then - mc mv "myminio/visual-semiotic-ai-analysis/label-studio-output/$file" \ - "myminio/visual-semiotic-ai-analysis/label-studio-output/$file.json" - fi - done + uv run python scripts/sync_label_studio_annotations.py diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 0edfdf9..a4d6b14 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ inputs.python-version }} + python-version: ${{ vars.PYTHON_VERSION }} - name: Install uv run: pip install uv diff --git a/data_store/dto/annotation.py b/data_store/dto/annotation.py index afeafcd..67c9ddf 100644 --- a/data_store/dto/annotation.py +++ b/data_store/dto/annotation.py @@ -13,17 +13,17 @@ from .point_of_view_enum import PointOfViewEnum class Annotation(TypeCheckingBaseModel): """Data Transfer Object representing an Annotation.""" - angle: AngleEnum + angle: AngleEnum | None angle_uncertainty: bool - colour: ColourEnum + colour: ColourEnum | None colour_uncertainty: bool - contact: ContactEnum + contact: ContactEnum | None contact_uncertainty: bool - depth: DepthEnum + depth: DepthEnum | None depth_uncertainty: bool - distance: DistanceEnum + distance: DistanceEnum | None distance_uncertainty: bool - lighting: LightingEnum + lighting: LightingEnum | None lighting_uncertainty: bool - point_of_view: PointOfViewEnum + point_of_view: PointOfViewEnum | None point_of_view_uncertainty: bool diff --git a/scripts/sync_label_studio_annotations.py b/scripts/sync_label_studio_annotations.py new file mode 100644 index 0000000..0ab0f3a --- /dev/null +++ b/scripts/sync_label_studio_annotations.py @@ -0,0 +1,129 @@ +"""Script to convert Label Studio output JSON files to an Annotation.""" + +import json +from uuid import UUID +from dotenv import load_dotenv +from python_repositories.adapters import MinioAdapter +from data_store.dto import Annotation +from data_store.repositories import AnnotationRepository + + +def list_label_studio_output_files(minio_adapter: MinioAdapter) -> list[str]: + """List the Label Studio output files in MinIO.""" + # Check input + if not isinstance(minio_adapter, MinioAdapter): + raise ValueError("minio_adapter must be an instance of MinioAdapter") + if not minio_adapter.is_connected: + raise ConnectionError("minio_adapter must be connected to MinIO") + # List objects with the Label Studio output prefix + prefix = "label-studio-output/" + return minio_adapter._list_objects(prefix) + + +def get_label_studio_output(minio_adapter: MinioAdapter, object_name: str) -> dict: + """Get a Label Studio output file from MinIO.""" + # Check input + if not isinstance(minio_adapter, MinioAdapter): + raise ValueError("minio_adapter must be an instance of MinioAdapter") + if not minio_adapter.is_connected: + raise ConnectionError("minio_adapter must be connected to MinIO") + if not isinstance(object_name, str) or len(object_name) == 0: + raise ValueError("object_name must be a non-empty string") + # Get object from MinIO + buffer = minio_adapter._get(object_name) + if buffer is None: + raise FileNotFoundError(f"File not found in MinIO: {object_name}") + # Convert buffer to JSON + data: dict = json.load(buffer) + return data + + +def convert_label_studio_output(label_studio_output: dict) -> tuple[Annotation, UUID]: + """Convert Label Studio output to Annotation and corresponding job ID.""" + # Check input + if not isinstance(label_studio_output, dict): + raise ValueError("label_studio_output must be a dictionary") + # Extract job id + image_path = label_studio_output["task"]["data"]["image"] + job_id_str = image_path.rsplit("/")[-1].split(".")[0] + job_id = UUID(job_id_str) + # Extract relevant information from the Label Studio output + annotation_data: dict[str, str | None | bool] = {} + for elem in label_studio_output["result"]: + # Extract key and value + key = elem["from_name"] + choices = elem["value"]["choices"] + # Check for presence of uncertainty flag + uncertainty_key = f"{key}_uncertainty" + uncertainty_value = bool("uncertain" in choices) + if uncertainty_value: + choices.remove("uncertain") + # If no choices left, set to None + value: str | None = choices[0] if choices else None + # Store in annotation data + annotation_data[key] = value + annotation_data[uncertainty_key] = uncertainty_value + # Create the Annotation object + annotation = Annotation(**annotation_data) # type: ignore[arg-type] + return annotation, job_id + + +def sync_label_studio_annotations( + minio_adapter: MinioAdapter, + annotation_repository: AnnotationRepository, +) -> None: + """Sync Label Studio output files to AnnotationRepository.""" + # Check inputs + if not isinstance(minio_adapter, MinioAdapter): + raise ValueError("minio_adapter must be an instance of MinioAdapter") + if not minio_adapter.is_connected: + raise ConnectionError("minio_adapter must be connected to MinIO") + if not isinstance(annotation_repository, AnnotationRepository): + raise ValueError( + "annotation_repository must be an instance of AnnotationRepository" + ) + if not annotation_repository.is_connected: + raise ConnectionError("annotation_repository must be connected to MinIO") + # List annotations in repository + annotation_ids = annotation_repository.list_all() + print(f"Found {len(annotation_ids)} annotations in repository.") + # Count Label Studio output files + label_studio_files = list_label_studio_output_files(minio_adapter) + print(f"Found {len(label_studio_files)} Label Studio output files.") + # Stop early if equal number of annotations and Label Studio files + if len(annotation_ids) == len(label_studio_files): + print("No new Label Studio output files to process. Exiting.") + return + # Process each Label Studio output file + for file_name in label_studio_files: + print(f"Processing file: {file_name}") + # Get Label Studio output + try: + label_studio_output = get_label_studio_output( + minio_adapter, + file_name, + ) + except Exception as exc: + print(f"Failed to get Label Studio output for {file_name}: {exc}") + continue + # Convert to Annotation + try: + annotation, job_id = convert_label_studio_output(label_studio_output) + except Exception as exc: + print(f"Failed to convert Label Studio output for {file_name}: {exc}") + continue + # Store Annotation in repository + try: + annotation_repository.put(annotation, job_id) + except Exception as exc: + print(f"Failed to store annotation for {file_name}: {exc}") + continue + print(f"Successfully processed and stored annotation for {file_name}.") + + +if __name__ == "__main__": + # Load environment variables from .env file + load_dotenv() + # Connect to data sources + with MinioAdapter() as minio_adapter, AnnotationRepository() as annotation_repo: + sync_label_studio_annotations(minio_adapter, annotation_repo)