added use of larger model for refining prompt

Reviewed-on: #17

.gitea/workflows/code-quality.yml

@@ -1,44 +0,0 @@
-name: Code Quality Pipeline
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-
-jobs:
-  code-quality:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ inputs.python-version }}
-
-      - name: Install uv
-        run: pip install uv
-
-      - name: Install dependencies
-        env:
-          UV_LINK_MODE: copy
-        run: uv sync --all-extras
-
-      - name: Type check with mypy
-        run: uv run mypy .
-
-      - name: Ruff lint
-        run: uv run ruff check .
-
-      - name: Ruff format
-        run: uv run ruff format --check .
-
-      - name: Pyupgrade check
-        run: uv run pyupgrade --py312-plus $(git ls-files '*.py') && git diff --exit-code
-
-      - name: Prettier format
-        run: |
-          npm install --save-dev --save-exact prettier
-          npx prettier --check .

.gitea/workflows/formatting-check.yml

@@ -0,0 +1,11 @@
+name: Formatting Check
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  formatting-check:
+    uses: brian/CI-templates/.gitea/workflows/web/[email protected]

.gitea/workflows/python-code-quality.yml

@@ -0,0 +1,13 @@
+name: Python Code Quality
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  python-code-quality:
+    uses: brian/CI-templates/.gitea/workflows/python/[email protected]
+    with:
+      python-version: ${{ vars.PYTHON_VERSION }}

.gitea/workflows/python-test.yml

@@ -0,0 +1,17 @@
+name: Python Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  python-test:
+    uses: brian/CI-templates/.gitea/workflows/python/[email protected]
+    with:
+      python-version: ${{ vars.PYTHON_VERSION }}
+      coverage-package: "data_store"
+      api-url: ${{ vars.API_URL }}
+    secrets:
+      CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}

.gitea/workflows/sync-annotations.yml

@@ -1,15 +1,15 @@
-name: Sync Label Studio
+name: Sync Label Studio Annotations
 
 on:
   schedule:
     # This job will run every day at 03:00 AM
-    - cron: "0/10 * * * *"
+    - cron: "0 3 * * *"
 
 jobs:
   sync_storage:
     runs-on: ubuntu-latest
     steps:
-      - name: Sync Label Studio Target Storage
+      - name: Sync Label Studio to target storage
         env:
           LABEL_STUDIO_API_TOKEN: ${{ secrets.LABEL_STUDIO_API_TOKEN }}
         run: |
@@ -17,12 +17,27 @@ jobs:
           -H "Authorization: Token $LABEL_STUDIO_API_TOKEN" \
           "https://label-studio.gt-proj.com/api/storages/export/s3/1/sync"
 
-      - name: Fix Label Studio output content type in MinIO
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ vars.PYTHON_VERSION }}
+
+      - name: Install uv
+        run: pip install uv
+
+      - name: Install dependencies
         env:
+          UV_LINK_MODE: copy
+        run: uv sync --all-extras
+
+      - name: Sync target storage to annotations
+        env:
+          MINIO_ENDPOINT: ${{ vars.MINIO_ENDPOINT }}
           MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
           MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }}
+          MINIO_BUCKET: ${{ vars.MINIO_BUCKET }}
         run: |
-          mc alias set myminio http://10.0.0.2:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY
-          mc mirror --recursive --overwrite --attr 'Content-Type=application/json' \
-          myminio/visual-semiotic-ai-analysis/label-studio-output \
-          myminio/visual-semiotic-ai-analysis/label-studio-output
+          uv run python scripts/sync_label_studio_annotations.py

.gitea/workflows/test.yml

@@ -1,47 +0,0 @@
-name: Test Python Package
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ inputs.python-version }}
-
-      - name: Install uv
-        run: pip install uv
-
-      - name: Install dependencies
-        env:
-          UV_LINK_MODE: copy
-        run: uv sync --all-extras
-
-      - name: Run pytest
-        env:
-          PYTHONPATH: .
-        run: uv run pytest --cov=data_store --cov-report=term-missing > coverage.txt
-
-      - name: Post coverage summary to PR
-        env:
-          API_URL: ${{ vars.API_URL }}
-          REPO_OWNER: ${{ github.repository_owner }}
-          REPO_NAME: ${{ github.event.repository.name }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
-        run: |
-          COVERAGE=$(cat coverage.txt)
-          COMMENT_BODY="**Test Coverage Report:**\n\`\`\`\n$COVERAGE\n\`\`\`"
-          curl -s -X POST "$API_URL/repos/$REPO_OWNER/$REPO_NAME/issues/$PR_NUMBER/comments" \
-            -H "Authorization: token $CI_RUNNER_TOKEN" \
-            -H "Content-Type: application/json" \
-            -d "{\"body\": \"$COMMENT_BODY\"}"

data_store/dto/annotation.py

@@ -13,17 +13,17 @@ from .point_of_view_enum import PointOfViewEnum
 class Annotation(TypeCheckingBaseModel):
     """Data Transfer Object representing an Annotation."""
 
-    angle: AngleEnum
+    angle: AngleEnum | None
     angle_uncertainty: bool
-    colour: ColourEnum
+    colour: ColourEnum | None
     colour_uncertainty: bool
-    contact: ContactEnum
+    contact: ContactEnum | None
     contact_uncertainty: bool
-    depth: DepthEnum
+    depth: DepthEnum | None
     depth_uncertainty: bool
-    distance: DistanceEnum
+    distance: DistanceEnum | None
     distance_uncertainty: bool
-    lighting: LightingEnum
+    lighting: LightingEnum | None
     lighting_uncertainty: bool
-    point_of_view: PointOfViewEnum
+    point_of_view: PointOfViewEnum | None
     point_of_view_uncertainty: bool

data_store/repositories/annotation_repository.py

@@ -58,19 +58,29 @@ class AnnotationRepository(MinioAdapter, AnnotationRepositoryInterface):
             data_str = buffer.read().decode(self.encoding)
             data = json.loads(data_str)
             annotation = Annotation(
-                angle=AngleEnum(data["angle"]),
+                angle=AngleEnum(data["angle"]) if data["angle"] is not None else None,
                 angle_uncertainty=bool(data["angle_uncertainty"]),
-                colour=ColourEnum(data["colour"]),
+                colour=ColourEnum(data["colour"])
+                if data["colour"] is not None
+                else None,
                 colour_uncertainty=bool(data["colour_uncertainty"]),
-                contact=ContactEnum(data["contact"]),
+                contact=ContactEnum(data["contact"])
+                if data["contact"] is not None
+                else None,
                 contact_uncertainty=bool(data["contact_uncertainty"]),
-                depth=DepthEnum(data["depth"]),
+                depth=DepthEnum(data["depth"]) if data["depth"] is not None else None,
                 depth_uncertainty=bool(data["depth_uncertainty"]),
-                distance=DistanceEnum(data["distance"]),
+                distance=DistanceEnum(data["distance"])
+                if data["distance"] is not None
+                else None,
                 distance_uncertainty=bool(data["distance_uncertainty"]),
-                lighting=LightingEnum(data["lighting"]),
+                lighting=LightingEnum(data["lighting"])
+                if data["lighting"] is not None
+                else None,
                 lighting_uncertainty=bool(data["lighting_uncertainty"]),
-                point_of_view=PointOfViewEnum(data["point_of_view"]),
+                point_of_view=PointOfViewEnum(data["point_of_view"])
+                if data["point_of_view"] is not None
+                else None,
                 point_of_view_uncertainty=bool(data["point_of_view_uncertainty"]),
             )
         except (KeyError, ValueError) as e:

prompt_accuracy_results-60_jobs.json

@@ -0,0 +1,93 @@
+{
+  "angle": {
+    "In the context of visual angle in the Kress and van Leeuwen framework, analyze the image to determine the angle of the subject.": 0.425531914893617,
+    "\"Analyze the image to determine the visual angle of the subject in the Kress and van Leeuwen framework.\"": 0.46808510638297873,
+    "Refined Prompt: \"In the context of visual angle analysis in the Kress and van Leeuwen framework, please provide the angle of the subject depicted in the image.\"": 0.44680851063829785,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the visual angle of the subject depicted.\"": 0.5106382978723404,
+    "\"In the context of visual angle analysis in the Kress and van Leeuwen framework, please provide the angle of the subject depicted in the image.\"": 0.46808510638297873,
+    "\"Refine the prompt by removing unnecessary words and focusing on the task of providing the visual angle of the subject in the image using the Kress and van Leeuwen framework.\"": 0.46808510638297873,
+    "Refined Prompt: \"Provide the visual angle of the subject depicted in the image using the Kress and van Leeuwen framework.\"": 0.425531914893617,
+    "\"Provide the visual angle of the subject depicted in the image using the Kress and van Leeuwen framework.\"": 0.3404255319148936,
+    "\"In the context of visual angle analysis in the Kress and van Leeuwen framework, please provide the exact angle measurement of the subject depicted in the image.\"": 0.5106382978723404,
+    "\"Refine the prompt by removing unnecessary words and focusing on providing the exact angle measurement of the subject in the image using the Kress and van Leeuwen framework.\"": 0.5106382978723404,
+    "\"Provide the exact visual angle measurement of the subject depicted in the image using the Kress and van Leeuwen framework.\"": 0.425531914893617,
+    "Refine the prompt by removing unnecessary words and focusing on providing the exact angle measurement of the subject in the image using the Kress and van Leeuwen framework.": 0.5106382978723404
+  },
+  "colour": {
+    "In the context of colour in the Kress and van Leeuwen framework, analyze the image to determine the colour saturation of the whole image.": 0.3220338983050847,
+    "\"Analyze the image for colour saturation using the Kress and van Leeuwen framework, considering the entire image.\"": 0.3220338983050847,
+    "To ensure accuracy in the analysis of colour saturation using the Kress and van Leeuwen framework for the entire image, please provide a prompt that clearly and succinctly requests this information.": 0.3050847457627119,
+    "\"Using the Kress and van Leeuwen framework, please provide an analysis of the colour saturation for the entire image.\"": 0.3220338983050847,
+    "\"Using the Kress and van Leeuwen framework, please provide an analysis of the color saturation for the entire image.\"": 0.288135593220339,
+    "Using the Kress and van Leeuwen framework, please provide an analysis of the color saturation for the entire image.": 0.3898305084745763,
+    "To ensure accuracy in the analysis of color saturation using the Kress and van Leeuwen framework for the entire image, please provide a prompt that clearly and succinctly requests this information.": 0.2711864406779661
+  },
+  "contact": {
+    "In the context of contact in the Kress and van Leeuwen framework, analyze the image to determine the contact between subject and viewer.": 0.6363636363636364,
+    "\"Analyze the image to determine the contact between the subject and the viewer in the context of the Kress and van Leeuwen framework.\"": 0.3181818181818182,
+    "\"Evaluate the contact between the subject and viewer in the context of the Kress and van Leeuwen framework by analyzing this image.\"": 0.5454545454545454,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the contact between the subject and viewer.\"": 0.4090909090909091,
+    "\"Using the Kress and van Leeuwen framework, analyze this image to determine the contact between the subject and viewer.\"": 0.6818181818181818,
+    "\"Analyze this image using the Kress and van Leeuwen framework to determine the contact between the subject and viewer.\"": 0.7272727272727273,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the level of contact between the subject and viewer.\"": 0.5909090909090909,
+    "\"Analyze this image using the Kress and van Leeuwen framework to determine the level of contact between the subject and viewer.\"": 0.45454545454545453,
+    "\"Using the Kress and van Leeuwen framework, analyze this image to determine the level of contact between the subject and viewer.\"": 0.5454545454545454,
+    "\"Analyze this image using the Kress and van Leeuwen framework by evaluating the contact between the subject and viewer.\"": 0.4090909090909091,
+    "Refined prompt: \"Analyze this image using the Kress and van Leeuwen framework by evaluating the level of contact between the subject and viewer.\"": 0.5,
+    "\"Analyze this image using the Kress and van Leeuwen framework by evaluating the level of contact between the subject and viewer.\"": 0.5,
+    "\"Evaluate the level of contact between the subject and viewer in the context of the Kress and van Leeuwen framework by analyzing this image.\"": 0.5
+  },
+  "depth": {
+    "In the context of depth in the Kress and van Leeuwen framework, analyze the image to determine the depth of the whole image.": 0.3793103448275862,
+    "\"Analyze the image to determine the depth of the scene in terms of foreground, midground, and background elements using the Kress and van Leeuwen framework.\"": 0.4482758620689655,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the depth of the foreground, midground, and background elements.\"": 0.4482758620689655,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the depth of each element in the scene, including foreground, midground, and background elements.\"": 0.4482758620689655,
+    "\"Using the Kress and van Leeuwen framework, analyze the image to determine the depth of each element in the scene, including foreground, midground, and background elements.\"": 0.39655172413793105,
+    "\"Using the Kress and van Leeuwen framework, analyze the depth of each element in the image, including foreground, midground, and background elements.\"": 0.39655172413793105
+  },
+  "distance": {
+    "In the context of spatial distance in the Kress and van Leeuwen framework, analyze the image to determine the spatial distance that the subject is portrayed at.": 0.4,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the spatial distance portrayed for the subject.\"": 0.37142857142857144,
+    "\"Using the Kress and van Leeuwen framework, determine the spatial distance portrayed for the subject in this image.\"": 0.4,
+    "The refined prompt is: \"Analyze the image using the Kress and van Leeuwen framework to determine the spatial distance portrayed for the subject.\"": 0.4857142857142857,
+    "The refined prompt is: \"Determine the spatial distance portrayed for the subject in this image using the Kress and van Leeuwen framework.\"": 0.4857142857142857,
+    "The refined prompt is: \"Using the Kress and van Leeuwen framework, analyze the image to determine the spatial distance portrayed for the subject.\"": 0.6,
+    "\"Using the Kress and van Leeuwen framework, analyze this image to determine the spatial distance portrayed for the subject.\"": 0.5428571428571428,
+    "The refined prompt is: \"Analyze this image using the Kress and van Leeuwen framework to determine the spatial distance portrayed for the subject.\"": 0.4857142857142857,
+    "\"Analyze this image using the Kress and van Leeuwen framework to determine the spatial distance portrayed for the subject.\"": 0.42857142857142855,
+    "\"Determine the spatial distance portrayed for the subject in this image using the Kress and van Leeuwen framework.\"": 0.4857142857142857
+  },
+  "lighting": {
+    "In the context of lighting in the Kress and van Leeuwen framework, analyze the image to determine the lighting of the whole image.": 0.3333333333333333,
+    "Refined Prompt: \"Analyze the image for its overall lighting using the Kress and van Leeuwen framework, and provide an accuracy estimate of the analysis.\"": 0.4074074074074074,
+    "\"Analyze the image for its overall lighting using the Kress and van Leeuwen framework, and provide an accuracy estimate of the analysis.\"": 0.35185185185185186,
+    "\"In the context of the Kress and van Leeuwen framework, analyze the image for its overall lighting and provide an accuracy estimate of the analysis.\"": 0.4074074074074074,
+    "\"Analyze the image for its overall lighting using the Kress and van Leeuwen framework and provide an accuracy estimate of the analysis.\"": 0.24074074074074073,
+    "\"Refined Prompt: \\\"Analyze the image for its overall lighting using the Kress and van Leeuwen framework, providing a detailed description of the lighting conditions and an accuracy estimate of the analysis.\\\"\"": 0.35185185185185186,
+    "\"Analyze the image for its overall lighting using the Kress and van Leeuwen framework, providing a detailed description of the lighting conditions and an accuracy estimate of the analysis.\"": 0.3148148148148148,
+    "\"Refine the prompt for analyzing the image's overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the lighting conditions and an accuracy estimate of the analysis.\"": 0.3148148148148148,
+    "Refine the prompt for analyzing the image's overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the lighting conditions and an accuracy estimate of the analysis.": 0.18518518518518517,
+    "\"Analyze the image for its overall lighting conditions using the Kress and van Leeuwen framework, providing a detailed description of the light sources, their intensities, and the resulting illumination, along with an accuracy estimate of the analysis.\"": 0.2962962962962963,
+    "Refine the prompt for analyzing the image's overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination, along with an accuracy estimate of the analysis.": 0.18518518518518517,
+    "\"Analyze the image for its overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination, along with an accuracy estimate of the analysis.\"": 0.2222222222222222,
+    "\"Refine the prompt for analyzing the image's overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination, along with an accuracy estimate of the analysis.\"": 0.3888888888888889,
+    "\"Analyze the image for its overall lighting conditions using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination, along with an accuracy estimate of the analysis.\"": 0.2037037037037037,
+    "To improve the accuracy of analyzing the image's overall lighting using the Kress and van Leeuwen framework, provide a detailed description of the light sources, their intensities, and the resulting illumination in the image, along with an estimated accuracy for the analysis.": 0.16666666666666666,
+    "\"Analyze the image for its overall lighting conditions using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination in the image, along with an estimated accuracy of the analysis.\"": 0.35185185185185186,
+    "\"Provide a detailed description of the light sources, their intensities, and the resulting illumination in the image using the Kress and van Leeuwen framework, along with an estimated accuracy for the analysis.\"": 0.2037037037037037,
+    "\"Analyze the image for its overall lighting conditions using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination in the image, along with an estimated accuracy for the analysis.\"": 0.37037037037037035,
+    "Refine the prompt for analyzing the image's overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination in the image, along with an estimated accuracy for the analysis.": 0.25925925925925924
+  },
+  "point_of_view": {
+    "In the context of point of view in the Kress and van Leeuwen framework, analyze the image to determine the orientation of the subject in relation to the viewer.": 0.64,
+    "Refined Prompt: \"Analyze the image using the Kress and van Leeuwen framework to determine the orientation of the subject in relation to the viewer's point of view.\"": 0.72,
+    "\"Using the Kress and van Leeuwen framework, analyze the image to determine the subject's orientation in relation to the viewer's point of view.\"": 0.6,
+    "Refined Prompt: \"Using the Kress and van Leeuwen framework, analyze the image to determine the subject's orientation in relation to the viewer's point of view.\"": 0.72,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the subject's orientation in relation to the viewer's point of view.\"": 0.64,
+    "\"In the context of point of view in the Kress and van Leeuwen framework, provide an analysis of the image to determine the orientation of the subject in relation to the viewer's perspective.\"": 0.52,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the orientation of the subject in relation to the viewer's point of view.\"": 0.56,
+    "\"Using the Kress and van Leeuwen framework, provide an analysis of the image to determine the orientation of the subject in relation to the viewer's point of view.\"": 0.68,
+    "\"Using the Kress and van Leeuwen framework, provide an analysis of the image to determine the subject's orientation in relation to the viewer's point of view.\"": 0.76,
+    "\"Using the Kress and van Leeuwen framework, provide a detailed analysis of the image to determine the orientation of the subject in relation to the viewer's point of view.\"": 0.68
+  }
+}

prompt_accuracy_results-7_jobs.json

@@ -0,0 +1,64 @@
+{
+  "angle": {
+    "In the context of visual angle in the Kress and van Leeuwen framework, analyze the image to determine the angle of the subject.": 0.3333333333333333,
+    "\"Analyze the image to determine the visual angle of the subject using the Kress and van Leeuwen framework, providing an accuracy of at least 0.4.\"": 0.16666666666666666,
+    "\"Analyze the image provided within the context of the Kress and van Leeuwen framework to accurately determine the visual angle of the subject, with an accuracy target of at least 0.4.\"": 0.3333333333333333,
+    "\"Analyze the image provided using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with an accuracy target of at least 0.4.\"": 0.0,
+    "\"Analyze the image provided within the context of the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with an accuracy target of at least 0.4.\"": 0.5,
+    "\"Analyze the image provided using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with a minimum accuracy target of 0.4.\"": 0.16666666666666666,
+    "Refined Prompt: \"Analyze the provided image using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with a minimum accuracy target of 0.4.\"": 0.16666666666666666,
+    "\"Analyze the provided image using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with a minimum accuracy target of 0.4.\"": 0.16666666666666666,
+    "\"Analyze the provided image using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject, targeting an accuracy of at least 0.5.\"": 0.0,
+    "\"Analyze the provided image using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with a minimum accuracy target of 0.5.\"": 0.0,
+    "\"Analyze the provided image using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject, targeting an accuracy of at least 0.4.\"": 0.6666666666666666
+  },
+  "colour": {
+    "In the context of colour in the Kress and van Leeuwen framework, analyze the image to determine the colour saturation of the whole image.": 0.2857142857142857,
+    "\"Analyze the image for color saturation using the Kress and van Leeuwen framework.\"": 0.42857142857142855,
+    "\"Analyze the image for color saturation using the Kress and van Leeuwen framework by considering all hues present in the image.\"": 0.2857142857142857,
+    "\"Using the Kress and van Leeuwen framework for color analysis, determine the overall saturation level of all hues present in the image.\"": 0.2857142857142857,
+    "\"Analyze the image for color saturation using the Kress and van Leeuwen framework by considering all hues present in the image and provide a numerical value representing the overall saturation level.\"": 0.5714285714285714,
+    "\"Using the Kress and van Leeuwen framework for color analysis, provide a numerical value representing the overall saturation level of all hues present in the image.\"": 0.2857142857142857,
+    "\"Using the Kress and van Leeuwen framework for color analysis, determine the overall saturation level of all hues present in the image by considering their respective saturation values.\"": 0.14285714285714285,
+    "\"Using the Kress and van Leeuwen framework for color analysis, provide a numerical value representing the overall saturation level of all hues present in the image by considering their respective saturation values.\"": 0.42857142857142855,
+    "\"Analyze the image using the Kress and van Leeuwen framework for color analysis by considering all hues present in the image to determine the overall saturation level as a numerical value.\"": 0.42857142857142855,
+    "\"Using the Kress and van Leeuwen framework for color analysis, determine the overall saturation level of all hues present in the image by considering their respective saturation values and provide a numerical value representing the overall saturation level.\"": 0.8571428571428571
+  },
+  "contact": {
+    "In the context of contact in the Kress and van Leeuwen framework, analyze the image to determine the contact between subject and viewer.": 0.5,
+    "\"Analyze the image in terms of the Kress and van Leeuwen framework by considering the contact between the subject and viewer. Please provide a detailed explanation of the contact points and their significance.\"": 0.0,
+    "\"Analyze the image using the Kress and van Leeuwen framework, focusing on the contact between the subject and viewer. Provide a detailed explanation of the contact points and their significance.\"": 1.0
+  },
+  "depth": {
+    "In the context of depth in the Kress and van Leeuwen framework, analyze the image to determine the depth of the whole image.": 0.7142857142857143,
+    "\"Analyze the image in the context of depth according to the Kress and van Leeuwen framework and provide an accurate assessment of the overall depth of the image.\"": 0.7142857142857143,
+    "\"Analyze the image for depth using the Kress and van Leeuwen framework and provide a precise measurement of the overall depth.\"": 0.5714285714285714,
+    "\"Analyze the image using the Kress and van Leeuwen framework to provide an accurate measurement of its overall depth.\"": 0.8571428571428571,
+    "To improve the accuracy of depth measurement using Kress and van Leeuwen framework, please provide a precise analysis of the image's overall depth based on this framework.": 0.5714285714285714,
+    "\"Analyze the image using the Kress and van Leeuwen framework and provide a precise measurement of its overall depth.\"": 0.8571428571428571,
+    "\"Analyze the image for depth using the Kress and van Leeuwen framework and provide a precise measurement of its overall depth.\"": 0.5714285714285714,
+    "\"Analyze the image using the Kress and van Leeuwen framework to provide a precise measurement of its overall depth.\"": 0.2857142857142857,
+    "To ensure accurate assessment of depth in the image using the Kress and van Leeuwen framework, please analyze the image for depth and provide a precise measurement based on this framework.": 0.42857142857142855,
+    "To provide an accurate measurement of the overall depth of the image using the Kress and van Leeuwen framework, please analyze the image for depth and provide a precise assessment based on this framework.": 0.2857142857142857
+  },
+  "distance": {
+    "In the context of spatial distance in the Kress and van Leeuwen framework, analyze the image to determine the spatial distance that the subject is portrayed at.": 0.5714285714285714,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the spatial distance at which the subject is portrayed.\"": 1.0
+  },
+  "lighting": {
+    "In the context of lighting in the Kress and van Leeuwen framework, analyze the image to determine the lighting of the whole image.": 0.14285714285714285,
+    "\"Analyze the image to determine the lighting conditions present in the scene using the Kress and van Leeuwen framework.\"": 0.42857142857142855,
+    "Refined Prompt: \"Analyze the image using the Kress and van Leeuwen framework to determine the lighting conditions present in the scene.\"": 0.2857142857142857,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the lighting conditions present in the scene.\"": 0.8571428571428571,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the type of lighting present in the scene.\"": 0.2857142857142857,
+    "Refined Prompt: \"Analyze the image using the Kress and van Leeuwen framework to determine the type of lighting present in the scene.\"": 0.2857142857142857
+  },
+  "point_of_view": {
+    "In the context of point of view in the Kress and van Leeuwen framework, analyze the image to determine the orientation of the subject in relation to the viewer.": 0.3333333333333333,
+    "\"Analyze the image to determine the orientation of the subject in relation to the viewer using the Kress and van Leeuwen framework.\"": 0.6666666666666666,
+    "Refined Prompt: \"Using the Kress and van Leeuwen framework, determine the orientation of the subject in relation to the viewer by analyzing this image.\"": 0.0,
+    "\"Analyze the image using the Kress and van Leeuwen framework to determine the orientation of the subject in relation to the viewer.\"": 0.3333333333333333,
+    "\"Using the Kress and van Leeuwen framework, determine the orientation of the subject in relation to the viewer by analyzing this image.\"": 1.0,
+    "\"Analyze this image using the Kress and van Leeuwen framework to determine the orientation of the subject in relation to the viewer.\"": 0.6666666666666666
+  }
+}

pyproject.toml

@@ -14,6 +14,7 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dependencies = [
+    "mlflow>=3.4.0",
     "pandas>=2.3.2",
     "pillow>=11.3.0",
     "pydantic>=2.11.9",
@@ -82,4 +83,5 @@ dev = [
     "ruff>=0.13.0",
     "safety>=3.2.11",
     "testcontainers>=4.13.0",
+    "types-requests>=2.32.4.20250913",
 ]

scripts/query_ai_model.py

@@ -0,0 +1,336 @@
+"""Script to query an AI model with a prompt and image and print the response."""
+
+import os
+import base64
+import logging
+import json
+from io import BytesIO
+import requests
+import structlog
+from dotenv import load_dotenv
+from PIL import Image
+from datetime import datetime
+import mlflow
+from python_utils import check_env
+from data_store.repositories import JobRepository
+from data_store.dto import (
+    AngleEnum,
+    ColourEnum,
+    ContactEnum,
+    DepthEnum,
+    DistanceEnum,
+    LightingEnum,
+    PointOfViewEnum,
+)
+
+RUN_NAME = f"query-ai-model_{datetime.now().strftime('%Y%m%d-%H%M%S')}"
+IMAGE_MODEL = "llava:13b"
+PROMPT_MODEL = "mistral-small:22b-instruct-2409-q4_K_M"
+CATEGORY_PROMPT_MAP = {
+    "angle": (
+        "In the context of visual angle in the Kress and van Leeuwen framework, "
+        "analyze the image to determine the angle of the subject."
+    ),
+    "colour": (
+        "In the context of colour in the Kress and van Leeuwen framework, "
+        "analyze the image to determine the colour saturation of the whole image."
+    ),
+    "contact": (
+        "In the context of contact in the Kress and van Leeuwen framework, "
+        "analyze the image to determine the contact between subject and viewer."
+    ),
+    "depth": (
+        "In the context of depth in the Kress and van Leeuwen framework, "
+        "analyze the image to determine the depth of the whole image."
+    ),
+    "distance": (
+        "In the context of spatial distance in the Kress and van Leeuwen framework, "
+        "analyze the image to determine the spatial distance that the subject is portrayed at."
+    ),
+    "lighting": (
+        "In the context of lighting in the Kress and van Leeuwen framework, "
+        "analyze the image to determine the lighting of the whole image."
+    ),
+    "point_of_view": (
+        "In the context of point of view in the Kress and van Leeuwen framework, "
+        "analyze the image to determine the orientation of the subject in relation to the viewer."
+    ),
+}
+
+
+def query_model_with_image(
+    prompt: str,
+    allowed_answers: list[str],
+    image: Image.Image,
+    model: str = "llava:13b",
+) -> str:
+    """
+    Query an Ollama instance with a prompt, image, and constrained answers.
+
+    Args:
+        prompt: The text prompt to send
+        allowed_answers: List of valid response options
+        image: PIL Image to analyze
+        model: The Ollama model to use (default: "llava:13b")
+
+    Returns:
+        The model's response text
+    """
+    # Ensure required env vars are set
+    check_env("OLLAMA_ENDPOINT")
+    # Read environment variables
+    ollama_endpoint = str(os.getenv("OLLAMA_ENDPOINT"))
+    # Handle RGBA images by converting to RGB
+    if image.mode == "RGBA":
+        # Convert RGBA to RGB by compositing against white background
+        rgb_image = Image.new("RGB", image.size, (255, 255, 255))
+        rgb_image.paste(image, mask=image.split()[-1])  # Use alpha as mask
+        image = rgb_image
+    # Convert PIL Image to base64
+    buffer = BytesIO()
+    image.save(buffer, format="JPEG")
+    image_b64 = base64.b64encode(buffer.getvalue()).decode()
+    # Build prompt with allowed answers
+    prompt_with_options = (
+        f"{prompt}\n"
+        f"Please choose one of the following options: {', '.join(allowed_answers)}.\n"
+        "Respond with only the chosen option."
+    )
+    # Prepare request payload
+    payload = {
+        "model": model,
+        "prompt": prompt_with_options,
+        "images": [image_b64],
+        "stream": False,
+    }
+    # Make request to Ollama
+    # N.B. the generate endpoint does not retain context between calls
+    response = requests.post(
+        f"{ollama_endpoint}/api/generate", json=payload, timeout=60
+    )
+    response.raise_for_status()
+    result = response.json()
+
+    return str(result.get("response", "").strip())
+
+
+def score_response(response: str, expected_answer: str) -> int:
+    """
+    Score the model's response based on allowed answers.
+
+    Args:
+        response: The model's response text
+        expected_answer: The expected answer
+
+    Returns:
+        1 if the response is in allowed answers, else 0
+    """
+    # Score response
+    return 1 if response.lower() == expected_answer.lower() else 0
+
+
+def allowed_answers_for_category(
+    category: str,
+) -> list[str]:
+    """
+    Get allowed answers based on the category.
+
+    Args:
+        category: The category to determine allowed answers
+
+    Returns:
+        List of allowed answers
+    """
+    match category.lower():
+        case "angle":
+            return [e.value for e in AngleEnum]
+        case "colour":
+            return [e.value for e in ColourEnum]
+        case "contact":
+            return [e.value for e in ContactEnum]
+        case "depth":
+            return [e.value for e in DepthEnum]
+        case "distance":
+            return [e.value for e in DistanceEnum]
+        case "lighting":
+            return [e.value for e in LightingEnum]
+        case "point_of_view":
+            return [e.value for e in PointOfViewEnum]
+        case _:
+            raise ValueError(f"Invalid category: {category}")
+
+
+def calculate_model_prompt_accuracy(
+    model: str,
+    prompt: str,
+    category: str,
+) -> float:
+    """
+    Calculate the accuracy of the prompt against the expected answers for the given category.
+    """
+    # Determine allowed answers based on category
+    allowed_answers = allowed_answers_for_category(category)
+    # Score each job in the repository)
+    jobs_skipped = 0
+    total_score = 0
+    with JobRepository() as job_repo:
+        job_ids = job_repo.list_all()
+        if not job_ids:
+            raise ValueError("No jobs found in JobRepository.")
+        total_jobs = len(job_ids)
+        # Get all responses for the prompt
+        for job_id in job_ids:
+            job = job_repo.get(job_id)
+            if not job:
+                jobs_skipped += 1
+                continue
+            # skip job if uncertainty flag is set for the category
+            if getattr(job.annotation, f"{category.lower()}_uncertainty"):
+                print(f"Job {job_id}: Skipping due to uncertainty flag.")
+                jobs_skipped += 1
+                continue
+            response = query_model_with_image(
+                model=model,
+                prompt=prompt,
+                allowed_answers=allowed_answers,
+                image=job.image,
+            )
+            # Get expected answer from job annotation
+            expected_answer = getattr(job.annotation, category.lower())
+            # Score the response
+            score = score_response(response, expected_answer)
+            print(
+                f"Job {job_id}: Response '{response}' - Expected answer '{expected_answer}' - Score: {score}"
+            )
+            total_score += score
+    # Calculate accuracy
+    jobs_evaluated = total_jobs - jobs_skipped
+    mlflow.log_metrics(
+        {
+            "total_jobs": total_jobs,
+            "jobs_skipped": jobs_skipped,
+            "jobs_evaluated": jobs_evaluated,
+        }
+    )
+    print(
+        f"Total Jobs: {total_jobs}, Jobs Skipped: {jobs_skipped}, Jobs Evaluated: {jobs_evaluated}"
+    )
+    if jobs_evaluated == 0:
+        raise ValueError("No jobs were evaluated. All jobs were skipped.")
+    accuracy = total_score / jobs_evaluated
+    return accuracy
+
+
+def refine_prompt(
+    model: str,
+    prompt_accuracy_map: dict[str, float],
+) -> str:
+    """
+    Refine the prompt to improve accuracy.
+
+    Args:
+        model: The Ollama model to use
+        prompt_accuracy_map: A mapping of prompts to their accuracies
+
+    Returns:
+        The refined prompt text
+    """
+    # Ensure required env vars are set
+    check_env("OLLAMA_ENDPOINT")
+    # Read environment variables
+    ollama_endpoint = str(os.getenv("OLLAMA_ENDPOINT"))
+    # Convert prompt accuracy map to json string for better readability
+    prompt_accuracy_json = json.dumps(prompt_accuracy_map, indent=2)
+    # Build refinement instruction
+    refinement_instruction = (
+        "Given the following prompts and accuracies:\n"
+        f"{prompt_accuracy_json}\n"
+        "Please refine the prompt to improve the accuracy.\n"
+        "The refined prompt should be clear, concise, and focused on obtaining accurate responses.\n"
+        "Respond with only the refined prompt."
+    )
+    # Prepare request payload
+    payload = {"model": model, "prompt": refinement_instruction, "stream": False}
+    # Make request to Ollama
+    # N.B. the generate endpoint does not retain context between calls
+    response = requests.post(
+        f"{ollama_endpoint}/api/generate", json=payload, timeout=60
+    )
+    response.raise_for_status()
+    result = response.json()
+
+    return str(result.get("response", "").strip())
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    # Ensure required env vars are set
+    check_env("OLLAMA_ENDPOINT")
+    # Configure structlog
+    structlog.configure(
+        processors=[
+            structlog.stdlib.filter_by_level,
+            structlog.stdlib.add_logger_name,
+            structlog.stdlib.add_log_level,
+            structlog.processors.TimeStamper(fmt="iso"),
+            structlog.processors.JSONRenderer(),
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+    # Set up basic logging configuration
+    logging.basicConfig(level=logging.ERROR)
+    # Connect to MLFlow
+    mlflow.set_tracking_uri("http://10.0.0.2:5000")
+    experiment = mlflow.get_experiment_by_name("visual-semiotic-ai-analysis")
+    experiment_id = experiment.experiment_id if experiment else None
+    # Calculate accuracy
+    for category, initial_prompt in CATEGORY_PROMPT_MAP.items():
+        step = 0
+        run = mlflow.start_run(
+            experiment_id=experiment_id,
+            run_name=RUN_NAME,
+            tags={
+                "image_model": IMAGE_MODEL,
+                "prompt_model": PROMPT_MODEL,
+                "category": category,
+            },
+        )
+        mlflow.log_param(f"step_{step}_prompt", initial_prompt)
+        print(f"Calculating accuracy for category: {category}")
+        accuracy = calculate_model_prompt_accuracy(IMAGE_MODEL, initial_prompt, category)
+        mlflow.log_metric("accuracy", accuracy, step=step)
+        # Log to MLFlow
+        prompt_accuracy_map = {initial_prompt: accuracy}
+        print(f"Prompt accuracy for category '{category}': {accuracy:.2%}")
+        # Loop to refine prompt based on accuracy
+        for i in range(99):
+            step += 1
+            try:
+                # Generate refined prompt
+                refined_prompt = refine_prompt(PROMPT_MODEL, prompt_accuracy_map)
+                mlflow.log_param(f"step_{step}_prompt", refined_prompt)
+                print(f"Refined prompt: {refined_prompt}")
+            except Exception as e:
+                print(f"Error refining prompt: {e}")
+                continue
+            try:
+                # Calculate accuracy for refined prompt
+                accuracy = calculate_model_prompt_accuracy(
+                    IMAGE_MODEL, refined_prompt, category
+                )
+                mlflow.log_metric("accuracy", accuracy, step=step)
+            except Exception as e:
+                print(f"Error calculating accuracy: {e}")
+                continue
+            # Update prompt accuracy map
+            # prompt_accuracy_map[refined_prompt] = accuracy
+            print(f"Prompt accuracy for category '{category}': {accuracy:.2%}")
+            # Stop if accuracy is 100%
+            if accuracy == 1.0:
+                print(
+                    f"Achieved 100% accuracy for category '{category}'. Stopping refinement."
+                )
+                break
+        mlflow.end_run()

scripts/sync_label_studio_annotations.py

@@ -0,0 +1,129 @@
+"""Script to convert Label Studio output JSON files to an Annotation."""
+
+import json
+from uuid import UUID
+from dotenv import load_dotenv
+from python_repositories.adapters import MinioAdapter
+from data_store.dto import Annotation
+from data_store.repositories import AnnotationRepository
+
+
+def list_label_studio_output_files(minio_adapter: MinioAdapter) -> list[str]:
+    """List the Label Studio output files in MinIO."""
+    # Check input
+    if not isinstance(minio_adapter, MinioAdapter):
+        raise ValueError("minio_adapter must be an instance of MinioAdapter")
+    if not minio_adapter.is_connected:
+        raise ConnectionError("minio_adapter must be connected to MinIO")
+    # List objects with the Label Studio output prefix
+    prefix = "label-studio-output/"
+    return minio_adapter._list_objects(prefix)
+
+
+def get_label_studio_output(minio_adapter: MinioAdapter, object_name: str) -> dict:
+    """Get a Label Studio output file from MinIO."""
+    # Check input
+    if not isinstance(minio_adapter, MinioAdapter):
+        raise ValueError("minio_adapter must be an instance of MinioAdapter")
+    if not minio_adapter.is_connected:
+        raise ConnectionError("minio_adapter must be connected to MinIO")
+    if not isinstance(object_name, str) or len(object_name) == 0:
+        raise ValueError("object_name must be a non-empty string")
+    # Get object from MinIO
+    buffer = minio_adapter._get(object_name)
+    if buffer is None:
+        raise FileNotFoundError(f"File not found in MinIO: {object_name}")
+    # Convert buffer to JSON
+    data: dict = json.load(buffer)
+    return data
+
+
+def convert_label_studio_output(label_studio_output: dict) -> tuple[Annotation, UUID]:
+    """Convert Label Studio output to Annotation and corresponding job ID."""
+    # Check input
+    if not isinstance(label_studio_output, dict):
+        raise ValueError("label_studio_output must be a dictionary")
+    # Extract job id
+    image_path = label_studio_output["task"]["data"]["image"]
+    job_id_str = image_path.rsplit("/")[-1].split(".")[0]
+    job_id = UUID(job_id_str)
+    # Extract relevant information from the Label Studio output
+    annotation_data: dict[str, str | None | bool] = {}
+    for elem in label_studio_output["result"]:
+        # Extract key and value
+        key = elem["from_name"]
+        choices = elem["value"]["choices"]
+        # Check for presence of uncertainty flag
+        uncertainty_key = f"{key}_uncertainty"
+        uncertainty_value = bool("uncertain" in choices)
+        if uncertainty_value:
+            choices.remove("uncertain")
+        # If no choices left, set to None
+        value: str | None = choices[0] if choices else None
+        # Store in annotation data
+        annotation_data[key] = value
+        annotation_data[uncertainty_key] = uncertainty_value
+    # Create the Annotation object
+    annotation = Annotation(**annotation_data)  # type: ignore[arg-type]
+    return annotation, job_id
+
+
+def sync_label_studio_annotations(
+    minio_adapter: MinioAdapter,
+    annotation_repository: AnnotationRepository,
+) -> None:
+    """Sync Label Studio output files to AnnotationRepository."""
+    # Check inputs
+    if not isinstance(minio_adapter, MinioAdapter):
+        raise ValueError("minio_adapter must be an instance of MinioAdapter")
+    if not minio_adapter.is_connected:
+        raise ConnectionError("minio_adapter must be connected to MinIO")
+    if not isinstance(annotation_repository, AnnotationRepository):
+        raise ValueError(
+            "annotation_repository must be an instance of AnnotationRepository"
+        )
+    if not annotation_repository.is_connected:
+        raise ConnectionError("annotation_repository must be connected to MinIO")
+    # List annotations in repository
+    annotation_ids = annotation_repository.list_all()
+    print(f"Found {len(annotation_ids)} annotations in repository.")
+    # Count Label Studio output files
+    label_studio_files = list_label_studio_output_files(minio_adapter)
+    print(f"Found {len(label_studio_files)} Label Studio output files.")
+    # Stop early if equal number of annotations and Label Studio files
+    if len(annotation_ids) == len(label_studio_files):
+        print("No new Label Studio output files to process. Exiting.")
+        return
+    # Process each Label Studio output file
+    for file_name in label_studio_files:
+        print(f"Processing file: {file_name}")
+        # Get Label Studio output
+        try:
+            label_studio_output = get_label_studio_output(
+                minio_adapter,
+                file_name,
+            )
+        except Exception as exc:
+            print(f"Failed to get Label Studio output for {file_name}: {exc}")
+            continue
+        # Convert to Annotation
+        try:
+            annotation, job_id = convert_label_studio_output(label_studio_output)
+        except Exception as exc:
+            print(f"Failed to convert Label Studio output for {file_name}: {exc}")
+            continue
+        # Store Annotation in repository
+        try:
+            annotation_repository.put(annotation, job_id)
+        except Exception as exc:
+            print(f"Failed to store annotation for {file_name}: {exc}")
+            continue
+        print(f"Successfully processed and stored annotation for {file_name}.")
+
+
+if __name__ == "__main__":
+    # Load environment variables from .env file
+    load_dotenv()
+    # Connect to data sources
+    with MinioAdapter() as minio_adapter, AnnotationRepository() as annotation_repo:
+        sync_label_studio_annotations(minio_adapter, annotation_repo)