From 604b00c8edea010c070e1211974352a483a8f0dd Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 20 Sep 2025 10:26:57 +0200 Subject: [PATCH 1/4] fixed bug that didn't allow choice to be None --- data_store/repositories/annotation_repository.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/data_store/repositories/annotation_repository.py b/data_store/repositories/annotation_repository.py index bdbaa2e..b2974e8 100644 --- a/data_store/repositories/annotation_repository.py +++ b/data_store/repositories/annotation_repository.py @@ -58,19 +58,19 @@ class AnnotationRepository(MinioAdapter, AnnotationRepositoryInterface): data_str = buffer.read().decode(self.encoding) data = json.loads(data_str) annotation = Annotation( - angle=AngleEnum(data["angle"]), + angle=AngleEnum(data["angle"]) if data["angle"] is not None else None, angle_uncertainty=bool(data["angle_uncertainty"]), - colour=ColourEnum(data["colour"]), + colour=ColourEnum(data["colour"]) if data["colour"] is not None else None, colour_uncertainty=bool(data["colour_uncertainty"]), - contact=ContactEnum(data["contact"]), + contact=ContactEnum(data["contact"]) if data["contact"] is not None else None, contact_uncertainty=bool(data["contact_uncertainty"]), - depth=DepthEnum(data["depth"]), + depth=DepthEnum(data["depth"]) if data["depth"] is not None else None, depth_uncertainty=bool(data["depth_uncertainty"]), - distance=DistanceEnum(data["distance"]), + distance=DistanceEnum(data["distance"]) if data["distance"] is not None else None, distance_uncertainty=bool(data["distance_uncertainty"]), - lighting=LightingEnum(data["lighting"]), + lighting=LightingEnum(data["lighting"]) if data["lighting"] is not None else None, lighting_uncertainty=bool(data["lighting_uncertainty"]), - point_of_view=PointOfViewEnum(data["point_of_view"]), + point_of_view=PointOfViewEnum(data["point_of_view"]) if data["point_of_view"] is not None else None, point_of_view_uncertainty=bool(data["point_of_view_uncertainty"]), ) except (KeyError, ValueError) as e: -- 2.54.0 From 3c8845aa6d00be79b29563acf0843e4203983eff Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 20 Sep 2025 10:29:52 +0200 Subject: [PATCH 2/4] added script to query ai model --- scripts/query_ai_model.py | 300 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 scripts/query_ai_model.py diff --git a/scripts/query_ai_model.py b/scripts/query_ai_model.py new file mode 100644 index 0000000..dcac18e --- /dev/null +++ b/scripts/query_ai_model.py @@ -0,0 +1,300 @@ +"""Script to query an AI model with a prompt and image and print the response.""" + +import os +import base64 +import logging +import json +from io import BytesIO +import requests +import structlog +from dotenv import load_dotenv +from PIL import Image +from python_utils import check_env +from data_store.repositories import JobRepository +from data_store.dto import ( + AngleEnum, + ColourEnum, + ContactEnum, + DepthEnum, + DistanceEnum, + LightingEnum, + PointOfViewEnum, +) + +MODEL = "llava:13b" +CATEGORY_PROMPT_MAP = { + "angle": ( + "In the context of visual angle in the Kress and van Leeuwen framework, " + "analyze the image to determine the angle of the subject." + ), + "colour": ( + "In the context of colour in the Kress and van Leeuwen framework, " + "analyze the image to determine the colour saturation of the whole image." + ), + "contact": ( + "In the context of contact in the Kress and van Leeuwen framework, " + "analyze the image to determine the contact between subject and viewer." + ), + "depth": ( + "In the context of depth in the Kress and van Leeuwen framework, " + "analyze the image to determine the depth of the whole image." + ), + "distance": ( + "In the context of spatial distance in the Kress and van Leeuwen framework, " + "analyze the image to determine the spatial distance that the subject is portrayed at." + ), + "lighting": ( + "In the context of lighting in the Kress and van Leeuwen framework, " + "analyze the image to determine the lighting of the whole image." + ), + "point_of_view": ( + "In the context of point of view in the Kress and van Leeuwen framework, " + "analyze the image to determine the orientation of the subject in relation to the viewer." + ), +} + + +def query_model_with_image( + prompt: str, + allowed_answers: list[str], + image: Image.Image, + model: str = "llava:13b" +) -> str: + """ + Query an Ollama instance with a prompt, image, and constrained answers. + + Args: + prompt: The text prompt to send + allowed_answers: List of valid response options + image: PIL Image to analyze + model: The Ollama model to use (default: "llava:13b") + + Returns: + The model's response text + """ + # Ensure required env vars are set + check_env("OLLAMA_ENDPOINT") + # Read environment variables + ollama_endpoint = str(os.getenv("OLLAMA_ENDPOINT")) + # Handle RGBA images by converting to RGB + if image.mode == 'RGBA': + # Convert RGBA to RGB by compositing against white background + rgb_image = Image.new('RGB', image.size, (255, 255, 255)) + rgb_image.paste(image, mask=image.split()[-1]) # Use alpha as mask + image = rgb_image + # Convert PIL Image to base64 + buffer = BytesIO() + image.save(buffer, format="JPEG") + image_b64 = base64.b64encode(buffer.getvalue()).decode() + # Build prompt with allowed answers + prompt_with_options = ( + f"{prompt}\n" + f"Please choose one of the following options: {', '.join(allowed_answers)}.\n" + "Respond with only the chosen option." + ) + # Prepare request payload + payload = { + "model": model, + "prompt": prompt_with_options, + "images": [image_b64], + "stream": False + } + # Make request to Ollama + # N.B. the generate endpoint does not retain context between calls + response = requests.post( + f"{ollama_endpoint}/api/generate", + json=payload, + timeout=60 + ) + response.raise_for_status() + result = response.json() + + return str(result.get("response", "").strip()) + + +def score_response(response: str, expected_answer: str) -> int: + """ + Score the model's response based on allowed answers. + + Args: + response: The model's response text + expected_answer: The expected answer + + Returns: + 1 if the response is in allowed answers, else 0 + """ + # Score response + return 1 if response.lower() == expected_answer.lower() else 0 + + +def allowed_answers_for_category( + category: str, +) -> list[str]: + """ + Get allowed answers based on the category. + + Args: + category: The category to determine allowed answers + + Returns: + List of allowed answers + """ + match category.lower(): + case "angle": + return [e.value for e in AngleEnum] + case "colour": + return [e.value for e in ColourEnum] + case "contact": + return [e.value for e in ContactEnum] + case "depth": + return [e.value for e in DepthEnum] + case "distance": + return [e.value for e in DistanceEnum] + case "lighting": + return [e.value for e in LightingEnum] + case "point_of_view": + return [e.value for e in PointOfViewEnum] + case _: + raise ValueError(f"Invalid category: {category}") + + +def calculate_model_prompt_accuracy( + model: str, + prompt: str, + category: str, +) -> float: + """ + Calculate the accuracy of the prompt against the expected answers for the given category. + """ + # Determine allowed answers based on category + allowed_answers = allowed_answers_for_category(category) + # Score each job in the repository) + jobs_skipped = 0 + total_score = 0 + with JobRepository() as job_repo: + job_ids = job_repo.list_all() + if not job_ids: + raise ValueError("No jobs found in JobRepository.") + total_jobs = len(job_ids) + # Get all responses for the prompt + for job_id in job_ids: + job = job_repo.get(job_id) + if not job: + jobs_skipped += 1 + continue + # skip job if uncertainty flag is set for the category + if getattr(job.annotation, f"{category.lower()}_uncertainty"): + print(f"Job {job_id}: Skipping due to uncertainty flag.") + jobs_skipped += 1 + continue + response = query_model_with_image( + model=model, + prompt=prompt, + allowed_answers=allowed_answers, + image=job.image, + ) + # Get expected answer from job annotation + expected_answer = getattr(job.annotation, category.lower()) + # Score the response + score = score_response(response, expected_answer) + print(f"Job {job_id}: Response '{response}' - Expected answer '{expected_answer}' - Score: {score}") + total_score += score + # Calculate accuracy + jobs_evaluated = total_jobs - jobs_skipped + print(f"Total Jobs: {total_jobs}, Jobs Skipped: {jobs_skipped}, Jobs Evaluated: {jobs_evaluated}") + if jobs_evaluated == 0: + raise ValueError("No jobs were evaluated. All jobs were skipped.") + accuracy = total_score / jobs_evaluated + return accuracy + + +def refine_prompt( + model: str, + prompt_accuracy_map: dict[str, float], +) -> str: + """ + Refine the prompt to improve accuracy. + + Args: + model: The Ollama model to use + prompt_accuracy_map: A mapping of prompts to their accuracies + + Returns: + The refined prompt text + """ + # Ensure required env vars are set + check_env("OLLAMA_ENDPOINT") + # Read environment variables + ollama_endpoint = str(os.getenv("OLLAMA_ENDPOINT")) + # Convert prompt accuracy map to json string for better readability + prompt_accuracy_json = json.dumps(prompt_accuracy_map, indent=2) + # Build refinement instruction + refinement_instruction = ( + "Given the following prompts and accuracies:\n" + f"{prompt_accuracy_json}\n" + "Please refine the prompt to improve the accuracy.\n" + "The refined prompt should be clear, concise, and focused on obtaining accurate responses.\n" + "Respond with only the refined prompt." + ) + # Prepare request payload + payload = { + "model": model, + "prompt": refinement_instruction, + "stream": False + } + # Make request to Ollama + # N.B. the generate endpoint does not retain context between calls + response = requests.post( + f"{ollama_endpoint}/api/generate", + json=payload, + timeout=60 + ) + response.raise_for_status() + result = response.json() + + return str(result.get("response", "").strip()) + + +if __name__ == "__main__": + load_dotenv() + # Ensure required env vars are set + check_env("OLLAMA_ENDPOINT") + # Configure structlog + structlog.configure( + processors=[ + structlog.stdlib.filter_by_level, + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + structlog.processors.TimeStamper(fmt="iso"), + structlog.processors.JSONRenderer(), + ], + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + cache_logger_on_first_use=True, + ) + # Set up basic logging configuration + logging.basicConfig(level=logging.ERROR) + # Calculate accuracy + category_events = {} + for category, initial_prompt in CATEGORY_PROMPT_MAP.items(): + print(f"Calculating accuracy for category: {category}") + accuracy = calculate_model_prompt_accuracy(MODEL, initial_prompt, category) + prompt_accuracy_map = {initial_prompt: accuracy} + print(f"Prompt accuracy for category '{category}': {accuracy:.2%}") + # loop to refine prompt based on accuracy + for i in range(30): + refined_prompt = refine_prompt(MODEL, prompt_accuracy_map) + print(f"Refined prompt: {refined_prompt}") + accuracy = calculate_model_prompt_accuracy(MODEL, refined_prompt, category) + prompt_accuracy_map[refined_prompt] = accuracy + print(f"Prompt accuracy for category '{category}': {accuracy:.2%}") + # Stop if accuracy is 100% + if accuracy == 1.0: + print(f"Achieved 100% accuracy for category '{category}'. Stopping refinement.") + break + print(f"Final prompt accuracy map for category '{category}': {prompt_accuracy_map}") + category_events[category] = prompt_accuracy_map + # save results to json file + with open("prompt_accuracy_results.json", "w", encoding="utf-8") as f: + json.dump(category_events, f, indent=2) + print("Prompt accuracy results saved to 'prompt_accuracy_results.json'.") -- 2.54.0 From 7e9c4474ac58154f83e705324f7a85412ebf4f08 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sat, 20 Sep 2025 10:29:59 +0200 Subject: [PATCH 3/4] ran a few query sessions --- prompt_accuracy_results-60_jobs.json | 93 ++++++++++++++++++++++++++++ prompt_accuracy_results-7_jobs.json | 64 +++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 prompt_accuracy_results-60_jobs.json create mode 100644 prompt_accuracy_results-7_jobs.json diff --git a/prompt_accuracy_results-60_jobs.json b/prompt_accuracy_results-60_jobs.json new file mode 100644 index 0000000..bc79bb6 --- /dev/null +++ b/prompt_accuracy_results-60_jobs.json @@ -0,0 +1,93 @@ +{ + "angle": { + "In the context of visual angle in the Kress and van Leeuwen framework, analyze the image to determine the angle of the subject.": 0.425531914893617, + "\"Analyze the image to determine the visual angle of the subject in the Kress and van Leeuwen framework.\"": 0.46808510638297873, + "Refined Prompt: \"In the context of visual angle analysis in the Kress and van Leeuwen framework, please provide the angle of the subject depicted in the image.\"": 0.44680851063829785, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the visual angle of the subject depicted.\"": 0.5106382978723404, + "\"In the context of visual angle analysis in the Kress and van Leeuwen framework, please provide the angle of the subject depicted in the image.\"": 0.46808510638297873, + "\"Refine the prompt by removing unnecessary words and focusing on the task of providing the visual angle of the subject in the image using the Kress and van Leeuwen framework.\"": 0.46808510638297873, + "Refined Prompt: \"Provide the visual angle of the subject depicted in the image using the Kress and van Leeuwen framework.\"": 0.425531914893617, + "\"Provide the visual angle of the subject depicted in the image using the Kress and van Leeuwen framework.\"": 0.3404255319148936, + "\"In the context of visual angle analysis in the Kress and van Leeuwen framework, please provide the exact angle measurement of the subject depicted in the image.\"": 0.5106382978723404, + "\"Refine the prompt by removing unnecessary words and focusing on providing the exact angle measurement of the subject in the image using the Kress and van Leeuwen framework.\"": 0.5106382978723404, + "\"Provide the exact visual angle measurement of the subject depicted in the image using the Kress and van Leeuwen framework.\"": 0.425531914893617, + "Refine the prompt by removing unnecessary words and focusing on providing the exact angle measurement of the subject in the image using the Kress and van Leeuwen framework.": 0.5106382978723404 + }, + "colour": { + "In the context of colour in the Kress and van Leeuwen framework, analyze the image to determine the colour saturation of the whole image.": 0.3220338983050847, + "\"Analyze the image for colour saturation using the Kress and van Leeuwen framework, considering the entire image.\"": 0.3220338983050847, + "To ensure accuracy in the analysis of colour saturation using the Kress and van Leeuwen framework for the entire image, please provide a prompt that clearly and succinctly requests this information.": 0.3050847457627119, + "\"Using the Kress and van Leeuwen framework, please provide an analysis of the colour saturation for the entire image.\"": 0.3220338983050847, + "\"Using the Kress and van Leeuwen framework, please provide an analysis of the color saturation for the entire image.\"": 0.288135593220339, + "Using the Kress and van Leeuwen framework, please provide an analysis of the color saturation for the entire image.": 0.3898305084745763, + "To ensure accuracy in the analysis of color saturation using the Kress and van Leeuwen framework for the entire image, please provide a prompt that clearly and succinctly requests this information.": 0.2711864406779661 + }, + "contact": { + "In the context of contact in the Kress and van Leeuwen framework, analyze the image to determine the contact between subject and viewer.": 0.6363636363636364, + "\"Analyze the image to determine the contact between the subject and the viewer in the context of the Kress and van Leeuwen framework.\"": 0.3181818181818182, + "\"Evaluate the contact between the subject and viewer in the context of the Kress and van Leeuwen framework by analyzing this image.\"": 0.5454545454545454, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the contact between the subject and viewer.\"": 0.4090909090909091, + "\"Using the Kress and van Leeuwen framework, analyze this image to determine the contact between the subject and viewer.\"": 0.6818181818181818, + "\"Analyze this image using the Kress and van Leeuwen framework to determine the contact between the subject and viewer.\"": 0.7272727272727273, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the level of contact between the subject and viewer.\"": 0.5909090909090909, + "\"Analyze this image using the Kress and van Leeuwen framework to determine the level of contact between the subject and viewer.\"": 0.45454545454545453, + "\"Using the Kress and van Leeuwen framework, analyze this image to determine the level of contact between the subject and viewer.\"": 0.5454545454545454, + "\"Analyze this image using the Kress and van Leeuwen framework by evaluating the contact between the subject and viewer.\"": 0.4090909090909091, + "Refined prompt: \"Analyze this image using the Kress and van Leeuwen framework by evaluating the level of contact between the subject and viewer.\"": 0.5, + "\"Analyze this image using the Kress and van Leeuwen framework by evaluating the level of contact between the subject and viewer.\"": 0.5, + "\"Evaluate the level of contact between the subject and viewer in the context of the Kress and van Leeuwen framework by analyzing this image.\"": 0.5 + }, + "depth": { + "In the context of depth in the Kress and van Leeuwen framework, analyze the image to determine the depth of the whole image.": 0.3793103448275862, + "\"Analyze the image to determine the depth of the scene in terms of foreground, midground, and background elements using the Kress and van Leeuwen framework.\"": 0.4482758620689655, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the depth of the foreground, midground, and background elements.\"": 0.4482758620689655, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the depth of each element in the scene, including foreground, midground, and background elements.\"": 0.4482758620689655, + "\"Using the Kress and van Leeuwen framework, analyze the image to determine the depth of each element in the scene, including foreground, midground, and background elements.\"": 0.39655172413793105, + "\"Using the Kress and van Leeuwen framework, analyze the depth of each element in the image, including foreground, midground, and background elements.\"": 0.39655172413793105 + }, + "distance": { + "In the context of spatial distance in the Kress and van Leeuwen framework, analyze the image to determine the spatial distance that the subject is portrayed at.": 0.4, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the spatial distance portrayed for the subject.\"": 0.37142857142857144, + "\"Using the Kress and van Leeuwen framework, determine the spatial distance portrayed for the subject in this image.\"": 0.4, + "The refined prompt is: \"Analyze the image using the Kress and van Leeuwen framework to determine the spatial distance portrayed for the subject.\"": 0.4857142857142857, + "The refined prompt is: \"Determine the spatial distance portrayed for the subject in this image using the Kress and van Leeuwen framework.\"": 0.4857142857142857, + "The refined prompt is: \"Using the Kress and van Leeuwen framework, analyze the image to determine the spatial distance portrayed for the subject.\"": 0.6, + "\"Using the Kress and van Leeuwen framework, analyze this image to determine the spatial distance portrayed for the subject.\"": 0.5428571428571428, + "The refined prompt is: \"Analyze this image using the Kress and van Leeuwen framework to determine the spatial distance portrayed for the subject.\"": 0.4857142857142857, + "\"Analyze this image using the Kress and van Leeuwen framework to determine the spatial distance portrayed for the subject.\"": 0.42857142857142855, + "\"Determine the spatial distance portrayed for the subject in this image using the Kress and van Leeuwen framework.\"": 0.4857142857142857 + }, + "lighting": { + "In the context of lighting in the Kress and van Leeuwen framework, analyze the image to determine the lighting of the whole image.": 0.3333333333333333, + "Refined Prompt: \"Analyze the image for its overall lighting using the Kress and van Leeuwen framework, and provide an accuracy estimate of the analysis.\"": 0.4074074074074074, + "\"Analyze the image for its overall lighting using the Kress and van Leeuwen framework, and provide an accuracy estimate of the analysis.\"": 0.35185185185185186, + "\"In the context of the Kress and van Leeuwen framework, analyze the image for its overall lighting and provide an accuracy estimate of the analysis.\"": 0.4074074074074074, + "\"Analyze the image for its overall lighting using the Kress and van Leeuwen framework and provide an accuracy estimate of the analysis.\"": 0.24074074074074073, + "\"Refined Prompt: \\\"Analyze the image for its overall lighting using the Kress and van Leeuwen framework, providing a detailed description of the lighting conditions and an accuracy estimate of the analysis.\\\"\"": 0.35185185185185186, + "\"Analyze the image for its overall lighting using the Kress and van Leeuwen framework, providing a detailed description of the lighting conditions and an accuracy estimate of the analysis.\"": 0.3148148148148148, + "\"Refine the prompt for analyzing the image's overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the lighting conditions and an accuracy estimate of the analysis.\"": 0.3148148148148148, + "Refine the prompt for analyzing the image's overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the lighting conditions and an accuracy estimate of the analysis.": 0.18518518518518517, + "\"Analyze the image for its overall lighting conditions using the Kress and van Leeuwen framework, providing a detailed description of the light sources, their intensities, and the resulting illumination, along with an accuracy estimate of the analysis.\"": 0.2962962962962963, + "Refine the prompt for analyzing the image's overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination, along with an accuracy estimate of the analysis.": 0.18518518518518517, + "\"Analyze the image for its overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination, along with an accuracy estimate of the analysis.\"": 0.2222222222222222, + "\"Refine the prompt for analyzing the image's overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination, along with an accuracy estimate of the analysis.\"": 0.3888888888888889, + "\"Analyze the image for its overall lighting conditions using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination, along with an accuracy estimate of the analysis.\"": 0.2037037037037037, + "To improve the accuracy of analyzing the image's overall lighting using the Kress and van Leeuwen framework, provide a detailed description of the light sources, their intensities, and the resulting illumination in the image, along with an estimated accuracy for the analysis.": 0.16666666666666666, + "\"Analyze the image for its overall lighting conditions using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination in the image, along with an estimated accuracy of the analysis.\"": 0.35185185185185186, + "\"Provide a detailed description of the light sources, their intensities, and the resulting illumination in the image using the Kress and van Leeuwen framework, along with an estimated accuracy for the analysis.\"": 0.2037037037037037, + "\"Analyze the image for its overall lighting conditions using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination in the image, along with an estimated accuracy for the analysis.\"": 0.37037037037037035, + "Refine the prompt for analyzing the image's overall lighting using the Kress and van Leeuwen framework by providing a detailed description of the light sources, their intensities, and the resulting illumination in the image, along with an estimated accuracy for the analysis.": 0.25925925925925924 + }, + "point_of_view": { + "In the context of point of view in the Kress and van Leeuwen framework, analyze the image to determine the orientation of the subject in relation to the viewer.": 0.64, + "Refined Prompt: \"Analyze the image using the Kress and van Leeuwen framework to determine the orientation of the subject in relation to the viewer's point of view.\"": 0.72, + "\"Using the Kress and van Leeuwen framework, analyze the image to determine the subject's orientation in relation to the viewer's point of view.\"": 0.6, + "Refined Prompt: \"Using the Kress and van Leeuwen framework, analyze the image to determine the subject's orientation in relation to the viewer's point of view.\"": 0.72, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the subject's orientation in relation to the viewer's point of view.\"": 0.64, + "\"In the context of point of view in the Kress and van Leeuwen framework, provide an analysis of the image to determine the orientation of the subject in relation to the viewer's perspective.\"": 0.52, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the orientation of the subject in relation to the viewer's point of view.\"": 0.56, + "\"Using the Kress and van Leeuwen framework, provide an analysis of the image to determine the orientation of the subject in relation to the viewer's point of view.\"": 0.68, + "\"Using the Kress and van Leeuwen framework, provide an analysis of the image to determine the subject's orientation in relation to the viewer's point of view.\"": 0.76, + "\"Using the Kress and van Leeuwen framework, provide a detailed analysis of the image to determine the orientation of the subject in relation to the viewer's point of view.\"": 0.68 + } +} \ No newline at end of file diff --git a/prompt_accuracy_results-7_jobs.json b/prompt_accuracy_results-7_jobs.json new file mode 100644 index 0000000..0ebcc94 --- /dev/null +++ b/prompt_accuracy_results-7_jobs.json @@ -0,0 +1,64 @@ +{ + "angle": { + "In the context of visual angle in the Kress and van Leeuwen framework, analyze the image to determine the angle of the subject.": 0.3333333333333333, + "\"Analyze the image to determine the visual angle of the subject using the Kress and van Leeuwen framework, providing an accuracy of at least 0.4.\"": 0.16666666666666666, + "\"Analyze the image provided within the context of the Kress and van Leeuwen framework to accurately determine the visual angle of the subject, with an accuracy target of at least 0.4.\"": 0.3333333333333333, + "\"Analyze the image provided using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with an accuracy target of at least 0.4.\"": 0.0, + "\"Analyze the image provided within the context of the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with an accuracy target of at least 0.4.\"": 0.5, + "\"Analyze the image provided using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with a minimum accuracy target of 0.4.\"": 0.16666666666666666, + "Refined Prompt: \"Analyze the provided image using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with a minimum accuracy target of 0.4.\"": 0.16666666666666666, + "\"Analyze the provided image using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with a minimum accuracy target of 0.4.\"": 0.16666666666666666, + "\"Analyze the provided image using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject, targeting an accuracy of at least 0.5.\"": 0.0, + "\"Analyze the provided image using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject with a minimum accuracy target of 0.5.\"": 0.0, + "\"Analyze the provided image using the Kress and van Leeuwen framework to accurately determine the visual angle of the subject, targeting an accuracy of at least 0.4.\"": 0.6666666666666666 + }, + "colour": { + "In the context of colour in the Kress and van Leeuwen framework, analyze the image to determine the colour saturation of the whole image.": 0.2857142857142857, + "\"Analyze the image for color saturation using the Kress and van Leeuwen framework.\"": 0.42857142857142855, + "\"Analyze the image for color saturation using the Kress and van Leeuwen framework by considering all hues present in the image.\"": 0.2857142857142857, + "\"Using the Kress and van Leeuwen framework for color analysis, determine the overall saturation level of all hues present in the image.\"": 0.2857142857142857, + "\"Analyze the image for color saturation using the Kress and van Leeuwen framework by considering all hues present in the image and provide a numerical value representing the overall saturation level.\"": 0.5714285714285714, + "\"Using the Kress and van Leeuwen framework for color analysis, provide a numerical value representing the overall saturation level of all hues present in the image.\"": 0.2857142857142857, + "\"Using the Kress and van Leeuwen framework for color analysis, determine the overall saturation level of all hues present in the image by considering their respective saturation values.\"": 0.14285714285714285, + "\"Using the Kress and van Leeuwen framework for color analysis, provide a numerical value representing the overall saturation level of all hues present in the image by considering their respective saturation values.\"": 0.42857142857142855, + "\"Analyze the image using the Kress and van Leeuwen framework for color analysis by considering all hues present in the image to determine the overall saturation level as a numerical value.\"": 0.42857142857142855, + "\"Using the Kress and van Leeuwen framework for color analysis, determine the overall saturation level of all hues present in the image by considering their respective saturation values and provide a numerical value representing the overall saturation level.\"": 0.8571428571428571 + }, + "contact": { + "In the context of contact in the Kress and van Leeuwen framework, analyze the image to determine the contact between subject and viewer.": 0.5, + "\"Analyze the image in terms of the Kress and van Leeuwen framework by considering the contact between the subject and viewer. Please provide a detailed explanation of the contact points and their significance.\"": 0.0, + "\"Analyze the image using the Kress and van Leeuwen framework, focusing on the contact between the subject and viewer. Provide a detailed explanation of the contact points and their significance.\"": 1.0 + }, + "depth": { + "In the context of depth in the Kress and van Leeuwen framework, analyze the image to determine the depth of the whole image.": 0.7142857142857143, + "\"Analyze the image in the context of depth according to the Kress and van Leeuwen framework and provide an accurate assessment of the overall depth of the image.\"": 0.7142857142857143, + "\"Analyze the image for depth using the Kress and van Leeuwen framework and provide a precise measurement of the overall depth.\"": 0.5714285714285714, + "\"Analyze the image using the Kress and van Leeuwen framework to provide an accurate measurement of its overall depth.\"": 0.8571428571428571, + "To improve the accuracy of depth measurement using Kress and van Leeuwen framework, please provide a precise analysis of the image's overall depth based on this framework.": 0.5714285714285714, + "\"Analyze the image using the Kress and van Leeuwen framework and provide a precise measurement of its overall depth.\"": 0.8571428571428571, + "\"Analyze the image for depth using the Kress and van Leeuwen framework and provide a precise measurement of its overall depth.\"": 0.5714285714285714, + "\"Analyze the image using the Kress and van Leeuwen framework to provide a precise measurement of its overall depth.\"": 0.2857142857142857, + "To ensure accurate assessment of depth in the image using the Kress and van Leeuwen framework, please analyze the image for depth and provide a precise measurement based on this framework.": 0.42857142857142855, + "To provide an accurate measurement of the overall depth of the image using the Kress and van Leeuwen framework, please analyze the image for depth and provide a precise assessment based on this framework.": 0.2857142857142857 + }, + "distance": { + "In the context of spatial distance in the Kress and van Leeuwen framework, analyze the image to determine the spatial distance that the subject is portrayed at.": 0.5714285714285714, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the spatial distance at which the subject is portrayed.\"": 1.0 + }, + "lighting": { + "In the context of lighting in the Kress and van Leeuwen framework, analyze the image to determine the lighting of the whole image.": 0.14285714285714285, + "\"Analyze the image to determine the lighting conditions present in the scene using the Kress and van Leeuwen framework.\"": 0.42857142857142855, + "Refined Prompt: \"Analyze the image using the Kress and van Leeuwen framework to determine the lighting conditions present in the scene.\"": 0.2857142857142857, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the lighting conditions present in the scene.\"": 0.8571428571428571, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the type of lighting present in the scene.\"": 0.2857142857142857, + "Refined Prompt: \"Analyze the image using the Kress and van Leeuwen framework to determine the type of lighting present in the scene.\"": 0.2857142857142857 + }, + "point_of_view": { + "In the context of point of view in the Kress and van Leeuwen framework, analyze the image to determine the orientation of the subject in relation to the viewer.": 0.3333333333333333, + "\"Analyze the image to determine the orientation of the subject in relation to the viewer using the Kress and van Leeuwen framework.\"": 0.6666666666666666, + "Refined Prompt: \"Using the Kress and van Leeuwen framework, determine the orientation of the subject in relation to the viewer by analyzing this image.\"": 0.0, + "\"Analyze the image using the Kress and van Leeuwen framework to determine the orientation of the subject in relation to the viewer.\"": 0.3333333333333333, + "\"Using the Kress and van Leeuwen framework, determine the orientation of the subject in relation to the viewer by analyzing this image.\"": 1.0, + "\"Analyze this image using the Kress and van Leeuwen framework to determine the orientation of the subject in relation to the viewer.\"": 0.6666666666666666 + } +} \ No newline at end of file -- 2.54.0 From cbac5ded35e0db7dda404c0959639ebe01ea9f21 Mon Sep 17 00:00:00 2001 From: Brian Bjarke Jensen Date: Sun, 21 Sep 2025 19:40:01 +0200 Subject: [PATCH 4/4] ruff format fixes --- .../repositories/annotation_repository.py | 20 +++++++--- scripts/query_ai_model.py | 38 +++++++++---------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/data_store/repositories/annotation_repository.py b/data_store/repositories/annotation_repository.py index b2974e8..2d6e582 100644 --- a/data_store/repositories/annotation_repository.py +++ b/data_store/repositories/annotation_repository.py @@ -60,17 +60,27 @@ class AnnotationRepository(MinioAdapter, AnnotationRepositoryInterface): annotation = Annotation( angle=AngleEnum(data["angle"]) if data["angle"] is not None else None, angle_uncertainty=bool(data["angle_uncertainty"]), - colour=ColourEnum(data["colour"]) if data["colour"] is not None else None, + colour=ColourEnum(data["colour"]) + if data["colour"] is not None + else None, colour_uncertainty=bool(data["colour_uncertainty"]), - contact=ContactEnum(data["contact"]) if data["contact"] is not None else None, + contact=ContactEnum(data["contact"]) + if data["contact"] is not None + else None, contact_uncertainty=bool(data["contact_uncertainty"]), depth=DepthEnum(data["depth"]) if data["depth"] is not None else None, depth_uncertainty=bool(data["depth_uncertainty"]), - distance=DistanceEnum(data["distance"]) if data["distance"] is not None else None, + distance=DistanceEnum(data["distance"]) + if data["distance"] is not None + else None, distance_uncertainty=bool(data["distance_uncertainty"]), - lighting=LightingEnum(data["lighting"]) if data["lighting"] is not None else None, + lighting=LightingEnum(data["lighting"]) + if data["lighting"] is not None + else None, lighting_uncertainty=bool(data["lighting_uncertainty"]), - point_of_view=PointOfViewEnum(data["point_of_view"]) if data["point_of_view"] is not None else None, + point_of_view=PointOfViewEnum(data["point_of_view"]) + if data["point_of_view"] is not None + else None, point_of_view_uncertainty=bool(data["point_of_view_uncertainty"]), ) except (KeyError, ValueError) as e: diff --git a/scripts/query_ai_model.py b/scripts/query_ai_model.py index dcac18e..46319a2 100644 --- a/scripts/query_ai_model.py +++ b/scripts/query_ai_model.py @@ -58,7 +58,7 @@ def query_model_with_image( prompt: str, allowed_answers: list[str], image: Image.Image, - model: str = "llava:13b" + model: str = "llava:13b", ) -> str: """ Query an Ollama instance with a prompt, image, and constrained answers. @@ -77,9 +77,9 @@ def query_model_with_image( # Read environment variables ollama_endpoint = str(os.getenv("OLLAMA_ENDPOINT")) # Handle RGBA images by converting to RGB - if image.mode == 'RGBA': + if image.mode == "RGBA": # Convert RGBA to RGB by compositing against white background - rgb_image = Image.new('RGB', image.size, (255, 255, 255)) + rgb_image = Image.new("RGB", image.size, (255, 255, 255)) rgb_image.paste(image, mask=image.split()[-1]) # Use alpha as mask image = rgb_image # Convert PIL Image to base64 @@ -97,14 +97,12 @@ def query_model_with_image( "model": model, "prompt": prompt_with_options, "images": [image_b64], - "stream": False + "stream": False, } # Make request to Ollama # N.B. the generate endpoint does not retain context between calls response = requests.post( - f"{ollama_endpoint}/api/generate", - json=payload, - timeout=60 + f"{ollama_endpoint}/api/generate", json=payload, timeout=60 ) response.raise_for_status() result = response.json() @@ -197,11 +195,15 @@ def calculate_model_prompt_accuracy( expected_answer = getattr(job.annotation, category.lower()) # Score the response score = score_response(response, expected_answer) - print(f"Job {job_id}: Response '{response}' - Expected answer '{expected_answer}' - Score: {score}") + print( + f"Job {job_id}: Response '{response}' - Expected answer '{expected_answer}' - Score: {score}" + ) total_score += score # Calculate accuracy jobs_evaluated = total_jobs - jobs_skipped - print(f"Total Jobs: {total_jobs}, Jobs Skipped: {jobs_skipped}, Jobs Evaluated: {jobs_evaluated}") + print( + f"Total Jobs: {total_jobs}, Jobs Skipped: {jobs_skipped}, Jobs Evaluated: {jobs_evaluated}" + ) if jobs_evaluated == 0: raise ValueError("No jobs were evaluated. All jobs were skipped.") accuracy = total_score / jobs_evaluated @@ -237,17 +239,11 @@ def refine_prompt( "Respond with only the refined prompt." ) # Prepare request payload - payload = { - "model": model, - "prompt": refinement_instruction, - "stream": False - } + payload = {"model": model, "prompt": refinement_instruction, "stream": False} # Make request to Ollama # N.B. the generate endpoint does not retain context between calls response = requests.post( - f"{ollama_endpoint}/api/generate", - json=payload, - timeout=60 + f"{ollama_endpoint}/api/generate", json=payload, timeout=60 ) response.raise_for_status() result = response.json() @@ -290,9 +286,13 @@ if __name__ == "__main__": print(f"Prompt accuracy for category '{category}': {accuracy:.2%}") # Stop if accuracy is 100% if accuracy == 1.0: - print(f"Achieved 100% accuracy for category '{category}'. Stopping refinement.") + print( + f"Achieved 100% accuracy for category '{category}'. Stopping refinement." + ) break - print(f"Final prompt accuracy map for category '{category}': {prompt_accuracy_map}") + print( + f"Final prompt accuracy map for category '{category}': {prompt_accuracy_map}" + ) category_events[category] = prompt_accuracy_map # save results to json file with open("prompt_accuracy_results.json", "w", encoding="utf-8") as f: -- 2.54.0