removed unused packages
This commit is contained in:
brian
+69
-61
@@ -4,29 +4,28 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from base64 import b64decode, b64encode
|
||||
from io import BytesIO
|
||||
|
||||
import dash_bootstrap_components as dbc
|
||||
from dash import ALL, Dash, Input, Output, State
|
||||
from dash_auth import BasicAuth
|
||||
from PIL import Image
|
||||
from pydantic import ValidationError
|
||||
from pymongo.collection import Collection
|
||||
|
||||
from shared.datastore import Datastore
|
||||
from shared.docstore import count_documents, get_visual_communication, upsert_annotation
|
||||
from shared.docstore.src.classes import ModelData, VisualCommunication
|
||||
from shared.docstore.src.exceptions import NoDocumentFoundException
|
||||
from shared.repositories import (
|
||||
ImageData,
|
||||
ImageRepository,
|
||||
VisualCommunicationData,
|
||||
VisualCommunicationRepository,
|
||||
)
|
||||
|
||||
from .layout import app_layout
|
||||
|
||||
|
||||
def init_app(
|
||||
mongo_collection: Collection,
|
||||
datastore: Datastore,
|
||||
) -> Dash:
|
||||
def init_app() -> Dash:
|
||||
"""Initialise web UI application."""
|
||||
assert isinstance(mongo_collection, Collection)
|
||||
assert isinstance(datastore, Datastore)
|
||||
assert datastore._client is not None
|
||||
# setup app
|
||||
app = Dash(
|
||||
name='visual_critical_discourse_analysis_web_ui',
|
||||
@@ -90,14 +89,10 @@ def init_app(
|
||||
if filename_list is not None:
|
||||
assert isinstance(filename_list, list)
|
||||
# get values from database
|
||||
num_total = count_documents(
|
||||
collection=mongo_collection,
|
||||
only_with_annotation=False,
|
||||
)
|
||||
num_handled = count_documents(
|
||||
collection=mongo_collection,
|
||||
only_with_annotation=True,
|
||||
)
|
||||
with VisualCommunicationRepository() as repo:
|
||||
name_list = repo.list_names()
|
||||
num_total = len(name_list)
|
||||
num_handled = 0
|
||||
limit = int(num_total / 20)
|
||||
label_str = f'{num_handled}/{num_total}' if num_handled >= limit else ''
|
||||
return num_handled, num_total, label_str
|
||||
@@ -125,38 +120,37 @@ def init_app(
|
||||
for content, filename in zip(content_list, filename_list):
|
||||
try:
|
||||
# decode image content
|
||||
image = VisualCommunication.decode_image(content=content)
|
||||
content_data = content.split(',')[-1]
|
||||
image = Image.open(BytesIO(b64decode(content_data)))
|
||||
# instantiate ImageData object
|
||||
image_data = ImageData(
|
||||
name=filename,
|
||||
image=image,
|
||||
)
|
||||
# instantiate VisualCommunicationData object
|
||||
vis_com_data = VisualCommunicationData(name=filename)
|
||||
except Exception:
|
||||
logging.debug('failed decoding %s', filename)
|
||||
failed_filename_list.append(filename)
|
||||
continue
|
||||
try:
|
||||
assert datastore._client is not None
|
||||
# instantiate to upload image to minio
|
||||
vis_com = VisualCommunication.from_name_and_image(
|
||||
name=filename,
|
||||
image=image,
|
||||
minio_client=datastore._client,
|
||||
)
|
||||
# save ImageData
|
||||
with ImageRepository() as repo:
|
||||
repo.put_data(image_data)
|
||||
# save VisualCommunicationData
|
||||
with VisualCommunicationRepository() as repo:
|
||||
repo.put_data(vis_com_data)
|
||||
except Exception as exc:
|
||||
logging.debug(exc)
|
||||
failed_filename_list.append(filename)
|
||||
continue
|
||||
try:
|
||||
# save to mongodb
|
||||
vis_com.save_to_mongo(collection=mongo_collection)
|
||||
except Exception as exc:
|
||||
logging.debug(exc)
|
||||
failed_filename_list.append(filename)
|
||||
# remove document from minio
|
||||
datastore._delete(
|
||||
object_name=vis_com.object_name,
|
||||
)
|
||||
assert (
|
||||
len(failed_filename_list) == 0
|
||||
), f"failed uploading:{
|
||||
'\n'.join(failed_filename_list)
|
||||
}"
|
||||
# cleanup failed uploads
|
||||
with ImageRepository() as repo:
|
||||
repo.remove_data(image_data.name)
|
||||
with VisualCommunicationRepository() as repo:
|
||||
repo.remove_data(vis_com_data.name)
|
||||
if len(failed_filename_list) == 0:
|
||||
err_msg = 'failed uploading:' + '\n'.join(failed_filename_list)
|
||||
raise FileNotFoundError(err_msg)
|
||||
except Exception as exc:
|
||||
logging.debug(exc)
|
||||
return None, str(exc).title()
|
||||
@@ -208,7 +202,7 @@ def init_app(
|
||||
logging.info(annotation_keys)
|
||||
for option, value in zip(annotation_keys, annotation_values):
|
||||
if value is None:
|
||||
raise ValueError(f"{option} is not set")
|
||||
raise ValueError(f'{option} is not set')
|
||||
# prepare data to save
|
||||
annotation_keys = [elem.replace(' ', '_') for elem in annotation_keys]
|
||||
annotation_values = [
|
||||
@@ -217,16 +211,21 @@ def init_app(
|
||||
annotation_map = {
|
||||
key: value for key, value in zip(annotation_keys, annotation_values)
|
||||
}
|
||||
# instantiate ModelOutputs object
|
||||
annotations = ModelData.from_annotations(**annotation_map)
|
||||
# save data to database
|
||||
upsert_annotation(
|
||||
collection=mongo_collection,
|
||||
vis_com_name=vis_com_name,
|
||||
annotations=annotations,
|
||||
)
|
||||
# get data
|
||||
with VisualCommunicationRepository() as repo:
|
||||
vis_com_data = repo.get_data(vis_com_name)
|
||||
# data already present, as it was loaded earlier to get here
|
||||
assert vis_com_data is not None
|
||||
# update annotations
|
||||
vis_com_data_updated = vis_com_data.model_copy(
|
||||
update={
|
||||
'annotations': annotation_map,
|
||||
},
|
||||
)
|
||||
# save updated data
|
||||
repo.put_data(vis_com_data_updated)
|
||||
except (ValueError, ValidationError) as exc:
|
||||
msg = f"failed saving annotation: {exc}"
|
||||
msg = f'failed saving annotation: {exc}'
|
||||
logging.warning(msg)
|
||||
response[0] = msg
|
||||
return tuple(response)
|
||||
@@ -234,21 +233,30 @@ def init_app(
|
||||
logging.info('trying to get new visual communication')
|
||||
try:
|
||||
# get data
|
||||
vis_com = get_visual_communication(
|
||||
collection=mongo_collection,
|
||||
with_annotation=False,
|
||||
)
|
||||
with VisualCommunicationRepository() as repo:
|
||||
name_list = repo.list_names()
|
||||
random_name = random.choice(name_list)
|
||||
vis_com = repo.get_data(random_name)
|
||||
# could not get here if data doesn't exist
|
||||
assert vis_com is not None
|
||||
# set variables
|
||||
vis_com_name = vis_com.name
|
||||
assert datastore._client is not None
|
||||
image_src = vis_com.webencoded_image(minio_client=datastore._client)
|
||||
with ImageRepository() as repo:
|
||||
image_data = repo.get_data(vis_com_name)
|
||||
# could not get here if data doesn't exist
|
||||
assert image_data is not None
|
||||
buffer = BytesIO()
|
||||
image_data.image.save(buffer, format='PNG')
|
||||
image_src = 'data:image/png;base64,' + b64encode(buffer.getvalue()).decode(
|
||||
'utf-8',
|
||||
)
|
||||
if vis_com.prediction is not None:
|
||||
# TODO: update to use optional predictions
|
||||
pass
|
||||
else:
|
||||
# reset annotations
|
||||
annotation_values = [None for elem in annotation_values]
|
||||
except NoDocumentFoundException:
|
||||
except Exception:
|
||||
msg = 'no unannotated data in database'
|
||||
logging.warning(msg)
|
||||
response[0] = msg
|
||||
|
||||
+2
-15
@@ -4,15 +4,13 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from shared.datastore import Datastore
|
||||
from shared.docstore import connect_mongodb
|
||||
from shared.utils import check_env, setup_logging
|
||||
|
||||
from .app import init_app
|
||||
|
||||
# ensure env vars set
|
||||
NECESSARY_ENV_VAR_LIST = {
|
||||
'MONGO_HOST',
|
||||
'MONGO_ENDPOINT',
|
||||
'MONGO_DB',
|
||||
'MONGO_COLLECTION',
|
||||
'MONGO_USER',
|
||||
@@ -23,24 +21,13 @@ NECESSARY_ENV_VAR_LIST = {
|
||||
'MINIO_ACCESS_KEY',
|
||||
'MINIO_SECRET_KEY',
|
||||
'MINIO_BUCKET_NAME',
|
||||
'MINIO_BUCKET_NAME_MODELS',
|
||||
}
|
||||
check_env(NECESSARY_ENV_VAR_LIST)
|
||||
|
||||
# setup logging stream handler
|
||||
setup_logging()
|
||||
|
||||
# connect to database
|
||||
collection, db, client = connect_mongodb()
|
||||
|
||||
# connect to minio
|
||||
datastore = Datastore()
|
||||
datastore.connect()
|
||||
|
||||
# initialise application
|
||||
app = init_app(
|
||||
mongo_collection=collection,
|
||||
datastore=datastore,
|
||||
)
|
||||
app = init_app()
|
||||
server = app.server
|
||||
server.config.update(SECRET_KEY=os.urandom(24))
|
||||
|
||||
Reference in New Issue
Block a user