ruff format
This commit is contained in:
@@ -25,7 +25,7 @@ class MinioAdapter(
|
|||||||
access_key_env_var_name: str = "MINIO_ACCESS_KEY"
|
access_key_env_var_name: str = "MINIO_ACCESS_KEY"
|
||||||
secret_key_env_var_name: str = "MINIO_SECRET_KEY"
|
secret_key_env_var_name: str = "MINIO_SECRET_KEY"
|
||||||
bucket_env_var_name: str = "MINIO_BUCKET"
|
bucket_env_var_name: str = "MINIO_BUCKET"
|
||||||
chunk_size: int = 5*2**20 # 5 MiB
|
chunk_size: int = 5 * 2**20 # 5 MiB
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
# Setup logger
|
# Setup logger
|
||||||
@@ -133,7 +133,9 @@ class MinioAdapter(
|
|||||||
length=num_bytes,
|
length=num_bytes,
|
||||||
part_size=self.chunk_size,
|
part_size=self.chunk_size,
|
||||||
)
|
)
|
||||||
self.logger.debug(f"Put object '{object_name}' into bucket '{self._bucket_name}'")
|
self.logger.debug(
|
||||||
|
f"Put object '{object_name}' into bucket '{self._bucket_name}'"
|
||||||
|
)
|
||||||
|
|
||||||
def _get(self, object_name: str) -> BytesIO | None:
|
def _get(self, object_name: str) -> BytesIO | None:
|
||||||
"""Get an object from the Minio bucket."""
|
"""Get an object from the Minio bucket."""
|
||||||
@@ -155,11 +157,15 @@ class MinioAdapter(
|
|||||||
while chunk := response.read(self.chunk_size):
|
while chunk := response.read(self.chunk_size):
|
||||||
buffer.write(chunk)
|
buffer.write(chunk)
|
||||||
buffer.seek(0)
|
buffer.seek(0)
|
||||||
self.logger.debug(f"Got object '{object_name}' from bucket '{self._bucket_name}'")
|
self.logger.debug(
|
||||||
|
f"Got object '{object_name}' from bucket '{self._bucket_name}'"
|
||||||
|
)
|
||||||
return buffer
|
return buffer
|
||||||
except S3Error as exc:
|
except S3Error as exc:
|
||||||
if exc.code == "NoSuchKey":
|
if exc.code == "NoSuchKey":
|
||||||
self.logger.warning(f"Object '{object_name}' not found in bucket '{self._bucket_name}'")
|
self.logger.warning(
|
||||||
|
f"Object '{object_name}' not found in bucket '{self._bucket_name}'"
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
self.logger.error(repr(exc))
|
self.logger.error(repr(exc))
|
||||||
except Exception as exc: # pylint: disable=broad-except
|
except Exception as exc: # pylint: disable=broad-except
|
||||||
@@ -180,7 +186,9 @@ class MinioAdapter(
|
|||||||
bucket_name=self._bucket_name,
|
bucket_name=self._bucket_name,
|
||||||
object_name=object_name,
|
object_name=object_name,
|
||||||
)
|
)
|
||||||
self.logger.debug(f"Deleted object '{object_name}' from bucket '{self._bucket_name}'")
|
self.logger.debug(
|
||||||
|
f"Deleted object '{object_name}' from bucket '{self._bucket_name}'"
|
||||||
|
)
|
||||||
|
|
||||||
def _list_objects(self, prefix: str = "") -> list[str]:
|
def _list_objects(self, prefix: str = "") -> list[str]:
|
||||||
"""List objects in the Minio bucket with an optional prefix."""
|
"""List objects in the Minio bucket with an optional prefix."""
|
||||||
@@ -197,6 +205,10 @@ class MinioAdapter(
|
|||||||
prefix=prefix,
|
prefix=prefix,
|
||||||
recursive=True,
|
recursive=True,
|
||||||
)
|
)
|
||||||
object_names = [obj.object_name for obj in objects if obj.object_name is not None]
|
object_names = [
|
||||||
self.logger.debug(f"Listed {len(object_names)} object(s) in bucket '{self._bucket_name}' with prefix '{prefix}'")
|
obj.object_name for obj in objects if obj.object_name is not None
|
||||||
|
]
|
||||||
|
self.logger.debug(
|
||||||
|
f"Listed {len(object_names)} object(s) in bucket '{self._bucket_name}' with prefix '{prefix}'"
|
||||||
|
)
|
||||||
return object_names
|
return object_names
|
||||||
|
|||||||
@@ -138,4 +138,4 @@ def raw_minio_client(minio_container: dict[str, str]) -> Generator[Minio]:
|
|||||||
# Cleanup
|
# Cleanup
|
||||||
objects = client.list_objects(bucket_name, recursive=True)
|
objects = client.list_objects(bucket_name, recursive=True)
|
||||||
for obj in objects:
|
for obj in objects:
|
||||||
client.remove_object(bucket_name, obj.object_name)
|
client.remove_object(bucket_name, obj.object_name)
|
||||||
|
|||||||
@@ -31,14 +31,14 @@ def same_data(
|
|||||||
# compare size
|
# compare size
|
||||||
if len(data_a_bytes) != len(data_b_bytes):
|
if len(data_a_bytes) != len(data_b_bytes):
|
||||||
logging.error(
|
logging.error(
|
||||||
'data has different length: %s and %s',
|
"data has different length: %s and %s",
|
||||||
len(data_a_bytes),
|
len(data_a_bytes),
|
||||||
len(data_b_bytes),
|
len(data_b_bytes),
|
||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
# compare content
|
# compare content
|
||||||
if data_a_bytes != data_b_bytes:
|
if data_a_bytes != data_b_bytes:
|
||||||
logging.error('data has different bytes')
|
logging.error("data has different bytes")
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -235,7 +235,16 @@ def test_should_log_warning_when_getting_nonexistent_object(
|
|||||||
# Arrange
|
# Arrange
|
||||||
adapter = MinioAdapter()
|
adapter = MinioAdapter()
|
||||||
adapter._client = MagicMock(spec=Minio)
|
adapter._client = MagicMock(spec=Minio)
|
||||||
adapter._client.get_object.side_effect = S3Error(code="NoSuchKey", message="", resource="", request_id="", host_id="", response="", bucket_name="test-bucket", object_name="missing-object")
|
adapter._client.get_object.side_effect = S3Error(
|
||||||
|
code="NoSuchKey",
|
||||||
|
message="",
|
||||||
|
resource="",
|
||||||
|
request_id="",
|
||||||
|
host_id="",
|
||||||
|
response="",
|
||||||
|
bucket_name="test-bucket",
|
||||||
|
object_name="missing-object",
|
||||||
|
)
|
||||||
adapter._bucket_name = "test-bucket"
|
adapter._bucket_name = "test-bucket"
|
||||||
object_name = "missing-object"
|
object_name = "missing-object"
|
||||||
# Act
|
# Act
|
||||||
@@ -243,7 +252,10 @@ def test_should_log_warning_when_getting_nonexistent_object(
|
|||||||
result = adapter._get(object_name)
|
result = adapter._get(object_name)
|
||||||
# Assert
|
# Assert
|
||||||
assert result is None
|
assert result is None
|
||||||
assert f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'" in caplog.text
|
assert (
|
||||||
|
f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'"
|
||||||
|
in caplog.text
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
|
def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
|
||||||
@@ -253,7 +265,16 @@ def test_should_log_error_when_getting_with_s3error_other_than_no_such_key(
|
|||||||
# Arrange
|
# Arrange
|
||||||
adapter = MinioAdapter()
|
adapter = MinioAdapter()
|
||||||
adapter._client = MagicMock(spec=Minio)
|
adapter._client = MagicMock(spec=Minio)
|
||||||
other_s3error = S3Error(code="UnhandledError", message="", resource="", request_id="", host_id="", response="", bucket_name="test-bucket", object_name="missing-object")
|
other_s3error = S3Error(
|
||||||
|
code="UnhandledError",
|
||||||
|
message="",
|
||||||
|
resource="",
|
||||||
|
request_id="",
|
||||||
|
host_id="",
|
||||||
|
response="",
|
||||||
|
bucket_name="test-bucket",
|
||||||
|
object_name="missing-object",
|
||||||
|
)
|
||||||
adapter._client.get_object.side_effect = other_s3error
|
adapter._client.get_object.side_effect = other_s3error
|
||||||
adapter._bucket_name = "test-bucket"
|
adapter._bucket_name = "test-bucket"
|
||||||
object_name = "missing-object"
|
object_name = "missing-object"
|
||||||
|
|||||||
Reference in New Issue
Block a user