381 lines
14 KiB
Python
381 lines
14 KiB
Python
"""Alibaba Cloud OSS SDK V2 adapter for the guarded ARR object-store port."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
from dataclasses import dataclass
|
|
from typing import Any, Mapping, Optional
|
|
from urllib.parse import urlparse
|
|
|
|
from arr_storage.contracts import BackendObject, valid_object_key
|
|
from arr_storage.remote import CloudClientError
|
|
|
|
|
|
BUCKET_RE = re.compile(r"^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$")
|
|
REGION_RE = re.compile(r"^[a-z0-9][a-z0-9-]{1,62}$")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AliyunOssConfig:
|
|
"""Non-secret OSS routing; credentials remain in the SDK provider chain."""
|
|
|
|
region: str
|
|
bucket: str
|
|
endpoint: Optional[str] = None
|
|
|
|
def __post_init__(self) -> None:
|
|
region = self.region.strip().lower()
|
|
bucket = self.bucket.strip().lower()
|
|
endpoint = self.endpoint.strip().rstrip("/") if self.endpoint else None
|
|
if not REGION_RE.fullmatch(region) or not BUCKET_RE.fullmatch(bucket):
|
|
raise ValueError("OSS region or bucket configuration is invalid")
|
|
if endpoint is not None:
|
|
parsed = urlparse(endpoint)
|
|
if parsed.scheme != "https" or not parsed.netloc or parsed.path not in {"", "/"}:
|
|
raise ValueError("ARR_OSS_ENDPOINT must be an HTTPS endpoint origin")
|
|
object.__setattr__(self, "region", region)
|
|
object.__setattr__(self, "bucket", bucket)
|
|
object.__setattr__(self, "endpoint", endpoint)
|
|
|
|
@classmethod
|
|
def from_environment(
|
|
cls,
|
|
environment: Optional[Mapping[str, str]] = None,
|
|
) -> "AliyunOssConfig":
|
|
values = environment if environment is not None else os.environ
|
|
region = str(values.get("ARR_OSS_REGION", "")).strip()
|
|
bucket = str(values.get("ARR_OSS_BUCKET", "")).strip()
|
|
endpoint = str(values.get("ARR_OSS_ENDPOINT", "")).strip() or None
|
|
if not region or not bucket:
|
|
raise ValueError("ARR_OSS_REGION and ARR_OSS_BUCKET are required")
|
|
return cls(region=region, bucket=bucket, endpoint=endpoint)
|
|
|
|
|
|
class _OssReader:
|
|
"""Small file-like wrapper that never leaks provider exception text."""
|
|
|
|
def __init__(self, body: Any) -> None:
|
|
self._body = body
|
|
self._buffer = bytearray()
|
|
self._eof = False
|
|
iterator_factory = getattr(body, "iter_bytes", None)
|
|
try:
|
|
self._iterator = (
|
|
iter(iterator_factory(block_size=1024 * 1024))
|
|
if callable(iterator_factory)
|
|
else None
|
|
)
|
|
except Exception:
|
|
raise OSError("OSS stream initialization failed") from None
|
|
|
|
def read(self, size: int = -1) -> bytes:
|
|
if size == 0:
|
|
return b""
|
|
if self._iterator is not None:
|
|
return self._read_iterator(size)
|
|
try:
|
|
value = self._body.read(size)
|
|
except TypeError:
|
|
try:
|
|
value = self._body.read()
|
|
except Exception:
|
|
raise OSError("OSS stream read failed") from None
|
|
except Exception:
|
|
raise OSError("OSS stream read failed") from None
|
|
if not isinstance(value, bytes):
|
|
raise OSError("OSS stream returned invalid bytes")
|
|
return value
|
|
|
|
def _read_iterator(self, size: int) -> bytes:
|
|
try:
|
|
if size is None or size < 0:
|
|
chunks = [bytes(self._buffer)] if self._buffer else []
|
|
self._buffer.clear()
|
|
while not self._eof:
|
|
chunk = self._next_chunk()
|
|
if chunk:
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
while len(self._buffer) < size and not self._eof:
|
|
chunk = self._next_chunk()
|
|
if chunk:
|
|
self._buffer.extend(chunk)
|
|
value = bytes(self._buffer[:size])
|
|
del self._buffer[:size]
|
|
return value
|
|
except OSError:
|
|
raise
|
|
except Exception:
|
|
raise OSError("OSS stream read failed") from None
|
|
|
|
def _next_chunk(self) -> bytes:
|
|
if self._eof or self._iterator is None:
|
|
return b""
|
|
try:
|
|
chunk = next(self._iterator)
|
|
except StopIteration:
|
|
self._eof = True
|
|
return b""
|
|
if not isinstance(chunk, bytes):
|
|
raise OSError("OSS stream returned invalid bytes")
|
|
return chunk
|
|
|
|
def close(self) -> None:
|
|
try:
|
|
close = getattr(self._body, "close", None)
|
|
if callable(close):
|
|
close()
|
|
except Exception:
|
|
raise OSError("OSS stream close failed") from None
|
|
|
|
def __enter__(self) -> "_OssReader":
|
|
return self
|
|
|
|
def __exit__(self, exc_type: object, exc: object, traceback: object) -> None:
|
|
self.close()
|
|
|
|
|
|
class AliyunOssV2Client:
|
|
"""CloudClientPort implemented with ``alibabacloud_oss_v2``.
|
|
|
|
The bucket is required to be unversioned because OSS ignores
|
|
``forbid_overwrite`` when versioning is enabled or suspended. This is a
|
|
fail-closed invariant for the staged/committed object workflow.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
config: AliyunOssConfig,
|
|
*,
|
|
sdk: Optional[Any] = None,
|
|
client: Optional[Any] = None,
|
|
) -> None:
|
|
if sdk is None:
|
|
try:
|
|
import alibabacloud_oss_v2 as sdk_module # type: ignore[import-not-found]
|
|
except ImportError:
|
|
raise CloudClientError("driver_unavailable") from None
|
|
sdk = sdk_module
|
|
self._sdk = sdk
|
|
self._config = config
|
|
self._http_session: Optional[Any] = None
|
|
if client is None:
|
|
http_session: Optional[Any] = None
|
|
try:
|
|
import requests
|
|
|
|
credentials = sdk.credentials.EnvironmentVariableCredentialsProvider()
|
|
sdk_config = sdk.config.load_default()
|
|
sdk_config.credentials_provider = credentials
|
|
sdk_config.region = config.region
|
|
if config.endpoint is not None:
|
|
sdk_config.endpoint = config.endpoint
|
|
# Never send OSS credentials through an ambient desktop/system
|
|
# proxy. Any deployment proxy must be configured explicitly.
|
|
http_session = requests.Session()
|
|
http_session.trust_env = False
|
|
sdk_config.http_client = sdk.transport.RequestsHttpClient(
|
|
session=http_session
|
|
)
|
|
client = sdk.Client(sdk_config)
|
|
except Exception as error:
|
|
if http_session is not None:
|
|
http_session.close()
|
|
self._raise_mapped(error)
|
|
self._http_session = http_session
|
|
self._client = client
|
|
|
|
@property
|
|
def bucket(self) -> str:
|
|
return self._config.bucket
|
|
|
|
def assert_immutable_writes_supported(self) -> None:
|
|
"""Require encrypted, unversioned storage with no anonymous writes."""
|
|
|
|
try:
|
|
result = self._client.get_bucket_info(
|
|
self._sdk.GetBucketInfoRequest(bucket=self.bucket)
|
|
)
|
|
except Exception as error:
|
|
self._raise_mapped(error)
|
|
info = getattr(result, "bucket_info", None)
|
|
if info is None:
|
|
raise CloudClientError("unavailable")
|
|
location = str(getattr(info, "location", "") or "").strip().lower()
|
|
if location not in {self._config.region, f"oss-{self._config.region}"}:
|
|
raise CloudClientError("region_mismatch")
|
|
acl = str(getattr(info, "acl", "") or "").strip().lower()
|
|
if acl not in {"private", "public-read"}:
|
|
raise CloudClientError("public_access_incompatible")
|
|
if getattr(info, "sse_rule", None) is None:
|
|
raise CloudClientError("encryption_incompatible")
|
|
status = str(getattr(info, "versioning", "") or "").strip().lower()
|
|
if status in {"enabled", "suspended"}:
|
|
raise CloudClientError("versioning_incompatible")
|
|
|
|
def upload_file_if_absent(
|
|
self,
|
|
object_key: str,
|
|
source: str,
|
|
mime_type: str,
|
|
metadata: Mapping[str, str],
|
|
) -> BackendObject:
|
|
self._validate_request(object_key, metadata)
|
|
try:
|
|
self._client.put_object_from_file(
|
|
self._sdk.PutObjectRequest(
|
|
bucket=self.bucket,
|
|
key=object_key,
|
|
acl=self._object_acl(object_key, metadata),
|
|
content_type=mime_type,
|
|
metadata=dict(metadata),
|
|
forbid_overwrite=True,
|
|
),
|
|
source,
|
|
)
|
|
except Exception as error:
|
|
self._raise_mapped(error)
|
|
return self.stat_object(object_key)
|
|
|
|
def stat_object(self, object_key: str) -> BackendObject:
|
|
self._validate_key(object_key)
|
|
try:
|
|
result = self._client.head_object(
|
|
self._sdk.HeadObjectRequest(bucket=self.bucket, key=object_key)
|
|
)
|
|
except Exception as error:
|
|
self._raise_mapped(error)
|
|
raw_metadata = getattr(result, "metadata", None) or {}
|
|
if not isinstance(raw_metadata, Mapping) or any(
|
|
not isinstance(key, str) or not isinstance(value, str)
|
|
for key, value in raw_metadata.items()
|
|
):
|
|
raise CloudClientError("unavailable")
|
|
metadata = dict(raw_metadata)
|
|
content_type = getattr(result, "content_type", None)
|
|
if isinstance(content_type, str) and content_type:
|
|
metadata.setdefault("arr-backend-content-type", content_type)
|
|
content_length = getattr(result, "content_length", None)
|
|
if not isinstance(content_length, int) or isinstance(content_length, bool) or content_length < 0:
|
|
raise CloudClientError("unavailable")
|
|
return BackendObject(
|
|
object_key=object_key,
|
|
byte_size=content_length,
|
|
metadata=metadata,
|
|
etag=self._optional_text(getattr(result, "etag", None)),
|
|
version_id=self._optional_text(getattr(result, "version_id", None)),
|
|
)
|
|
|
|
def stream_object(self, object_key: str) -> _OssReader:
|
|
self._validate_key(object_key)
|
|
try:
|
|
result = self._client.get_object(
|
|
self._sdk.GetObjectRequest(bucket=self.bucket, key=object_key)
|
|
)
|
|
except Exception as error:
|
|
self._raise_mapped(error)
|
|
body = getattr(result, "body", None)
|
|
if body is None or not callable(getattr(body, "read", None)):
|
|
raise CloudClientError("unavailable")
|
|
return _OssReader(body)
|
|
|
|
def copy_object_if_absent(
|
|
self,
|
|
source_key: str,
|
|
destination_key: str,
|
|
metadata: Mapping[str, str],
|
|
) -> BackendObject:
|
|
self._validate_key(source_key)
|
|
self._validate_request(destination_key, metadata)
|
|
try:
|
|
self._client.copy_object(
|
|
self._sdk.CopyObjectRequest(
|
|
bucket=self.bucket,
|
|
key=destination_key,
|
|
source_bucket=self.bucket,
|
|
source_key=source_key,
|
|
acl=self._object_acl(destination_key, metadata),
|
|
metadata=dict(metadata),
|
|
metadata_directive="REPLACE",
|
|
content_type=metadata.get("arr-mime-type"),
|
|
forbid_overwrite=True,
|
|
)
|
|
)
|
|
except Exception as error:
|
|
self._raise_mapped(error)
|
|
return self.stat_object(destination_key)
|
|
|
|
def delete_exact_object(self, object_key: str) -> None:
|
|
self._validate_key(object_key)
|
|
try:
|
|
self._client.delete_object(
|
|
self._sdk.DeleteObjectRequest(bucket=self.bucket, key=object_key)
|
|
)
|
|
except Exception as error:
|
|
self._raise_mapped(error)
|
|
|
|
def close(self) -> None:
|
|
close = getattr(self._client, "close", None)
|
|
if callable(close):
|
|
try:
|
|
close()
|
|
except Exception:
|
|
pass
|
|
if self._http_session is not None:
|
|
try:
|
|
self._http_session.close()
|
|
except Exception:
|
|
pass
|
|
self._http_session = None
|
|
|
|
@staticmethod
|
|
def _optional_text(value: Any) -> Optional[str]:
|
|
return value if isinstance(value, str) and value else None
|
|
|
|
@staticmethod
|
|
def _object_acl(object_key: str, metadata: Mapping[str, str]) -> str:
|
|
del object_key, metadata
|
|
return "private"
|
|
|
|
@staticmethod
|
|
def _validate_key(object_key: str) -> None:
|
|
if not valid_object_key(object_key):
|
|
raise CloudClientError("invalid_request")
|
|
|
|
@classmethod
|
|
def _validate_request(cls, object_key: str, metadata: Mapping[str, str]) -> None:
|
|
cls._validate_key(object_key)
|
|
if any(
|
|
not isinstance(key, str)
|
|
or not key
|
|
or not isinstance(value, str)
|
|
for key, value in metadata.items()
|
|
):
|
|
raise CloudClientError("invalid_request")
|
|
|
|
@staticmethod
|
|
def _raise_mapped(error: BaseException) -> None:
|
|
unwrapped: Any = error
|
|
unwrap = getattr(error, "unwrap", None)
|
|
if callable(unwrap):
|
|
try:
|
|
unwrapped = unwrap()
|
|
except Exception:
|
|
unwrapped = error
|
|
status = getattr(unwrapped, "status_code", None)
|
|
code = str(getattr(unwrapped, "code", "") or "")
|
|
if status == 404 or code in {"NoSuchKey", "NoSuchBucket", "NotFound"}:
|
|
kind = "not_found"
|
|
elif status in {409, 412} or code in {
|
|
"FileAlreadyExists",
|
|
"FileImmutable",
|
|
"PreconditionFailed",
|
|
}:
|
|
kind = "conflict"
|
|
elif status in {400, 401, 403}:
|
|
kind = "invalid_request" if status == 400 else "forbidden"
|
|
else:
|
|
kind = "unavailable"
|
|
raise CloudClientError(kind) from None
|