feat: prepare ARR for controlled public deployment
This commit is contained in:
Wyndham ARR
379
arr_storage/aliyun_oss_v2.py
Normal file
379
arr_storage/aliyun_oss_v2.py
Normal file
@@ -0,0 +1,379 @@
|
||||
"""Alibaba Cloud OSS SDK V2 adapter for the guarded ARR object-store port."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from arr_storage.contracts import BackendObject, valid_object_key
|
||||
from arr_storage.remote import CloudClientError
|
||||
|
||||
|
||||
BUCKET_RE = re.compile(r"^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$")
|
||||
REGION_RE = re.compile(r"^[a-z0-9][a-z0-9-]{1,62}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AliyunOssConfig:
|
||||
"""Non-secret OSS routing; credentials remain in the SDK provider chain."""
|
||||
|
||||
region: str
|
||||
bucket: str
|
||||
endpoint: Optional[str] = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
region = self.region.strip().lower()
|
||||
bucket = self.bucket.strip().lower()
|
||||
endpoint = self.endpoint.strip().rstrip("/") if self.endpoint else None
|
||||
if not REGION_RE.fullmatch(region) or not BUCKET_RE.fullmatch(bucket):
|
||||
raise ValueError("OSS region or bucket configuration is invalid")
|
||||
if endpoint is not None:
|
||||
parsed = urlparse(endpoint)
|
||||
if parsed.scheme != "https" or not parsed.netloc or parsed.path not in {"", "/"}:
|
||||
raise ValueError("ARR_OSS_ENDPOINT must be an HTTPS endpoint origin")
|
||||
object.__setattr__(self, "region", region)
|
||||
object.__setattr__(self, "bucket", bucket)
|
||||
object.__setattr__(self, "endpoint", endpoint)
|
||||
|
||||
@classmethod
|
||||
def from_environment(
|
||||
cls,
|
||||
environment: Optional[Mapping[str, str]] = None,
|
||||
) -> "AliyunOssConfig":
|
||||
values = environment if environment is not None else os.environ
|
||||
region = str(values.get("ARR_OSS_REGION", "")).strip()
|
||||
bucket = str(values.get("ARR_OSS_BUCKET", "")).strip()
|
||||
endpoint = str(values.get("ARR_OSS_ENDPOINT", "")).strip() or None
|
||||
if not region or not bucket:
|
||||
raise ValueError("ARR_OSS_REGION and ARR_OSS_BUCKET are required")
|
||||
return cls(region=region, bucket=bucket, endpoint=endpoint)
|
||||
|
||||
|
||||
class _OssReader:
|
||||
"""Small file-like wrapper that never leaks provider exception text."""
|
||||
|
||||
def __init__(self, body: Any) -> None:
|
||||
self._body = body
|
||||
self._buffer = bytearray()
|
||||
self._eof = False
|
||||
iterator_factory = getattr(body, "iter_bytes", None)
|
||||
try:
|
||||
self._iterator = (
|
||||
iter(iterator_factory(block_size=1024 * 1024))
|
||||
if callable(iterator_factory)
|
||||
else None
|
||||
)
|
||||
except Exception:
|
||||
raise OSError("OSS stream initialization failed") from None
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
if size == 0:
|
||||
return b""
|
||||
if self._iterator is not None:
|
||||
return self._read_iterator(size)
|
||||
try:
|
||||
value = self._body.read(size)
|
||||
except TypeError:
|
||||
try:
|
||||
value = self._body.read()
|
||||
except Exception:
|
||||
raise OSError("OSS stream read failed") from None
|
||||
except Exception:
|
||||
raise OSError("OSS stream read failed") from None
|
||||
if not isinstance(value, bytes):
|
||||
raise OSError("OSS stream returned invalid bytes")
|
||||
return value
|
||||
|
||||
def _read_iterator(self, size: int) -> bytes:
|
||||
try:
|
||||
if size is None or size < 0:
|
||||
chunks = [bytes(self._buffer)] if self._buffer else []
|
||||
self._buffer.clear()
|
||||
while not self._eof:
|
||||
chunk = self._next_chunk()
|
||||
if chunk:
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
while len(self._buffer) < size and not self._eof:
|
||||
chunk = self._next_chunk()
|
||||
if chunk:
|
||||
self._buffer.extend(chunk)
|
||||
value = bytes(self._buffer[:size])
|
||||
del self._buffer[:size]
|
||||
return value
|
||||
except OSError:
|
||||
raise
|
||||
except Exception:
|
||||
raise OSError("OSS stream read failed") from None
|
||||
|
||||
def _next_chunk(self) -> bytes:
|
||||
if self._eof or self._iterator is None:
|
||||
return b""
|
||||
try:
|
||||
chunk = next(self._iterator)
|
||||
except StopIteration:
|
||||
self._eof = True
|
||||
return b""
|
||||
if not isinstance(chunk, bytes):
|
||||
raise OSError("OSS stream returned invalid bytes")
|
||||
return chunk
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
close = getattr(self._body, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
except Exception:
|
||||
raise OSError("OSS stream close failed") from None
|
||||
|
||||
def __enter__(self) -> "_OssReader":
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type: object, exc: object, traceback: object) -> None:
|
||||
self.close()
|
||||
|
||||
|
||||
class AliyunOssV2Client:
|
||||
"""CloudClientPort implemented with ``alibabacloud_oss_v2``.
|
||||
|
||||
The bucket is required to be unversioned because OSS ignores
|
||||
``forbid_overwrite`` when versioning is enabled or suspended. This is a
|
||||
fail-closed invariant for the staged/committed object workflow.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AliyunOssConfig,
|
||||
*,
|
||||
sdk: Optional[Any] = None,
|
||||
client: Optional[Any] = None,
|
||||
) -> None:
|
||||
if sdk is None:
|
||||
try:
|
||||
import alibabacloud_oss_v2 as sdk_module # type: ignore[import-not-found]
|
||||
except ImportError:
|
||||
raise CloudClientError("driver_unavailable") from None
|
||||
sdk = sdk_module
|
||||
self._sdk = sdk
|
||||
self._config = config
|
||||
self._http_session: Optional[Any] = None
|
||||
if client is None:
|
||||
http_session: Optional[Any] = None
|
||||
try:
|
||||
import requests
|
||||
|
||||
credentials = sdk.credentials.EnvironmentVariableCredentialsProvider()
|
||||
sdk_config = sdk.config.load_default()
|
||||
sdk_config.credentials_provider = credentials
|
||||
sdk_config.region = config.region
|
||||
if config.endpoint is not None:
|
||||
sdk_config.endpoint = config.endpoint
|
||||
# Never send OSS credentials through an ambient desktop/system
|
||||
# proxy. Any deployment proxy must be configured explicitly.
|
||||
http_session = requests.Session()
|
||||
http_session.trust_env = False
|
||||
sdk_config.http_client = sdk.transport.RequestsHttpClient(
|
||||
session=http_session
|
||||
)
|
||||
client = sdk.Client(sdk_config)
|
||||
except Exception as error:
|
||||
if http_session is not None:
|
||||
http_session.close()
|
||||
self._raise_mapped(error)
|
||||
self._http_session = http_session
|
||||
self._client = client
|
||||
|
||||
@property
|
||||
def bucket(self) -> str:
|
||||
return self._config.bucket
|
||||
|
||||
def assert_immutable_writes_supported(self) -> None:
|
||||
"""Reject buckets that cannot safely hold private immutable ARR data."""
|
||||
|
||||
try:
|
||||
result = self._client.get_bucket_info(
|
||||
self._sdk.GetBucketInfoRequest(bucket=self.bucket)
|
||||
)
|
||||
except Exception as error:
|
||||
self._raise_mapped(error)
|
||||
info = getattr(result, "bucket_info", None)
|
||||
if info is None:
|
||||
raise CloudClientError("unavailable")
|
||||
location = str(getattr(info, "location", "") or "").strip().lower()
|
||||
if location not in {self._config.region, f"oss-{self._config.region}"}:
|
||||
raise CloudClientError("region_mismatch")
|
||||
acl = str(getattr(info, "acl", "") or "").strip().lower()
|
||||
# A public-read bucket is an approved deployment choice for this
|
||||
# integration. Every ARR object is still written with an explicit
|
||||
# private object ACL, which overrides the bucket ACL. Never accept a
|
||||
# bucket that grants anonymous writes.
|
||||
if acl not in {"private", "public-read"}:
|
||||
raise CloudClientError("public_access_incompatible")
|
||||
if getattr(info, "sse_rule", None) is None:
|
||||
raise CloudClientError("encryption_incompatible")
|
||||
status = str(getattr(info, "versioning", "") or "").strip().lower()
|
||||
if status in {"enabled", "suspended"}:
|
||||
raise CloudClientError("versioning_incompatible")
|
||||
|
||||
def upload_file_if_absent(
|
||||
self,
|
||||
object_key: str,
|
||||
source: str,
|
||||
mime_type: str,
|
||||
metadata: Mapping[str, str],
|
||||
) -> BackendObject:
|
||||
self._validate_request(object_key, metadata)
|
||||
try:
|
||||
self._client.put_object_from_file(
|
||||
self._sdk.PutObjectRequest(
|
||||
bucket=self.bucket,
|
||||
key=object_key,
|
||||
acl="private",
|
||||
content_type=mime_type,
|
||||
metadata=dict(metadata),
|
||||
forbid_overwrite=True,
|
||||
),
|
||||
source,
|
||||
)
|
||||
except Exception as error:
|
||||
self._raise_mapped(error)
|
||||
return self.stat_object(object_key)
|
||||
|
||||
def stat_object(self, object_key: str) -> BackendObject:
|
||||
self._validate_key(object_key)
|
||||
try:
|
||||
result = self._client.head_object(
|
||||
self._sdk.HeadObjectRequest(bucket=self.bucket, key=object_key)
|
||||
)
|
||||
except Exception as error:
|
||||
self._raise_mapped(error)
|
||||
raw_metadata = getattr(result, "metadata", None) or {}
|
||||
if not isinstance(raw_metadata, Mapping) or any(
|
||||
not isinstance(key, str) or not isinstance(value, str)
|
||||
for key, value in raw_metadata.items()
|
||||
):
|
||||
raise CloudClientError("unavailable")
|
||||
metadata = dict(raw_metadata)
|
||||
content_type = getattr(result, "content_type", None)
|
||||
if isinstance(content_type, str) and content_type:
|
||||
metadata.setdefault("arr-backend-content-type", content_type)
|
||||
content_length = getattr(result, "content_length", None)
|
||||
if not isinstance(content_length, int) or isinstance(content_length, bool) or content_length < 0:
|
||||
raise CloudClientError("unavailable")
|
||||
return BackendObject(
|
||||
object_key=object_key,
|
||||
byte_size=content_length,
|
||||
metadata=metadata,
|
||||
etag=self._optional_text(getattr(result, "etag", None)),
|
||||
version_id=self._optional_text(getattr(result, "version_id", None)),
|
||||
)
|
||||
|
||||
def stream_object(self, object_key: str) -> _OssReader:
|
||||
self._validate_key(object_key)
|
||||
try:
|
||||
result = self._client.get_object(
|
||||
self._sdk.GetObjectRequest(bucket=self.bucket, key=object_key)
|
||||
)
|
||||
except Exception as error:
|
||||
self._raise_mapped(error)
|
||||
body = getattr(result, "body", None)
|
||||
if body is None or not callable(getattr(body, "read", None)):
|
||||
raise CloudClientError("unavailable")
|
||||
return _OssReader(body)
|
||||
|
||||
def copy_object_if_absent(
|
||||
self,
|
||||
source_key: str,
|
||||
destination_key: str,
|
||||
metadata: Mapping[str, str],
|
||||
) -> BackendObject:
|
||||
self._validate_key(source_key)
|
||||
self._validate_request(destination_key, metadata)
|
||||
try:
|
||||
self._client.copy_object(
|
||||
self._sdk.CopyObjectRequest(
|
||||
bucket=self.bucket,
|
||||
key=destination_key,
|
||||
source_bucket=self.bucket,
|
||||
source_key=source_key,
|
||||
acl="private",
|
||||
metadata=dict(metadata),
|
||||
metadata_directive="REPLACE",
|
||||
content_type=metadata.get("arr-mime-type"),
|
||||
forbid_overwrite=True,
|
||||
)
|
||||
)
|
||||
except Exception as error:
|
||||
self._raise_mapped(error)
|
||||
return self.stat_object(destination_key)
|
||||
|
||||
def delete_exact_object(self, object_key: str) -> None:
|
||||
self._validate_key(object_key)
|
||||
try:
|
||||
self._client.delete_object(
|
||||
self._sdk.DeleteObjectRequest(bucket=self.bucket, key=object_key)
|
||||
)
|
||||
except Exception as error:
|
||||
self._raise_mapped(error)
|
||||
|
||||
def close(self) -> None:
|
||||
close = getattr(self._client, "close", None)
|
||||
if callable(close):
|
||||
try:
|
||||
close()
|
||||
except Exception:
|
||||
pass
|
||||
if self._http_session is not None:
|
||||
try:
|
||||
self._http_session.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._http_session = None
|
||||
|
||||
@staticmethod
|
||||
def _optional_text(value: Any) -> Optional[str]:
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
@staticmethod
|
||||
def _validate_key(object_key: str) -> None:
|
||||
if not valid_object_key(object_key):
|
||||
raise CloudClientError("invalid_request")
|
||||
|
||||
@classmethod
|
||||
def _validate_request(cls, object_key: str, metadata: Mapping[str, str]) -> None:
|
||||
cls._validate_key(object_key)
|
||||
if any(
|
||||
not isinstance(key, str)
|
||||
or not key
|
||||
or not isinstance(value, str)
|
||||
for key, value in metadata.items()
|
||||
):
|
||||
raise CloudClientError("invalid_request")
|
||||
|
||||
@staticmethod
|
||||
def _raise_mapped(error: BaseException) -> None:
|
||||
unwrapped: Any = error
|
||||
unwrap = getattr(error, "unwrap", None)
|
||||
if callable(unwrap):
|
||||
try:
|
||||
unwrapped = unwrap()
|
||||
except Exception:
|
||||
unwrapped = error
|
||||
status = getattr(unwrapped, "status_code", None)
|
||||
code = str(getattr(unwrapped, "code", "") or "")
|
||||
if status == 404 or code in {"NoSuchKey", "NoSuchBucket", "NotFound"}:
|
||||
kind = "not_found"
|
||||
elif status in {409, 412} or code in {
|
||||
"FileAlreadyExists",
|
||||
"FileImmutable",
|
||||
"PreconditionFailed",
|
||||
}:
|
||||
kind = "conflict"
|
||||
elif status in {400, 401, 403}:
|
||||
kind = "invalid_request" if status == 400 else "forbidden"
|
||||
else:
|
||||
kind = "unavailable"
|
||||
raise CloudClientError(kind) from None
|
||||
Reference in New Issue
Block a user