feat: sync latest ARR implementation
This commit is contained in:
Wyndham ARR
@@ -1,76 +1,20 @@
|
||||
# ARR private object exchange
|
||||
# ARR guarded object storage
|
||||
|
||||
`arr_storage` is the ARR-owned binary exchange boundary for the Opera XML flow. It is infrastructure only: it does not parse XML, invoke an Agent, generate reports, or write business facts.
|
||||
`ManagedObjectStore` owns an immutable staged-to-committed workflow for source and generated artifacts. Canonical keys
|
||||
contain only opaque job/attempt identity and role-controlled filenames; user filenames never enter object keys.
|
||||
|
||||
## Frozen object contract
|
||||
Every object carries job, attempt, state, role, SHA-256, exact byte size and MIME metadata. Upload snapshots and hashes
|
||||
the local file, conditionally creates a staged object, conditionally copies to the committed key, verifies committed
|
||||
bytes, and deletes only the exact staged object.
|
||||
|
||||
Every object is private and immutable. The only accepted key shape is:
|
||||
ARR2 OSS requirements:
|
||||
|
||||
```text
|
||||
{prefix}/jobs/{opaque_job_id}/attempts/{0001..9999}/{staged|committed}/{role}/{canonical_filename}
|
||||
```
|
||||
- configured region matches the bucket and server-side encryption is enabled;
|
||||
- versioning is neither Enabled nor Suspended;
|
||||
- bucket ACL is `private` or `public-read`, never `public-read-write`;
|
||||
- every ARR-managed object is explicitly written with object ACL `private`;
|
||||
- credentials come from the SDK environment/RAM/STS chain and ambient desktop proxies are disabled.
|
||||
|
||||
The default prefix is `arr`. Object-key filenames are controlled (`source.xml`, `daily-report.xlsx`, `result.json`, `structured-result.json`, `exception-report.xlsx`), so an uploaded user filename cannot leak guest or booking data through an OSS key. The delivery filename remains separate and is validated before it can enter `ArtifactRef`.
|
||||
|
||||
Objects carry Schema 1.0 metadata for job, attempt, role, state, SHA-256, exact byte size, and MIME type. ETag is stored only as provider evidence; it is never treated as a content hash. A cloud adapter should preserve a provider version ID when bucket versioning exposes one.
|
||||
|
||||
Upload is two-step:
|
||||
|
||||
1. copy the input into a private snapshot while computing SHA-256 and enforcing the role limit;
|
||||
2. create `staged` with an if-absent condition, server-side copy it to a fresh `committed` key, download/hash-check the committed object, then delete only the exact staged object.
|
||||
|
||||
No code path overwrites a committed key. An idempotent retry succeeds only when the existing object has the same metadata and bytes. Only committed keys implement `ArtifactStore.materialize()` and may enter a `DeliveryEnvelope`.
|
||||
|
||||
## Cloud adapter boundary
|
||||
|
||||
`CloudObjectBackend` accepts a narrow `CloudClientPort`. `AliyunOssV2Client` implements it with the Aliyun OSS Python SDK V2: conditional upload, head, streamed read, conditional server-side copy, exact-key delete, normalized custom metadata/content type and safe provider error classes. Startup queries bucket info and fails closed unless the region matches, the bucket does not allow anonymous writes, server-side encryption is configured and versioning is off. A `public-read` bucket is supported by explicit deployment decision; every ARR put/copy sets object ACL `private`, which prevents new processing outputs from inheriting public read access.
|
||||
|
||||
Install `requirements-oss.txt`, then inject `ARR_OSS_REGION`, `ARR_OSS_BUCKET` and optional HTTPS `ARR_OSS_ENDPOINT`. Credentials use the SDK environment credential provider (RAM/STS variables); no access key is accepted from an Agent message or source file. The adapter can be present while live readiness remains false when deployment values are absent.
|
||||
|
||||
The SDK transport uses a dedicated HTTP session with ambient desktop/system proxies disabled, so OSS credentials are not silently routed through an unrelated proxy. A deployment that requires an outbound proxy must add and review an explicit transport configuration rather than relying on inherited OS settings.
|
||||
|
||||
Provider requirements:
|
||||
|
||||
- bucket ACL is `private` or the explicitly approved `public-read` mode, TLS is required, and server-side encryption is enabled; `public-read-write` is always rejected;
|
||||
- ARR and the runtime fetch provider use separate least-privilege identities;
|
||||
- ARR can put/head/get/copy exact `arr/jobs/` keys; the runtime can only read the resolved committed source object;
|
||||
- credentials come from platform secret/instance-role facilities, never prompts, object metadata, URLs, source files, `.env.example`, or logs;
|
||||
- bucket versioning status must be `Off`; the adapter fails closed for `Enabled` or `Suspended`, because the required forbid-overwrite condition is not honored in those modes;
|
||||
- conditional create must be real provider-side `If-None-Match`/forbid-overwrite behavior, not a head-then-overwrite sequence.
|
||||
|
||||
## Agent source fetch
|
||||
|
||||
ARR uploads the XML itself and records the committed object before Agent dispatch. The program message contains one attachment-shaped descriptor with bucket, endpoint, exact object key, hash and byte size. Those values are routing and integrity metadata, not download credentials. The installed `fetch_oss_file` provider owns its OSS credential and must be restricted to read-only access to the ARR source prefix.
|
||||
|
||||
The Agent must not use an OSS SDK, URL or user-supplied key, and it must hash/size-check the materialized file before invoking the Skill.
|
||||
|
||||
## Optional short-lived fetch authorization
|
||||
|
||||
`InMemoryReadGrantBroker` and `PostgresReadGrantBroker` remain available for a future runtime that supports opaque, job-bound, single-read grants. They are not required by the currently installed `fetch_oss_file` contract, which consumes the attachment-shaped OSS descriptor directly.
|
||||
|
||||
## Lifecycle
|
||||
|
||||
- incomplete `staged` objects: provider lifecycle deletion after 24 hours;
|
||||
- committed source XML and Agent outputs: no automatic deletion within their business month; retain according to the approved data-retention policy after that month;
|
||||
- generated monthly/channel downloads: governed by their report archive policy, not by the Agent exchange prefix;
|
||||
- short-lived grants: at most 300 seconds and one materialization;
|
||||
- logs/outbox: opaque IDs, state, SHA-256, byte size and safe error code only; never signed URLs, raw filenames, XML bytes, guest fields, or credentials.
|
||||
|
||||
Local wiring:
|
||||
|
||||
```python
|
||||
from pathlib import Path
|
||||
|
||||
from arr_storage import FilesystemObjectBackend, ManagedObjectStore
|
||||
|
||||
backend = FilesystemObjectBackend(Path("/private/arr-object-store"))
|
||||
store = ManagedObjectStore(backend)
|
||||
committed = store.upload_committed(
|
||||
job_id="job-001",
|
||||
attempt_no=1,
|
||||
role="source_xml",
|
||||
source=Path("/private/upload/input.xml"),
|
||||
original_filename="source.xml",
|
||||
)
|
||||
source_ref = committed.to_artifact_ref()
|
||||
```
|
||||
The active flow materializes the committed source into a private temporary directory before processing. No public URL,
|
||||
Provider, fetch grant or external downloader is used. Legacy grant/output-exchange modules remain inactive compatibility
|
||||
code.
|
||||
|
||||
@@ -191,7 +191,7 @@ class AliyunOssV2Client:
|
||||
return self._config.bucket
|
||||
|
||||
def assert_immutable_writes_supported(self) -> None:
|
||||
"""Reject buckets that cannot safely hold private immutable ARR data."""
|
||||
"""Require encrypted, unversioned storage with no anonymous writes."""
|
||||
|
||||
try:
|
||||
result = self._client.get_bucket_info(
|
||||
@@ -206,10 +206,6 @@ class AliyunOssV2Client:
|
||||
if location not in {self._config.region, f"oss-{self._config.region}"}:
|
||||
raise CloudClientError("region_mismatch")
|
||||
acl = str(getattr(info, "acl", "") or "").strip().lower()
|
||||
# A public-read bucket is an approved deployment choice for this
|
||||
# integration. Every ARR object is still written with an explicit
|
||||
# private object ACL, which overrides the bucket ACL. Never accept a
|
||||
# bucket that grants anonymous writes.
|
||||
if acl not in {"private", "public-read"}:
|
||||
raise CloudClientError("public_access_incompatible")
|
||||
if getattr(info, "sse_rule", None) is None:
|
||||
@@ -231,7 +227,7 @@ class AliyunOssV2Client:
|
||||
self._sdk.PutObjectRequest(
|
||||
bucket=self.bucket,
|
||||
key=object_key,
|
||||
acl="private",
|
||||
acl=self._object_acl(object_key, metadata),
|
||||
content_type=mime_type,
|
||||
metadata=dict(metadata),
|
||||
forbid_overwrite=True,
|
||||
@@ -299,7 +295,7 @@ class AliyunOssV2Client:
|
||||
key=destination_key,
|
||||
source_bucket=self.bucket,
|
||||
source_key=source_key,
|
||||
acl="private",
|
||||
acl=self._object_acl(destination_key, metadata),
|
||||
metadata=dict(metadata),
|
||||
metadata_directive="REPLACE",
|
||||
content_type=metadata.get("arr-mime-type"),
|
||||
@@ -337,6 +333,11 @@ class AliyunOssV2Client:
|
||||
def _optional_text(value: Any) -> Optional[str]:
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
@staticmethod
|
||||
def _object_acl(object_key: str, metadata: Mapping[str, str]) -> str:
|
||||
del object_key, metadata
|
||||
return "private"
|
||||
|
||||
@staticmethod
|
||||
def _validate_key(object_key: str) -> None:
|
||||
if not valid_object_key(object_key):
|
||||
|
||||
@@ -20,6 +20,7 @@ OBJECT_METADATA_SCHEMA = "1.0"
|
||||
OBJECT_STATES = frozenset({"staged", "committed"})
|
||||
CANONICAL_OBJECT_FILENAMES = {
|
||||
"source_xml": "source.xml",
|
||||
"booking_source": "booking-source.xlsx",
|
||||
"daily_report": "daily-report.xlsx",
|
||||
"result_json": "result.json",
|
||||
"structured_result_json": "structured-result.json",
|
||||
|
||||
Reference in New Issue
Block a user