feat: prepare ARR for controlled public deployment
This commit is contained in:
Wyndham ARR
184
arr_processing/registration.py
Normal file
184
arr_processing/registration.py
Normal file
@@ -0,0 +1,184 @@
|
||||
"""ARR-side registration of authenticated remote file handles into private storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Dict, Mapping, Protocol
|
||||
|
||||
from arr_ingestion.contracts import (
|
||||
ARTIFACT_SIZE_LIMITS,
|
||||
DELIVERY_SCHEMA_VERSION,
|
||||
ArtifactRef,
|
||||
DeliveryEnvelope,
|
||||
)
|
||||
from arr_processing.contracts import ProcessingResult, ResultArtifactHandle
|
||||
from arr_processing.errors import ProcessingError, ProcessingTransportError
|
||||
from arr_storage.contracts import StoredObject
|
||||
from arr_storage.store import ManagedObjectStore
|
||||
|
||||
|
||||
class RemoteFilePort(Protocol):
|
||||
"""Official runtime/file API seam; it receives only one opaque file handle."""
|
||||
|
||||
def materialize(self, file_handle: str, destination: Path, max_bytes: int) -> None:
|
||||
...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RegisteredDelivery:
|
||||
envelope: DeliveryEnvelope
|
||||
raw_envelope: bytes
|
||||
output_objects: Mapping[str, StoredObject]
|
||||
|
||||
|
||||
class ProcessingOutputRegistrar:
|
||||
def __init__(
|
||||
self,
|
||||
remote_files: RemoteFilePort,
|
||||
object_store: ManagedObjectStore,
|
||||
) -> None:
|
||||
self._remote_files = remote_files
|
||||
self._object_store = object_store
|
||||
|
||||
def register(
|
||||
self,
|
||||
result: ProcessingResult,
|
||||
source: ArtifactRef,
|
||||
) -> RegisteredDelivery:
|
||||
if source.role != "source_xml":
|
||||
raise ProcessingError(
|
||||
"PROCESSING_SOURCE_INVALID", "processing source artifact is invalid"
|
||||
)
|
||||
verified_source = self._object_store.inspect_committed(
|
||||
source.object_key,
|
||||
source.original_filename,
|
||||
).to_artifact_ref()
|
||||
if verified_source != source:
|
||||
raise ProcessingError(
|
||||
"PROCESSING_SOURCE_INVALID", "processing source identity does not match"
|
||||
)
|
||||
handles = [
|
||||
artifact.file_handle
|
||||
for artifact in result.artifacts.values()
|
||||
if artifact is not None
|
||||
]
|
||||
if len(handles) != len(set(handles)):
|
||||
raise ProcessingError(
|
||||
"PROCESSING_ARTIFACT_INVALID", "processing artifact handles are not unique"
|
||||
)
|
||||
|
||||
stored: Dict[str, StoredObject] = {}
|
||||
with tempfile.TemporaryDirectory(prefix="arr-processing-output-") as temporary:
|
||||
root = Path(temporary)
|
||||
for role in sorted(result.artifacts):
|
||||
artifact = result.artifacts[role]
|
||||
if artifact is None:
|
||||
continue
|
||||
destination = root / role / artifact.original_filename
|
||||
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
try:
|
||||
self._remote_files.materialize(
|
||||
artifact.file_handle,
|
||||
destination,
|
||||
ARTIFACT_SIZE_LIMITS[role],
|
||||
)
|
||||
except ProcessingTransportError as error:
|
||||
raise ProcessingError(
|
||||
error.code,
|
||||
"remote processing artifact could not be fetched",
|
||||
retryable=error.retryable,
|
||||
) from None
|
||||
self._verify_materialized(destination, artifact)
|
||||
stored[role] = self._object_store.upload_committed(
|
||||
job_id=result.job_id,
|
||||
attempt_no=result.attempt_no,
|
||||
role=role,
|
||||
source=destination,
|
||||
original_filename=artifact.original_filename,
|
||||
expected_sha256=artifact.sha256,
|
||||
expected_byte_size=artifact.byte_size,
|
||||
)
|
||||
|
||||
artifacts: Dict[str, object] = {
|
||||
"source_xml": source.to_dict(),
|
||||
"daily_report": None,
|
||||
"result_json": None,
|
||||
"structured_result_json": None,
|
||||
"exception_report": None,
|
||||
}
|
||||
for role, value in stored.items():
|
||||
artifacts[role] = value.to_artifact_ref().to_dict()
|
||||
payload = {
|
||||
"delivery_schema_version": DELIVERY_SCHEMA_VERSION,
|
||||
"delivery_id": result.delivery_id,
|
||||
"job_id": result.job_id,
|
||||
"attempt_no": result.attempt_no,
|
||||
"status": result.status,
|
||||
"processor_version": result.processor_version,
|
||||
"rule_set_sha256": result.rule_set_sha256,
|
||||
"result_schema_version": result.result_schema_version,
|
||||
"business_date": (
|
||||
result.business_date.isoformat() if result.business_date is not None else None
|
||||
),
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
envelope = DeliveryEnvelope.from_dict(payload)
|
||||
raw = (
|
||||
json.dumps(
|
||||
envelope.to_dict(),
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
+ "\n"
|
||||
).encode("utf-8")
|
||||
return RegisteredDelivery(envelope, raw, stored)
|
||||
|
||||
@staticmethod
|
||||
def _verify_materialized(destination: Path, artifact: ResultArtifactHandle) -> None:
|
||||
try:
|
||||
file_stat = destination.lstat()
|
||||
except OSError:
|
||||
raise ProcessingError(
|
||||
"PROCESSING_ARTIFACT_NOT_FOUND", "remote processing artifact is unavailable"
|
||||
) from None
|
||||
if destination.is_symlink() or not stat.S_ISREG(file_stat.st_mode):
|
||||
raise ProcessingError(
|
||||
"PROCESSING_ARTIFACT_INVALID", "remote processing artifact is not a regular file"
|
||||
)
|
||||
if file_stat.st_size != artifact.byte_size:
|
||||
raise ProcessingError(
|
||||
"PROCESSING_ARTIFACT_HASH_MISMATCH", "remote artifact identity does not match"
|
||||
)
|
||||
digest = hashlib.sha256()
|
||||
descriptor = -1
|
||||
try:
|
||||
descriptor = os.open(
|
||||
destination,
|
||||
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
|
||||
)
|
||||
with os.fdopen(descriptor, "rb") as source:
|
||||
descriptor = -1
|
||||
while True:
|
||||
chunk = source.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
except OSError:
|
||||
raise ProcessingError(
|
||||
"PROCESSING_ARTIFACT_INVALID", "remote processing artifact could not be read"
|
||||
) from None
|
||||
finally:
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
if digest.hexdigest() != artifact.sha256:
|
||||
raise ProcessingError(
|
||||
"PROCESSING_ARTIFACT_HASH_MISMATCH", "remote artifact identity does not match"
|
||||
)
|
||||
Reference in New Issue
Block a user