feat: prepare ARR for controlled public deployment
This commit is contained in:
Wyndham ARR
209
arr_ingestion/repository.py
Normal file
209
arr_ingestion/repository.py
Normal file
@@ -0,0 +1,209 @@
|
||||
"""Repository contracts and an offline reference implementation for ARR ingestion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from typing import Any, Dict, Mapping, Optional, Protocol, Tuple
|
||||
|
||||
from arr_ingestion.contracts import ArtifactRef, IngestionError
|
||||
from arr_ingestion.validation import VerifiedDelivery
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JobRegistration:
|
||||
job_id: str
|
||||
source: ArtifactRef
|
||||
processor_version: str
|
||||
rule_set_sha256: str
|
||||
attempt_no: int
|
||||
idempotency_key: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IngestionOutcome:
|
||||
status: str
|
||||
job_id: str
|
||||
business_date: Optional[date]
|
||||
daily_version_id: Optional[int]
|
||||
version_no: Optional[int]
|
||||
|
||||
|
||||
class IngestionRepository(Protocol):
|
||||
def register_job(self, registration: JobRegistration) -> None:
|
||||
...
|
||||
|
||||
def commit_delivery(self, delivery: VerifiedDelivery) -> IngestionOutcome:
|
||||
...
|
||||
|
||||
|
||||
@dataclass
|
||||
class _MemoryJob:
|
||||
registration: JobRegistration
|
||||
status: str = "queued"
|
||||
failure_code: Optional[str] = None
|
||||
business_date: Optional[date] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _MemoryDailyVersion:
|
||||
id: int
|
||||
business_date: date
|
||||
version_no: int
|
||||
identity: Tuple[str, date, str, str]
|
||||
status: str
|
||||
records: Tuple[Mapping[str, Any], ...]
|
||||
channels: Tuple[Mapping[str, Any], ...]
|
||||
|
||||
|
||||
class InMemoryIngestionRepository:
|
||||
"""Thread-safe vertical-slice repository; never used as the production fact store."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._jobs: Dict[str, _MemoryJob] = {}
|
||||
self._callbacks: Dict[str, Tuple[str, IngestionOutcome]] = {}
|
||||
self._versions: Dict[int, _MemoryDailyVersion] = {}
|
||||
self._identity_versions: Dict[Tuple[str, date, str, str], int] = {}
|
||||
self._current: Dict[date, int] = {}
|
||||
self._next_version_id = 1
|
||||
|
||||
def register_job(self, registration: JobRegistration) -> None:
|
||||
if registration.source.role != "source_xml" or registration.attempt_no < 1:
|
||||
raise IngestionError("JOB_INVALID", "processing job registration is invalid")
|
||||
with self._lock:
|
||||
existing = self._jobs.get(registration.job_id)
|
||||
if existing is not None:
|
||||
if existing.registration != registration:
|
||||
raise IngestionError("JOB_CONFLICT", "processing job identity conflicts")
|
||||
return
|
||||
self._jobs[registration.job_id] = _MemoryJob(registration=registration)
|
||||
|
||||
def commit_delivery(self, delivery: VerifiedDelivery) -> IngestionOutcome:
|
||||
envelope = delivery.envelope
|
||||
with self._lock:
|
||||
existing_callback = self._callbacks.get(envelope.delivery_id)
|
||||
if existing_callback is not None:
|
||||
existing_hash, outcome = existing_callback
|
||||
if existing_hash != delivery.envelope_sha256:
|
||||
raise IngestionError("CALLBACK_CONFLICT", "delivery identifier conflicts")
|
||||
return outcome
|
||||
|
||||
job = self._jobs.get(envelope.job_id)
|
||||
if job is None:
|
||||
raise IngestionError("JOB_NOT_FOUND", "processing job was not registered")
|
||||
registration = job.registration
|
||||
source = envelope.artifacts["source_xml"]
|
||||
if (
|
||||
source is None
|
||||
or source.sha256 != registration.source.sha256
|
||||
or source.byte_size != registration.source.byte_size
|
||||
or source.object_key != registration.source.object_key
|
||||
or envelope.attempt_no != registration.attempt_no
|
||||
or envelope.processor_version != registration.processor_version
|
||||
or envelope.rule_set_sha256 != registration.rule_set_sha256
|
||||
):
|
||||
raise IngestionError("JOB_DELIVERY_MISMATCH", "delivery does not match its job")
|
||||
|
||||
if envelope.status == "failed":
|
||||
job.status = "failed"
|
||||
errors = delivery.structured_payload.get("errors")
|
||||
first_error = errors[0] if isinstance(errors, list) and errors else {}
|
||||
code = first_error.get("code") if isinstance(first_error, Mapping) else None
|
||||
job.failure_code = str(code or "PROCESSING_FAILED")
|
||||
outcome = IngestionOutcome(
|
||||
status="recorded_failure",
|
||||
job_id=envelope.job_id,
|
||||
business_date=envelope.business_date,
|
||||
daily_version_id=None,
|
||||
version_no=None,
|
||||
)
|
||||
self._callbacks[envelope.delivery_id] = (
|
||||
delivery.envelope_sha256,
|
||||
outcome,
|
||||
)
|
||||
return outcome
|
||||
|
||||
if envelope.business_date is None:
|
||||
raise IngestionError("RESULT_CONTRACT_INVALID", "successful delivery has no date")
|
||||
identity = (
|
||||
registration.source.sha256,
|
||||
envelope.business_date,
|
||||
envelope.processor_version,
|
||||
envelope.rule_set_sha256,
|
||||
)
|
||||
existing_version_id = self._identity_versions.get(identity)
|
||||
if existing_version_id is not None:
|
||||
version = self._versions[existing_version_id]
|
||||
job.status = "succeeded"
|
||||
job.business_date = envelope.business_date
|
||||
outcome = IngestionOutcome(
|
||||
status="already_committed",
|
||||
job_id=envelope.job_id,
|
||||
business_date=envelope.business_date,
|
||||
daily_version_id=version.id,
|
||||
version_no=version.version_no,
|
||||
)
|
||||
self._callbacks[envelope.delivery_id] = (
|
||||
delivery.envelope_sha256,
|
||||
outcome,
|
||||
)
|
||||
return outcome
|
||||
|
||||
version_no = 1 + max(
|
||||
(
|
||||
version.version_no
|
||||
for version in self._versions.values()
|
||||
if version.business_date == envelope.business_date
|
||||
),
|
||||
default=0,
|
||||
)
|
||||
version_id = self._next_version_id
|
||||
self._next_version_id += 1
|
||||
previous_id = self._current.get(envelope.business_date)
|
||||
if previous_id is not None:
|
||||
self._versions[previous_id].status = "superseded"
|
||||
records = delivery.structured_payload.get("records")
|
||||
channels = delivery.structured_payload.get("channels")
|
||||
version = _MemoryDailyVersion(
|
||||
id=version_id,
|
||||
business_date=envelope.business_date,
|
||||
version_no=version_no,
|
||||
identity=identity,
|
||||
status="active",
|
||||
records=tuple(copy.deepcopy(records if isinstance(records, list) else [])),
|
||||
channels=tuple(copy.deepcopy(channels if isinstance(channels, list) else [])),
|
||||
)
|
||||
self._versions[version_id] = version
|
||||
self._identity_versions[identity] = version_id
|
||||
self._current[envelope.business_date] = version_id
|
||||
job.status = "succeeded"
|
||||
job.business_date = envelope.business_date
|
||||
outcome = IngestionOutcome(
|
||||
status="committed",
|
||||
job_id=envelope.job_id,
|
||||
business_date=envelope.business_date,
|
||||
daily_version_id=version_id,
|
||||
version_no=version_no,
|
||||
)
|
||||
self._callbacks[envelope.delivery_id] = (
|
||||
delivery.envelope_sha256,
|
||||
outcome,
|
||||
)
|
||||
return outcome
|
||||
|
||||
def current_version_id(self, business_date: date) -> Optional[int]:
|
||||
with self._lock:
|
||||
return self._current.get(business_date)
|
||||
|
||||
def version_status(self, version_id: int) -> Optional[str]:
|
||||
with self._lock:
|
||||
version = self._versions.get(version_id)
|
||||
return version.status if version else None
|
||||
|
||||
def version_records(self, version_id: int) -> Tuple[Mapping[str, Any], ...]:
|
||||
with self._lock:
|
||||
version = self._versions.get(version_id)
|
||||
return tuple(copy.deepcopy(version.records)) if version else tuple()
|
||||
Reference in New Issue
Block a user