210 lines
8.1 KiB
Python
210 lines
8.1 KiB
Python
"""Repository contracts and an offline reference implementation for ARR ingestion."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import threading
|
|
from dataclasses import dataclass
|
|
from datetime import date
|
|
from typing import Any, Dict, Mapping, Optional, Protocol, Tuple
|
|
|
|
from arr_ingestion.contracts import ArtifactRef, IngestionError
|
|
from arr_ingestion.validation import VerifiedDelivery
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class JobRegistration:
|
|
job_id: str
|
|
source: ArtifactRef
|
|
processor_version: str
|
|
rule_set_sha256: str
|
|
attempt_no: int
|
|
idempotency_key: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class IngestionOutcome:
|
|
status: str
|
|
job_id: str
|
|
business_date: Optional[date]
|
|
daily_version_id: Optional[int]
|
|
version_no: Optional[int]
|
|
|
|
|
|
class IngestionRepository(Protocol):
|
|
def register_job(self, registration: JobRegistration) -> None:
|
|
...
|
|
|
|
def commit_delivery(self, delivery: VerifiedDelivery) -> IngestionOutcome:
|
|
...
|
|
|
|
|
|
@dataclass
|
|
class _MemoryJob:
|
|
registration: JobRegistration
|
|
status: str = "queued"
|
|
failure_code: Optional[str] = None
|
|
business_date: Optional[date] = None
|
|
|
|
|
|
@dataclass
|
|
class _MemoryDailyVersion:
|
|
id: int
|
|
business_date: date
|
|
version_no: int
|
|
identity: Tuple[str, date, str, str]
|
|
status: str
|
|
records: Tuple[Mapping[str, Any], ...]
|
|
channels: Tuple[Mapping[str, Any], ...]
|
|
|
|
|
|
class InMemoryIngestionRepository:
|
|
"""Thread-safe vertical-slice repository; never used as the production fact store."""
|
|
|
|
def __init__(self) -> None:
|
|
self._lock = threading.RLock()
|
|
self._jobs: Dict[str, _MemoryJob] = {}
|
|
self._callbacks: Dict[str, Tuple[str, IngestionOutcome]] = {}
|
|
self._versions: Dict[int, _MemoryDailyVersion] = {}
|
|
self._identity_versions: Dict[Tuple[str, date, str, str], int] = {}
|
|
self._current: Dict[date, int] = {}
|
|
self._next_version_id = 1
|
|
|
|
def register_job(self, registration: JobRegistration) -> None:
|
|
if registration.source.role != "source_xml" or registration.attempt_no < 1:
|
|
raise IngestionError("JOB_INVALID", "processing job registration is invalid")
|
|
with self._lock:
|
|
existing = self._jobs.get(registration.job_id)
|
|
if existing is not None:
|
|
if existing.registration != registration:
|
|
raise IngestionError("JOB_CONFLICT", "processing job identity conflicts")
|
|
return
|
|
self._jobs[registration.job_id] = _MemoryJob(registration=registration)
|
|
|
|
def commit_delivery(self, delivery: VerifiedDelivery) -> IngestionOutcome:
|
|
envelope = delivery.envelope
|
|
with self._lock:
|
|
existing_callback = self._callbacks.get(envelope.delivery_id)
|
|
if existing_callback is not None:
|
|
existing_hash, outcome = existing_callback
|
|
if existing_hash != delivery.envelope_sha256:
|
|
raise IngestionError("CALLBACK_CONFLICT", "delivery identifier conflicts")
|
|
return outcome
|
|
|
|
job = self._jobs.get(envelope.job_id)
|
|
if job is None:
|
|
raise IngestionError("JOB_NOT_FOUND", "processing job was not registered")
|
|
registration = job.registration
|
|
source = envelope.artifacts["source_xml"]
|
|
if (
|
|
source is None
|
|
or source.sha256 != registration.source.sha256
|
|
or source.byte_size != registration.source.byte_size
|
|
or source.object_key != registration.source.object_key
|
|
or envelope.attempt_no != registration.attempt_no
|
|
or envelope.processor_version != registration.processor_version
|
|
or envelope.rule_set_sha256 != registration.rule_set_sha256
|
|
):
|
|
raise IngestionError("JOB_DELIVERY_MISMATCH", "delivery does not match its job")
|
|
|
|
if envelope.status == "failed":
|
|
job.status = "failed"
|
|
errors = delivery.structured_payload.get("errors")
|
|
first_error = errors[0] if isinstance(errors, list) and errors else {}
|
|
code = first_error.get("code") if isinstance(first_error, Mapping) else None
|
|
job.failure_code = str(code or "PROCESSING_FAILED")
|
|
outcome = IngestionOutcome(
|
|
status="recorded_failure",
|
|
job_id=envelope.job_id,
|
|
business_date=envelope.business_date,
|
|
daily_version_id=None,
|
|
version_no=None,
|
|
)
|
|
self._callbacks[envelope.delivery_id] = (
|
|
delivery.envelope_sha256,
|
|
outcome,
|
|
)
|
|
return outcome
|
|
|
|
if envelope.business_date is None:
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "successful delivery has no date")
|
|
identity = (
|
|
registration.source.sha256,
|
|
envelope.business_date,
|
|
envelope.processor_version,
|
|
envelope.rule_set_sha256,
|
|
)
|
|
existing_version_id = self._identity_versions.get(identity)
|
|
if existing_version_id is not None:
|
|
version = self._versions[existing_version_id]
|
|
job.status = "succeeded"
|
|
job.business_date = envelope.business_date
|
|
outcome = IngestionOutcome(
|
|
status="already_committed",
|
|
job_id=envelope.job_id,
|
|
business_date=envelope.business_date,
|
|
daily_version_id=version.id,
|
|
version_no=version.version_no,
|
|
)
|
|
self._callbacks[envelope.delivery_id] = (
|
|
delivery.envelope_sha256,
|
|
outcome,
|
|
)
|
|
return outcome
|
|
|
|
version_no = 1 + max(
|
|
(
|
|
version.version_no
|
|
for version in self._versions.values()
|
|
if version.business_date == envelope.business_date
|
|
),
|
|
default=0,
|
|
)
|
|
version_id = self._next_version_id
|
|
self._next_version_id += 1
|
|
previous_id = self._current.get(envelope.business_date)
|
|
if previous_id is not None:
|
|
self._versions[previous_id].status = "superseded"
|
|
records = delivery.structured_payload.get("records")
|
|
channels = delivery.structured_payload.get("channels")
|
|
version = _MemoryDailyVersion(
|
|
id=version_id,
|
|
business_date=envelope.business_date,
|
|
version_no=version_no,
|
|
identity=identity,
|
|
status="active",
|
|
records=tuple(copy.deepcopy(records if isinstance(records, list) else [])),
|
|
channels=tuple(copy.deepcopy(channels if isinstance(channels, list) else [])),
|
|
)
|
|
self._versions[version_id] = version
|
|
self._identity_versions[identity] = version_id
|
|
self._current[envelope.business_date] = version_id
|
|
job.status = "succeeded"
|
|
job.business_date = envelope.business_date
|
|
outcome = IngestionOutcome(
|
|
status="committed",
|
|
job_id=envelope.job_id,
|
|
business_date=envelope.business_date,
|
|
daily_version_id=version_id,
|
|
version_no=version_no,
|
|
)
|
|
self._callbacks[envelope.delivery_id] = (
|
|
delivery.envelope_sha256,
|
|
outcome,
|
|
)
|
|
return outcome
|
|
|
|
def current_version_id(self, business_date: date) -> Optional[int]:
|
|
with self._lock:
|
|
return self._current.get(business_date)
|
|
|
|
def version_status(self, version_id: int) -> Optional[str]:
|
|
with self._lock:
|
|
version = self._versions.get(version_id)
|
|
return version.status if version else None
|
|
|
|
def version_records(self, version_id: int) -> Tuple[Mapping[str, Any], ...]:
|
|
with self._lock:
|
|
version = self._versions.get(version_id)
|
|
return tuple(copy.deepcopy(version.records)) if version else tuple()
|