"""Repository contracts and an offline reference implementation for ARR ingestion.""" from __future__ import annotations import copy import threading from dataclasses import dataclass from datetime import date from typing import Any, Dict, Mapping, Optional, Protocol, Tuple from arr_ingestion.contracts import ArtifactRef, IngestionError from arr_ingestion.validation import VerifiedDelivery @dataclass(frozen=True) class JobRegistration: job_id: str source: ArtifactRef processor_version: str rule_set_sha256: str attempt_no: int idempotency_key: str uploaded_filename: Optional[str] = None @dataclass(frozen=True) class IngestionOutcome: status: str job_id: str business_date: Optional[date] daily_version_id: Optional[int] version_no: Optional[int] class IngestionRepository(Protocol): def register_job(self, registration: JobRegistration) -> None: ... def mark_running(self, job_id: str, attempt_no: int) -> None: ... def record_failure(self, job_id: str, attempt_no: int, failure_code: str) -> None: ... def commit_delivery(self, delivery: VerifiedDelivery) -> IngestionOutcome: ... @dataclass class _MemoryJob: registration: JobRegistration status: str = "queued" failure_code: Optional[str] = None business_date: Optional[date] = None @dataclass class _MemoryDailyVersion: id: int business_date: date version_no: int identity: Tuple[str, date, str, str] status: str records: Tuple[Mapping[str, Any], ...] channels: Tuple[Mapping[str, Any], ...] class InMemoryIngestionRepository: """Thread-safe vertical-slice repository; never used as the production fact store.""" def __init__(self) -> None: self._lock = threading.RLock() self._jobs: Dict[str, _MemoryJob] = {} self._callbacks: Dict[str, Tuple[str, IngestionOutcome]] = {} self._versions: Dict[int, _MemoryDailyVersion] = {} self._identity_versions: Dict[Tuple[str, date, str, str], int] = {} self._current: Dict[date, int] = {} self._next_version_id = 1 def register_job(self, registration: JobRegistration) -> None: if registration.source.role != "source_xml" or registration.attempt_no < 1: raise IngestionError("JOB_INVALID", "processing job registration is invalid") with self._lock: existing = self._jobs.get(registration.job_id) if existing is not None: if existing.registration != registration: raise IngestionError("JOB_CONFLICT", "processing job identity conflicts") return self._jobs[registration.job_id] = _MemoryJob(registration=registration) def mark_running(self, job_id: str, attempt_no: int) -> None: with self._lock: job = self._jobs.get(job_id) if job is None: raise IngestionError("JOB_NOT_FOUND", "processing job was not registered") if attempt_no != job.registration.attempt_no: raise IngestionError("JOB_NOT_FOUND", "processing attempt was not registered") if job.status in {"succeeded", "failed"}: raise IngestionError("JOB_TERMINAL", "processing job is already terminal") job.status = "running" def record_failure(self, job_id: str, attempt_no: int, failure_code: str) -> None: with self._lock: job = self._jobs.get(job_id) if job is None: raise IngestionError("JOB_NOT_FOUND", "processing job was not registered") if attempt_no != job.registration.attempt_no: raise IngestionError("JOB_NOT_FOUND", "processing attempt was not registered") if job.status == "succeeded": raise IngestionError("JOB_TERMINAL", "processing job is already terminal") job.status = "failed" job.failure_code = failure_code def commit_delivery(self, delivery: VerifiedDelivery) -> IngestionOutcome: envelope = delivery.envelope with self._lock: existing_callback = self._callbacks.get(envelope.delivery_id) if existing_callback is not None: existing_hash, outcome = existing_callback if existing_hash != delivery.envelope_sha256: raise IngestionError("CALLBACK_CONFLICT", "delivery identifier conflicts") return outcome job = self._jobs.get(envelope.job_id) if job is None: raise IngestionError("JOB_NOT_FOUND", "processing job was not registered") registration = job.registration source = envelope.artifacts["source_xml"] if ( source is None or source.sha256 != registration.source.sha256 or source.byte_size != registration.source.byte_size or source.object_key != registration.source.object_key or envelope.attempt_no != registration.attempt_no or envelope.processor_version != registration.processor_version or envelope.rule_set_sha256 != registration.rule_set_sha256 ): raise IngestionError("JOB_DELIVERY_MISMATCH", "delivery does not match its job") if envelope.status == "failed": job.status = "failed" errors = delivery.structured_payload.get("errors") first_error = errors[0] if isinstance(errors, list) and errors else {} code = first_error.get("code") if isinstance(first_error, Mapping) else None job.failure_code = str(code or "PROCESSING_FAILED") outcome = IngestionOutcome( status="recorded_failure", job_id=envelope.job_id, business_date=envelope.business_date, daily_version_id=None, version_no=None, ) self._callbacks[envelope.delivery_id] = ( delivery.envelope_sha256, outcome, ) return outcome if envelope.business_date is None: raise IngestionError("RESULT_CONTRACT_INVALID", "successful delivery has no date") identity = ( registration.source.sha256, envelope.business_date, envelope.processor_version, envelope.rule_set_sha256, ) existing_version_id = self._identity_versions.get(identity) if existing_version_id is not None: version = self._versions[existing_version_id] job.status = "succeeded" job.business_date = envelope.business_date outcome = IngestionOutcome( status="already_committed", job_id=envelope.job_id, business_date=envelope.business_date, daily_version_id=version.id, version_no=version.version_no, ) self._callbacks[envelope.delivery_id] = ( delivery.envelope_sha256, outcome, ) return outcome version_no = 1 + max( ( version.version_no for version in self._versions.values() if version.business_date == envelope.business_date ), default=0, ) version_id = self._next_version_id self._next_version_id += 1 previous_id = self._current.get(envelope.business_date) if previous_id is not None: self._versions[previous_id].status = "superseded" records = delivery.structured_payload.get("records") channels = delivery.structured_payload.get("channels") version = _MemoryDailyVersion( id=version_id, business_date=envelope.business_date, version_no=version_no, identity=identity, status="active", records=tuple(copy.deepcopy(records if isinstance(records, list) else [])), channels=tuple(copy.deepcopy(channels if isinstance(channels, list) else [])), ) self._versions[version_id] = version self._identity_versions[identity] = version_id self._current[envelope.business_date] = version_id job.status = "succeeded" job.business_date = envelope.business_date outcome = IngestionOutcome( status="committed", job_id=envelope.job_id, business_date=envelope.business_date, daily_version_id=version_id, version_no=version_no, ) self._callbacks[envelope.delivery_id] = ( delivery.envelope_sha256, outcome, ) return outcome def current_version_id(self, business_date: date) -> Optional[int]: with self._lock: return self._current.get(business_date) def version_status(self, version_id: int) -> Optional[str]: with self._lock: version = self._versions.get(version_id) return version.status if version else None def version_records(self, version_id: int) -> Tuple[Mapping[str, Any], ...]: with self._lock: version = self._versions.get(version_id) return tuple(copy.deepcopy(version.records)) if version else tuple()