306 lines
9.4 KiB
Python
306 lines
9.4 KiB
Python
"""Strict contracts for one-call ARR direct structured-result ingestion."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from datetime import date, datetime
|
|
from typing import Any, Dict, Mapping, Optional
|
|
|
|
from arr_ingestion.contracts import (
|
|
OPAQUE_ID_RE,
|
|
PROCESSOR_VERSION_RE,
|
|
RESULT_SCHEMA_VERSION,
|
|
SHA256_RE,
|
|
ArtifactRef,
|
|
IngestionError,
|
|
)
|
|
|
|
|
|
DIRECT_CONTRACT_VERSION = "arr-direct-ingestion-1"
|
|
MAX_DIRECT_PAYLOAD_BYTES = 3 * 1024 * 1024
|
|
DIRECT_REQUEST_FIELDS = {
|
|
"submission_grant",
|
|
"job_id",
|
|
"attempt_no",
|
|
"payload",
|
|
}
|
|
DIRECT_RECEIPT_FIELDS = {
|
|
"contract_version",
|
|
"status",
|
|
"job_id",
|
|
"attempt_no",
|
|
"business_date",
|
|
"daily_version_id",
|
|
"version_no",
|
|
"record_count",
|
|
}
|
|
GRANT_TOKEN_RE = re.compile(r"^[A-Za-z0-9_-]{32,128}$")
|
|
|
|
|
|
def _fail(code: str, message: str) -> None:
|
|
raise IngestionError(code, message)
|
|
|
|
|
|
def canonical_json_bytes(value: Mapping[str, Any]) -> bytes:
|
|
"""Return the one canonical encoding used for hashing and persistence."""
|
|
|
|
try:
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
allow_nan=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
except (TypeError, ValueError):
|
|
_fail(
|
|
"DIRECT_SUBMISSION_INVALID",
|
|
"direct result payload is not strict JSON",
|
|
)
|
|
|
|
|
|
def _strict_mapping(
|
|
value: Any,
|
|
expected: set[str],
|
|
label: str,
|
|
*,
|
|
error_code: str = "DIRECT_SUBMISSION_INVALID",
|
|
) -> Mapping[str, Any]:
|
|
if not isinstance(value, Mapping) or set(value) != expected:
|
|
_fail(
|
|
error_code,
|
|
f"{label} field set is invalid",
|
|
)
|
|
return value
|
|
|
|
|
|
def _parse_business_date(
|
|
value: Any,
|
|
*,
|
|
error_code: str = "DIRECT_SUBMISSION_INVALID",
|
|
error_message: str = "direct result business date is invalid",
|
|
) -> date:
|
|
if not isinstance(value, str):
|
|
_fail(error_code, error_message)
|
|
try:
|
|
return date.fromisoformat(value)
|
|
except ValueError:
|
|
_fail(error_code, error_message)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DirectSubmissionRequest:
|
|
"""Validated transport metadata plus a detached strict-JSON payload."""
|
|
|
|
submission_grant: str = field(repr=False)
|
|
job_id: str
|
|
attempt_no: int
|
|
payload: Mapping[str, Any] = field(repr=False)
|
|
payload_bytes: bytes = field(repr=False)
|
|
payload_sha256: str
|
|
business_date: date
|
|
processor_version: str
|
|
rule_set_sha256: str
|
|
result_schema_version: str
|
|
record_count: int
|
|
|
|
@classmethod
|
|
def from_dict(cls, raw: Any) -> "DirectSubmissionRequest":
|
|
values = _strict_mapping(raw, DIRECT_REQUEST_FIELDS, "direct submission")
|
|
grant = values.get("submission_grant")
|
|
job_id = values.get("job_id")
|
|
attempt_no = values.get("attempt_no")
|
|
payload = values.get("payload")
|
|
if not isinstance(grant, str) or not GRANT_TOKEN_RE.fullmatch(grant):
|
|
_fail(
|
|
"SUBMISSION_GRANT_INVALID",
|
|
"submission grant is invalid",
|
|
)
|
|
if not isinstance(job_id, str) or not OPAQUE_ID_RE.fullmatch(job_id):
|
|
_fail(
|
|
"DIRECT_SUBMISSION_INVALID",
|
|
"direct submission job identifier is invalid",
|
|
)
|
|
if (
|
|
not isinstance(attempt_no, int)
|
|
or isinstance(attempt_no, bool)
|
|
or not 1 <= attempt_no <= 9999
|
|
):
|
|
_fail(
|
|
"DIRECT_SUBMISSION_INVALID",
|
|
"direct submission attempt number is invalid",
|
|
)
|
|
if not isinstance(payload, Mapping):
|
|
_fail(
|
|
"DIRECT_SUBMISSION_INVALID",
|
|
"direct result payload must be an object",
|
|
)
|
|
payload_bytes = canonical_json_bytes(payload)
|
|
if not 2 <= len(payload_bytes) <= MAX_DIRECT_PAYLOAD_BYTES:
|
|
_fail(
|
|
"DIRECT_PAYLOAD_TOO_LARGE",
|
|
"direct result payload exceeds its size limit",
|
|
)
|
|
detached = json.loads(payload_bytes.decode("utf-8"))
|
|
if (
|
|
detached.get("status") != "success"
|
|
or detached.get("activation_eligible") is not True
|
|
):
|
|
_fail(
|
|
"DIRECT_RESULT_NOT_ACTIVATABLE",
|
|
"direct result is not eligible for Finance activation",
|
|
)
|
|
processor_version = detached.get("processor_version")
|
|
rule_set_sha256 = detached.get("rule_set_sha256")
|
|
result_schema_version = detached.get("result_schema_version")
|
|
records = detached.get("records")
|
|
if (
|
|
not isinstance(processor_version, str)
|
|
or not PROCESSOR_VERSION_RE.fullmatch(processor_version)
|
|
or not isinstance(rule_set_sha256, str)
|
|
or not SHA256_RE.fullmatch(rule_set_sha256)
|
|
or result_schema_version != RESULT_SCHEMA_VERSION
|
|
or not isinstance(records, list)
|
|
):
|
|
_fail(
|
|
"DIRECT_SUBMISSION_INVALID",
|
|
"direct result identity is invalid",
|
|
)
|
|
return cls(
|
|
submission_grant=grant,
|
|
job_id=job_id,
|
|
attempt_no=attempt_no,
|
|
payload=detached,
|
|
payload_bytes=payload_bytes,
|
|
payload_sha256=hashlib.sha256(payload_bytes).hexdigest(),
|
|
business_date=_parse_business_date(detached.get("business_date")),
|
|
processor_version=processor_version,
|
|
rule_set_sha256=rule_set_sha256,
|
|
result_schema_version=result_schema_version,
|
|
record_count=len(records),
|
|
)
|
|
|
|
@property
|
|
def grant_sha256(self) -> str:
|
|
return hashlib.sha256(self.submission_grant.encode("utf-8")).hexdigest()
|
|
|
|
@property
|
|
def submission_key(self) -> str:
|
|
identity = f"{self.job_id}\x00{self.attempt_no}".encode("utf-8")
|
|
return "direct:" + hashlib.sha256(identity).hexdigest()
|
|
|
|
def payload_json(self) -> str:
|
|
return self.payload_bytes.decode("utf-8")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DirectSubmissionReceipt:
|
|
status: str
|
|
job_id: str
|
|
attempt_no: int
|
|
business_date: date
|
|
daily_version_id: int
|
|
version_no: int
|
|
record_count: int
|
|
|
|
@classmethod
|
|
def from_dict(cls, raw: Any) -> "DirectSubmissionReceipt":
|
|
values = _strict_mapping(
|
|
raw,
|
|
DIRECT_RECEIPT_FIELDS,
|
|
"stored direct receipt",
|
|
error_code="DATABASE_STATE_INVALID",
|
|
)
|
|
if values.get("contract_version") != DIRECT_CONTRACT_VERSION:
|
|
_fail(
|
|
"DATABASE_STATE_INVALID",
|
|
"stored direct receipt contract is invalid",
|
|
)
|
|
status = values.get("status")
|
|
job_id = values.get("job_id")
|
|
attempt_no = values.get("attempt_no")
|
|
daily_version_id = values.get("daily_version_id")
|
|
version_no = values.get("version_no")
|
|
record_count = values.get("record_count")
|
|
if (
|
|
status not in {"committed", "already_committed"}
|
|
or not isinstance(job_id, str)
|
|
or not OPAQUE_ID_RE.fullmatch(job_id)
|
|
or not isinstance(attempt_no, int)
|
|
or isinstance(attempt_no, bool)
|
|
or attempt_no < 1
|
|
or not isinstance(daily_version_id, int)
|
|
or isinstance(daily_version_id, bool)
|
|
or daily_version_id < 1
|
|
or not isinstance(version_no, int)
|
|
or isinstance(version_no, bool)
|
|
or version_no < 1
|
|
or not isinstance(record_count, int)
|
|
or isinstance(record_count, bool)
|
|
or record_count < 0
|
|
):
|
|
_fail(
|
|
"DATABASE_STATE_INVALID",
|
|
"stored direct receipt is invalid",
|
|
)
|
|
return cls(
|
|
status=str(status),
|
|
job_id=job_id,
|
|
attempt_no=attempt_no,
|
|
business_date=_parse_business_date(
|
|
values.get("business_date"),
|
|
error_code="DATABASE_STATE_INVALID",
|
|
error_message="stored direct receipt business date is invalid",
|
|
),
|
|
daily_version_id=daily_version_id,
|
|
version_no=version_no,
|
|
record_count=record_count,
|
|
)
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
"contract_version": DIRECT_CONTRACT_VERSION,
|
|
"status": self.status,
|
|
"job_id": self.job_id,
|
|
"attempt_no": self.attempt_no,
|
|
"business_date": self.business_date.isoformat(),
|
|
"daily_version_id": self.daily_version_id,
|
|
"version_no": self.version_no,
|
|
"record_count": self.record_count,
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SubmissionGrant:
|
|
submission_grant: str = field(repr=False)
|
|
job_id: str
|
|
attempt_no: int
|
|
expires_at: datetime
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
"contract_version": DIRECT_CONTRACT_VERSION,
|
|
"submission_grant": self.submission_grant,
|
|
"job_id": self.job_id,
|
|
"attempt_no": self.attempt_no,
|
|
"expires_at": self.expires_at.isoformat(),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReceivedDirectSubmission:
|
|
submission_id: int
|
|
status: str
|
|
request: DirectSubmissionRequest
|
|
source: ArtifactRef
|
|
receipt: Optional[DirectSubmissionReceipt] = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VerifiedDirectSubmission:
|
|
submission: ReceivedDirectSubmission
|
|
replay_payload_sha256: str
|