feat: prepare ARR for controlled public deployment
This commit is contained in:
Wyndham ARR
303
arr_processing/contracts.py
Normal file
303
arr_processing/contracts.py
Normal file
@@ -0,0 +1,303 @@
|
||||
"""Strict, path-free contracts for ARR remote processing requests and results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from typing import Any, Dict, Mapping, Optional
|
||||
|
||||
from arr_ingestion.contracts import (
|
||||
ARTIFACT_SIZE_LIMITS,
|
||||
OPAQUE_ID_RE,
|
||||
PROCESSOR_VERSION_RE,
|
||||
RESULT_SCHEMA_VERSION,
|
||||
ROLE_CONTRACTS,
|
||||
SHA256_RE,
|
||||
)
|
||||
from arr_processing.errors import ProcessingError
|
||||
from arr_ingestion.direct_contracts import GRANT_TOKEN_RE
|
||||
from arr_storage.contracts import valid_delivery_filename
|
||||
|
||||
|
||||
PROCESSING_REQUEST_VERSION = "arr-opera-daily-request-1"
|
||||
PROCESSING_RESULT_VERSION = "arr-opera-daily-result-1"
|
||||
REQUEST_FIELDS = {
|
||||
"contract_version",
|
||||
"job_id",
|
||||
"attempt_no",
|
||||
"source_file_id",
|
||||
"processor_version",
|
||||
"rule_set_sha256",
|
||||
}
|
||||
RESULT_FIELDS = {
|
||||
"contract_version",
|
||||
"delivery_id",
|
||||
"job_id",
|
||||
"attempt_no",
|
||||
"remote_run_id",
|
||||
"status",
|
||||
"business_date",
|
||||
"processor_version",
|
||||
"rule_set_sha256",
|
||||
"result_schema_version",
|
||||
"artifacts",
|
||||
}
|
||||
RESULT_ARTIFACT_ROLES = {
|
||||
"daily_report",
|
||||
"result_json",
|
||||
"structured_result_json",
|
||||
"exception_report",
|
||||
}
|
||||
HANDLE_FIELDS = {
|
||||
"file_handle",
|
||||
"original_filename",
|
||||
"sha256",
|
||||
"byte_size",
|
||||
"mime_type",
|
||||
}
|
||||
|
||||
|
||||
def _fail(code: str, message: str) -> None:
|
||||
raise ProcessingError(code, message)
|
||||
|
||||
|
||||
def _strict_mapping(value: Any, fields: set[str], label: str) -> Mapping[str, Any]:
|
||||
if not isinstance(value, Mapping) or set(value) != fields:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", f"{label} field set is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _opaque(value: Any, label: str) -> str:
|
||||
if not isinstance(value, str) or not OPAQUE_ID_RE.fullmatch(value):
|
||||
_fail("PROCESSING_CONTRACT_INVALID", f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _attempt(value: Any) -> int:
|
||||
if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= 9999:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "attempt number is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _processor(value: Any) -> str:
|
||||
if not isinstance(value, str) or not PROCESSOR_VERSION_RE.fullmatch(value):
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "processor version is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _sha256(value: Any, label: str) -> str:
|
||||
if not isinstance(value, str) or not SHA256_RE.fullmatch(value):
|
||||
_fail("PROCESSING_CONTRACT_INVALID", f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def canonical_json_bytes(value: Mapping[str, Any]) -> bytes:
|
||||
try:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
except (TypeError, ValueError):
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "processing payload is not strict JSON")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProcessingRequest:
|
||||
job_id: str
|
||||
attempt_no: int
|
||||
source_file_id: str
|
||||
processor_version: str
|
||||
rule_set_sha256: str
|
||||
submission_grant: Optional[str] = field(
|
||||
default=None,
|
||||
repr=False,
|
||||
compare=False,
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_opaque(self.job_id, "job identifier")
|
||||
_attempt(self.attempt_no)
|
||||
_opaque(self.source_file_id, "source file identifier")
|
||||
_processor(self.processor_version)
|
||||
_sha256(self.rule_set_sha256, "rule set identity")
|
||||
if (
|
||||
self.submission_grant is not None
|
||||
and (
|
||||
not isinstance(self.submission_grant, str)
|
||||
or not GRANT_TOKEN_RE.fullmatch(self.submission_grant)
|
||||
)
|
||||
):
|
||||
_fail(
|
||||
"PROCESSING_CONTRACT_INVALID",
|
||||
"direct submission grant is invalid",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, payload: Any) -> "ProcessingRequest":
|
||||
values = _strict_mapping(payload, REQUEST_FIELDS, "processing request")
|
||||
if values.get("contract_version") != PROCESSING_REQUEST_VERSION:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "request contract version is unsupported")
|
||||
return cls(
|
||||
job_id=_opaque(values.get("job_id"), "job identifier"),
|
||||
attempt_no=_attempt(values.get("attempt_no")),
|
||||
source_file_id=_opaque(values.get("source_file_id"), "source file identifier"),
|
||||
processor_version=_processor(values.get("processor_version")),
|
||||
rule_set_sha256=_sha256(values.get("rule_set_sha256"), "rule set identity"),
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"contract_version": PROCESSING_REQUEST_VERSION,
|
||||
"job_id": self.job_id,
|
||||
"attempt_no": self.attempt_no,
|
||||
"source_file_id": self.source_file_id,
|
||||
"processor_version": self.processor_version,
|
||||
"rule_set_sha256": self.rule_set_sha256,
|
||||
}
|
||||
|
||||
def canonical_bytes(self) -> bytes:
|
||||
return canonical_json_bytes(self.to_dict())
|
||||
|
||||
def sha256(self) -> str:
|
||||
return hashlib.sha256(self.canonical_bytes()).hexdigest()
|
||||
|
||||
def message(self) -> str:
|
||||
return self.canonical_bytes().decode("utf-8")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResultArtifactHandle:
|
||||
role: str
|
||||
file_handle: str
|
||||
original_filename: str
|
||||
sha256: str
|
||||
byte_size: int
|
||||
mime_type: str
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, role: str, payload: Any) -> "ResultArtifactHandle":
|
||||
if role not in RESULT_ARTIFACT_ROLES:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "result artifact role is invalid")
|
||||
values = _strict_mapping(payload, HANDLE_FIELDS, f"{role} artifact")
|
||||
file_handle = _opaque(values.get("file_handle"), "artifact file handle")
|
||||
original_filename = values.get("original_filename")
|
||||
sha256 = _sha256(values.get("sha256"), "artifact hash")
|
||||
byte_size = values.get("byte_size")
|
||||
mime_type = values.get("mime_type")
|
||||
if (
|
||||
not valid_delivery_filename(role, original_filename)
|
||||
or not isinstance(byte_size, int)
|
||||
or isinstance(byte_size, bool)
|
||||
or not 0 <= byte_size <= ARTIFACT_SIZE_LIMITS[role]
|
||||
or mime_type != ROLE_CONTRACTS[role][2]
|
||||
):
|
||||
_fail("PROCESSING_CONTRACT_INVALID", f"{role} artifact metadata is invalid")
|
||||
return cls(
|
||||
role=role,
|
||||
file_handle=file_handle,
|
||||
original_filename=str(original_filename),
|
||||
sha256=sha256,
|
||||
byte_size=byte_size,
|
||||
mime_type=str(mime_type),
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"file_handle": self.file_handle,
|
||||
"original_filename": self.original_filename,
|
||||
"sha256": self.sha256,
|
||||
"byte_size": self.byte_size,
|
||||
"mime_type": self.mime_type,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProcessingResult:
|
||||
delivery_id: str
|
||||
job_id: str
|
||||
attempt_no: int
|
||||
remote_run_id: str
|
||||
status: str
|
||||
business_date: Optional[date]
|
||||
processor_version: str
|
||||
rule_set_sha256: str
|
||||
result_schema_version: str
|
||||
artifacts: Mapping[str, Optional[ResultArtifactHandle]]
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, payload: Any) -> "ProcessingResult":
|
||||
values = _strict_mapping(payload, RESULT_FIELDS, "processing result")
|
||||
if values.get("contract_version") != PROCESSING_RESULT_VERSION:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "result contract version is unsupported")
|
||||
status = values.get("status")
|
||||
if status not in {"success", "failed"}:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "result status is invalid")
|
||||
raw_date = values.get("business_date")
|
||||
parsed_date: Optional[date]
|
||||
if raw_date is None:
|
||||
parsed_date = None
|
||||
elif isinstance(raw_date, str):
|
||||
try:
|
||||
parsed_date = date.fromisoformat(raw_date)
|
||||
except ValueError:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "business date is invalid")
|
||||
else:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "business date is invalid")
|
||||
|
||||
raw_artifacts = _strict_mapping(
|
||||
values.get("artifacts"), RESULT_ARTIFACT_ROLES, "result artifacts"
|
||||
)
|
||||
artifacts: Dict[str, Optional[ResultArtifactHandle]] = {}
|
||||
for role in sorted(RESULT_ARTIFACT_ROLES):
|
||||
raw = raw_artifacts.get(role)
|
||||
artifacts[role] = None if raw is None else ResultArtifactHandle.from_dict(role, raw)
|
||||
if artifacts["result_json"] is None or artifacts["structured_result_json"] is None:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "required result artifact is missing")
|
||||
if status == "success":
|
||||
if (
|
||||
parsed_date is None
|
||||
or artifacts["daily_report"] is None
|
||||
or artifacts["exception_report"] is not None
|
||||
):
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "success result artifact shape is invalid")
|
||||
elif artifacts["daily_report"] is not None or artifacts["exception_report"] is None:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "failed result artifact shape is invalid")
|
||||
|
||||
result_schema_version = values.get("result_schema_version")
|
||||
if result_schema_version != RESULT_SCHEMA_VERSION:
|
||||
_fail("PROCESSING_CONTRACT_INVALID", "result schema version is unsupported")
|
||||
return cls(
|
||||
delivery_id=_opaque(values.get("delivery_id"), "delivery identifier"),
|
||||
job_id=_opaque(values.get("job_id"), "job identifier"),
|
||||
attempt_no=_attempt(values.get("attempt_no")),
|
||||
remote_run_id=_opaque(values.get("remote_run_id"), "remote run identifier"),
|
||||
status=str(status),
|
||||
business_date=parsed_date,
|
||||
processor_version=_processor(values.get("processor_version")),
|
||||
rule_set_sha256=_sha256(values.get("rule_set_sha256"), "rule set identity"),
|
||||
result_schema_version=str(result_schema_version),
|
||||
artifacts=artifacts,
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"contract_version": PROCESSING_RESULT_VERSION,
|
||||
"delivery_id": self.delivery_id,
|
||||
"job_id": self.job_id,
|
||||
"attempt_no": self.attempt_no,
|
||||
"remote_run_id": self.remote_run_id,
|
||||
"status": self.status,
|
||||
"business_date": self.business_date.isoformat() if self.business_date else None,
|
||||
"processor_version": self.processor_version,
|
||||
"rule_set_sha256": self.rule_set_sha256,
|
||||
"result_schema_version": self.result_schema_version,
|
||||
"artifacts": {
|
||||
role: artifact.to_dict() if artifact is not None else None
|
||||
for role, artifact in self.artifacts.items()
|
||||
},
|
||||
}
|
||||
Reference in New Issue
Block a user