feat: prepare ARR for controlled public deployment
This commit is contained in:
Wyndham ARR
273
arr_ingestion/contracts.py
Normal file
273
arr_ingestion/contracts.py
Normal file
@@ -0,0 +1,273 @@
|
||||
"""Strict, privacy-safe contracts for ARR processing deliveries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any, Dict, Mapping, Optional
|
||||
|
||||
|
||||
DELIVERY_SCHEMA_VERSION = "1.0"
|
||||
RESULT_SCHEMA_VERSION = "3.0"
|
||||
XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
OPAQUE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
|
||||
PROCESSOR_VERSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$")
|
||||
|
||||
TOP_LEVEL_FIELDS = {
|
||||
"delivery_schema_version",
|
||||
"delivery_id",
|
||||
"job_id",
|
||||
"attempt_no",
|
||||
"status",
|
||||
"processor_version",
|
||||
"rule_set_sha256",
|
||||
"result_schema_version",
|
||||
"business_date",
|
||||
"artifacts",
|
||||
}
|
||||
ARTIFACT_FIELDS = {
|
||||
"object_key",
|
||||
"original_filename",
|
||||
"sha256",
|
||||
"byte_size",
|
||||
"mime_type",
|
||||
}
|
||||
ARTIFACT_ROLES = {
|
||||
"source_xml",
|
||||
"daily_report",
|
||||
"result_json",
|
||||
"structured_result_json",
|
||||
"exception_report",
|
||||
}
|
||||
ROLE_CONTRACTS = {
|
||||
"source_xml": ("opera_xml", ".xml", "application/xml"),
|
||||
"daily_report": ("daily_xlsx", ".xlsx", XLSX_MIME),
|
||||
"result_json": ("result_json", ".json", "application/json"),
|
||||
"structured_result_json": (
|
||||
"structured_result_json",
|
||||
".json",
|
||||
"application/json",
|
||||
),
|
||||
"exception_report": ("exception_xlsx", ".xlsx", XLSX_MIME),
|
||||
}
|
||||
ARTIFACT_SIZE_LIMITS = {
|
||||
"source_xml": 100 * 1024 * 1024,
|
||||
"daily_report": 100 * 1024 * 1024,
|
||||
"result_json": 5 * 1024 * 1024,
|
||||
"structured_result_json": 50 * 1024 * 1024,
|
||||
"exception_report": 20 * 1024 * 1024,
|
||||
}
|
||||
|
||||
|
||||
class IngestionError(RuntimeError):
|
||||
"""A stable, non-sensitive validation or persistence error."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
code: str,
|
||||
safe_message: str,
|
||||
*,
|
||||
retryable: bool = False,
|
||||
):
|
||||
super().__init__(safe_message)
|
||||
self.code = code
|
||||
self.safe_message = safe_message
|
||||
self.retryable = retryable
|
||||
|
||||
|
||||
def _fail(code: str, safe_message: str) -> None:
|
||||
raise IngestionError(code, safe_message)
|
||||
|
||||
|
||||
def _strict_mapping(value: Any, expected: set[str], label: str) -> Mapping[str, Any]:
|
||||
if not isinstance(value, Mapping) or set(value) != expected:
|
||||
_fail("DELIVERY_INVALID", f"{label} field set is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _valid_object_key(value: Any) -> bool:
|
||||
if not isinstance(value, str) or not value or len(value) > 1024:
|
||||
return False
|
||||
if (
|
||||
value.startswith("/")
|
||||
or value.endswith("/")
|
||||
or "\\" in value
|
||||
or "?" in value
|
||||
or "#" in value
|
||||
or "//" in value
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in value)
|
||||
):
|
||||
return False
|
||||
path = PurePosixPath(value)
|
||||
return not path.is_absolute() and ".." not in path.parts and "." not in path.parts
|
||||
|
||||
|
||||
def _valid_basename(value: Any) -> bool:
|
||||
return (
|
||||
isinstance(value, str)
|
||||
and 0 < len(value) <= 255
|
||||
and PurePosixPath(value).name == value
|
||||
and value not in {".", ".."}
|
||||
and "/" not in value
|
||||
and "\\" not in value
|
||||
and not any(ord(char) < 32 or ord(char) == 127 for char in value)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArtifactRef:
|
||||
role: str
|
||||
file_kind: str
|
||||
object_key: str
|
||||
original_filename: str
|
||||
sha256: str
|
||||
byte_size: int
|
||||
mime_type: str
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, role: str, payload: Any) -> "ArtifactRef":
|
||||
if role not in ROLE_CONTRACTS:
|
||||
_fail("DELIVERY_INVALID", "artifact role is invalid")
|
||||
values = _strict_mapping(payload, ARTIFACT_FIELDS, f"{role} artifact")
|
||||
file_kind, extension, expected_mime = ROLE_CONTRACTS[role]
|
||||
object_key = values.get("object_key")
|
||||
original_filename = values.get("original_filename")
|
||||
sha256 = values.get("sha256")
|
||||
byte_size = values.get("byte_size")
|
||||
mime_type = values.get("mime_type")
|
||||
if (
|
||||
not _valid_object_key(object_key)
|
||||
or not _valid_basename(original_filename)
|
||||
or not str(original_filename).lower().endswith(extension)
|
||||
or not isinstance(sha256, str)
|
||||
or not SHA256_RE.fullmatch(sha256)
|
||||
or not isinstance(byte_size, int)
|
||||
or isinstance(byte_size, bool)
|
||||
or byte_size < 0
|
||||
or mime_type != expected_mime
|
||||
):
|
||||
_fail("ARTIFACT_REFERENCE_INVALID", f"{role} metadata is invalid")
|
||||
return cls(
|
||||
role=role,
|
||||
file_kind=file_kind,
|
||||
object_key=str(object_key),
|
||||
original_filename=str(original_filename),
|
||||
sha256=sha256,
|
||||
byte_size=byte_size,
|
||||
mime_type=str(mime_type),
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"object_key": self.object_key,
|
||||
"original_filename": self.original_filename,
|
||||
"sha256": self.sha256,
|
||||
"byte_size": self.byte_size,
|
||||
"mime_type": self.mime_type,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeliveryEnvelope:
|
||||
delivery_id: str
|
||||
job_id: str
|
||||
attempt_no: int
|
||||
status: str
|
||||
processor_version: str
|
||||
rule_set_sha256: str
|
||||
result_schema_version: str
|
||||
business_date: Optional[date]
|
||||
artifacts: Mapping[str, Optional[ArtifactRef]]
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, payload: Any) -> "DeliveryEnvelope":
|
||||
values = _strict_mapping(payload, TOP_LEVEL_FIELDS, "delivery")
|
||||
if values.get("delivery_schema_version") != DELIVERY_SCHEMA_VERSION:
|
||||
_fail("DELIVERY_INVALID", "delivery schema version is unsupported")
|
||||
delivery_id = values.get("delivery_id")
|
||||
job_id = values.get("job_id")
|
||||
attempt_no = values.get("attempt_no")
|
||||
status = values.get("status")
|
||||
processor_version = values.get("processor_version")
|
||||
rule_set_sha256 = values.get("rule_set_sha256")
|
||||
result_schema_version = values.get("result_schema_version")
|
||||
if not isinstance(delivery_id, str) or not OPAQUE_ID_RE.fullmatch(delivery_id):
|
||||
_fail("DELIVERY_INVALID", "delivery identifier is invalid")
|
||||
if not isinstance(job_id, str) or not OPAQUE_ID_RE.fullmatch(job_id):
|
||||
_fail("DELIVERY_INVALID", "job identifier is invalid")
|
||||
if not isinstance(attempt_no, int) or isinstance(attempt_no, bool) or attempt_no < 1:
|
||||
_fail("DELIVERY_INVALID", "attempt number is invalid")
|
||||
if status not in {"success", "failed"}:
|
||||
_fail("DELIVERY_INVALID", "delivery status is invalid")
|
||||
if (
|
||||
not isinstance(processor_version, str)
|
||||
or not PROCESSOR_VERSION_RE.fullmatch(processor_version)
|
||||
):
|
||||
_fail("DELIVERY_INVALID", "processor version is invalid")
|
||||
if not isinstance(rule_set_sha256, str) or not SHA256_RE.fullmatch(rule_set_sha256):
|
||||
_fail("DELIVERY_INVALID", "rule set identity is invalid")
|
||||
if result_schema_version != RESULT_SCHEMA_VERSION:
|
||||
_fail("DELIVERY_INVALID", "result schema version is unsupported")
|
||||
|
||||
raw_business_date = values.get("business_date")
|
||||
parsed_date: Optional[date]
|
||||
if raw_business_date is None:
|
||||
parsed_date = None
|
||||
elif isinstance(raw_business_date, str):
|
||||
try:
|
||||
parsed_date = date.fromisoformat(raw_business_date)
|
||||
except ValueError:
|
||||
_fail("DELIVERY_INVALID", "business date is invalid")
|
||||
else:
|
||||
_fail("DELIVERY_INVALID", "business date is invalid")
|
||||
|
||||
raw_artifacts = _strict_mapping(values.get("artifacts"), ARTIFACT_ROLES, "artifact")
|
||||
artifacts: Dict[str, Optional[ArtifactRef]] = {}
|
||||
for role in sorted(ARTIFACT_ROLES):
|
||||
raw_artifact = raw_artifacts.get(role)
|
||||
artifacts[role] = (
|
||||
None if raw_artifact is None else ArtifactRef.from_dict(role, raw_artifact)
|
||||
)
|
||||
always_required = ("source_xml", "result_json", "structured_result_json")
|
||||
if any(artifacts[role] is None for role in always_required):
|
||||
_fail("DELIVERY_INVALID", "required delivery artifact is missing")
|
||||
if status == "success":
|
||||
if (
|
||||
parsed_date is None
|
||||
or artifacts["daily_report"] is None
|
||||
or artifacts["exception_report"] is not None
|
||||
):
|
||||
_fail("DELIVERY_INVALID", "success delivery artifact shape is invalid")
|
||||
elif artifacts["daily_report"] is not None or artifacts["exception_report"] is None:
|
||||
_fail("DELIVERY_INVALID", "failed delivery artifact shape is invalid")
|
||||
return cls(
|
||||
delivery_id=delivery_id,
|
||||
job_id=job_id,
|
||||
attempt_no=attempt_no,
|
||||
status=status,
|
||||
processor_version=processor_version,
|
||||
rule_set_sha256=rule_set_sha256,
|
||||
result_schema_version=result_schema_version,
|
||||
business_date=parsed_date,
|
||||
artifacts=artifacts,
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"delivery_schema_version": DELIVERY_SCHEMA_VERSION,
|
||||
"delivery_id": self.delivery_id,
|
||||
"job_id": self.job_id,
|
||||
"attempt_no": self.attempt_no,
|
||||
"status": self.status,
|
||||
"processor_version": self.processor_version,
|
||||
"rule_set_sha256": self.rule_set_sha256,
|
||||
"result_schema_version": self.result_schema_version,
|
||||
"business_date": self.business_date.isoformat() if self.business_date else None,
|
||||
"artifacts": {
|
||||
role: reference.to_dict() if reference is not None else None
|
||||
for role, reference in self.artifacts.items()
|
||||
},
|
||||
}
|
||||
Reference in New Issue
Block a user