276 lines
9.7 KiB
Python
276 lines
9.7 KiB
Python
"""Strict, privacy-safe contracts for ARR processing deliveries."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
from datetime import date
|
|
from pathlib import PurePosixPath
|
|
from typing import Any, Dict, Mapping, Optional
|
|
|
|
|
|
DELIVERY_SCHEMA_VERSION = "1.0"
|
|
RESULT_SCHEMA_VERSION = "3.0"
|
|
XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
OPAQUE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
|
|
PROCESSOR_VERSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$")
|
|
|
|
TOP_LEVEL_FIELDS = {
|
|
"delivery_schema_version",
|
|
"delivery_id",
|
|
"job_id",
|
|
"attempt_no",
|
|
"status",
|
|
"processor_version",
|
|
"rule_set_sha256",
|
|
"result_schema_version",
|
|
"business_date",
|
|
"artifacts",
|
|
}
|
|
ARTIFACT_FIELDS = {
|
|
"object_key",
|
|
"original_filename",
|
|
"sha256",
|
|
"byte_size",
|
|
"mime_type",
|
|
}
|
|
ARTIFACT_ROLES = {
|
|
"source_xml",
|
|
"daily_report",
|
|
"result_json",
|
|
"structured_result_json",
|
|
"exception_report",
|
|
}
|
|
ROLE_CONTRACTS = {
|
|
"source_xml": ("opera_xml", ".xml", "application/xml"),
|
|
"booking_source": ("booking_excel", ".xlsx", XLSX_MIME),
|
|
"daily_report": ("daily_xlsx", ".xlsx", XLSX_MIME),
|
|
"result_json": ("result_json", ".json", "application/json"),
|
|
"structured_result_json": (
|
|
"structured_result_json",
|
|
".json",
|
|
"application/json",
|
|
),
|
|
"exception_report": ("exception_xlsx", ".xlsx", XLSX_MIME),
|
|
}
|
|
ARTIFACT_SIZE_LIMITS = {
|
|
"source_xml": 100 * 1024 * 1024,
|
|
"booking_source": 25 * 1024 * 1024,
|
|
"daily_report": 100 * 1024 * 1024,
|
|
"result_json": 5 * 1024 * 1024,
|
|
"structured_result_json": 50 * 1024 * 1024,
|
|
"exception_report": 20 * 1024 * 1024,
|
|
}
|
|
|
|
|
|
class IngestionError(RuntimeError):
|
|
"""A stable, non-sensitive validation or persistence error."""
|
|
|
|
def __init__(
|
|
self,
|
|
code: str,
|
|
safe_message: str,
|
|
*,
|
|
retryable: bool = False,
|
|
):
|
|
super().__init__(safe_message)
|
|
self.code = code
|
|
self.safe_message = safe_message
|
|
self.retryable = retryable
|
|
|
|
|
|
def _fail(code: str, safe_message: str) -> None:
|
|
raise IngestionError(code, safe_message)
|
|
|
|
|
|
def _strict_mapping(value: Any, expected: set[str], label: str) -> Mapping[str, Any]:
|
|
if not isinstance(value, Mapping) or set(value) != expected:
|
|
_fail("DELIVERY_INVALID", f"{label} field set is invalid")
|
|
return value
|
|
|
|
|
|
def _valid_object_key(value: Any) -> bool:
|
|
if not isinstance(value, str) or not value or len(value) > 1024:
|
|
return False
|
|
if (
|
|
value.startswith("/")
|
|
or value.endswith("/")
|
|
or "\\" in value
|
|
or "?" in value
|
|
or "#" in value
|
|
or "//" in value
|
|
or any(ord(char) < 32 or ord(char) == 127 for char in value)
|
|
):
|
|
return False
|
|
path = PurePosixPath(value)
|
|
return not path.is_absolute() and ".." not in path.parts and "." not in path.parts
|
|
|
|
|
|
def _valid_basename(value: Any) -> bool:
|
|
return (
|
|
isinstance(value, str)
|
|
and 0 < len(value) <= 255
|
|
and PurePosixPath(value).name == value
|
|
and value not in {".", ".."}
|
|
and "/" not in value
|
|
and "\\" not in value
|
|
and not any(ord(char) < 32 or ord(char) == 127 for char in value)
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ArtifactRef:
|
|
role: str
|
|
file_kind: str
|
|
object_key: str
|
|
original_filename: str
|
|
sha256: str
|
|
byte_size: int
|
|
mime_type: str
|
|
|
|
@classmethod
|
|
def from_dict(cls, role: str, payload: Any) -> "ArtifactRef":
|
|
if role not in ROLE_CONTRACTS:
|
|
_fail("DELIVERY_INVALID", "artifact role is invalid")
|
|
values = _strict_mapping(payload, ARTIFACT_FIELDS, f"{role} artifact")
|
|
file_kind, extension, expected_mime = ROLE_CONTRACTS[role]
|
|
object_key = values.get("object_key")
|
|
original_filename = values.get("original_filename")
|
|
sha256 = values.get("sha256")
|
|
byte_size = values.get("byte_size")
|
|
mime_type = values.get("mime_type")
|
|
if (
|
|
not _valid_object_key(object_key)
|
|
or not _valid_basename(original_filename)
|
|
or not str(original_filename).lower().endswith(extension)
|
|
or not isinstance(sha256, str)
|
|
or not SHA256_RE.fullmatch(sha256)
|
|
or not isinstance(byte_size, int)
|
|
or isinstance(byte_size, bool)
|
|
or byte_size < 0
|
|
or mime_type != expected_mime
|
|
):
|
|
_fail("ARTIFACT_REFERENCE_INVALID", f"{role} metadata is invalid")
|
|
return cls(
|
|
role=role,
|
|
file_kind=file_kind,
|
|
object_key=str(object_key),
|
|
original_filename=str(original_filename),
|
|
sha256=sha256,
|
|
byte_size=byte_size,
|
|
mime_type=str(mime_type),
|
|
)
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
"object_key": self.object_key,
|
|
"original_filename": self.original_filename,
|
|
"sha256": self.sha256,
|
|
"byte_size": self.byte_size,
|
|
"mime_type": self.mime_type,
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DeliveryEnvelope:
|
|
delivery_id: str
|
|
job_id: str
|
|
attempt_no: int
|
|
status: str
|
|
processor_version: str
|
|
rule_set_sha256: str
|
|
result_schema_version: str
|
|
business_date: Optional[date]
|
|
artifacts: Mapping[str, Optional[ArtifactRef]]
|
|
|
|
@classmethod
|
|
def from_dict(cls, payload: Any) -> "DeliveryEnvelope":
|
|
values = _strict_mapping(payload, TOP_LEVEL_FIELDS, "delivery")
|
|
if values.get("delivery_schema_version") != DELIVERY_SCHEMA_VERSION:
|
|
_fail("DELIVERY_INVALID", "delivery schema version is unsupported")
|
|
delivery_id = values.get("delivery_id")
|
|
job_id = values.get("job_id")
|
|
attempt_no = values.get("attempt_no")
|
|
status = values.get("status")
|
|
processor_version = values.get("processor_version")
|
|
rule_set_sha256 = values.get("rule_set_sha256")
|
|
result_schema_version = values.get("result_schema_version")
|
|
if not isinstance(delivery_id, str) or not OPAQUE_ID_RE.fullmatch(delivery_id):
|
|
_fail("DELIVERY_INVALID", "delivery identifier is invalid")
|
|
if not isinstance(job_id, str) or not OPAQUE_ID_RE.fullmatch(job_id):
|
|
_fail("DELIVERY_INVALID", "job identifier is invalid")
|
|
if not isinstance(attempt_no, int) or isinstance(attempt_no, bool) or attempt_no < 1:
|
|
_fail("DELIVERY_INVALID", "attempt number is invalid")
|
|
if status not in {"success", "failed"}:
|
|
_fail("DELIVERY_INVALID", "delivery status is invalid")
|
|
if (
|
|
not isinstance(processor_version, str)
|
|
or not PROCESSOR_VERSION_RE.fullmatch(processor_version)
|
|
):
|
|
_fail("DELIVERY_INVALID", "processor version is invalid")
|
|
if not isinstance(rule_set_sha256, str) or not SHA256_RE.fullmatch(rule_set_sha256):
|
|
_fail("DELIVERY_INVALID", "rule set identity is invalid")
|
|
if result_schema_version != RESULT_SCHEMA_VERSION:
|
|
_fail("DELIVERY_INVALID", "result schema version is unsupported")
|
|
|
|
raw_business_date = values.get("business_date")
|
|
parsed_date: Optional[date]
|
|
if raw_business_date is None:
|
|
parsed_date = None
|
|
elif isinstance(raw_business_date, str):
|
|
try:
|
|
parsed_date = date.fromisoformat(raw_business_date)
|
|
except ValueError:
|
|
_fail("DELIVERY_INVALID", "business date is invalid")
|
|
else:
|
|
_fail("DELIVERY_INVALID", "business date is invalid")
|
|
|
|
raw_artifacts = _strict_mapping(values.get("artifacts"), ARTIFACT_ROLES, "artifact")
|
|
artifacts: Dict[str, Optional[ArtifactRef]] = {}
|
|
for role in sorted(ARTIFACT_ROLES):
|
|
raw_artifact = raw_artifacts.get(role)
|
|
artifacts[role] = (
|
|
None if raw_artifact is None else ArtifactRef.from_dict(role, raw_artifact)
|
|
)
|
|
always_required = ("source_xml", "result_json", "structured_result_json")
|
|
if any(artifacts[role] is None for role in always_required):
|
|
_fail("DELIVERY_INVALID", "required delivery artifact is missing")
|
|
if status == "success":
|
|
if (
|
|
parsed_date is None
|
|
or artifacts["daily_report"] is None
|
|
or artifacts["exception_report"] is not None
|
|
):
|
|
_fail("DELIVERY_INVALID", "success delivery artifact shape is invalid")
|
|
elif artifacts["daily_report"] is not None or artifacts["exception_report"] is None:
|
|
_fail("DELIVERY_INVALID", "failed delivery artifact shape is invalid")
|
|
return cls(
|
|
delivery_id=delivery_id,
|
|
job_id=job_id,
|
|
attempt_no=attempt_no,
|
|
status=status,
|
|
processor_version=processor_version,
|
|
rule_set_sha256=rule_set_sha256,
|
|
result_schema_version=result_schema_version,
|
|
business_date=parsed_date,
|
|
artifacts=artifacts,
|
|
)
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
"delivery_schema_version": DELIVERY_SCHEMA_VERSION,
|
|
"delivery_id": self.delivery_id,
|
|
"job_id": self.job_id,
|
|
"attempt_no": self.attempt_no,
|
|
"status": self.status,
|
|
"processor_version": self.processor_version,
|
|
"rule_set_sha256": self.rule_set_sha256,
|
|
"result_schema_version": self.result_schema_version,
|
|
"business_date": self.business_date.isoformat() if self.business_date else None,
|
|
"artifacts": {
|
|
role: reference.to_dict() if reference is not None else None
|
|
for role, reference in self.artifacts.items()
|
|
},
|
|
}
|