304 lines
11 KiB
Python
304 lines
11 KiB
Python
"""Strict, path-free contracts for ARR remote processing requests and results."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from dataclasses import dataclass, field
|
|
from datetime import date
|
|
from typing import Any, Dict, Mapping, Optional
|
|
|
|
from arr_ingestion.contracts import (
|
|
ARTIFACT_SIZE_LIMITS,
|
|
OPAQUE_ID_RE,
|
|
PROCESSOR_VERSION_RE,
|
|
RESULT_SCHEMA_VERSION,
|
|
ROLE_CONTRACTS,
|
|
SHA256_RE,
|
|
)
|
|
from arr_processing.errors import ProcessingError
|
|
from arr_ingestion.direct_contracts import GRANT_TOKEN_RE
|
|
from arr_storage.contracts import valid_delivery_filename
|
|
|
|
|
|
PROCESSING_REQUEST_VERSION = "arr-opera-daily-request-1"
|
|
PROCESSING_RESULT_VERSION = "arr-opera-daily-result-1"
|
|
REQUEST_FIELDS = {
|
|
"contract_version",
|
|
"job_id",
|
|
"attempt_no",
|
|
"source_file_id",
|
|
"processor_version",
|
|
"rule_set_sha256",
|
|
}
|
|
RESULT_FIELDS = {
|
|
"contract_version",
|
|
"delivery_id",
|
|
"job_id",
|
|
"attempt_no",
|
|
"remote_run_id",
|
|
"status",
|
|
"business_date",
|
|
"processor_version",
|
|
"rule_set_sha256",
|
|
"result_schema_version",
|
|
"artifacts",
|
|
}
|
|
RESULT_ARTIFACT_ROLES = {
|
|
"daily_report",
|
|
"result_json",
|
|
"structured_result_json",
|
|
"exception_report",
|
|
}
|
|
HANDLE_FIELDS = {
|
|
"file_handle",
|
|
"original_filename",
|
|
"sha256",
|
|
"byte_size",
|
|
"mime_type",
|
|
}
|
|
|
|
|
|
def _fail(code: str, message: str) -> None:
|
|
raise ProcessingError(code, message)
|
|
|
|
|
|
def _strict_mapping(value: Any, fields: set[str], label: str) -> Mapping[str, Any]:
|
|
if not isinstance(value, Mapping) or set(value) != fields:
|
|
_fail("PROCESSING_CONTRACT_INVALID", f"{label} field set is invalid")
|
|
return value
|
|
|
|
|
|
def _opaque(value: Any, label: str) -> str:
|
|
if not isinstance(value, str) or not OPAQUE_ID_RE.fullmatch(value):
|
|
_fail("PROCESSING_CONTRACT_INVALID", f"{label} is invalid")
|
|
return value
|
|
|
|
|
|
def _attempt(value: Any) -> int:
|
|
if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= 9999:
|
|
_fail("PROCESSING_CONTRACT_INVALID", "attempt number is invalid")
|
|
return value
|
|
|
|
|
|
def _processor(value: Any) -> str:
|
|
if not isinstance(value, str) or not PROCESSOR_VERSION_RE.fullmatch(value):
|
|
_fail("PROCESSING_CONTRACT_INVALID", "processor version is invalid")
|
|
return value
|
|
|
|
|
|
def _sha256(value: Any, label: str) -> str:
|
|
if not isinstance(value, str) or not SHA256_RE.fullmatch(value):
|
|
_fail("PROCESSING_CONTRACT_INVALID", f"{label} is invalid")
|
|
return value
|
|
|
|
|
|
def canonical_json_bytes(value: Mapping[str, Any]) -> bytes:
|
|
try:
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
allow_nan=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
except (TypeError, ValueError):
|
|
_fail("PROCESSING_CONTRACT_INVALID", "processing payload is not strict JSON")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ProcessingRequest:
|
|
job_id: str
|
|
attempt_no: int
|
|
source_file_id: str
|
|
processor_version: str
|
|
rule_set_sha256: str
|
|
submission_grant: Optional[str] = field(
|
|
default=None,
|
|
repr=False,
|
|
compare=False,
|
|
)
|
|
|
|
def __post_init__(self) -> None:
|
|
_opaque(self.job_id, "job identifier")
|
|
_attempt(self.attempt_no)
|
|
_opaque(self.source_file_id, "source file identifier")
|
|
_processor(self.processor_version)
|
|
_sha256(self.rule_set_sha256, "rule set identity")
|
|
if (
|
|
self.submission_grant is not None
|
|
and (
|
|
not isinstance(self.submission_grant, str)
|
|
or not GRANT_TOKEN_RE.fullmatch(self.submission_grant)
|
|
)
|
|
):
|
|
_fail(
|
|
"PROCESSING_CONTRACT_INVALID",
|
|
"direct submission grant is invalid",
|
|
)
|
|
|
|
@classmethod
|
|
def from_dict(cls, payload: Any) -> "ProcessingRequest":
|
|
values = _strict_mapping(payload, REQUEST_FIELDS, "processing request")
|
|
if values.get("contract_version") != PROCESSING_REQUEST_VERSION:
|
|
_fail("PROCESSING_CONTRACT_INVALID", "request contract version is unsupported")
|
|
return cls(
|
|
job_id=_opaque(values.get("job_id"), "job identifier"),
|
|
attempt_no=_attempt(values.get("attempt_no")),
|
|
source_file_id=_opaque(values.get("source_file_id"), "source file identifier"),
|
|
processor_version=_processor(values.get("processor_version")),
|
|
rule_set_sha256=_sha256(values.get("rule_set_sha256"), "rule set identity"),
|
|
)
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
"contract_version": PROCESSING_REQUEST_VERSION,
|
|
"job_id": self.job_id,
|
|
"attempt_no": self.attempt_no,
|
|
"source_file_id": self.source_file_id,
|
|
"processor_version": self.processor_version,
|
|
"rule_set_sha256": self.rule_set_sha256,
|
|
}
|
|
|
|
def canonical_bytes(self) -> bytes:
|
|
return canonical_json_bytes(self.to_dict())
|
|
|
|
def sha256(self) -> str:
|
|
return hashlib.sha256(self.canonical_bytes()).hexdigest()
|
|
|
|
def message(self) -> str:
|
|
return self.canonical_bytes().decode("utf-8")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ResultArtifactHandle:
|
|
role: str
|
|
file_handle: str
|
|
original_filename: str
|
|
sha256: str
|
|
byte_size: int
|
|
mime_type: str
|
|
|
|
@classmethod
|
|
def from_dict(cls, role: str, payload: Any) -> "ResultArtifactHandle":
|
|
if role not in RESULT_ARTIFACT_ROLES:
|
|
_fail("PROCESSING_CONTRACT_INVALID", "result artifact role is invalid")
|
|
values = _strict_mapping(payload, HANDLE_FIELDS, f"{role} artifact")
|
|
file_handle = _opaque(values.get("file_handle"), "artifact file handle")
|
|
original_filename = values.get("original_filename")
|
|
sha256 = _sha256(values.get("sha256"), "artifact hash")
|
|
byte_size = values.get("byte_size")
|
|
mime_type = values.get("mime_type")
|
|
if (
|
|
not valid_delivery_filename(role, original_filename)
|
|
or not isinstance(byte_size, int)
|
|
or isinstance(byte_size, bool)
|
|
or not 0 <= byte_size <= ARTIFACT_SIZE_LIMITS[role]
|
|
or mime_type != ROLE_CONTRACTS[role][2]
|
|
):
|
|
_fail("PROCESSING_CONTRACT_INVALID", f"{role} artifact metadata is invalid")
|
|
return cls(
|
|
role=role,
|
|
file_handle=file_handle,
|
|
original_filename=str(original_filename),
|
|
sha256=sha256,
|
|
byte_size=byte_size,
|
|
mime_type=str(mime_type),
|
|
)
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
"file_handle": self.file_handle,
|
|
"original_filename": self.original_filename,
|
|
"sha256": self.sha256,
|
|
"byte_size": self.byte_size,
|
|
"mime_type": self.mime_type,
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ProcessingResult:
|
|
delivery_id: str
|
|
job_id: str
|
|
attempt_no: int
|
|
remote_run_id: str
|
|
status: str
|
|
business_date: Optional[date]
|
|
processor_version: str
|
|
rule_set_sha256: str
|
|
result_schema_version: str
|
|
artifacts: Mapping[str, Optional[ResultArtifactHandle]]
|
|
|
|
@classmethod
|
|
def from_dict(cls, payload: Any) -> "ProcessingResult":
|
|
values = _strict_mapping(payload, RESULT_FIELDS, "processing result")
|
|
if values.get("contract_version") != PROCESSING_RESULT_VERSION:
|
|
_fail("PROCESSING_CONTRACT_INVALID", "result contract version is unsupported")
|
|
status = values.get("status")
|
|
if status not in {"success", "failed"}:
|
|
_fail("PROCESSING_CONTRACT_INVALID", "result status is invalid")
|
|
raw_date = values.get("business_date")
|
|
parsed_date: Optional[date]
|
|
if raw_date is None:
|
|
parsed_date = None
|
|
elif isinstance(raw_date, str):
|
|
try:
|
|
parsed_date = date.fromisoformat(raw_date)
|
|
except ValueError:
|
|
_fail("PROCESSING_CONTRACT_INVALID", "business date is invalid")
|
|
else:
|
|
_fail("PROCESSING_CONTRACT_INVALID", "business date is invalid")
|
|
|
|
raw_artifacts = _strict_mapping(
|
|
values.get("artifacts"), RESULT_ARTIFACT_ROLES, "result artifacts"
|
|
)
|
|
artifacts: Dict[str, Optional[ResultArtifactHandle]] = {}
|
|
for role in sorted(RESULT_ARTIFACT_ROLES):
|
|
raw = raw_artifacts.get(role)
|
|
artifacts[role] = None if raw is None else ResultArtifactHandle.from_dict(role, raw)
|
|
if artifacts["result_json"] is None or artifacts["structured_result_json"] is None:
|
|
_fail("PROCESSING_CONTRACT_INVALID", "required result artifact is missing")
|
|
if status == "success":
|
|
if (
|
|
parsed_date is None
|
|
or artifacts["daily_report"] is None
|
|
or artifacts["exception_report"] is not None
|
|
):
|
|
_fail("PROCESSING_CONTRACT_INVALID", "success result artifact shape is invalid")
|
|
elif artifacts["daily_report"] is not None or artifacts["exception_report"] is None:
|
|
_fail("PROCESSING_CONTRACT_INVALID", "failed result artifact shape is invalid")
|
|
|
|
result_schema_version = values.get("result_schema_version")
|
|
if result_schema_version != RESULT_SCHEMA_VERSION:
|
|
_fail("PROCESSING_CONTRACT_INVALID", "result schema version is unsupported")
|
|
return cls(
|
|
delivery_id=_opaque(values.get("delivery_id"), "delivery identifier"),
|
|
job_id=_opaque(values.get("job_id"), "job identifier"),
|
|
attempt_no=_attempt(values.get("attempt_no")),
|
|
remote_run_id=_opaque(values.get("remote_run_id"), "remote run identifier"),
|
|
status=str(status),
|
|
business_date=parsed_date,
|
|
processor_version=_processor(values.get("processor_version")),
|
|
rule_set_sha256=_sha256(values.get("rule_set_sha256"), "rule set identity"),
|
|
result_schema_version=str(result_schema_version),
|
|
artifacts=artifacts,
|
|
)
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
"contract_version": PROCESSING_RESULT_VERSION,
|
|
"delivery_id": self.delivery_id,
|
|
"job_id": self.job_id,
|
|
"attempt_no": self.attempt_no,
|
|
"remote_run_id": self.remote_run_id,
|
|
"status": self.status,
|
|
"business_date": self.business_date.isoformat() if self.business_date else None,
|
|
"processor_version": self.processor_version,
|
|
"rule_set_sha256": self.rule_set_sha256,
|
|
"result_schema_version": self.result_schema_version,
|
|
"artifacts": {
|
|
role: artifact.to_dict() if artifact is not None else None
|
|
for role, artifact in self.artifacts.items()
|
|
},
|
|
}
|