268 lines
9.4 KiB
Python
268 lines
9.4 KiB
Python
"""ARR-owned validation and deterministic replay for direct MCP results."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import hmac
|
|
import subprocess
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Mapping, Optional
|
|
|
|
from arr_ingestion.artifacts import ArtifactStore
|
|
from arr_ingestion.contracts import (
|
|
ARTIFACT_SIZE_LIMITS,
|
|
RESULT_SCHEMA_VERSION,
|
|
ROLE_CONTRACTS,
|
|
ArtifactRef,
|
|
DeliveryEnvelope,
|
|
IngestionError,
|
|
)
|
|
from arr_ingestion.direct_contracts import (
|
|
DirectSubmissionRequest,
|
|
ReceivedDirectSubmission,
|
|
VerifiedDirectSubmission,
|
|
canonical_json_bytes,
|
|
)
|
|
from arr_ingestion.validation import (
|
|
INNER_ARTIFACT_FIELDS,
|
|
ProcessorPolicy,
|
|
_validate_structured_payload,
|
|
sha256_file,
|
|
strict_json_file,
|
|
)
|
|
|
|
|
|
def _artifact_from_inner(role: str, raw: Any) -> Optional[ArtifactRef]:
|
|
if raw is None:
|
|
return None
|
|
if not isinstance(raw, Mapping) or set(raw) != INNER_ARTIFACT_FIELDS:
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
f"{role} processor artifact contract is invalid",
|
|
)
|
|
expected_kind, extension, expected_mime = ROLE_CONTRACTS[role]
|
|
if raw.get("file_kind") != expected_kind:
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
f"{role} processor artifact kind is invalid",
|
|
)
|
|
filename = raw.get("original_filename")
|
|
if not isinstance(filename, str):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
f"{role} processor artifact name is invalid",
|
|
)
|
|
return ArtifactRef.from_dict(
|
|
role,
|
|
{
|
|
"object_key": f"direct/replay/{role}/{role}{extension}",
|
|
"original_filename": filename,
|
|
"sha256": raw.get("sha256"),
|
|
"byte_size": raw.get("byte_size"),
|
|
"mime_type": expected_mime,
|
|
},
|
|
)
|
|
|
|
|
|
def _validate_payload_shape(
|
|
request: DirectSubmissionRequest,
|
|
source: ArtifactRef,
|
|
) -> None:
|
|
payload = request.payload
|
|
artifacts = payload.get("artifacts")
|
|
if not isinstance(artifacts, Mapping):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"structured artifact contract is invalid",
|
|
)
|
|
daily = _artifact_from_inner("daily_report", artifacts.get("daily_report"))
|
|
result = _artifact_from_inner("result_json", artifacts.get("result_json"))
|
|
exception = _artifact_from_inner(
|
|
"exception_report",
|
|
artifacts.get("exception_report"),
|
|
)
|
|
if daily is None or result is None or exception is not None:
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"direct success artifacts are invalid",
|
|
)
|
|
structured = ArtifactRef(
|
|
role="structured_result_json",
|
|
file_kind=ROLE_CONTRACTS["structured_result_json"][0],
|
|
object_key="direct/replay/structured_result_json/structured-result.json",
|
|
original_filename="structured-result.json",
|
|
sha256=request.payload_sha256,
|
|
byte_size=len(request.payload_bytes),
|
|
mime_type=ROLE_CONTRACTS["structured_result_json"][2],
|
|
)
|
|
envelope = DeliveryEnvelope(
|
|
delivery_id=request.submission_key,
|
|
job_id=request.job_id,
|
|
attempt_no=request.attempt_no,
|
|
status="success",
|
|
processor_version=request.processor_version,
|
|
rule_set_sha256=request.rule_set_sha256,
|
|
result_schema_version=RESULT_SCHEMA_VERSION,
|
|
business_date=request.business_date,
|
|
artifacts={
|
|
"source_xml": source,
|
|
"daily_report": daily,
|
|
"result_json": result,
|
|
"structured_result_json": structured,
|
|
"exception_report": None,
|
|
},
|
|
)
|
|
_validate_structured_payload(payload, envelope)
|
|
|
|
|
|
def _normalized_for_replay(payload: Mapping[str, Any]) -> bytes:
|
|
"""Normalize only nondeterministic XLSX container byte metadata."""
|
|
|
|
normalized: Dict[str, Any] = copy.deepcopy(dict(payload))
|
|
artifacts = normalized.get("artifacts")
|
|
if not isinstance(artifacts, dict):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"structured artifact contract is invalid",
|
|
)
|
|
daily = artifacts.get("daily_report")
|
|
if not isinstance(daily, dict):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"daily result artifact contract is invalid",
|
|
)
|
|
daily["sha256"] = "0" * 64
|
|
daily["byte_size"] = 0
|
|
return canonical_json_bytes(normalized)
|
|
|
|
|
|
class DirectResultValidator:
|
|
"""Re-run the frozen processor from the committed source XML and compare facts."""
|
|
|
|
def __init__(self, store: ArtifactStore, policy: ProcessorPolicy) -> None:
|
|
self._store = store
|
|
self._policy = policy
|
|
self._processor = policy.skill_root / "scripts" / "process_daily.py"
|
|
if not self._processor.is_file():
|
|
raise ValueError("direct processor is unavailable")
|
|
|
|
def validate(
|
|
self,
|
|
submission: ReceivedDirectSubmission,
|
|
) -> VerifiedDirectSubmission:
|
|
request = submission.request
|
|
source = submission.source
|
|
if (
|
|
request.processor_version != self._policy.processor_version
|
|
or request.rule_set_sha256 != self._policy.rule_set_sha256
|
|
):
|
|
raise IngestionError(
|
|
"PROCESSOR_NOT_ALLOWED",
|
|
"direct result processor identity is not approved",
|
|
)
|
|
if (
|
|
source.role != "source_xml"
|
|
or source.original_filename != "source.xml"
|
|
or source.byte_size > ARTIFACT_SIZE_LIMITS["source_xml"]
|
|
):
|
|
raise IngestionError(
|
|
"JOB_DELIVERY_MISMATCH",
|
|
"direct result source identity is invalid",
|
|
)
|
|
_validate_payload_shape(request, source)
|
|
|
|
with tempfile.TemporaryDirectory(prefix="arr-direct-replay-") as temporary:
|
|
root = Path(temporary)
|
|
source_path = root / "source.xml"
|
|
output_dir = root / "output"
|
|
output_dir.mkdir(mode=0o700)
|
|
self._store.materialize(
|
|
source.object_key,
|
|
source_path,
|
|
ARTIFACT_SIZE_LIMITS["source_xml"],
|
|
)
|
|
if (
|
|
not source_path.is_file()
|
|
or source_path.stat().st_size != source.byte_size
|
|
or sha256_file(source_path) != source.sha256
|
|
):
|
|
raise IngestionError(
|
|
"ARTIFACT_HASH_MISMATCH",
|
|
"direct source artifact identity does not match",
|
|
)
|
|
result_path = output_dir / "result.json"
|
|
structured_path = output_dir / "structured-result.json"
|
|
try:
|
|
completed = subprocess.run(
|
|
[
|
|
self._policy.python_binary,
|
|
str(self._processor),
|
|
"--xml",
|
|
str(source_path),
|
|
"--output-dir",
|
|
str(output_dir),
|
|
"--result-json",
|
|
str(result_path),
|
|
"--structured-result-json",
|
|
str(structured_path),
|
|
],
|
|
cwd=self._policy.skill_root,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=180,
|
|
check=False,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
raise IngestionError(
|
|
"DIRECT_REPLAY_UNAVAILABLE",
|
|
"independent direct result replay did not complete",
|
|
retryable=True,
|
|
) from None
|
|
if completed.returncode == 4:
|
|
raise IngestionError(
|
|
"DIRECT_REPLAY_UNAVAILABLE",
|
|
"independent direct result replay did not complete",
|
|
retryable=True,
|
|
)
|
|
if completed.returncode != 0:
|
|
raise IngestionError(
|
|
"DIRECT_RESULT_MISMATCH",
|
|
"independent replay rejected the direct result",
|
|
)
|
|
try:
|
|
replay_payload = strict_json_file(
|
|
structured_path,
|
|
"direct replay structured result",
|
|
)
|
|
except IngestionError:
|
|
raise IngestionError(
|
|
"DIRECT_REPLAY_UNAVAILABLE",
|
|
"independent direct result replay was incomplete",
|
|
retryable=True,
|
|
) from None
|
|
replay_request = DirectSubmissionRequest.from_dict(
|
|
{
|
|
"submission_grant": request.submission_grant,
|
|
"job_id": request.job_id,
|
|
"attempt_no": request.attempt_no,
|
|
"payload": replay_payload,
|
|
}
|
|
)
|
|
_validate_payload_shape(replay_request, source)
|
|
if not hmac.compare_digest(
|
|
_normalized_for_replay(request.payload),
|
|
_normalized_for_replay(replay_payload),
|
|
):
|
|
raise IngestionError(
|
|
"DIRECT_RESULT_MISMATCH",
|
|
"direct result does not match independent replay",
|
|
)
|
|
return VerifiedDirectSubmission(
|
|
submission=submission,
|
|
replay_payload_sha256=hashlib.sha256(
|
|
canonical_json_bytes(replay_payload)
|
|
).hexdigest(),
|
|
)
|