"""ARR-owned validation and deterministic replay for direct MCP results.""" from __future__ import annotations import copy import hashlib import hmac import subprocess import tempfile from pathlib import Path from typing import Any, Dict, Mapping, Optional from arr_ingestion.artifacts import ArtifactStore from arr_ingestion.contracts import ( ARTIFACT_SIZE_LIMITS, RESULT_SCHEMA_VERSION, ROLE_CONTRACTS, ArtifactRef, DeliveryEnvelope, IngestionError, ) from arr_ingestion.direct_contracts import ( DirectSubmissionRequest, ReceivedDirectSubmission, VerifiedDirectSubmission, canonical_json_bytes, ) from arr_ingestion.validation import ( INNER_ARTIFACT_FIELDS, ProcessorPolicy, _validate_structured_payload, sha256_file, strict_json_file, ) def _artifact_from_inner(role: str, raw: Any) -> Optional[ArtifactRef]: if raw is None: return None if not isinstance(raw, Mapping) or set(raw) != INNER_ARTIFACT_FIELDS: raise IngestionError( "RESULT_CONTRACT_INVALID", f"{role} processor artifact contract is invalid", ) expected_kind, extension, expected_mime = ROLE_CONTRACTS[role] if raw.get("file_kind") != expected_kind: raise IngestionError( "RESULT_CONTRACT_INVALID", f"{role} processor artifact kind is invalid", ) filename = raw.get("original_filename") if not isinstance(filename, str): raise IngestionError( "RESULT_CONTRACT_INVALID", f"{role} processor artifact name is invalid", ) return ArtifactRef.from_dict( role, { "object_key": f"direct/replay/{role}/{role}{extension}", "original_filename": filename, "sha256": raw.get("sha256"), "byte_size": raw.get("byte_size"), "mime_type": expected_mime, }, ) def _validate_payload_shape( request: DirectSubmissionRequest, source: ArtifactRef, ) -> None: payload = request.payload artifacts = payload.get("artifacts") if not isinstance(artifacts, Mapping): raise IngestionError( "RESULT_CONTRACT_INVALID", "structured artifact contract is invalid", ) daily = _artifact_from_inner("daily_report", artifacts.get("daily_report")) result = _artifact_from_inner("result_json", artifacts.get("result_json")) exception = _artifact_from_inner( "exception_report", artifacts.get("exception_report"), ) if daily is None or result is None or exception is not None: raise IngestionError( "RESULT_CONTRACT_INVALID", "direct success artifacts are invalid", ) structured = ArtifactRef( role="structured_result_json", file_kind=ROLE_CONTRACTS["structured_result_json"][0], object_key="direct/replay/structured_result_json/structured-result.json", original_filename="structured-result.json", sha256=request.payload_sha256, byte_size=len(request.payload_bytes), mime_type=ROLE_CONTRACTS["structured_result_json"][2], ) envelope = DeliveryEnvelope( delivery_id=request.submission_key, job_id=request.job_id, attempt_no=request.attempt_no, status="success", processor_version=request.processor_version, rule_set_sha256=request.rule_set_sha256, result_schema_version=RESULT_SCHEMA_VERSION, business_date=request.business_date, artifacts={ "source_xml": source, "daily_report": daily, "result_json": result, "structured_result_json": structured, "exception_report": None, }, ) _validate_structured_payload(payload, envelope) def _normalized_for_replay(payload: Mapping[str, Any]) -> bytes: """Normalize only nondeterministic XLSX container byte metadata.""" normalized: Dict[str, Any] = copy.deepcopy(dict(payload)) artifacts = normalized.get("artifacts") if not isinstance(artifacts, dict): raise IngestionError( "RESULT_CONTRACT_INVALID", "structured artifact contract is invalid", ) daily = artifacts.get("daily_report") if not isinstance(daily, dict): raise IngestionError( "RESULT_CONTRACT_INVALID", "daily result artifact contract is invalid", ) daily["sha256"] = "0" * 64 daily["byte_size"] = 0 return canonical_json_bytes(normalized) class DirectResultValidator: """Re-run the frozen processor from the committed source XML and compare facts.""" def __init__(self, store: ArtifactStore, policy: ProcessorPolicy) -> None: self._store = store self._policy = policy self._processor = policy.skill_root / "scripts" / "process_daily.py" if not self._processor.is_file(): raise ValueError("direct processor is unavailable") def validate( self, submission: ReceivedDirectSubmission, ) -> VerifiedDirectSubmission: request = submission.request source = submission.source if ( request.processor_version != self._policy.processor_version or request.rule_set_sha256 != self._policy.rule_set_sha256 ): raise IngestionError( "PROCESSOR_NOT_ALLOWED", "direct result processor identity is not approved", ) if ( source.role != "source_xml" or source.original_filename != "source.xml" or source.byte_size > ARTIFACT_SIZE_LIMITS["source_xml"] ): raise IngestionError( "JOB_DELIVERY_MISMATCH", "direct result source identity is invalid", ) _validate_payload_shape(request, source) with tempfile.TemporaryDirectory(prefix="arr-direct-replay-") as temporary: root = Path(temporary) source_path = root / "source.xml" output_dir = root / "output" output_dir.mkdir(mode=0o700) self._store.materialize( source.object_key, source_path, ARTIFACT_SIZE_LIMITS["source_xml"], ) if ( not source_path.is_file() or source_path.stat().st_size != source.byte_size or sha256_file(source_path) != source.sha256 ): raise IngestionError( "ARTIFACT_HASH_MISMATCH", "direct source artifact identity does not match", ) result_path = output_dir / "result.json" structured_path = output_dir / "structured-result.json" try: completed = subprocess.run( [ self._policy.python_binary, str(self._processor), "--xml", str(source_path), "--output-dir", str(output_dir), "--result-json", str(result_path), "--structured-result-json", str(structured_path), ], cwd=self._policy.skill_root, capture_output=True, text=True, timeout=180, check=False, ) except (OSError, subprocess.TimeoutExpired): raise IngestionError( "DIRECT_REPLAY_UNAVAILABLE", "independent direct result replay did not complete", retryable=True, ) from None if completed.returncode == 4: raise IngestionError( "DIRECT_REPLAY_UNAVAILABLE", "independent direct result replay did not complete", retryable=True, ) if completed.returncode != 0: raise IngestionError( "DIRECT_RESULT_MISMATCH", "independent replay rejected the direct result", ) try: replay_payload = strict_json_file( structured_path, "direct replay structured result", ) except IngestionError: raise IngestionError( "DIRECT_REPLAY_UNAVAILABLE", "independent direct result replay was incomplete", retryable=True, ) from None replay_request = DirectSubmissionRequest.from_dict( { "submission_grant": request.submission_grant, "job_id": request.job_id, "attempt_no": request.attempt_no, "payload": replay_payload, } ) _validate_payload_shape(replay_request, source) if not hmac.compare_digest( _normalized_for_replay(request.payload), _normalized_for_replay(replay_payload), ): raise IngestionError( "DIRECT_RESULT_MISMATCH", "direct result does not match independent replay", ) return VerifiedDirectSubmission( submission=submission, replay_payload_sha256=hashlib.sha256( canonical_json_bytes(replay_payload) ).hexdigest(), )