522 lines
19 KiB
Python
522 lines
19 KiB
Python
"""ARR-side artifact verification and independent deterministic replay."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
|
|
|
|
from arr_ingestion.artifacts import ArtifactStore
|
|
from arr_ingestion.contracts import (
|
|
ARTIFACT_SIZE_LIMITS,
|
|
ArtifactRef,
|
|
DeliveryEnvelope,
|
|
IngestionError,
|
|
ROLE_CONTRACTS,
|
|
SHA256_RE,
|
|
)
|
|
STRUCTURED_TOP_FIELDS = {
|
|
"result_schema_version",
|
|
"status",
|
|
"activation_eligible",
|
|
"ingestion_mode",
|
|
"business_date",
|
|
"processor_version",
|
|
"rule_set_sha256",
|
|
"source_rows",
|
|
"removed_by_rate_code",
|
|
"removed_as_duplicates",
|
|
"output_rows",
|
|
"outcome_counts",
|
|
"channels",
|
|
"artifacts",
|
|
"records",
|
|
"errors",
|
|
}
|
|
STRUCTURED_ARTIFACT_FIELDS = {
|
|
"source_xml",
|
|
"daily_report",
|
|
"result_json",
|
|
"exception_report",
|
|
}
|
|
INNER_ARTIFACT_FIELDS = {
|
|
"file_kind",
|
|
"original_filename",
|
|
"sha256",
|
|
"byte_size",
|
|
"mime_type",
|
|
}
|
|
OUTCOME_FIELDS = {
|
|
"duplicate",
|
|
"excluded_rate_code",
|
|
"price_unmatched",
|
|
"retained",
|
|
"validation_failed",
|
|
}
|
|
RECORD_FIELDS = {
|
|
"source_sequence",
|
|
"source_location",
|
|
"source_worksheet",
|
|
"source_row_no",
|
|
"outcome",
|
|
"decision_codes",
|
|
"duplicate_of_source_sequence",
|
|
"adults",
|
|
"children",
|
|
"block_code",
|
|
"no_of_rooms",
|
|
"company_name",
|
|
"company_key",
|
|
"confirmation_no",
|
|
"disp_room_no",
|
|
"effective_rate_amount",
|
|
"full_name",
|
|
"res_comment",
|
|
"group_code_key",
|
|
"booking_source_match_status",
|
|
"trace_text",
|
|
"products",
|
|
"rate_code",
|
|
"normalized_rate_code",
|
|
"room_category_label",
|
|
"arrival",
|
|
"departure",
|
|
"nights",
|
|
"real_price",
|
|
"total_price",
|
|
"kb_amount",
|
|
"channel_key",
|
|
"pricing_method",
|
|
}
|
|
ERROR_FIELDS = {
|
|
"code",
|
|
"stage",
|
|
"source_location",
|
|
"company_name",
|
|
"rate_code",
|
|
"effective_rate_amount",
|
|
"confirmation_no",
|
|
"message",
|
|
}
|
|
RESULT_FIELDS = {
|
|
"version",
|
|
"status",
|
|
"business_date",
|
|
"message",
|
|
"metrics",
|
|
"outputs",
|
|
"errors",
|
|
}
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _reject_constant(_value: str) -> None:
|
|
raise ValueError("non-finite JSON number")
|
|
|
|
|
|
def _unique_object(pairs: Sequence[Tuple[str, Any]]) -> Dict[str, Any]:
|
|
result: Dict[str, Any] = {}
|
|
for key, value in pairs:
|
|
if key in result:
|
|
raise ValueError("duplicate JSON key")
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def strict_json_bytes(raw: bytes, label: str) -> Dict[str, Any]:
|
|
try:
|
|
value = json.loads(
|
|
raw.decode("utf-8"),
|
|
object_pairs_hook=_unique_object,
|
|
parse_constant=_reject_constant,
|
|
)
|
|
except (UnicodeDecodeError, ValueError, json.JSONDecodeError):
|
|
raise IngestionError("DELIVERY_JSON_INVALID", f"{label} JSON is invalid") from None
|
|
if not isinstance(value, dict):
|
|
raise IngestionError("DELIVERY_JSON_INVALID", f"{label} JSON must be an object")
|
|
return value
|
|
|
|
|
|
def strict_json_file(path: Path, label: str) -> Dict[str, Any]:
|
|
try:
|
|
return strict_json_bytes(path.read_bytes(), label)
|
|
except OSError:
|
|
raise IngestionError("ARTIFACT_UNREADABLE", f"{label} artifact is unreadable") from None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ProcessorPolicy:
|
|
processor_version: str
|
|
rule_set_sha256: str
|
|
skill_root: Path
|
|
python_binary: str = sys.executable
|
|
|
|
def __post_init__(self) -> None:
|
|
if not self.processor_version or not SHA256_RE.fullmatch(self.rule_set_sha256):
|
|
raise ValueError("processor policy identity is invalid")
|
|
resolved = self.skill_root.resolve()
|
|
object.__setattr__(self, "skill_root", resolved)
|
|
if not (resolved / "scripts" / "process_daily.py").is_file():
|
|
raise ValueError("processor executable is unavailable")
|
|
if not (resolved / "scripts" / "validate_daily.py").is_file():
|
|
raise ValueError("processor validator is unavailable")
|
|
if not (resolved / "references" / "价格对照.xlsx").is_file():
|
|
raise ValueError("processor price reference is unavailable")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VerifiedDelivery:
|
|
envelope: DeliveryEnvelope
|
|
envelope_sha256: str
|
|
result_payload: Mapping[str, Any]
|
|
structured_payload: Mapping[str, Any]
|
|
|
|
|
|
def _require_exact_mapping(value: Any, fields: set[str], label: str) -> Mapping[str, Any]:
|
|
if not isinstance(value, Mapping) or set(value) != fields:
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", f"{label} contract is invalid")
|
|
return value
|
|
|
|
|
|
def _require_nonnegative_integer(value: Any, label: str) -> int:
|
|
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", f"{label} must be a non-negative integer")
|
|
return value
|
|
|
|
|
|
def _inner_artifact(
|
|
role: str,
|
|
value: Any,
|
|
reference: Optional[ArtifactRef],
|
|
) -> Optional[Mapping[str, Any]]:
|
|
if value is None:
|
|
if reference is not None:
|
|
raise IngestionError(
|
|
"RESULT_ARTIFACT_MISMATCH", f"{role} processor artifact is missing"
|
|
)
|
|
return None
|
|
fields = _require_exact_mapping(value, INNER_ARTIFACT_FIELDS, f"{role} artifact")
|
|
if reference is None:
|
|
raise IngestionError(
|
|
"RESULT_ARTIFACT_MISMATCH", f"{role} processor artifact was not delivered"
|
|
)
|
|
expected_kind, _extension, expected_mime = ROLE_CONTRACTS[role]
|
|
if (
|
|
fields.get("file_kind") != expected_kind
|
|
or fields.get("original_filename") != reference.original_filename
|
|
or fields.get("sha256") != reference.sha256
|
|
or fields.get("byte_size") != reference.byte_size
|
|
or fields.get("mime_type") != expected_mime
|
|
):
|
|
raise IngestionError(
|
|
"RESULT_ARTIFACT_MISMATCH", f"{role} processor artifact metadata does not match delivery"
|
|
)
|
|
return fields
|
|
|
|
|
|
def _validate_result_payload(
|
|
payload: Mapping[str, Any],
|
|
envelope: DeliveryEnvelope,
|
|
structured: Mapping[str, Any],
|
|
) -> None:
|
|
_require_exact_mapping(payload, RESULT_FIELDS, "result")
|
|
expected_date = envelope.business_date.isoformat() if envelope.business_date else None
|
|
if (
|
|
payload.get("version") != "3.0"
|
|
or payload.get("status") != envelope.status
|
|
or payload.get("business_date") != expected_date
|
|
or not isinstance(payload.get("message"), str)
|
|
):
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "result identity is invalid")
|
|
metrics = _require_exact_mapping(
|
|
payload.get("metrics"),
|
|
{
|
|
"source_rows",
|
|
"removed_by_rate_code",
|
|
"removed_as_duplicates",
|
|
"output_rows",
|
|
"channels",
|
|
},
|
|
"result metrics",
|
|
)
|
|
for field in (
|
|
"source_rows",
|
|
"removed_by_rate_code",
|
|
"removed_as_duplicates",
|
|
"output_rows",
|
|
):
|
|
_require_nonnegative_integer(metrics.get(field), f"metrics.{field}")
|
|
if metrics.get(field) != structured.get(field):
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "result counters do not reconcile")
|
|
if metrics.get("channels") != structured.get("channels"):
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "result channels do not reconcile")
|
|
outputs = _require_exact_mapping(
|
|
payload.get("outputs"),
|
|
{"daily_report", "structured_result", "exception_report"},
|
|
"result outputs",
|
|
)
|
|
daily_ref = envelope.artifacts["daily_report"]
|
|
structured_ref = envelope.artifacts["structured_result_json"]
|
|
exception_ref = envelope.artifacts["exception_report"]
|
|
if (
|
|
outputs.get("daily_report")
|
|
!= (daily_ref.original_filename if daily_ref is not None else None)
|
|
or outputs.get("structured_result")
|
|
!= (
|
|
structured_ref.original_filename
|
|
if structured_ref is not None
|
|
else None
|
|
)
|
|
or outputs.get("exception_report")
|
|
!= (exception_ref.original_filename if exception_ref is not None else None)
|
|
):
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "result output names are invalid")
|
|
errors = payload.get("errors")
|
|
if not isinstance(errors, list) or errors != structured.get("errors"):
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "result errors do not reconcile")
|
|
|
|
|
|
def _validate_structured_payload(
|
|
payload: Mapping[str, Any], envelope: DeliveryEnvelope
|
|
) -> None:
|
|
_require_exact_mapping(payload, STRUCTURED_TOP_FIELDS, "structured result")
|
|
expected_date = envelope.business_date.isoformat() if envelope.business_date else None
|
|
if (
|
|
payload.get("result_schema_version") != "3.0"
|
|
or payload.get("status") != envelope.status
|
|
or payload.get("activation_eligible") != (envelope.status == "success")
|
|
or payload.get("ingestion_mode") != "opera_xml"
|
|
or payload.get("business_date") != expected_date
|
|
or payload.get("processor_version") != envelope.processor_version
|
|
or payload.get("rule_set_sha256") != envelope.rule_set_sha256
|
|
):
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "structured result identity is invalid")
|
|
counts: Dict[str, int] = {}
|
|
for field in (
|
|
"source_rows",
|
|
"removed_by_rate_code",
|
|
"removed_as_duplicates",
|
|
"output_rows",
|
|
):
|
|
counts[field] = _require_nonnegative_integer(payload.get(field), field)
|
|
outcomes = _require_exact_mapping(payload.get("outcome_counts"), OUTCOME_FIELDS, "outcome counts")
|
|
for outcome in OUTCOME_FIELDS:
|
|
_require_nonnegative_integer(outcomes.get(outcome), f"outcome_counts.{outcome}")
|
|
records = payload.get("records")
|
|
if not isinstance(records, list) or len(records) != counts["source_rows"]:
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "structured record count is invalid")
|
|
actual_outcomes = {outcome: 0 for outcome in OUTCOME_FIELDS}
|
|
for index, record in enumerate(records, 1):
|
|
values = _require_exact_mapping(record, RECORD_FIELDS, "structured record")
|
|
if values.get("source_sequence") != index or values.get("source_location") != f"reservation[{index}]":
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "structured source order is invalid")
|
|
if values.get("source_worksheet") is not None or values.get("source_row_no") is not None:
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID", "direct XML source coordinates are invalid"
|
|
)
|
|
outcome = values.get("outcome")
|
|
if outcome not in OUTCOME_FIELDS:
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "structured outcome is invalid")
|
|
actual_outcomes[str(outcome)] += 1
|
|
duplicate_of = values.get("duplicate_of_source_sequence")
|
|
if (
|
|
outcome == "duplicate"
|
|
and (
|
|
not isinstance(duplicate_of, int)
|
|
or isinstance(duplicate_of, bool)
|
|
or duplicate_of < 1
|
|
or duplicate_of >= index
|
|
)
|
|
) or (outcome != "duplicate" and duplicate_of is not None):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID", "structured duplicate lineage is invalid"
|
|
)
|
|
decisions = values.get("decision_codes")
|
|
if (
|
|
not isinstance(decisions, list)
|
|
or not decisions
|
|
or any(not isinstance(code, str) or not code for code in decisions)
|
|
or len(decisions) != len(set(decisions))
|
|
):
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "structured decisions are invalid")
|
|
group_code = values.get("group_code_key")
|
|
booking_status = values.get("booking_source_match_status")
|
|
if group_code is None:
|
|
if booking_status != "missing_group_code":
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID", "booking source status is invalid"
|
|
)
|
|
elif (
|
|
not isinstance(group_code, str)
|
|
or not group_code
|
|
or group_code != group_code.strip().upper()
|
|
or booking_status != "not_checked"
|
|
):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID", "structured Group Code is invalid"
|
|
)
|
|
if dict(outcomes) != actual_outcomes:
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "structured outcomes do not reconcile")
|
|
if (
|
|
counts["removed_by_rate_code"] != outcomes.get("excluded_rate_code")
|
|
or counts["removed_as_duplicates"] != outcomes.get("duplicate")
|
|
or counts["output_rows"] != outcomes.get("retained")
|
|
):
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "structured counters do not reconcile")
|
|
channels = payload.get("channels")
|
|
errors = payload.get("errors")
|
|
if not isinstance(channels, list) or not isinstance(errors, list):
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "structured arrays are invalid")
|
|
channel_names: set[str] = set()
|
|
channel_rows = 0
|
|
for channel in channels:
|
|
values = _require_exact_mapping(
|
|
channel, {"worksheet", "rows"}, "structured channel"
|
|
)
|
|
worksheet = values.get("worksheet")
|
|
rows = _require_nonnegative_integer(
|
|
values.get("rows"), "structured channel rows"
|
|
)
|
|
if (
|
|
not isinstance(worksheet, str)
|
|
or not worksheet.strip()
|
|
or worksheet in channel_names
|
|
):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID", "structured channel is invalid"
|
|
)
|
|
channel_names.add(worksheet)
|
|
channel_rows += rows
|
|
for error in errors:
|
|
_require_exact_mapping(error, ERROR_FIELDS, "structured error")
|
|
if envelope.status == "success":
|
|
if (
|
|
errors
|
|
or outcomes.get("validation_failed") != 0
|
|
or outcomes.get("price_unmatched") != 0
|
|
or counts["source_rows"]
|
|
!= counts["removed_by_rate_code"]
|
|
+ counts["removed_as_duplicates"]
|
|
+ counts["output_rows"]
|
|
or channel_rows != counts["output_rows"]
|
|
):
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "successful result is not balanced")
|
|
elif counts["output_rows"] != 0 or channels or not errors:
|
|
raise IngestionError("RESULT_CONTRACT_INVALID", "failed result shape is invalid")
|
|
|
|
artifacts = _require_exact_mapping(
|
|
payload.get("artifacts"), STRUCTURED_ARTIFACT_FIELDS, "structured artifacts"
|
|
)
|
|
_inner_artifact("source_xml", artifacts.get("source_xml"), envelope.artifacts["source_xml"])
|
|
_inner_artifact("daily_report", artifacts.get("daily_report"), envelope.artifacts["daily_report"])
|
|
_inner_artifact("result_json", artifacts.get("result_json"), envelope.artifacts["result_json"])
|
|
_inner_artifact(
|
|
"exception_report",
|
|
artifacts.get("exception_report"),
|
|
envelope.artifacts["exception_report"],
|
|
)
|
|
|
|
|
|
class DeliveryValidator:
|
|
def __init__(self, store: ArtifactStore, policy: ProcessorPolicy) -> None:
|
|
self._store = store
|
|
self._policy = policy
|
|
|
|
def validate(self, raw_envelope: bytes) -> VerifiedDelivery:
|
|
envelope_payload = strict_json_bytes(raw_envelope, "delivery envelope")
|
|
envelope = DeliveryEnvelope.from_dict(envelope_payload)
|
|
if (
|
|
envelope.processor_version != self._policy.processor_version
|
|
or envelope.rule_set_sha256 != self._policy.rule_set_sha256
|
|
):
|
|
raise IngestionError(
|
|
"PROCESSOR_NOT_ALLOWED", "delivery processor identity is not approved"
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix="arr-ingestion-") as temporary:
|
|
root = Path(temporary)
|
|
paths: Dict[str, Path] = {}
|
|
for role, reference in envelope.artifacts.items():
|
|
if reference is None:
|
|
continue
|
|
limit = ARTIFACT_SIZE_LIMITS[role]
|
|
if reference.byte_size > limit:
|
|
raise IngestionError(
|
|
"ARTIFACT_TOO_LARGE", "delivery artifact exceeds its size limit"
|
|
)
|
|
destination = root / role / reference.original_filename
|
|
self._store.materialize(reference.object_key, destination, limit)
|
|
if (
|
|
not destination.is_file()
|
|
or destination.stat().st_size != reference.byte_size
|
|
or sha256_file(destination) != reference.sha256
|
|
):
|
|
raise IngestionError(
|
|
"ARTIFACT_HASH_MISMATCH", "delivery artifact identity does not match"
|
|
)
|
|
paths[role] = destination
|
|
|
|
result_payload = strict_json_file(paths["result_json"], "result")
|
|
structured_payload = strict_json_file(
|
|
paths["structured_result_json"], "structured result"
|
|
)
|
|
_validate_structured_payload(structured_payload, envelope)
|
|
_validate_result_payload(result_payload, envelope, structured_payload)
|
|
if envelope.status == "success":
|
|
self._run_independent_validator(paths)
|
|
return VerifiedDelivery(
|
|
envelope=envelope,
|
|
envelope_sha256=hashlib.sha256(raw_envelope).hexdigest(),
|
|
result_payload=copy.deepcopy(result_payload),
|
|
structured_payload=copy.deepcopy(structured_payload),
|
|
)
|
|
|
|
def _run_independent_validator(
|
|
self,
|
|
paths: Mapping[str, Path],
|
|
) -> None:
|
|
validator = self._policy.skill_root / "scripts" / "validate_daily.py"
|
|
price_reference = self._policy.skill_root / "references" / "价格对照.xlsx"
|
|
try:
|
|
completed = subprocess.run(
|
|
[
|
|
self._policy.python_binary,
|
|
str(validator),
|
|
"--xml",
|
|
str(paths["source_xml"].resolve()),
|
|
"--daily",
|
|
str(paths["daily_report"].resolve()),
|
|
"--result-json",
|
|
str(paths["result_json"].resolve()),
|
|
"--structured-result-json",
|
|
str(paths["structured_result_json"].resolve()),
|
|
"--price-reference",
|
|
str(price_reference.resolve()),
|
|
],
|
|
cwd=self._policy.skill_root,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120,
|
|
check=False,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
raise IngestionError(
|
|
"OUTPUT_VALIDATION_FAILED", "independent result validation did not complete"
|
|
) from None
|
|
if completed.returncode != 0:
|
|
raise IngestionError(
|
|
"OUTPUT_VALIDATION_FAILED", "independent result validation rejected the delivery"
|
|
)
|