219 lines
9.5 KiB
Python
219 lines
9.5 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import copy
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from openpyxl import load_workbook
|
|
|
|
from arr_ingestion.contracts import IngestionError
|
|
from arr_ingestion.validation import DeliveryValidator, ProcessorPolicy
|
|
from tests.test_arr_opera_daily_ingest import (
|
|
reservation,
|
|
run_processor,
|
|
xml_document,
|
|
)
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
SKILL_ROOT = PROJECT_ROOT / "arr-opera-daily-ingest"
|
|
SCRIPTS = SKILL_ROOT / "scripts"
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
import process_daily as processor # noqa: E402
|
|
|
|
|
|
class MemoryStore:
|
|
def __init__(self, objects: dict[str, bytes]):
|
|
self.objects = objects
|
|
self.fetch_count = 0
|
|
|
|
def materialize(self, object_key: str, destination: Path, max_bytes: int) -> None:
|
|
self.fetch_count += 1
|
|
if object_key not in self.objects:
|
|
raise IngestionError("ARTIFACT_NOT_FOUND", "delivery artifact is unavailable")
|
|
value = self.objects[object_key]
|
|
if len(value) > max_bytes:
|
|
raise IngestionError("ARTIFACT_TOO_LARGE", "delivery artifact exceeds its size limit")
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(value)
|
|
|
|
|
|
def artifact_ref(key: str, filename: str, value: bytes, mime_type: str) -> dict[str, object]:
|
|
return {
|
|
"object_key": key,
|
|
"original_filename": filename,
|
|
"sha256": hashlib.sha256(value).hexdigest(),
|
|
"byte_size": len(value),
|
|
"mime_type": mime_type,
|
|
}
|
|
|
|
|
|
def build_delivery(xml_text: str, root: Path) -> tuple[bytes, MemoryStore, dict[str, bytes]]:
|
|
exit_code, xml_path, output_dir, result, _structured = run_processor(xml_text, root)
|
|
status = "success" if exit_code == 0 else "failed"
|
|
values = {
|
|
"source_xml": xml_path.read_bytes(),
|
|
"result_json": (output_dir / "result.json").read_bytes(),
|
|
"structured_result_json": (output_dir / "structured-result.json").read_bytes(),
|
|
}
|
|
if status == "success":
|
|
values["daily_report"] = (output_dir / result["outputs"]["daily_report"]).read_bytes()
|
|
else:
|
|
values["exception_report"] = (output_dir / result["outputs"]["exception_report"]).read_bytes()
|
|
keys = {role: f"arr/jobs/job-001/{role}/{role}" for role in values}
|
|
objects = {keys[role]: value for role, value in values.items()}
|
|
xlsx_mime = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
envelope = {
|
|
"delivery_schema_version": "1.0",
|
|
"delivery_id": "delivery-001",
|
|
"job_id": "job-001",
|
|
"attempt_no": 1,
|
|
"status": status,
|
|
"processor_version": processor.PROCESSOR_VERSION,
|
|
"rule_set_sha256": processor.rule_set_sha256(),
|
|
"result_schema_version": "3.0",
|
|
"business_date": result["business_date"],
|
|
"artifacts": {
|
|
"source_xml": artifact_ref(keys["source_xml"], xml_path.name, values["source_xml"], "application/xml"),
|
|
"daily_report": (
|
|
artifact_ref(
|
|
keys["daily_report"],
|
|
result["outputs"]["daily_report"],
|
|
values["daily_report"],
|
|
xlsx_mime,
|
|
)
|
|
if status == "success"
|
|
else None
|
|
),
|
|
"result_json": artifact_ref(
|
|
keys["result_json"], "result.json", values["result_json"], "application/json"
|
|
),
|
|
"structured_result_json": artifact_ref(
|
|
keys["structured_result_json"],
|
|
"structured-result.json",
|
|
values["structured_result_json"],
|
|
"application/json",
|
|
),
|
|
"exception_report": (
|
|
artifact_ref(
|
|
keys["exception_report"],
|
|
result["outputs"]["exception_report"],
|
|
values["exception_report"],
|
|
xlsx_mime,
|
|
)
|
|
if status == "failed"
|
|
else None
|
|
),
|
|
},
|
|
}
|
|
return json.dumps(envelope, ensure_ascii=False).encode("utf-8"), MemoryStore(objects), values
|
|
|
|
|
|
def policy() -> ProcessorPolicy:
|
|
return ProcessorPolicy(
|
|
processor_version=processor.PROCESSOR_VERSION,
|
|
rule_set_sha256=processor.rule_set_sha256(),
|
|
skill_root=SKILL_ROOT,
|
|
python_binary=sys.executable,
|
|
)
|
|
|
|
|
|
class DeliveryValidationTests(unittest.TestCase):
|
|
def success_xml(self) -> str:
|
|
return xml_document(
|
|
reservation(1, rate_code="NOT-ALLOWED", rate_amount="INVALID"),
|
|
reservation(2, res_comment="SYN-GROUP", rooms="2", departure="2026-07-30"),
|
|
)
|
|
|
|
def test_success_delivery_is_independently_replayed_from_object_artifacts(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
raw, store, _values = build_delivery(self.success_xml(), Path(temp_dir))
|
|
verified = DeliveryValidator(store, policy()).validate(raw)
|
|
self.assertEqual(verified.envelope.status, "success")
|
|
self.assertEqual(verified.structured_payload["output_rows"], 1)
|
|
self.assertEqual(store.fetch_count, 4)
|
|
|
|
def test_declared_hash_mismatch_is_rejected_before_replay(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
raw, store, _values = build_delivery(self.success_xml(), Path(temp_dir))
|
|
envelope = json.loads(raw)
|
|
envelope["artifacts"]["daily_report"]["sha256"] = "0" * 64
|
|
tampered = json.dumps(envelope).encode("utf-8")
|
|
with self.assertRaisesRegex(IngestionError, "identity does not match") as raised:
|
|
DeliveryValidator(store, policy()).validate(tampered)
|
|
self.assertEqual(raised.exception.code, "ARTIFACT_HASH_MISMATCH")
|
|
|
|
def test_semantically_tampered_daily_is_rejected_even_when_hashes_match(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir)
|
|
raw, store, _values = build_delivery(self.success_xml(), root / "processor")
|
|
envelope = json.loads(raw)
|
|
reference = envelope["artifacts"]["daily_report"]
|
|
key = reference["object_key"]
|
|
workbook_path = root / "tampered.xlsx"
|
|
workbook_path.write_bytes(store.objects[key])
|
|
workbook = load_workbook(workbook_path, data_only=False)
|
|
try:
|
|
total_col = processor.DAILY_HEADERS.index("TOTAL PRICE") + 1
|
|
workbook.active.cell(2, total_col).value = 999999
|
|
workbook.save(workbook_path)
|
|
finally:
|
|
workbook.close()
|
|
changed = workbook_path.read_bytes()
|
|
store.objects[key] = changed
|
|
reference["sha256"] = hashlib.sha256(changed).hexdigest()
|
|
reference["byte_size"] = len(changed)
|
|
structured_key = envelope["artifacts"]["structured_result_json"]["object_key"]
|
|
structured = json.loads(store.objects[structured_key])
|
|
structured["artifacts"]["daily_report"]["sha256"] = reference["sha256"]
|
|
structured["artifacts"]["daily_report"]["byte_size"] = reference["byte_size"]
|
|
structured_bytes = (json.dumps(structured, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
|
|
store.objects[structured_key] = structured_bytes
|
|
structured_ref = envelope["artifacts"]["structured_result_json"]
|
|
structured_ref["sha256"] = hashlib.sha256(structured_bytes).hexdigest()
|
|
structured_ref["byte_size"] = len(structured_bytes)
|
|
|
|
with self.assertRaisesRegex(IngestionError, "rejected the delivery") as raised:
|
|
DeliveryValidator(store, policy()).validate(json.dumps(envelope).encode("utf-8"))
|
|
self.assertEqual(raised.exception.code, "OUTPUT_VALIDATION_FAILED")
|
|
|
|
def test_failed_delivery_is_retained_but_never_declared_activation_eligible(self):
|
|
failure_xml = xml_document(
|
|
reservation(1, rate_code="GRPA4", rate_amount="1800", res_comment="SYN-FAIL")
|
|
)
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
raw, store, _values = build_delivery(failure_xml, Path(temp_dir))
|
|
verified = DeliveryValidator(store, policy()).validate(raw)
|
|
self.assertEqual(verified.envelope.status, "failed")
|
|
self.assertFalse(verified.structured_payload["activation_eligible"])
|
|
self.assertEqual(verified.structured_payload["output_rows"], 0)
|
|
|
|
envelope = json.loads(raw)
|
|
key = envelope["artifacts"]["structured_result_json"]["object_key"]
|
|
payload = json.loads(store.objects[key])
|
|
payload["activation_eligible"] = True
|
|
changed = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
|
|
store.objects[key] = changed
|
|
envelope["artifacts"]["structured_result_json"]["sha256"] = hashlib.sha256(changed).hexdigest()
|
|
envelope["artifacts"]["structured_result_json"]["byte_size"] = len(changed)
|
|
with self.assertRaisesRegex(IngestionError, "identity is invalid"):
|
|
DeliveryValidator(store, policy()).validate(json.dumps(envelope).encode("utf-8"))
|
|
|
|
def test_duplicate_json_keys_are_rejected(self):
|
|
raw = b'{"delivery_schema_version":"1.0","delivery_schema_version":"1.0"}'
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
store = MemoryStore({})
|
|
with self.assertRaisesRegex(IngestionError, "JSON is invalid"):
|
|
DeliveryValidator(store, policy()).validate(raw)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|