292 lines
13 KiB
Python
292 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import copy
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from openpyxl import load_workbook
|
|
|
|
from arr_ingestion.contracts import IngestionError
|
|
from arr_ingestion.validation import DeliveryValidator, ProcessorPolicy
|
|
from tests.test_arr_opera_daily_ingest import (
|
|
manual_override_manifest,
|
|
reservation,
|
|
run_processor,
|
|
xml_document,
|
|
)
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
SKILL_ROOT = PROJECT_ROOT / "arr-opera-daily-ingest"
|
|
SCRIPTS = SKILL_ROOT / "scripts"
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
import process_daily as processor # noqa: E402
|
|
|
|
|
|
class MemoryStore:
|
|
def __init__(self, objects: dict[str, bytes]):
|
|
self.objects = objects
|
|
self.fetch_count = 0
|
|
|
|
def materialize(self, object_key: str, destination: Path, max_bytes: int) -> None:
|
|
self.fetch_count += 1
|
|
if object_key not in self.objects:
|
|
raise IngestionError("ARTIFACT_NOT_FOUND", "delivery artifact is unavailable")
|
|
value = self.objects[object_key]
|
|
if len(value) > max_bytes:
|
|
raise IngestionError("ARTIFACT_TOO_LARGE", "delivery artifact exceeds its size limit")
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(value)
|
|
|
|
|
|
def artifact_ref(key: str, filename: str, value: bytes, mime_type: str) -> dict[str, object]:
|
|
return {
|
|
"object_key": key,
|
|
"original_filename": filename,
|
|
"sha256": hashlib.sha256(value).hexdigest(),
|
|
"byte_size": len(value),
|
|
"mime_type": mime_type,
|
|
}
|
|
|
|
|
|
def build_delivery(
|
|
xml_text: str,
|
|
root: Path,
|
|
*,
|
|
manual_override: dict[str, object] | None = None,
|
|
review_job_id: str | None = None,
|
|
review_case_id: str | None = None,
|
|
) -> tuple[bytes, MemoryStore, dict[str, bytes]]:
|
|
_exit_code, xml_path, output_dir, result, structured = run_processor(
|
|
xml_text,
|
|
root,
|
|
manual_override=manual_override,
|
|
review_job_id=review_job_id,
|
|
review_case_id=review_case_id,
|
|
)
|
|
status = result["status"]
|
|
values = {
|
|
"source_xml": xml_path.read_bytes(),
|
|
"result_json": (output_dir / "result.json").read_bytes(),
|
|
"structured_result_json": (output_dir / "structured-result.json").read_bytes(),
|
|
}
|
|
if status == "success":
|
|
values["daily_report"] = (output_dir / result["outputs"]["daily_report"]).read_bytes()
|
|
elif status == "failed":
|
|
values["exception_report"] = (output_dir / result["outputs"]["exception_report"]).read_bytes()
|
|
if structured["artifacts"]["manual_override_json"] is not None:
|
|
values["manual_override_json"] = (root / "manual-override.json").read_bytes()
|
|
keys = {role: f"arr/jobs/job-001/{role}/{role}" for role in values}
|
|
objects = {keys[role]: value for role, value in values.items()}
|
|
xlsx_mime = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
envelope = {
|
|
"delivery_schema_version": "1.0",
|
|
"delivery_id": "delivery-001",
|
|
"job_id": "job-001",
|
|
"attempt_no": 1,
|
|
"status": status,
|
|
"processor_version": processor.PROCESSOR_VERSION,
|
|
"rule_set_sha256": processor.rule_set_sha256(),
|
|
"result_schema_version": processor.RESULT_VERSION,
|
|
"business_date": result["business_date"],
|
|
"artifacts": {
|
|
"source_xml": artifact_ref(keys["source_xml"], xml_path.name, values["source_xml"], "application/xml"),
|
|
"daily_report": (
|
|
artifact_ref(
|
|
keys["daily_report"],
|
|
result["outputs"]["daily_report"],
|
|
values["daily_report"],
|
|
xlsx_mime,
|
|
)
|
|
if status == "success"
|
|
else None
|
|
),
|
|
"result_json": artifact_ref(
|
|
keys["result_json"], "result.json", values["result_json"], "application/json"
|
|
),
|
|
"structured_result_json": artifact_ref(
|
|
keys["structured_result_json"],
|
|
"structured-result.json",
|
|
values["structured_result_json"],
|
|
"application/json",
|
|
),
|
|
"exception_report": (
|
|
artifact_ref(
|
|
keys["exception_report"],
|
|
result["outputs"]["exception_report"],
|
|
values["exception_report"],
|
|
xlsx_mime,
|
|
)
|
|
if status == "failed"
|
|
else None
|
|
),
|
|
"manual_override_json": (
|
|
artifact_ref(
|
|
keys["manual_override_json"],
|
|
"manual-override.json",
|
|
values["manual_override_json"],
|
|
"application/json",
|
|
)
|
|
if "manual_override_json" in values
|
|
else None
|
|
),
|
|
},
|
|
}
|
|
return json.dumps(envelope, ensure_ascii=False).encode("utf-8"), MemoryStore(objects), values
|
|
|
|
|
|
def policy() -> ProcessorPolicy:
|
|
return ProcessorPolicy(
|
|
processor_version=processor.PROCESSOR_VERSION,
|
|
rule_set_sha256=processor.rule_set_sha256(),
|
|
skill_root=SKILL_ROOT,
|
|
python_binary=sys.executable,
|
|
)
|
|
|
|
|
|
class DeliveryValidationTests(unittest.TestCase):
|
|
def success_xml(self) -> str:
|
|
return xml_document(
|
|
reservation(1, rate_code="NOT-ALLOWED", rate_amount="INVALID"),
|
|
reservation(2, res_comment="SYN-GROUP", rooms="2", departure="2026-07-30"),
|
|
)
|
|
|
|
def test_success_delivery_is_independently_replayed_from_object_artifacts(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
raw, store, _values = build_delivery(self.success_xml(), Path(temp_dir))
|
|
verified = DeliveryValidator(store, policy()).validate(raw)
|
|
self.assertEqual(verified.envelope.status, "success")
|
|
self.assertEqual(verified.structured_payload["output_rows"], 1)
|
|
self.assertEqual(store.fetch_count, 4)
|
|
|
|
def test_declared_hash_mismatch_is_rejected_before_replay(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
raw, store, _values = build_delivery(self.success_xml(), Path(temp_dir))
|
|
envelope = json.loads(raw)
|
|
envelope["artifacts"]["daily_report"]["sha256"] = "0" * 64
|
|
tampered = json.dumps(envelope).encode("utf-8")
|
|
with self.assertRaisesRegex(IngestionError, "identity does not match") as raised:
|
|
DeliveryValidator(store, policy()).validate(tampered)
|
|
self.assertEqual(raised.exception.code, "ARTIFACT_HASH_MISMATCH")
|
|
|
|
def test_semantically_tampered_daily_is_rejected_even_when_hashes_match(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir)
|
|
raw, store, _values = build_delivery(self.success_xml(), root / "processor")
|
|
envelope = json.loads(raw)
|
|
reference = envelope["artifacts"]["daily_report"]
|
|
key = reference["object_key"]
|
|
workbook_path = root / "tampered.xlsx"
|
|
workbook_path.write_bytes(store.objects[key])
|
|
workbook = load_workbook(workbook_path, data_only=False)
|
|
try:
|
|
total_col = processor.DAILY_HEADERS.index("TOTAL PRICE") + 1
|
|
workbook.active.cell(2, total_col).value = 999999
|
|
workbook.save(workbook_path)
|
|
finally:
|
|
workbook.close()
|
|
changed = workbook_path.read_bytes()
|
|
store.objects[key] = changed
|
|
reference["sha256"] = hashlib.sha256(changed).hexdigest()
|
|
reference["byte_size"] = len(changed)
|
|
structured_key = envelope["artifacts"]["structured_result_json"]["object_key"]
|
|
structured = json.loads(store.objects[structured_key])
|
|
structured["artifacts"]["daily_report"]["sha256"] = reference["sha256"]
|
|
structured["artifacts"]["daily_report"]["byte_size"] = reference["byte_size"]
|
|
structured_bytes = (json.dumps(structured, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
|
|
store.objects[structured_key] = structured_bytes
|
|
structured_ref = envelope["artifacts"]["structured_result_json"]
|
|
structured_ref["sha256"] = hashlib.sha256(structured_bytes).hexdigest()
|
|
structured_ref["byte_size"] = len(structured_bytes)
|
|
|
|
with self.assertRaisesRegex(IngestionError, "rejected the delivery") as raised:
|
|
DeliveryValidator(store, policy()).validate(json.dumps(envelope).encode("utf-8"))
|
|
self.assertEqual(raised.exception.code, "OUTPUT_VALIDATION_FAILED")
|
|
|
|
def test_failed_delivery_is_retained_but_never_declared_activation_eligible(self):
|
|
failure_xml = xml_document(
|
|
reservation(1, departure="2026-07-26", res_comment="SYN-FAIL")
|
|
)
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
raw, store, _values = build_delivery(failure_xml, Path(temp_dir))
|
|
verified = DeliveryValidator(store, policy()).validate(raw)
|
|
self.assertEqual(verified.envelope.status, "failed")
|
|
self.assertFalse(verified.structured_payload["activation_eligible"])
|
|
self.assertEqual(verified.structured_payload["output_rows"], 0)
|
|
|
|
envelope = json.loads(raw)
|
|
key = envelope["artifacts"]["structured_result_json"]["object_key"]
|
|
payload = json.loads(store.objects[key])
|
|
payload["activation_eligible"] = True
|
|
changed = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
|
|
store.objects[key] = changed
|
|
envelope["artifacts"]["structured_result_json"]["sha256"] = hashlib.sha256(changed).hexdigest()
|
|
envelope["artifacts"]["structured_result_json"]["byte_size"] = len(changed)
|
|
with self.assertRaisesRegex(IngestionError, "identity is invalid"):
|
|
DeliveryValidator(store, policy()).validate(json.dumps(envelope).encode("utf-8"))
|
|
|
|
def test_review_delivery_is_replayed_without_daily_or_finance_artifacts(self):
|
|
review_xml = xml_document(
|
|
reservation(1, rate_code="GRPA1", rate_amount="900"),
|
|
reservation(2, rate_code="GRPA1", rate_amount="1800"),
|
|
)
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
raw, store, _values = build_delivery(review_xml, Path(temp_dir))
|
|
verified = DeliveryValidator(store, policy()).validate(raw)
|
|
self.assertEqual(verified.envelope.status, "review_required")
|
|
self.assertEqual(verified.structured_payload["candidate_rows"], 1)
|
|
self.assertEqual(verified.structured_payload["review_required_rows"], 1)
|
|
self.assertEqual(verified.structured_payload["review_issue_count"], 1)
|
|
self.assertIsNone(verified.envelope.artifacts["daily_report"])
|
|
self.assertIsNone(verified.envelope.artifacts["exception_report"])
|
|
self.assertIsNone(verified.envelope.artifacts["manual_override_json"])
|
|
self.assertEqual(store.fetch_count, 3)
|
|
|
|
def test_final_manual_delivery_replays_frozen_manifest(self):
|
|
source = xml_document(
|
|
reservation(1, rate_code="GRPA1", rate_amount="900"),
|
|
reservation(2, rate_code="GRPA1", rate_amount="1800"),
|
|
)
|
|
review_case_id = "dailyreview-0123456789abcdef0123456789abcdef"
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir)
|
|
review = run_processor(source, root / "review")
|
|
manifest = manual_override_manifest(
|
|
review[1],
|
|
review[4]["review_issues"],
|
|
job_id="job-001",
|
|
review_case_id=review_case_id,
|
|
real_price="0.00",
|
|
)
|
|
raw, store, _values = build_delivery(
|
|
source,
|
|
root / "final",
|
|
manual_override=manifest,
|
|
review_job_id="job-001",
|
|
review_case_id=review_case_id,
|
|
)
|
|
verified = DeliveryValidator(store, policy()).validate(raw)
|
|
self.assertEqual(verified.envelope.status, "success")
|
|
self.assertEqual(verified.structured_payload["review_case_id"], review_case_id)
|
|
self.assertEqual(verified.structured_payload["manually_priced_rows"], 1)
|
|
self.assertIsNotNone(verified.envelope.artifacts["manual_override_json"])
|
|
self.assertEqual(store.fetch_count, 5)
|
|
|
|
def test_duplicate_json_keys_are_rejected(self):
|
|
raw = b'{"delivery_schema_version":"1.0","delivery_schema_version":"1.0"}'
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
store = MemoryStore({})
|
|
with self.assertRaisesRegex(IngestionError, "JSON is invalid"):
|
|
DeliveryValidator(store, policy()).validate(raw)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|