"""Offline source-field review acceptance; every reservation is synthetic.""" import copy from concurrent.futures import ThreadPoolExecutor from datetime import date from decimal import Decimal import hashlib import json import os from pathlib import Path import stat import tempfile import threading import time import unittest from unittest.mock import patch from openpyxl import load_workbook from arr_ingestion.postgres import DatabaseConfig, PostgresIngestionRepository from arr_ingestion.repository import InMemoryIngestionRepository from arr_ingestion.service import IngestionService from arr_ingestion.validation import DeliveryValidator from arr_processing.local import LocalDailyProcessor from arr_processing.policy import load_processor_policy from arr_storage.filesystem import FilesystemObjectBackend from arr_storage.store import ManagedObjectStore from arr_web.app import PortalApplication from arr_web.arr_data_executor import DirectARRExecutor from arr_web.arr_data_review import DataFieldReviews from arr_web.arr_downloads import PersistentARRDownloads from arr_web.contracts import PortalError from arr_web.programmatic import ProgrammaticUploadCoordinator from arr_web.repository import PostgresPortalRepository from arr_web import arr_data_review as review_module from integrations.ohip.arr_data import ARRDataSource, FIELDS from integrations.ohip.collect_arr_source import CollectionError from tests.test_arr_web import TEST_CREDENTIALS, FakeRepository, login from tests.test_ohip_arr_data import DAY, HOTEL, REQUEST, FIXTURE, SimulatedOHIP from tests.local_postgres import TemporaryPostgres from tests import test_arr_download_postgres_integration as pg_helpers PROJECT = Path(__file__).resolve().parents[1] ACTOR = "operator@example.test" SOURCE_MANIFEST = "c" * 64 REANALYSIS_MANIFEST = "e" * 64 REANALYSIS_POLICY = "oracle-optional-association/v1" CONTEXT = {"hotel_id": HOTEL, "source_kind": "test_transport"} def source_document(count=1): cases = json.loads(FIXTURE.read_text())["normal_cases"][:count] rows = [] for sequence, case in enumerate(cases, 1): fields = {} for field in FIELDS: value = copy.deepcopy(case["record"][field]) if field == "RES_COMMENT": value = [value] if value else [] elif field == "PRODUCTS": value = [] fields[field] = {"state": "empty" if value in ("", []) else "available", "value": value} rows.append({"source_sequence": sequence, "reservation_id": f"synthetic-{sequence}", "fields": fields, "related": {}}) return {"version": "arr-ohip-data/v1", "hotel_id": HOTEL, "report_date": DAY, "source_kind": "test_transport", "status": "collected", "collection_complete": True, "input_complete": True, "field_names": list(FIELDS), "trace_included": False, "atomic_snapshot": False, "records": rows} def raw(document): return json.dumps(document, ensure_ascii=False, separators=(",", ":")).encode() def gap(document, field, sequence=1, state="missing"): document["records"][sequence - 1]["fields"][field] = { "state": state, "value": None, "reason": "synthetic_field_gap"} document["status"], document["input_complete"] = "collected_with_gaps", False def optional_omission_source(document): result = copy.deepcopy(document) for row in result["records"]: for field, reason, empty_value in (("BLOCK_CODE", "missing_reservation_block", ""), ("PRODUCTS", "missing_reservation_packages", [])): observation = row["fields"][field] if observation.get("state") == "missing" and observation.get("reason") == reason: row["fields"][field] = {"state": "empty", "value": empty_value} unresolved = any(value["state"] != "available" and not (field in review_module.OPTIONAL and value["state"] == "empty") for row in result["records"] for field, value in row["fields"].items()) result["input_complete"] = not unresolved result["status"] = "collected_with_gaps" if unresolved else "collected" result["optional_omission_policy"] = REANALYSIS_POLICY return result class DataFieldReviewTests(unittest.TestCase): @classmethod def setUpClass(cls): cls.policy = load_processor_policy(PROJECT) def setUp(self): temporary = tempfile.TemporaryDirectory(prefix="arr-data-review-") self.addCleanup(temporary.cleanup) self.root = Path(temporary.name) / "reviews" self.service = DataFieldReviews(root=self.root, policy=self.policy, context=CONTEXT) self.document = source_document() def prepare(self, document=None, request_id=REQUEST): return self.service.prepare(request_id, raw(self.document if document is None else document), SOURCE_MANIFEST, DAY) def update(self, review, field, value, sequence=1): return self.service.update(review["request_id"], f"{sequence}:{field}", review["revision"], value, ACTOR) def assert_error(self, code, callback, status=400): with self.assertRaises(PortalError) as raised: callback() self.assertEqual(raised.exception.code, "ARR_DATA_REVIEW_" + code) self.assertEqual(raised.exception.status, status) def omission_gaps(self, document=None, sequence=1): document = self.document if document is None else document for field, reason in (("BLOCK_CODE", "missing_reservation_block"), ("PRODUCTS", "missing_reservation_packages")): gap(document, field, sequence=sequence) document["records"][sequence - 1]["fields"][field]["reason"] = reason def reanalyze(self, review, document=None, **kwargs): return self.service.apply_source_reanalysis(REQUEST, raw(optional_omission_source(self.document) if document is None else document), REANALYSIS_MANIFEST, REANALYSIS_POLICY, expected_revision=review["revision"], **kwargs) def test_source_reanalysis_removes_only_25_optional_gaps_without_confirming_a_room(self): template = self.document["records"][0] self.document["records"] = [] for sequence in range(1, 19): row = copy.deepcopy(template) row["source_sequence"], row["reservation_id"] = sequence, f"synthetic-{sequence}" row["fields"]["CONFIRMATION_NO"]["value"] = f"SYNTHETIC-{sequence:03d}" row["fields"]["DISP_ROOM_NO"]["value"] = f"ROOM-{sequence:03d}" self.document["records"].append(row) gap(self.document, "BLOCK_CODE", sequence) row["fields"]["BLOCK_CODE"]["reason"] = "missing_reservation_block" if sequence <= 7: gap(self.document, "PRODUCTS", sequence) row["fields"]["PRODUCTS"]["reason"] = "missing_reservation_packages" gap(self.document, "DISP_ROOM_NO") original = raw(self.document) review = self.prepare() self.assertEqual(review["pending_count"], 26) revised = self.reanalyze(review) self.assertEqual((revised["revision"], revised["pending_count"], revised["total_count"]), (1, 1, 1)) self.assertEqual([item["field"] for item in revised["items"]], ["DISP_ROOM_NO"]) self.assertFalse(revised["can_finalize"]) self.assertIsNone(self.service.payload(REQUEST)) self.assertEqual(self.service.original(REQUEST), (original, SOURCE_MANIFEST)) directory = self.root / REQUEST _, _, state = self.service._read(directory) self.assertEqual(state["decisions"], {}) self.assertEqual([event["action"] for event in state["events"]], ["source_reanalysis"]) self.assertNotIn("actor", state["events"][0]) receipt = json.loads((directory / state["source_reanalysis"]["receipt_file"]).read_bytes()) self.assertEqual(len(receipt["changes"]), 25) self.assertEqual(receipt["parent_original_sha256"], hashlib.sha256(original).hexdigest()) for path in directory.iterdir(): self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600) def test_reanalysis_preserves_staff_decisions_and_final_manifest_has_both_source_bindings(self): self.omission_gaps() gap(self.document, "DISP_ROOM_NO") before = raw(self.document) first = self.prepare() staff = self.update(first, "BLOCK_CODE", "STAFF-VERIFIED-BLOCK") payload = raw(optional_omission_source(self.document)) revised = self.reanalyze(staff) confirmed = next(item for item in revised["items"] if item["field"] == "BLOCK_CODE") self.assertEqual((confirmed["value"], confirmed["source_state"], confirmed["confirmed"]), ("STAFF-VERIFIED-BLOCK", "available", True)) self.assertEqual(revised["pending_count"], 1) restored = DataFieldReviews(root=self.root, policy=self.policy, context=CONTEXT) self.assertEqual(restored.get(REQUEST), revised) revised = restored.update(REQUEST, "1:DISP_ROOM_NO", revised["revision"], "STAFF-ROOM", ACTOR) # A lost maintenance response can be retried even after a later staff save. self.assertEqual(restored.apply_source_reanalysis(REQUEST, payload, REANALYSIS_MANIFEST, REANALYSIS_POLICY, expected_revision=staff["revision"]), revised) frozen = restored.finalize(REQUEST, revised["revision"], ACTOR) result, binding = restored.payload(REQUEST) self.assertEqual(restored.original(REQUEST), (before, SOURCE_MANIFEST)) derived = json.loads(result) fields = derived["records"][0]["fields"] self.assertEqual(fields["BLOCK_CODE"]["value"], "STAFF-VERIFIED-BLOCK") self.assertEqual(fields["PRODUCTS"], {"state": "empty", "value": []}) audit = derived["manual_data_review"]["manifest"] self.assertEqual(audit["original_sha256"], hashlib.sha256(before).hexdigest()) self.assertEqual(audit["source_manifest_sha256"], SOURCE_MANIFEST) self.assertEqual(audit["source_reanalysis"]["policy_id"], REANALYSIS_POLICY) self.assertEqual(audit["source_reanalysis"]["reanalyzed_sha256"], hashlib.sha256(payload).hexdigest()) self.assertEqual(audit["source_reanalysis"]["reanalysis_manifest_sha256"], REANALYSIS_MANIFEST) self.assertEqual([event["action"] for event in audit["events"]], ["set_field", "source_reanalysis", "set_field", "finalize"]) self.assertEqual({change["actor"] for change in audit["changes"]}, {ACTOR}) self.assertRegex(binding, r"^[0-9a-f]{64}$") self.assertEqual(restored.finalize(REQUEST, revised["revision"], ACTOR), frozen) self.assert_error("FROZEN", lambda: restored.apply_source_reanalysis(REQUEST, payload, REANALYSIS_MANIFEST, REANALYSIS_POLICY, expected_revision=frozen["revision"]), 409) def test_reanalysis_rejects_other_source_changes_and_leaves_revision_unchanged(self): self.document = source_document(2) self.omission_gaps() review = self.prepare() valid = optional_omission_source(self.document) mutations = [ lambda data: data.update(hotel_id="OTHER"), lambda data: data.update(report_date="2026-09-16"), lambda data: data.update(source_kind="OTHER"), lambda data: data.update(collection_complete=False), lambda data: data.update(input_complete=False), lambda data: data.update(status="collected_with_gaps"), lambda data: data.update(optional_omission_policy="OTHER"), lambda data: data.update(extra="not allowed"), lambda data: data["records"].reverse(), lambda data: data["records"][0].update(reservation_id="OTHER"), lambda data: data["records"][0]["related"].update(extra="not allowed"), lambda data: data["records"][0]["fields"]["FULL_NAME"].update(value="OTHER"), lambda data: data["records"][0]["fields"]["BLOCK_CODE"].update(reason="not allowed"), ] for change in mutations: with self.subTest(change=change): document = copy.deepcopy(valid) change(document) with self.assertRaises(CollectionError): self.reanalyze(review, document) self.assertEqual(self.service.get(REQUEST), review) self.assertEqual(list((self.root / REQUEST).glob("*reanalysis*")), []) def test_reanalysis_never_clears_failed_ambiguous_available_or_unrecognized_observations(self): for index, (state, reason) in enumerate((("failed", "missing_reservation_packages"), ("ambiguous", "missing_reservation_packages"), ("available", "missing_reservation_packages"), ("missing", "synthetic_field_gap")), 1): with self.subTest(state=state, reason=reason): document = source_document() self.omission_gaps(document) observation = document["records"][0]["fields"]["PRODUCTS"] observation["state"], observation["reason"] = state, reason if state == "available": observation["value"] = [{"package": {"packageCode": "EXISTING"}}] request_id = f"{index:032x}" review = self.prepare(document, request_id) proposed = optional_omission_source(document) proposed["records"][0]["fields"]["PRODUCTS"] = {"state": "empty", "value": []} proposed["input_complete"], proposed["status"] = True, "collected" with self.assertRaisesRegex(CollectionError, "data_review_reanalysis_observation_invalid"): self.service.apply_source_reanalysis(request_id, raw(proposed), REANALYSIS_MANIFEST, REANALYSIS_POLICY, expected_revision=review["revision"]) self.assertEqual(self.service.get(request_id), review) def test_reanalysis_is_revision_guarded_and_rejects_other_policy_or_evidence(self): self.omission_gaps() review = self.prepare() payload = raw(optional_omission_source(self.document)) staff = self.update(review, "BLOCK_CODE", "STAFF") self.assert_error("CONFLICT", lambda: self.reanalyze(review), 409) with self.assertRaisesRegex(CollectionError, "data_review_reanalysis_policy_invalid"): self.service.apply_source_reanalysis(REQUEST, payload, REANALYSIS_MANIFEST, "OTHER", expected_revision=staff["revision"]) with self.assertRaisesRegex(CollectionError, "data_review_reanalysis_manifest_invalid"): self.service.apply_source_reanalysis(REQUEST, payload, "invalid", REANALYSIS_POLICY, expected_revision=staff["revision"]) current = self.reanalyze(staff) self.assertEqual(self.reanalyze(staff), current) self.assert_error("CONFLICT", lambda: self.service.apply_source_reanalysis(REQUEST, payload, "f" * 64, REANALYSIS_POLICY, expected_revision=current["revision"]), 409) self.assertEqual(self.service.get(REQUEST), current) def test_reanalysis_publication_recovers_after_crash_without_publishing_partial_source(self): self.omission_gaps() review = self.prepare() real_atomic = review_module.atomic_json def interrupted(path, value, **kwargs): if path.name == "state.json": raise OSError("synthetic reanalysis pointer interruption") return real_atomic(path, value, **kwargs) with patch.object(review_module, "atomic_json", side_effect=interrupted): with self.assertRaisesRegex(OSError, "synthetic reanalysis pointer interruption"): self.reanalyze(review) self.assertEqual(self.service.get(REQUEST), review) staff = self.update(review, "BLOCK_CODE", "PRESERVED-AFTER-INTERRUPTION") restored = DataFieldReviews(root=self.root, policy=self.policy, context=CONTEXT) result = restored.apply_source_reanalysis(REQUEST, raw(optional_omission_source(self.document)), REANALYSIS_MANIFEST, REANALYSIS_POLICY, expected_revision=staff["revision"]) self.assertEqual(result["revision"], staff["revision"] + 1) self.assertEqual(result["pending_count"], 0) self.assertEqual(result["items"][0]["value"], "PRESERVED-AFTER-INTERRUPTION") _, _, state = restored._read(self.root / REQUEST) self.assertEqual([event["action"] for event in state["events"]], ["set_field", "source_reanalysis"]) def test_reanalysis_with_no_pending_items_still_requires_finalize_and_recovers_its_intent(self): self.omission_gaps() original = raw(self.document) revised = self.reanalyze(self.prepare()) self.assertTrue(revised["can_finalize"]) self.assertEqual(revised["items"], []) self.assertEqual(self.prepare(), revised, "executor cannot bypass an unfinalized reanalysis") self.assertIsNone(self.service.payload(REQUEST)) real_atomic = review_module.atomic_json def interrupted(path, value, **kwargs): if path.name == "state.json": raise OSError("synthetic finalize pointer interruption") return real_atomic(path, value, **kwargs) with patch.object(review_module, "atomic_json", side_effect=interrupted): with self.assertRaisesRegex(OSError, "synthetic finalize pointer interruption"): self.service.finalize(REQUEST, revised["revision"], ACTOR) frozen_bytes = (self.root / REQUEST / "reviewed-source.json").read_bytes() self.assertEqual(self.service.get(REQUEST), revised) self.assertIsNone(self.service.payload(REQUEST)) self.assert_error("FROZEN", lambda: self.reanalyze(revised), 409) restored = DataFieldReviews(root=self.root, policy=self.policy, context=CONTEXT) final = restored.finalize(REQUEST, revised["revision"], "different-operator@example.test") self.assertEqual(final["status"], "finalized") self.assertEqual(restored.payload(REQUEST)[0], frozen_bytes) self.assertEqual(restored.original(REQUEST), (original, SOURCE_MANIFEST)) audit = json.loads(frozen_bytes)["manual_data_review"]["manifest"] self.assertEqual(audit["changes"], []) self.assertEqual([event["action"] for event in audit["events"]], ["source_reanalysis", "finalize"]) self.assertEqual(audit["events"][-1]["actor"], ACTOR) def test_reanalysis_and_staff_save_share_one_revision_winner(self): self.omission_gaps() gap(self.document, "DISP_ROOM_NO") review = self.prepare() start = threading.Barrier(2) def mutate(reanalysis): start.wait(timeout=3) try: return self.reanalyze(review) if reanalysis else self.update(review, "DISP_ROOM_NO", "STAFF-ROOM") except PortalError as error: return error with ThreadPoolExecutor(max_workers=2) as pool: results = list(pool.map(mutate, [True, False])) winners = [result for result in results if isinstance(result, dict)] conflicts = [result for result in results if isinstance(result, PortalError)] self.assertEqual((len(winners), len(conflicts)), (1, 1)) self.assertEqual((conflicts[0].code, conflicts[0].status), ("ARR_DATA_REVIEW_CONFLICT", 409)) self.assertEqual(self.service.get(REQUEST), winners[0]) def test_reanalysis_payload_and_receipt_tampering_are_rejected(self): self.omission_gaps() revised = self.reanalyze(self.prepare()) directory = self.root / REQUEST _, _, state = self.service._read(directory) reference = state["source_reanalysis"] for name in (reference["file"], reference["receipt_file"]): with self.subTest(name=name): path = directory / name before = path.read_bytes() path.write_bytes(before + b" ") with self.assertRaisesRegex(CollectionError, "data_review_reanalysis_changed"): self.service.get(REQUEST) path.write_bytes(before) self.assertEqual(self.service.get(REQUEST), revised) def test_complete_source_and_gaps_only_on_excluded_rows_need_no_review(self): self.assertIsNone(self.prepare()) excluded = source_document() excluded["records"][0]["fields"]["RATE_CODE"]["value"] = "BAR" for field in set(FIELDS) - {"RATE_CODE"}: gap(excluded, field, state="ambiguous") self.assertIsNone(self.prepare(excluded, "b" * 32)) def test_only_current_candidate_gaps_are_editable(self): document = source_document(2) gap(document, "BLOCK_CODE") document["records"][1]["fields"]["RATE_CODE"]["value"] = "BAR" gap(document, "FULL_NAME", sequence=2) review = self.prepare(document) self.assertEqual([item["item_id"] for item in review["items"]], ["1:BLOCK_CODE"]) self.assertEqual(review["pending_count"], 1) self.assertIsNone(self.service.payload(REQUEST)) self.assert_error("INVALID", lambda: self.update(review, "FULL_NAME", "changed")) self.assert_error("INVALID", lambda: self.update(review, "FULL_NAME", "changed", sequence=2)) self.assertEqual(self.service.get(REQUEST)["revision"], review["revision"]) def test_invalid_required_business_values_are_reviewed(self): invalid = {"FULL_NAME": " ", "DISP_ROOM_NO": "", "ADULTS": -1, "CHILDREN": -1, "NO_OF_ROOMS": 0, "EFFECTIVE_RATE_AMOUNT": "-1", "ARRIVAL": "invalid-date", "DEPARTURE": "invalid-date"} for field, value in invalid.items(): self.document["records"][0]["fields"][field] = {"state": "available", "value": value} review = self.prepare() self.assertEqual({item["field"] for item in review["items"]}, set(invalid)) self.assertEqual(review["pending_count"], len(invalid)) def test_review_exposes_source_failure_separately_from_optional_absence(self): gap(self.document, "DISP_ROOM_NO") gap(self.document, "BLOCK_CODE") gap(self.document, "PRODUCTS", state="failed") gap(self.document, "RES_COMMENT", state="ambiguous") before = raw(self.document) review = self.prepare() items = {item["field"]: item for item in review["items"]} self.assertEqual({field: (item["source_state"], item["can_be_empty"]) for field, item in items.items()}, { "DISP_ROOM_NO": ("missing", False), "BLOCK_CODE": ("missing", True), "PRODUCTS": ("failed", True), "RES_COMMENT": ("ambiguous", True), }) self.assertEqual(review["pending_count"], 4) self.assertFalse(review["can_finalize"]) self.assertEqual(self.service.original(REQUEST)[0], before) confirmed = self.update(review, "BLOCK_CODE", "") item = next(item for item in confirmed["items"] if item["field"] == "BLOCK_CODE") self.assertEqual((item["source_state"], item["confirmed"]), ("empty", True)) self.assertEqual(confirmed["pending_count"], 3) def test_departure_before_arrival_is_an_editable_issue(self): self.document["records"][0]["fields"]["DEPARTURE"]["value"] = "2026-09-14" review = self.prepare() self.assertIsNotNone(review) self.assertEqual([item["field"] for item in review["items"]], ["DEPARTURE"]) review = self.update(review, "DEPARTURE", "2026-09-16") self.assertTrue(review["can_finalize"]) def test_unknown_rate_is_resolved_before_other_candidate_gaps(self): gap(self.document, "RATE_CODE") gap(self.document, "FULL_NAME") gap(self.document, "BLOCK_CODE") review = self.prepare() self.assertEqual([item["field"] for item in review["items"]], ["RATE_CODE"]) review = self.update(review, "RATE_CODE", " grpa1 ") self.assertEqual({item["field"] for item in review["items"]}, {"RATE_CODE", "FULL_NAME", "BLOCK_CODE"}) self.assertEqual(review["pending_count"], 2) self.assertEqual(next(item for item in review["items"] if item["field"] == "RATE_CODE")["value"], "GRPA1") def test_resolving_unknown_rate_as_excluded_does_not_require_guest_fields(self): for field in FIELDS: gap(self.document, field) review = self.prepare() self.assertEqual([item["field"] for item in review["items"]], ["RATE_CODE"]) review = self.update(review, "RATE_CODE", "BAR") self.assertEqual(review["pending_count"], 0) self.assertTrue(review["can_finalize"]) self.assertEqual([item["field"] for item in review["items"]], ["RATE_CODE"]) self.assertEqual(self.service.finalize(REQUEST, review["revision"], ACTOR)["status"], "finalized") def test_optional_empty_values_require_explicit_confirmation(self): for field in ("BLOCK_CODE", "RES_COMMENT", "PRODUCTS", "ROOM_CATEGORY_LABEL"): gap(self.document, field) review = self.prepare() self.assertEqual(review["pending_count"], 4) self.assert_error("INCOMPLETE", lambda: self.service.finalize(REQUEST, review["revision"], ACTOR), 409) for field in ("BLOCK_CODE", "RES_COMMENT", "PRODUCTS", "ROOM_CATEGORY_LABEL"): review = self.update(review, field, [] if field == "PRODUCTS" else "") self.assertEqual(review["pending_count"], 0) self.assertTrue(all(item["confirmed"] for item in review["items"])) self.service.finalize(REQUEST, review["revision"], ACTOR) derived = json.loads(self.service.payload(REQUEST)[0]) for field in ("BLOCK_CODE", "RES_COMMENT", "PRODUCTS", "ROOM_CATEGORY_LABEL"): observation = derived["records"][0]["fields"][field] self.assertEqual(observation["state"], "empty") self.assertEqual(observation["origin"], "manual_review") def test_required_blank_and_invalid_numeric_inputs_leave_revision_unchanged(self): for field in ("FULL_NAME", "ADULTS", "CHILDREN", "NO_OF_ROOMS", "EFFECTIVE_RATE_AMOUNT"): gap(self.document, field) review = self.prepare() invalid = {"FULL_NAME": ("", " ", None, True), "ADULTS": (True, False, "1.5", "-1", 1.5, "NaN", ""), "CHILDREN": (True, "0.5", "-1"), "NO_OF_ROOMS": (True, "0", "1.5", "-1"), "EFFECTIVE_RATE_AMOUNT": (True, "-1", "NaN", "Infinity", "1.001")} for field, values in invalid.items(): for value in values: with self.subTest(field=field, value=value): self.assert_error("INVALID", lambda: self.update(review, field, value)) self.assertEqual(self.service.get(REQUEST)["revision"], review["revision"]) def test_strings_convert_to_native_schema_and_package_order_is_preserved(self): values = {"ADULTS": "2", "CHILDREN": "0", "NO_OF_ROOMS": "1", "EFFECTIVE_RATE_AMOUNT": "900.00", "PRODUCTS": ["PKG2", "PKG1", "PKG2"], "RES_COMMENT": "manual synthetic note"} for field in values: gap(self.document, field) review = self.prepare() for field, value in values.items(): review = self.update(review, field, value) self.service.finalize(REQUEST, review["revision"], ACTOR) derived = json.loads(self.service.payload(REQUEST)[0]) fields = derived["records"][0]["fields"] for field in ("ADULTS", "CHILDREN", "NO_OF_ROOMS"): self.assertIs(type(fields[field]["value"]), int) self.assertEqual(fields["EFFECTIVE_RATE_AMOUNT"]["value"], "900.00") self.assertEqual(fields["RES_COMMENT"]["value"], ["manual synthetic note"]) self.assertEqual([item["package"]["packageCode"] for item in fields["PRODUCTS"]["value"]], ["PKG2", "PKG1", "PKG2"]) for field in set(FIELDS) - set(values): self.assertEqual(fields[field], self.document["records"][0]["fields"][field]) def test_dates_must_match_report_day_and_stay_bounds(self): for field in ("ARRIVAL", "DEPARTURE"): gap(self.document, field) review = self.prepare() for field, value in (("ARRIVAL", "2026-09-16"), ("ARRIVAL", "2026-9-15"), ("ARRIVAL", "2026-02-30"), ("DEPARTURE", "2026-09-14")): with self.subTest(field=field, value=value): self.assert_error("INVALID", lambda: self.update(review, field, value)) review = self.update(review, "ARRIVAL", DAY) review = self.update(review, "DEPARTURE", DAY) self.assertTrue(review["can_finalize"]) def test_revision_conflict_and_final_frozen_idempotent_source_and_audit(self): gap(self.document, "BLOCK_CODE") before = raw(self.document) review = self.prepare() original = self.root / REQUEST / "original.json" review = self.update(review, "BLOCK_CODE", "SYNTHETIC-BLOCK") self.assert_error("CONFLICT", lambda: self.service.update(REQUEST, "1:BLOCK_CODE", 0, "OTHER", ACTOR), 409) review = self.update(review, "BLOCK_CODE", "") frozen = self.service.finalize(REQUEST, review["revision"], ACTOR) payload, binding = self.service.payload(REQUEST) self.assertEqual(original.read_bytes(), before) self.assertRegex(binding, r"^[0-9a-f]{64}$") self.assertEqual(self.service.finalize(REQUEST, review["revision"], ACTOR), frozen) self.assertEqual(self.service.payload(REQUEST), (payload, binding)) self.assert_error("FROZEN", lambda: self.update(frozen, "BLOCK_CODE", "OTHER"), 409) manifest = json.loads(payload)["manual_data_review"]["manifest"] self.assertEqual(manifest["original_sha256"], hashlib.sha256(before).hexdigest()) self.assertEqual(manifest["source_manifest_sha256"], SOURCE_MANIFEST) self.assertEqual([event["action"] for event in manifest["events"]], ["set_field", "set_field", "finalize"]) self.assertEqual([event["actor"] for event in manifest["events"]], [ACTOR] * 3) self.assertEqual(manifest["changes"][0]["original_observation"], self.document["records"][0]["fields"]["BLOCK_CODE"]) self.assertEqual(len(list((self.root / REQUEST).glob("revision-*.json"))), 4) for path in (self.root, self.root / REQUEST): self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o700) for path in (self.root / REQUEST).iterdir(): if path.is_file(): self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600) def test_restart_preserves_decisions_and_rejects_context_or_source_drift(self): gap(self.document, "BLOCK_CODE") review = self.update(self.prepare(), "BLOCK_CODE", "") restored = DataFieldReviews(root=self.root, policy=self.policy, context=CONTEXT) self.assertEqual(restored.get(REQUEST), review) changed_context = DataFieldReviews(root=self.root, policy=self.policy, context={**CONTEXT, "hotel_id": "OTHER"}) with self.assertRaisesRegex(CollectionError, "data_review_identity_changed"): changed_context.get(REQUEST) changed = copy.deepcopy(self.document) changed["records"][0]["fields"]["FULL_NAME"]["value"] = "another synthetic name" with self.assertRaisesRegex(CollectionError, "data_review_prepare_conflict"): self.prepare(changed) with self.assertRaisesRegex(CollectionError, "data_review_prepare_conflict"): restored.prepare(REQUEST, raw(self.document), "d" * 64, DAY) self.assertEqual(restored.get(REQUEST), review) def test_tampered_original_and_frozen_bytes_are_rejected(self): gap(self.document, "BLOCK_CODE") review = self.update(self.prepare(), "BLOCK_CODE", "") self.service.finalize(REQUEST, review["revision"], ACTOR) derived = self.root / REQUEST / "reviewed-source.json" derived.write_bytes(derived.read_bytes() + b" ") with self.assertRaisesRegex(CollectionError, "data_review_frozen_source_changed"): self.service.payload(REQUEST) original = self.root / REQUEST / "original.json" original.write_bytes(original.read_bytes() + b" ") with self.assertRaisesRegex(CollectionError, "data_review_original_changed"): self.service.get(REQUEST) def test_incomplete_collection_cannot_become_a_manual_review(self): gap(self.document, "BLOCK_CODE") self.document["collection_complete"] = False with self.assertRaisesRegex(CollectionError, "data_review_requires_complete_collection"): self.prepare() def test_initial_review_publication_recovers_without_losing_source(self): gap(self.document, "DISP_ROOM_NO") real_write = review_module.atomic_json def interrupted(path, document, **kwargs): if path.name == "state.json": raise OSError("simulated exit before initial pointer") return real_write(path, document, **kwargs) with patch.object(review_module, "atomic_json", side_effect=interrupted): with self.assertRaises(OSError): self.prepare() before = (self.root / REQUEST / "original.json").read_bytes() review = self.prepare() self.assertEqual(review["revision"], 0) self.assertEqual(review["pending_count"], 1) self.assertEqual((self.root / REQUEST / "original.json").read_bytes(), before) def test_interrupted_finalization_replays_same_frozen_bytes_and_timestamp(self): gap(self.document, "BLOCK_CODE") review = self.update(self.prepare(), "BLOCK_CODE", "") original_atomic = review_module.atomic_json def lose_pointer(path, value, **kwargs): if Path(path).name == "state.json": raise OSError("synthetic process exit before pointer publication") return original_atomic(path, value, **kwargs) with patch.object(review_module, "atomic_json", side_effect=lose_pointer): with self.assertRaisesRegex(OSError, "synthetic process exit"): self.service.finalize(REQUEST, review["revision"], ACTOR) frozen_path = self.root / REQUEST / "reviewed-source.json" frozen_bytes = frozen_path.read_bytes() self.assertEqual(self.service.get(REQUEST), review) self.assertIsNone(self.service.payload(REQUEST)) self.assert_error("FROZEN", lambda: self.update(review, "BLOCK_CODE", "OTHER"), 409) restored = DataFieldReviews(root=self.root, policy=self.policy, context=CONTEXT) completed = restored.finalize(REQUEST, review["revision"], "second-operator@example.test") self.assertEqual(completed["status"], "finalized") self.assertEqual(completed["revision"], review["revision"] + 1) self.assertEqual(frozen_path.read_bytes(), frozen_bytes) self.assertEqual(restored.payload(REQUEST)[0], frozen_bytes) final_events = [event for event in json.loads(frozen_bytes)["manual_data_review"]["manifest"]["events"] if event["action"] == "finalize"] self.assertEqual(len(final_events), 1) self.assertEqual(final_events[0]["actor"], ACTOR) self.assertEqual(restored.finalize(REQUEST, review["revision"], ACTOR), completed) def test_concurrent_saves_allow_one_revision_winner(self): gap(self.document, "BLOCK_CODE") review = self.prepare() start = threading.Barrier(2) def save(value): start.wait(timeout=3) try: return self.update(review, "BLOCK_CODE", value) except PortalError as error: return error with ThreadPoolExecutor(max_workers=2) as pool: results = list(pool.map(save, ["SYNTHETIC-FIRST", "SYNTHETIC-SECOND"])) winners = [result for result in results if isinstance(result, dict)] conflicts = [result for result in results if isinstance(result, PortalError)] self.assertEqual(len(winners), 1) self.assertEqual(len(conflicts), 1) self.assertEqual((conflicts[0].code, conflicts[0].status), ("ARR_DATA_REVIEW_CONFLICT", 409)) self.assertEqual(self.service.get(REQUEST), winners[0]) self.assertEqual(winners[0]["revision"], review["revision"] + 1) class DirectFieldReviewTests(unittest.TestCase): @classmethod def setUpClass(cls): cls.policy = load_processor_policy(PROJECT) def setUp(self): temporary = tempfile.TemporaryDirectory(prefix="arr-direct-field-review-") self.addCleanup(temporary.cleanup) self.files = Path(temporary.name) self.transport = SimulatedOHIP(1) room_stay = self.transport.rows[0]["roomStay"] room_stay.pop("roomId") room_stay["currentRoomInfo"].pop("roomId") room_stay["roomRates"][0].pop("roomId") self.transport.calendar_rooms = [] self.source = ARRDataSource(self.files / "source", HOTEL, transport_factory=lambda: self.transport) self.store = ManagedObjectStore(FilesystemObjectBackend(self.files / "objects", create=True)) self.repository = InMemoryIngestionRepository() self.processor = LocalDailyProcessor(self.policy) self.ingestion = IngestionService(DeliveryValidator(self.store, self.policy), self.repository) self.configure_executor() self.coordinator = ProgrammaticUploadCoordinator(self.store, self.repository, self.ingestion, self.processor, self.policy.processor_version, self.policy.rule_set_sha256) def configure_executor(self): self.executor = DirectARRExecutor(root=self.files / "executor", source=self.source, policy=self.policy, object_store=self.store, repository=self.repository, ingestion=self.ingestion, processor=self.processor) def execute(self): return self.executor.execute(request_id=REQUEST, from_date=date.fromisoformat(DAY), to_date=date.fromisoformat(DAY), report_stage=lambda _: None) def complete_fields(self): review = self.executor.get_data_review(REQUEST) self.assertEqual([item["field"] for item in review["items"]], ["DISP_ROOM_NO"]) review = self.executor.update_data_review_item(REQUEST, review["items"][0]["item_id"], review["revision"], "SIM-ROOM-001", ACTOR) return self.executor.finalize_data_review(REQUEST, review["revision"], ACTOR) def wait_task(self, queue): deadline = time.monotonic() + 30 while time.monotonic() < deadline: task = queue.get(REQUEST) if task["status"] not in {"queued", "downloading", "processing"}: return task time.sleep(.01) self.fail("synthetic download worker did not settle") def test_no_processor_or_finance_until_fields_complete_then_original_rules_commit(self): with patch.object(LocalDailyProcessor, "run", side_effect=AssertionError("processor called before completion")) as process: outcome = self.execute() process.assert_not_called() self.assertEqual(outcome.status, "needs_data_review") self.assertIsNone(outcome.job_id) self.assertIsNone(self.repository.current_version_id(date.fromisoformat(DAY))) self.assertFalse(list((self.files / "executor" / "handoffs").rglob("delivery.json"))) original = next((self.files / "source").rglob("arr-data.json")) before, calls = original.read_bytes(), len(self.transport.calls) self.assertEqual(self.execute(), outcome) self.assertEqual(len(self.transport.calls), calls) self.complete_fields() result = self.execute() self.assertEqual(result.status, "succeeded") self.assertEqual(len(self.transport.calls), calls) self.assertEqual(original.read_bytes(), before) version = self.repository.current_version_id(date.fromisoformat(DAY)) rows = self.repository.version_records(version) self.assertEqual(len(rows), 1) self.assertEqual(Decimal(str(rows[0]["total_price"])), Decimal("1800")) self.assertEqual(self.execute(), result) self.assertEqual(len(self.repository._versions), 1) self.assertFalse(list(self.files.rglob("*.xml"))) def test_field_completion_preserves_existing_missing_price_review_flow(self): old = self.transport.edit def edit(operation, document, request): document = old(operation, document, request) if operation == "searchRateInfo": document["data"]["detail"]["totalRateAmount"] = 8765 return document self.transport.edit = edit self.assertEqual(self.execute().status, "needs_data_review") calls = len(self.transport.calls) self.complete_fields() outcome = self.execute() self.assertEqual(outcome.status, "needs_review") self.assertIsNotNone(outcome.job_id) self.assertIsNone(self.repository.current_version_id(date.fromisoformat(DAY))) review = self.coordinator.get_price_review(outcome.job_id, 50, 0) self.assertEqual(len(review["items"]), 1) review = self.coordinator.update_price_review_item(outcome.job_id, review["items"][0]["item_id"], review["case_id"], review["revision"], "100", ACTOR) result = self.coordinator.finalize_price_review(outcome.job_id, review["case_id"], review["revision"], ACTOR) self.assertEqual(result["status"], "succeeded") self.assertEqual(len(self.transport.calls), calls) version = self.repository.current_version_id(date.fromisoformat(DAY)) self.assertEqual(Decimal(str(self.repository.version_records(version)[0]["total_price"])), Decimal("100")) def test_queue_restart_finalizes_and_resumes_same_request_without_http(self): queue = PersistentARRDownloads(self.files / "queue", self.executor) try: queue.create(DAY, REQUEST) task = self.wait_task(queue) self.assertEqual(task["status"], "needs_data_review") self.assertIsNone(task["job_id"]) self.assertFalse(task["can_retry"]) self.assertEqual(queue.create(DAY, "b" * 32)["request_id"], REQUEST) calls = len(self.transport.calls) finally: queue.close(wait=True) self.configure_executor() queue = PersistentARRDownloads(self.files / "queue", self.executor) try: self.assertEqual(queue.get(REQUEST)["status"], "needs_data_review") review = queue.get_data_review(REQUEST) review = queue.update_data_review_item(REQUEST, review["items"][0]["item_id"], review["revision"], "SIM-ROOM-001", ACTOR) queue.finalize_data_review(REQUEST, review["revision"], ACTOR) task = self.wait_task(queue) self.assertEqual(task["status"], "succeeded", task) self.assertEqual(task["request_id"], REQUEST) self.assertEqual(len(self.transport.calls), calls) self.assertEqual(len(self.repository._versions), 1) finally: queue.close(wait=True) def test_finalized_review_recovers_queue_after_lost_confirmation_response(self): queue = PersistentARRDownloads(self.files / "queue", self.executor) try: queue.create(DAY, REQUEST) self.assertEqual(self.wait_task(queue)["status"], "needs_data_review") calls = len(self.transport.calls) # Persist the operator's confirmation, then lose the queue update. self.complete_fields() finally: queue.close(wait=True) self.configure_executor() with patch.object(self.source, "fetch", side_effect=AssertionError("must use collected checkpoint")): queue = PersistentARRDownloads(self.files / "queue", self.executor) try: recovered = queue.get(REQUEST) self.assertIn(recovered["status"], {"queued", "downloading", "processing", "succeeded"}) task = self.wait_task(queue) self.assertEqual(task["status"], "succeeded", task) self.assertEqual(task["request_id"], REQUEST) self.assertEqual(len(self.repository._versions), 1) self.assertEqual(queue.get(REQUEST), task) self.assertEqual(self.execute().status, "succeeded") self.assertEqual(len(self.repository._versions), 1) self.assertEqual(len(self.transport.calls), calls) finally: queue.close(wait=True) @unittest.skipUnless(os.environ.get("ARR_TEST_LOCAL_POSTGRES") == "1", "explicit disposable PostgreSQL opt-in required") class DirectFieldReviewPostgresTests(unittest.TestCase): @classmethod def setUpClass(cls): cls.database = TemporaryPostgres().__enter__() cls.addClassCleanup(cls.database.__exit__, None, None, None) cls.policy = load_processor_policy(PROJECT) def setUp(self): self.database.reset_database() DirectFieldReviewTests.setUp(self) self.repository = PostgresIngestionRepository(DatabaseConfig("owned-fixture"), connect=self.database.connect) self.ingestion = IngestionService(DeliveryValidator(self.store, self.policy), self.repository) self.configure_executor() configure_executor = DirectFieldReviewTests.configure_executor execute = DirectFieldReviewTests.execute wait_task = DirectFieldReviewTests.wait_task sql = pg_helpers.PostgresDownloadIntegrationTests.sql count = pg_helpers.PostgresDownloadIntegrationTests.count daily_event_count = pg_helpers.PostgresDownloadIntegrationTests.daily_event_count monthly_worker = pg_helpers.PostgresDownloadIntegrationTests.monthly_worker def test_web_field_confirmation_gates_finance_outbox_and_monthly_publication(self): queue = PersistentARRDownloads(self.files / "queue", self.executor) try: app = PortalApplication(arr_downloads=queue, credentials=TEST_CREDENTIALS, repository=PostgresPortalRepository("owned-fixture", connect=self.database.connect)) _, headers = login(app) body = raw({"request_id": REQUEST, "report_date": DAY}) self.assertEqual(app.handle("POST", "/api/arr-downloads", headers, body).status, 202) task = self.wait_task(queue) self.assertEqual(task["status"], "needs_data_review", task) self.assertIsNone(task["job_id"]) for table in ("finance.daily_versions", "finance.current_daily_versions", "finance.daily_records", "ingestion.processing_runs", "ingestion.outbox_events", "reporting.monthly_runs"): self.assertEqual(self.count(table), 0, table) self.assertEqual(self.monthly_worker().process_next().status, "idle") original = next((self.files / "source").rglob("arr-data.json")) before, calls = original.read_bytes(), len(self.transport.calls) path = f"/api/arr-downloads/{REQUEST}/data-review" response = app.handle("GET", path, headers) self.assertEqual(response.status, 200) review = json.loads(response.body)["data"] self.assertEqual([item["field"] for item in review["items"]], ["DISP_ROOM_NO"]) response = app.handle("POST", path + "/items/1:DISP_ROOM_NO", headers, raw({"revision": review["revision"], "value": "SIM-ROOM-001"})) self.assertEqual(response.status, 200) review = json.loads(response.body)["data"] self.assertEqual(self.count("finance.daily_versions"), 0) self.assertEqual(self.count("ingestion.outbox_events"), 0) confirmation = raw({"revision": review["revision"]}) response = app.handle("POST", path + "/finalize", headers, confirmation) self.assertEqual(response.status, 202) task = self.wait_task(queue) self.assertEqual(task["status"], "succeeded", task) self.assertEqual(task["request_id"], REQUEST) self.assertEqual(len(self.transport.calls), calls) self.assertEqual(original.read_bytes(), before) self.assertEqual(self.count("finance.daily_versions"), 1) self.assertEqual(self.count("finance.current_daily_versions"), 1) self.assertEqual(self.daily_event_count(), 1) total, price, rooms, nights = self.sql( "SELECT total_price,real_price,no_of_rooms,nights FROM finance.daily_records")[0] self.assertEqual(total, Decimal("1800.00")) self.assertEqual(total, price * rooms * nights) self.assertEqual(self.sql("SELECT result_schema_version FROM ingestion.processing_runs"), [("5.0",)]) key, size = self.sql("""SELECT a.object_key,a.byte_size FROM finance.daily_versions v JOIN ingestion.artifacts a ON a.id=v.daily_report_artifact_id""")[0] daily = self.files / "verified-daily.xlsx" self.store.materialize(key, daily, size) workbook = load_workbook(daily, data_only=False) try: sheet = workbook[workbook.sheetnames[0]] self.assertEqual(sheet.cell(2, 19).value, 1800) self.assertEqual(sheet.cell(2, 18).value * sheet.cell(2, 11).value * sheet.cell(2, 17).value, 1800) finally: workbook.close() self.assertEqual(self.monthly_worker().process_next().status, "published") self.assertEqual(self.monthly_worker().process_next().status, "idle") self.assertEqual(self.count("reporting.monthly_runs"), 1) self.assertEqual(self.sql("SELECT publish_status FROM ingestion.outbox_events WHERE event_type='arr.daily_version_committed'"), [("published",)]) key, size = self.sql("""SELECT a.object_key,a.byte_size FROM reporting.monthly_runs r JOIN ingestion.artifacts a ON a.id=r.workbook_artifact_id""")[0] monthly = self.files / "verified-monthly.xlsx" self.store.materialize(key, monthly, size) workbook = load_workbook(monthly, data_only=False) try: formula_cells = [cell for sheet in workbook for row in sheet for cell in row if cell.data_type == "f"] self.assertEqual(len(formula_cells), 1) cell = formula_cells[0] self.assertEqual(cell.value, f"=R{cell.row}*C{cell.row}*G{cell.row}") self.assertEqual(cell.parent.cell(cell.row, 18).value * cell.parent.cell(cell.row, 3).value * cell.parent.cell(cell.row, 7).value, 1800) finally: workbook.close() self.assertEqual(app.handle("POST", path + "/finalize", headers, confirmation).status, 202) self.assertEqual(queue.get(REQUEST)["status"], "succeeded") self.assertEqual(self.execute().status, "succeeded") self.assertEqual(len(self.transport.calls), calls) self.assertEqual(self.count("finance.daily_versions"), 1) self.assertEqual(self.daily_event_count(), 1) self.assertEqual(self.count("reporting.monthly_runs"), 1) self.assertFalse(list(self.files.rglob("*.xml"))) finally: queue.close(wait=True) class FakeDataReviewDownloads: ready = True def __init__(self): self.calls = [] def latest(self): return None def get(self, request_id): return {"request_id": request_id, "report_date": DAY, "status": "needs_data_review", "job_id": None, "can_retry": False} def get_data_review(self, request_id): self.calls.append(("get", request_id)) return {"request_id": request_id, "status": "editing", "revision": 0, "items": []} def update_data_review_item(self, request_id, item_id, revision, value, actor): self.calls.append(("update", request_id, item_id, revision, value, actor)) return {"request_id": request_id, "revision": revision + 1, "items": []} def finalize_data_review(self, request_id, revision, actor): self.calls.append(("finalize", request_id, revision, actor)) return {"request_id": request_id, "status": "queued"} class DataReviewRoutesTests(unittest.TestCase): def setUp(self): self.downloads = FakeDataReviewDownloads() self.app = PortalApplication(repository=FakeRepository(), arr_downloads=self.downloads, credentials=TEST_CREDENTIALS) _, self.headers = login(self.app) self.path = f"/api/arr-downloads/{REQUEST}/data-review" def post(self, suffix, payload, headers=None): return self.app.handle("POST", self.path + suffix, self.headers if headers is None else headers, json.dumps(payload).encode()) def test_auth_csrf_session_actor_and_strict_review_requests(self): self.assertEqual(self.app.handle("GET", self.path, {}).status, 401) self.assertEqual(self.post("/items/1:BLOCK_CODE", {"revision": 0, "value": ""}, {}).status, 401) self.assertEqual(self.post("/items/1:BLOCK_CODE", {"revision": 0, "value": ""}, {"Cookie": self.headers["Cookie"]}).status, 403) self.assertEqual(self.downloads.calls, []) self.assertEqual(self.app.handle("GET", self.path, self.headers).status, 200) self.assertEqual(self.post("/items/1:BLOCK_CODE", {"revision": 0, "value": ""}).status, 200) self.assertEqual(self.post("/finalize", {"revision": 1}).status, 202) self.assertEqual(self.downloads.calls, [("get", REQUEST), ("update", REQUEST, "1:BLOCK_CODE", 0, "", TEST_CREDENTIALS.username), ("finalize", REQUEST, 1, TEST_CREDENTIALS.username)]) for suffix, payload in (("/items/1:BLOCK_CODE", {"revision": True, "value": ""}), ("/items/1:BLOCK_CODE", {"revision": 1, "value": "", "actor": "forged"}), ("/finalize", {"revision": 1, "actor": "forged"}), ("/finalize", {"revision": "1"})): with self.subTest(suffix=suffix, payload=payload): self.assertEqual(self.post(suffix, payload).status, 400) self.assertEqual(len(self.downloads.calls), 3) if __name__ == "__main__": unittest.main()