691 lines
37 KiB
Python
691 lines
37 KiB
Python
"""Offline source-field review acceptance; every reservation is synthetic."""
|
|
|
|
import copy
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import date
|
|
from decimal import Decimal
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import stat
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from openpyxl import load_workbook
|
|
|
|
from arr_ingestion.postgres import DatabaseConfig, PostgresIngestionRepository
|
|
from arr_ingestion.repository import InMemoryIngestionRepository
|
|
from arr_ingestion.service import IngestionService
|
|
from arr_ingestion.validation import DeliveryValidator
|
|
from arr_processing.local import LocalDailyProcessor
|
|
from arr_processing.policy import load_processor_policy
|
|
from arr_storage.filesystem import FilesystemObjectBackend
|
|
from arr_storage.store import ManagedObjectStore
|
|
from arr_web.app import PortalApplication
|
|
from arr_web.arr_data_executor import DirectARRExecutor
|
|
from arr_web.arr_data_review import DataFieldReviews
|
|
from arr_web.arr_downloads import PersistentARRDownloads
|
|
from arr_web.contracts import PortalError
|
|
from arr_web.programmatic import ProgrammaticUploadCoordinator
|
|
from arr_web.repository import PostgresPortalRepository
|
|
from arr_web import arr_data_review as review_module
|
|
from integrations.ohip.arr_data import ARRDataSource, FIELDS
|
|
from integrations.ohip.collect_arr_source import CollectionError
|
|
from tests.test_arr_web import TEST_CREDENTIALS, FakeRepository, login
|
|
from tests.test_ohip_arr_data import DAY, HOTEL, REQUEST, FIXTURE, SimulatedOHIP
|
|
from tests.local_postgres import TemporaryPostgres
|
|
from tests import test_arr_download_postgres_integration as pg_helpers
|
|
|
|
|
|
PROJECT = Path(__file__).resolve().parents[1]
|
|
ACTOR = "operator@example.test"
|
|
SOURCE_MANIFEST = "c" * 64
|
|
CONTEXT = {"hotel_id": HOTEL, "source_kind": "test_transport"}
|
|
|
|
|
|
def source_document(count=1):
|
|
cases = json.loads(FIXTURE.read_text())["normal_cases"][:count]
|
|
rows = []
|
|
for sequence, case in enumerate(cases, 1):
|
|
fields = {}
|
|
for field in FIELDS:
|
|
value = copy.deepcopy(case["record"][field])
|
|
if field == "RES_COMMENT":
|
|
value = [value] if value else []
|
|
elif field == "PRODUCTS":
|
|
value = []
|
|
fields[field] = {"state": "empty" if value in ("", []) else "available", "value": value}
|
|
rows.append({"source_sequence": sequence, "reservation_id": f"synthetic-{sequence}",
|
|
"fields": fields, "related": {}})
|
|
return {"version": "arr-ohip-data/v1", "hotel_id": HOTEL, "report_date": DAY,
|
|
"source_kind": "test_transport", "status": "collected", "collection_complete": True,
|
|
"input_complete": True, "field_names": list(FIELDS), "trace_included": False,
|
|
"atomic_snapshot": False, "records": rows}
|
|
|
|
|
|
def raw(document):
|
|
return json.dumps(document, ensure_ascii=False, separators=(",", ":")).encode()
|
|
|
|
|
|
def gap(document, field, sequence=1, state="missing"):
|
|
document["records"][sequence - 1]["fields"][field] = {
|
|
"state": state, "value": None, "reason": "synthetic_field_gap"}
|
|
document["status"], document["input_complete"] = "collected_with_gaps", False
|
|
|
|
|
|
class DataFieldReviewTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.policy = load_processor_policy(PROJECT)
|
|
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory(prefix="arr-data-review-")
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name) / "reviews"
|
|
self.service = DataFieldReviews(root=self.root, policy=self.policy, context=CONTEXT)
|
|
self.document = source_document()
|
|
|
|
def prepare(self, document=None, request_id=REQUEST):
|
|
return self.service.prepare(request_id, raw(self.document if document is None else document),
|
|
SOURCE_MANIFEST, DAY)
|
|
|
|
def update(self, review, field, value, sequence=1):
|
|
return self.service.update(review["request_id"], f"{sequence}:{field}",
|
|
review["revision"], value, ACTOR)
|
|
|
|
def assert_error(self, code, callback, status=400):
|
|
with self.assertRaises(PortalError) as raised:
|
|
callback()
|
|
self.assertEqual(raised.exception.code, "ARR_DATA_REVIEW_" + code)
|
|
self.assertEqual(raised.exception.status, status)
|
|
|
|
def test_complete_source_and_gaps_only_on_excluded_rows_need_no_review(self):
|
|
self.assertIsNone(self.prepare())
|
|
excluded = source_document()
|
|
excluded["records"][0]["fields"]["RATE_CODE"]["value"] = "BAR"
|
|
for field in set(FIELDS) - {"RATE_CODE"}:
|
|
gap(excluded, field, state="ambiguous")
|
|
self.assertIsNone(self.prepare(excluded, "b" * 32))
|
|
|
|
def test_only_current_candidate_gaps_are_editable(self):
|
|
document = source_document(2)
|
|
gap(document, "BLOCK_CODE")
|
|
document["records"][1]["fields"]["RATE_CODE"]["value"] = "BAR"
|
|
gap(document, "FULL_NAME", sequence=2)
|
|
review = self.prepare(document)
|
|
self.assertEqual([item["item_id"] for item in review["items"]], ["1:BLOCK_CODE"])
|
|
self.assertEqual(review["pending_count"], 1)
|
|
self.assertIsNone(self.service.payload(REQUEST))
|
|
self.assert_error("INVALID", lambda: self.update(review, "FULL_NAME", "changed"))
|
|
self.assert_error("INVALID", lambda: self.update(review, "FULL_NAME", "changed", sequence=2))
|
|
self.assertEqual(self.service.get(REQUEST)["revision"], review["revision"])
|
|
|
|
def test_invalid_required_business_values_are_reviewed(self):
|
|
invalid = {"FULL_NAME": " ", "DISP_ROOM_NO": "", "ADULTS": -1,
|
|
"CHILDREN": -1, "NO_OF_ROOMS": 0, "EFFECTIVE_RATE_AMOUNT": "-1",
|
|
"ARRIVAL": "invalid-date", "DEPARTURE": "invalid-date"}
|
|
for field, value in invalid.items():
|
|
self.document["records"][0]["fields"][field] = {"state": "available", "value": value}
|
|
review = self.prepare()
|
|
self.assertEqual({item["field"] for item in review["items"]}, set(invalid))
|
|
self.assertEqual(review["pending_count"], len(invalid))
|
|
|
|
def test_departure_before_arrival_is_an_editable_issue(self):
|
|
self.document["records"][0]["fields"]["DEPARTURE"]["value"] = "2026-09-14"
|
|
review = self.prepare()
|
|
self.assertIsNotNone(review)
|
|
self.assertEqual([item["field"] for item in review["items"]], ["DEPARTURE"])
|
|
review = self.update(review, "DEPARTURE", "2026-09-16")
|
|
self.assertTrue(review["can_finalize"])
|
|
|
|
def test_unknown_rate_is_resolved_before_other_candidate_gaps(self):
|
|
gap(self.document, "RATE_CODE")
|
|
gap(self.document, "FULL_NAME")
|
|
gap(self.document, "BLOCK_CODE")
|
|
review = self.prepare()
|
|
self.assertEqual([item["field"] for item in review["items"]], ["RATE_CODE"])
|
|
review = self.update(review, "RATE_CODE", " grpa1 ")
|
|
self.assertEqual({item["field"] for item in review["items"]},
|
|
{"RATE_CODE", "FULL_NAME", "BLOCK_CODE"})
|
|
self.assertEqual(review["pending_count"], 2)
|
|
self.assertEqual(next(item for item in review["items"] if item["field"] == "RATE_CODE")["value"], "GRPA1")
|
|
|
|
def test_resolving_unknown_rate_as_excluded_does_not_require_guest_fields(self):
|
|
for field in FIELDS:
|
|
gap(self.document, field)
|
|
review = self.prepare()
|
|
self.assertEqual([item["field"] for item in review["items"]], ["RATE_CODE"])
|
|
review = self.update(review, "RATE_CODE", "BAR")
|
|
self.assertEqual(review["pending_count"], 0)
|
|
self.assertTrue(review["can_finalize"])
|
|
self.assertEqual([item["field"] for item in review["items"]], ["RATE_CODE"])
|
|
self.assertEqual(self.service.finalize(REQUEST, review["revision"], ACTOR)["status"], "finalized")
|
|
|
|
def test_optional_empty_values_require_explicit_confirmation(self):
|
|
for field in ("BLOCK_CODE", "RES_COMMENT", "PRODUCTS", "ROOM_CATEGORY_LABEL"):
|
|
gap(self.document, field)
|
|
review = self.prepare()
|
|
self.assertEqual(review["pending_count"], 4)
|
|
self.assert_error("INCOMPLETE", lambda: self.service.finalize(REQUEST, review["revision"], ACTOR), 409)
|
|
for field in ("BLOCK_CODE", "RES_COMMENT", "PRODUCTS", "ROOM_CATEGORY_LABEL"):
|
|
review = self.update(review, field, [] if field == "PRODUCTS" else "")
|
|
self.assertEqual(review["pending_count"], 0)
|
|
self.assertTrue(all(item["confirmed"] for item in review["items"]))
|
|
self.service.finalize(REQUEST, review["revision"], ACTOR)
|
|
derived = json.loads(self.service.payload(REQUEST)[0])
|
|
for field in ("BLOCK_CODE", "RES_COMMENT", "PRODUCTS", "ROOM_CATEGORY_LABEL"):
|
|
observation = derived["records"][0]["fields"][field]
|
|
self.assertEqual(observation["state"], "empty")
|
|
self.assertEqual(observation["origin"], "manual_review")
|
|
|
|
def test_required_blank_and_invalid_numeric_inputs_leave_revision_unchanged(self):
|
|
for field in ("FULL_NAME", "ADULTS", "CHILDREN", "NO_OF_ROOMS", "EFFECTIVE_RATE_AMOUNT"):
|
|
gap(self.document, field)
|
|
review = self.prepare()
|
|
invalid = {"FULL_NAME": ("", " ", None, True),
|
|
"ADULTS": (True, False, "1.5", "-1", 1.5, "NaN", ""),
|
|
"CHILDREN": (True, "0.5", "-1"),
|
|
"NO_OF_ROOMS": (True, "0", "1.5", "-1"),
|
|
"EFFECTIVE_RATE_AMOUNT": (True, "-1", "NaN", "Infinity", "1.001")}
|
|
for field, values in invalid.items():
|
|
for value in values:
|
|
with self.subTest(field=field, value=value):
|
|
self.assert_error("INVALID", lambda: self.update(review, field, value))
|
|
self.assertEqual(self.service.get(REQUEST)["revision"], review["revision"])
|
|
|
|
def test_strings_convert_to_native_schema_and_package_order_is_preserved(self):
|
|
values = {"ADULTS": "2", "CHILDREN": "0", "NO_OF_ROOMS": "1",
|
|
"EFFECTIVE_RATE_AMOUNT": "900.00", "PRODUCTS": ["PKG2", "PKG1", "PKG2"],
|
|
"RES_COMMENT": "manual synthetic note"}
|
|
for field in values:
|
|
gap(self.document, field)
|
|
review = self.prepare()
|
|
for field, value in values.items():
|
|
review = self.update(review, field, value)
|
|
self.service.finalize(REQUEST, review["revision"], ACTOR)
|
|
derived = json.loads(self.service.payload(REQUEST)[0])
|
|
fields = derived["records"][0]["fields"]
|
|
for field in ("ADULTS", "CHILDREN", "NO_OF_ROOMS"):
|
|
self.assertIs(type(fields[field]["value"]), int)
|
|
self.assertEqual(fields["EFFECTIVE_RATE_AMOUNT"]["value"], "900.00")
|
|
self.assertEqual(fields["RES_COMMENT"]["value"], ["manual synthetic note"])
|
|
self.assertEqual([item["package"]["packageCode"] for item in fields["PRODUCTS"]["value"]],
|
|
["PKG2", "PKG1", "PKG2"])
|
|
for field in set(FIELDS) - set(values):
|
|
self.assertEqual(fields[field], self.document["records"][0]["fields"][field])
|
|
|
|
def test_dates_must_match_report_day_and_stay_bounds(self):
|
|
for field in ("ARRIVAL", "DEPARTURE"):
|
|
gap(self.document, field)
|
|
review = self.prepare()
|
|
for field, value in (("ARRIVAL", "2026-09-16"), ("ARRIVAL", "2026-9-15"),
|
|
("ARRIVAL", "2026-02-30"), ("DEPARTURE", "2026-09-14")):
|
|
with self.subTest(field=field, value=value):
|
|
self.assert_error("INVALID", lambda: self.update(review, field, value))
|
|
review = self.update(review, "ARRIVAL", DAY)
|
|
review = self.update(review, "DEPARTURE", DAY)
|
|
self.assertTrue(review["can_finalize"])
|
|
|
|
def test_revision_conflict_and_final_frozen_idempotent_source_and_audit(self):
|
|
gap(self.document, "BLOCK_CODE")
|
|
before = raw(self.document)
|
|
review = self.prepare()
|
|
original = self.root / REQUEST / "original.json"
|
|
review = self.update(review, "BLOCK_CODE", "SYNTHETIC-BLOCK")
|
|
self.assert_error("CONFLICT", lambda: self.service.update(REQUEST, "1:BLOCK_CODE", 0, "OTHER", ACTOR), 409)
|
|
review = self.update(review, "BLOCK_CODE", "")
|
|
frozen = self.service.finalize(REQUEST, review["revision"], ACTOR)
|
|
payload, binding = self.service.payload(REQUEST)
|
|
self.assertEqual(original.read_bytes(), before)
|
|
self.assertRegex(binding, r"^[0-9a-f]{64}$")
|
|
self.assertEqual(self.service.finalize(REQUEST, review["revision"], ACTOR), frozen)
|
|
self.assertEqual(self.service.payload(REQUEST), (payload, binding))
|
|
self.assert_error("FROZEN", lambda: self.update(frozen, "BLOCK_CODE", "OTHER"), 409)
|
|
manifest = json.loads(payload)["manual_data_review"]["manifest"]
|
|
self.assertEqual(manifest["original_sha256"], hashlib.sha256(before).hexdigest())
|
|
self.assertEqual(manifest["source_manifest_sha256"], SOURCE_MANIFEST)
|
|
self.assertEqual([event["action"] for event in manifest["events"]], ["set_field", "set_field", "finalize"])
|
|
self.assertEqual([event["actor"] for event in manifest["events"]], [ACTOR] * 3)
|
|
self.assertEqual(manifest["changes"][0]["original_observation"],
|
|
self.document["records"][0]["fields"]["BLOCK_CODE"])
|
|
self.assertEqual(len(list((self.root / REQUEST).glob("revision-*.json"))), 4)
|
|
for path in (self.root, self.root / REQUEST):
|
|
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o700)
|
|
for path in (self.root / REQUEST).iterdir():
|
|
if path.is_file():
|
|
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600)
|
|
|
|
def test_restart_preserves_decisions_and_rejects_context_or_source_drift(self):
|
|
gap(self.document, "BLOCK_CODE")
|
|
review = self.update(self.prepare(), "BLOCK_CODE", "")
|
|
restored = DataFieldReviews(root=self.root, policy=self.policy, context=CONTEXT)
|
|
self.assertEqual(restored.get(REQUEST), review)
|
|
changed_context = DataFieldReviews(root=self.root, policy=self.policy,
|
|
context={**CONTEXT, "hotel_id": "OTHER"})
|
|
with self.assertRaisesRegex(CollectionError, "data_review_identity_changed"):
|
|
changed_context.get(REQUEST)
|
|
changed = copy.deepcopy(self.document)
|
|
changed["records"][0]["fields"]["FULL_NAME"]["value"] = "another synthetic name"
|
|
with self.assertRaisesRegex(CollectionError, "data_review_prepare_conflict"):
|
|
self.prepare(changed)
|
|
with self.assertRaisesRegex(CollectionError, "data_review_prepare_conflict"):
|
|
restored.prepare(REQUEST, raw(self.document), "d" * 64, DAY)
|
|
self.assertEqual(restored.get(REQUEST), review)
|
|
|
|
def test_tampered_original_and_frozen_bytes_are_rejected(self):
|
|
gap(self.document, "BLOCK_CODE")
|
|
review = self.update(self.prepare(), "BLOCK_CODE", "")
|
|
self.service.finalize(REQUEST, review["revision"], ACTOR)
|
|
derived = self.root / REQUEST / "reviewed-source.json"
|
|
derived.write_bytes(derived.read_bytes() + b" ")
|
|
with self.assertRaisesRegex(CollectionError, "data_review_frozen_source_changed"):
|
|
self.service.payload(REQUEST)
|
|
original = self.root / REQUEST / "original.json"
|
|
original.write_bytes(original.read_bytes() + b" ")
|
|
with self.assertRaisesRegex(CollectionError, "data_review_original_changed"):
|
|
self.service.get(REQUEST)
|
|
|
|
def test_incomplete_collection_cannot_become_a_manual_review(self):
|
|
gap(self.document, "BLOCK_CODE")
|
|
self.document["collection_complete"] = False
|
|
with self.assertRaisesRegex(CollectionError, "data_review_requires_complete_collection"):
|
|
self.prepare()
|
|
|
|
def test_initial_review_publication_recovers_without_losing_source(self):
|
|
gap(self.document, "DISP_ROOM_NO")
|
|
real_write = review_module.atomic_json
|
|
def interrupted(path, document, **kwargs):
|
|
if path.name == "state.json":
|
|
raise OSError("simulated exit before initial pointer")
|
|
return real_write(path, document, **kwargs)
|
|
with patch.object(review_module, "atomic_json", side_effect=interrupted):
|
|
with self.assertRaises(OSError):
|
|
self.prepare()
|
|
before = (self.root / REQUEST / "original.json").read_bytes()
|
|
review = self.prepare()
|
|
self.assertEqual(review["revision"], 0)
|
|
self.assertEqual(review["pending_count"], 1)
|
|
self.assertEqual((self.root / REQUEST / "original.json").read_bytes(), before)
|
|
|
|
def test_interrupted_finalization_replays_same_frozen_bytes_and_timestamp(self):
|
|
gap(self.document, "BLOCK_CODE")
|
|
review = self.update(self.prepare(), "BLOCK_CODE", "")
|
|
original_atomic = review_module.atomic_json
|
|
def lose_pointer(path, value, **kwargs):
|
|
if Path(path).name == "state.json":
|
|
raise OSError("synthetic process exit before pointer publication")
|
|
return original_atomic(path, value, **kwargs)
|
|
with patch.object(review_module, "atomic_json", side_effect=lose_pointer):
|
|
with self.assertRaisesRegex(OSError, "synthetic process exit"):
|
|
self.service.finalize(REQUEST, review["revision"], ACTOR)
|
|
frozen_path = self.root / REQUEST / "reviewed-source.json"
|
|
frozen_bytes = frozen_path.read_bytes()
|
|
self.assertEqual(self.service.get(REQUEST), review)
|
|
self.assertIsNone(self.service.payload(REQUEST))
|
|
self.assert_error("FROZEN", lambda: self.update(review, "BLOCK_CODE", "OTHER"), 409)
|
|
restored = DataFieldReviews(root=self.root, policy=self.policy, context=CONTEXT)
|
|
completed = restored.finalize(REQUEST, review["revision"], "second-operator@example.test")
|
|
self.assertEqual(completed["status"], "finalized")
|
|
self.assertEqual(completed["revision"], review["revision"] + 1)
|
|
self.assertEqual(frozen_path.read_bytes(), frozen_bytes)
|
|
self.assertEqual(restored.payload(REQUEST)[0], frozen_bytes)
|
|
final_events = [event for event in json.loads(frozen_bytes)["manual_data_review"]["manifest"]["events"]
|
|
if event["action"] == "finalize"]
|
|
self.assertEqual(len(final_events), 1)
|
|
self.assertEqual(final_events[0]["actor"], ACTOR)
|
|
self.assertEqual(restored.finalize(REQUEST, review["revision"], ACTOR), completed)
|
|
|
|
def test_concurrent_saves_allow_one_revision_winner(self):
|
|
gap(self.document, "BLOCK_CODE")
|
|
review = self.prepare()
|
|
start = threading.Barrier(2)
|
|
def save(value):
|
|
start.wait(timeout=3)
|
|
try:
|
|
return self.update(review, "BLOCK_CODE", value)
|
|
except PortalError as error:
|
|
return error
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
results = list(pool.map(save, ["SYNTHETIC-FIRST", "SYNTHETIC-SECOND"]))
|
|
winners = [result for result in results if isinstance(result, dict)]
|
|
conflicts = [result for result in results if isinstance(result, PortalError)]
|
|
self.assertEqual(len(winners), 1)
|
|
self.assertEqual(len(conflicts), 1)
|
|
self.assertEqual((conflicts[0].code, conflicts[0].status), ("ARR_DATA_REVIEW_CONFLICT", 409))
|
|
self.assertEqual(self.service.get(REQUEST), winners[0])
|
|
self.assertEqual(winners[0]["revision"], review["revision"] + 1)
|
|
|
|
|
|
class DirectFieldReviewTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.policy = load_processor_policy(PROJECT)
|
|
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory(prefix="arr-direct-field-review-")
|
|
self.addCleanup(temporary.cleanup)
|
|
self.files = Path(temporary.name)
|
|
self.transport = SimulatedOHIP(1)
|
|
room_stay = self.transport.rows[0]["roomStay"]
|
|
room_stay.pop("roomId")
|
|
room_stay["currentRoomInfo"].pop("roomId")
|
|
room_stay["roomRates"][0].pop("roomId")
|
|
self.transport.calendar_rooms = []
|
|
self.source = ARRDataSource(self.files / "source", HOTEL, transport_factory=lambda: self.transport)
|
|
self.store = ManagedObjectStore(FilesystemObjectBackend(self.files / "objects", create=True))
|
|
self.repository = InMemoryIngestionRepository()
|
|
self.processor = LocalDailyProcessor(self.policy)
|
|
self.ingestion = IngestionService(DeliveryValidator(self.store, self.policy), self.repository)
|
|
self.configure_executor()
|
|
self.coordinator = ProgrammaticUploadCoordinator(self.store, self.repository, self.ingestion,
|
|
self.processor, self.policy.processor_version, self.policy.rule_set_sha256)
|
|
|
|
def configure_executor(self):
|
|
self.executor = DirectARRExecutor(root=self.files / "executor", source=self.source, policy=self.policy,
|
|
object_store=self.store, repository=self.repository, ingestion=self.ingestion, processor=self.processor)
|
|
|
|
def execute(self):
|
|
return self.executor.execute(request_id=REQUEST, from_date=date.fromisoformat(DAY),
|
|
to_date=date.fromisoformat(DAY), report_stage=lambda _: None)
|
|
|
|
def complete_fields(self):
|
|
review = self.executor.get_data_review(REQUEST)
|
|
self.assertEqual([item["field"] for item in review["items"]], ["DISP_ROOM_NO"])
|
|
review = self.executor.update_data_review_item(REQUEST, review["items"][0]["item_id"],
|
|
review["revision"], "SIM-ROOM-001", ACTOR)
|
|
return self.executor.finalize_data_review(REQUEST, review["revision"], ACTOR)
|
|
|
|
def wait_task(self, queue):
|
|
deadline = time.monotonic() + 30
|
|
while time.monotonic() < deadline:
|
|
task = queue.get(REQUEST)
|
|
if task["status"] not in {"queued", "downloading", "processing"}:
|
|
return task
|
|
time.sleep(.01)
|
|
self.fail("synthetic download worker did not settle")
|
|
|
|
def test_no_processor_or_finance_until_fields_complete_then_original_rules_commit(self):
|
|
with patch.object(LocalDailyProcessor, "run", side_effect=AssertionError("processor called before completion")) as process:
|
|
outcome = self.execute()
|
|
process.assert_not_called()
|
|
self.assertEqual(outcome.status, "needs_data_review")
|
|
self.assertIsNone(outcome.job_id)
|
|
self.assertIsNone(self.repository.current_version_id(date.fromisoformat(DAY)))
|
|
self.assertFalse(list((self.files / "executor" / "handoffs").rglob("delivery.json")))
|
|
original = next((self.files / "source").rglob("arr-data.json"))
|
|
before, calls = original.read_bytes(), len(self.transport.calls)
|
|
self.assertEqual(self.execute(), outcome)
|
|
self.assertEqual(len(self.transport.calls), calls)
|
|
self.complete_fields()
|
|
result = self.execute()
|
|
self.assertEqual(result.status, "succeeded")
|
|
self.assertEqual(len(self.transport.calls), calls)
|
|
self.assertEqual(original.read_bytes(), before)
|
|
version = self.repository.current_version_id(date.fromisoformat(DAY))
|
|
rows = self.repository.version_records(version)
|
|
self.assertEqual(len(rows), 1)
|
|
self.assertEqual(Decimal(str(rows[0]["total_price"])), Decimal("1800"))
|
|
self.assertEqual(self.execute(), result)
|
|
self.assertEqual(len(self.repository._versions), 1)
|
|
self.assertFalse(list(self.files.rglob("*.xml")))
|
|
|
|
def test_field_completion_preserves_existing_missing_price_review_flow(self):
|
|
old = self.transport.edit
|
|
def edit(operation, document, request):
|
|
document = old(operation, document, request)
|
|
if operation == "searchRateInfo":
|
|
document["data"]["detail"]["totalRateAmount"] = 8765
|
|
return document
|
|
self.transport.edit = edit
|
|
self.assertEqual(self.execute().status, "needs_data_review")
|
|
calls = len(self.transport.calls)
|
|
self.complete_fields()
|
|
outcome = self.execute()
|
|
self.assertEqual(outcome.status, "needs_review")
|
|
self.assertIsNotNone(outcome.job_id)
|
|
self.assertIsNone(self.repository.current_version_id(date.fromisoformat(DAY)))
|
|
review = self.coordinator.get_price_review(outcome.job_id, 50, 0)
|
|
self.assertEqual(len(review["items"]), 1)
|
|
review = self.coordinator.update_price_review_item(outcome.job_id, review["items"][0]["item_id"],
|
|
review["case_id"], review["revision"], "100", ACTOR)
|
|
result = self.coordinator.finalize_price_review(outcome.job_id, review["case_id"], review["revision"], ACTOR)
|
|
self.assertEqual(result["status"], "succeeded")
|
|
self.assertEqual(len(self.transport.calls), calls)
|
|
version = self.repository.current_version_id(date.fromisoformat(DAY))
|
|
self.assertEqual(Decimal(str(self.repository.version_records(version)[0]["total_price"])), Decimal("100"))
|
|
|
|
def test_queue_restart_finalizes_and_resumes_same_request_without_http(self):
|
|
queue = PersistentARRDownloads(self.files / "queue", self.executor)
|
|
try:
|
|
queue.create(DAY, REQUEST)
|
|
task = self.wait_task(queue)
|
|
self.assertEqual(task["status"], "needs_data_review")
|
|
self.assertIsNone(task["job_id"])
|
|
self.assertFalse(task["can_retry"])
|
|
self.assertEqual(queue.create(DAY, "b" * 32)["request_id"], REQUEST)
|
|
calls = len(self.transport.calls)
|
|
finally:
|
|
queue.close(wait=True)
|
|
self.configure_executor()
|
|
queue = PersistentARRDownloads(self.files / "queue", self.executor)
|
|
try:
|
|
self.assertEqual(queue.get(REQUEST)["status"], "needs_data_review")
|
|
review = queue.get_data_review(REQUEST)
|
|
review = queue.update_data_review_item(REQUEST, review["items"][0]["item_id"],
|
|
review["revision"], "SIM-ROOM-001", ACTOR)
|
|
queue.finalize_data_review(REQUEST, review["revision"], ACTOR)
|
|
task = self.wait_task(queue)
|
|
self.assertEqual(task["status"], "succeeded", task)
|
|
self.assertEqual(task["request_id"], REQUEST)
|
|
self.assertEqual(len(self.transport.calls), calls)
|
|
self.assertEqual(len(self.repository._versions), 1)
|
|
finally:
|
|
queue.close(wait=True)
|
|
|
|
def test_finalized_review_recovers_queue_after_lost_confirmation_response(self):
|
|
queue = PersistentARRDownloads(self.files / "queue", self.executor)
|
|
try:
|
|
queue.create(DAY, REQUEST)
|
|
self.assertEqual(self.wait_task(queue)["status"], "needs_data_review")
|
|
calls = len(self.transport.calls)
|
|
# Persist the operator's confirmation, then lose the queue update.
|
|
self.complete_fields()
|
|
finally:
|
|
queue.close(wait=True)
|
|
self.configure_executor()
|
|
with patch.object(self.source, "fetch", side_effect=AssertionError("must use collected checkpoint")):
|
|
queue = PersistentARRDownloads(self.files / "queue", self.executor)
|
|
try:
|
|
recovered = queue.get(REQUEST)
|
|
self.assertIn(recovered["status"], {"queued", "downloading", "processing", "succeeded"})
|
|
task = self.wait_task(queue)
|
|
self.assertEqual(task["status"], "succeeded", task)
|
|
self.assertEqual(task["request_id"], REQUEST)
|
|
self.assertEqual(len(self.repository._versions), 1)
|
|
self.assertEqual(queue.get(REQUEST), task)
|
|
self.assertEqual(self.execute().status, "succeeded")
|
|
self.assertEqual(len(self.repository._versions), 1)
|
|
self.assertEqual(len(self.transport.calls), calls)
|
|
finally:
|
|
queue.close(wait=True)
|
|
|
|
|
|
@unittest.skipUnless(os.environ.get("ARR_TEST_LOCAL_POSTGRES") == "1",
|
|
"explicit disposable PostgreSQL opt-in required")
|
|
class DirectFieldReviewPostgresTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.database = TemporaryPostgres().__enter__()
|
|
cls.addClassCleanup(cls.database.__exit__, None, None, None)
|
|
cls.policy = load_processor_policy(PROJECT)
|
|
|
|
def setUp(self):
|
|
self.database.reset_database()
|
|
DirectFieldReviewTests.setUp(self)
|
|
self.repository = PostgresIngestionRepository(DatabaseConfig("owned-fixture"), connect=self.database.connect)
|
|
self.ingestion = IngestionService(DeliveryValidator(self.store, self.policy), self.repository)
|
|
self.configure_executor()
|
|
|
|
configure_executor = DirectFieldReviewTests.configure_executor
|
|
execute = DirectFieldReviewTests.execute
|
|
wait_task = DirectFieldReviewTests.wait_task
|
|
sql = pg_helpers.PostgresDownloadIntegrationTests.sql
|
|
count = pg_helpers.PostgresDownloadIntegrationTests.count
|
|
daily_event_count = pg_helpers.PostgresDownloadIntegrationTests.daily_event_count
|
|
monthly_worker = pg_helpers.PostgresDownloadIntegrationTests.monthly_worker
|
|
|
|
def test_web_field_confirmation_gates_finance_outbox_and_monthly_publication(self):
|
|
queue = PersistentARRDownloads(self.files / "queue", self.executor)
|
|
try:
|
|
app = PortalApplication(arr_downloads=queue, credentials=TEST_CREDENTIALS,
|
|
repository=PostgresPortalRepository("owned-fixture", connect=self.database.connect))
|
|
_, headers = login(app)
|
|
body = raw({"request_id": REQUEST, "report_date": DAY})
|
|
self.assertEqual(app.handle("POST", "/api/arr-downloads", headers, body).status, 202)
|
|
task = self.wait_task(queue)
|
|
self.assertEqual(task["status"], "needs_data_review", task)
|
|
self.assertIsNone(task["job_id"])
|
|
for table in ("finance.daily_versions", "finance.current_daily_versions", "finance.daily_records",
|
|
"ingestion.processing_runs", "ingestion.outbox_events", "reporting.monthly_runs"):
|
|
self.assertEqual(self.count(table), 0, table)
|
|
self.assertEqual(self.monthly_worker().process_next().status, "idle")
|
|
original = next((self.files / "source").rglob("arr-data.json"))
|
|
before, calls = original.read_bytes(), len(self.transport.calls)
|
|
|
|
path = f"/api/arr-downloads/{REQUEST}/data-review"
|
|
response = app.handle("GET", path, headers)
|
|
self.assertEqual(response.status, 200)
|
|
review = json.loads(response.body)["data"]
|
|
self.assertEqual([item["field"] for item in review["items"]], ["DISP_ROOM_NO"])
|
|
response = app.handle("POST", path + "/items/1:DISP_ROOM_NO", headers,
|
|
raw({"revision": review["revision"], "value": "SIM-ROOM-001"}))
|
|
self.assertEqual(response.status, 200)
|
|
review = json.loads(response.body)["data"]
|
|
self.assertEqual(self.count("finance.daily_versions"), 0)
|
|
self.assertEqual(self.count("ingestion.outbox_events"), 0)
|
|
confirmation = raw({"revision": review["revision"]})
|
|
response = app.handle("POST", path + "/finalize", headers, confirmation)
|
|
self.assertEqual(response.status, 202)
|
|
task = self.wait_task(queue)
|
|
self.assertEqual(task["status"], "succeeded", task)
|
|
self.assertEqual(task["request_id"], REQUEST)
|
|
self.assertEqual(len(self.transport.calls), calls)
|
|
self.assertEqual(original.read_bytes(), before)
|
|
self.assertEqual(self.count("finance.daily_versions"), 1)
|
|
self.assertEqual(self.count("finance.current_daily_versions"), 1)
|
|
self.assertEqual(self.daily_event_count(), 1)
|
|
total, price, rooms, nights = self.sql(
|
|
"SELECT total_price,real_price,no_of_rooms,nights FROM finance.daily_records")[0]
|
|
self.assertEqual(total, Decimal("1800.00"))
|
|
self.assertEqual(total, price * rooms * nights)
|
|
self.assertEqual(self.sql("SELECT result_schema_version FROM ingestion.processing_runs"), [("5.0",)])
|
|
|
|
key, size = self.sql("""SELECT a.object_key,a.byte_size FROM finance.daily_versions v
|
|
JOIN ingestion.artifacts a ON a.id=v.daily_report_artifact_id""")[0]
|
|
daily = self.files / "verified-daily.xlsx"
|
|
self.store.materialize(key, daily, size)
|
|
workbook = load_workbook(daily, data_only=False)
|
|
try:
|
|
sheet = workbook[workbook.sheetnames[0]]
|
|
self.assertEqual(sheet.cell(2, 19).value, 1800)
|
|
self.assertEqual(sheet.cell(2, 18).value * sheet.cell(2, 11).value * sheet.cell(2, 17).value, 1800)
|
|
finally:
|
|
workbook.close()
|
|
|
|
self.assertEqual(self.monthly_worker().process_next().status, "published")
|
|
self.assertEqual(self.monthly_worker().process_next().status, "idle")
|
|
self.assertEqual(self.count("reporting.monthly_runs"), 1)
|
|
self.assertEqual(self.sql("SELECT publish_status FROM ingestion.outbox_events WHERE event_type='arr.daily_version_committed'"),
|
|
[("published",)])
|
|
key, size = self.sql("""SELECT a.object_key,a.byte_size FROM reporting.monthly_runs r
|
|
JOIN ingestion.artifacts a ON a.id=r.workbook_artifact_id""")[0]
|
|
monthly = self.files / "verified-monthly.xlsx"
|
|
self.store.materialize(key, monthly, size)
|
|
workbook = load_workbook(monthly, data_only=False)
|
|
try:
|
|
formula_cells = [cell for sheet in workbook for row in sheet for cell in row if cell.data_type == "f"]
|
|
self.assertEqual(len(formula_cells), 1)
|
|
cell = formula_cells[0]
|
|
self.assertEqual(cell.value, f"=R{cell.row}*C{cell.row}*G{cell.row}")
|
|
self.assertEqual(cell.parent.cell(cell.row, 18).value * cell.parent.cell(cell.row, 3).value
|
|
* cell.parent.cell(cell.row, 7).value, 1800)
|
|
finally:
|
|
workbook.close()
|
|
self.assertEqual(app.handle("POST", path + "/finalize", headers, confirmation).status, 202)
|
|
self.assertEqual(queue.get(REQUEST)["status"], "succeeded")
|
|
self.assertEqual(self.execute().status, "succeeded")
|
|
self.assertEqual(len(self.transport.calls), calls)
|
|
self.assertEqual(self.count("finance.daily_versions"), 1)
|
|
self.assertEqual(self.daily_event_count(), 1)
|
|
self.assertEqual(self.count("reporting.monthly_runs"), 1)
|
|
self.assertFalse(list(self.files.rglob("*.xml")))
|
|
finally:
|
|
queue.close(wait=True)
|
|
|
|
|
|
class FakeDataReviewDownloads:
|
|
ready = True
|
|
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
def latest(self):
|
|
return None
|
|
|
|
def get(self, request_id):
|
|
return {"request_id": request_id, "report_date": DAY, "status": "needs_data_review",
|
|
"job_id": None, "can_retry": False}
|
|
|
|
def get_data_review(self, request_id):
|
|
self.calls.append(("get", request_id))
|
|
return {"request_id": request_id, "status": "editing", "revision": 0, "items": []}
|
|
|
|
def update_data_review_item(self, request_id, item_id, revision, value, actor):
|
|
self.calls.append(("update", request_id, item_id, revision, value, actor))
|
|
return {"request_id": request_id, "revision": revision + 1, "items": []}
|
|
|
|
def finalize_data_review(self, request_id, revision, actor):
|
|
self.calls.append(("finalize", request_id, revision, actor))
|
|
return {"request_id": request_id, "status": "queued"}
|
|
|
|
|
|
class DataReviewRoutesTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.downloads = FakeDataReviewDownloads()
|
|
self.app = PortalApplication(repository=FakeRepository(), arr_downloads=self.downloads,
|
|
credentials=TEST_CREDENTIALS)
|
|
_, self.headers = login(self.app)
|
|
self.path = f"/api/arr-downloads/{REQUEST}/data-review"
|
|
|
|
def post(self, suffix, payload, headers=None):
|
|
return self.app.handle("POST", self.path + suffix, self.headers if headers is None else headers,
|
|
json.dumps(payload).encode())
|
|
|
|
def test_auth_csrf_session_actor_and_strict_review_requests(self):
|
|
self.assertEqual(self.app.handle("GET", self.path, {}).status, 401)
|
|
self.assertEqual(self.post("/items/1:BLOCK_CODE", {"revision": 0, "value": ""}, {}).status, 401)
|
|
self.assertEqual(self.post("/items/1:BLOCK_CODE", {"revision": 0, "value": ""},
|
|
{"Cookie": self.headers["Cookie"]}).status, 403)
|
|
self.assertEqual(self.downloads.calls, [])
|
|
self.assertEqual(self.app.handle("GET", self.path, self.headers).status, 200)
|
|
self.assertEqual(self.post("/items/1:BLOCK_CODE", {"revision": 0, "value": ""}).status, 200)
|
|
self.assertEqual(self.post("/finalize", {"revision": 1}).status, 202)
|
|
self.assertEqual(self.downloads.calls, [("get", REQUEST),
|
|
("update", REQUEST, "1:BLOCK_CODE", 0, "", TEST_CREDENTIALS.username),
|
|
("finalize", REQUEST, 1, TEST_CREDENTIALS.username)])
|
|
for suffix, payload in (("/items/1:BLOCK_CODE", {"revision": True, "value": ""}),
|
|
("/items/1:BLOCK_CODE", {"revision": 1, "value": "", "actor": "forged"}),
|
|
("/finalize", {"revision": 1, "actor": "forged"}),
|
|
("/finalize", {"revision": "1"})):
|
|
with self.subTest(suffix=suffix, payload=payload):
|
|
self.assertEqual(self.post(suffix, payload).status, 400)
|
|
self.assertEqual(len(self.downloads.calls), 3)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|