Exclude PM room types from shared daily report processing

This commit is contained in:
Wyndham ARR committed 2026-10-08 22:30:04 +08:00
1 parent d3902dd857
commit f46efd6d9a
32 files changed
+968 -69

No files matched your search

+11
View File
@@ -19,6 +19,17 @@ test('cancelled source rows need no fields and the exclusion stays visible befor
assert.equal(h.calls.every(call=>call.method==='GET'),true);
});
test('PM exclusions stay visible without requiring field completion',async()=>{
const h=harness(()=>response(200,{...review([]),excluded_cancelled_count:6,excluded_pm_count:19}));
await h.acceptARRDownloadTask(task,{sync:false});
const progress=h.element('#arr-data-review-progress').textContent;
assert.match(progress,/data_review\.excluded_cancelled/);
assert.match(progress,/data_review\.excluded_pm/);
assert.equal(h.element('#arr-data-review-body').innerHTML,'');
assert.equal(h.element('#arr-data-review-finalize').disabled,false);
assert.equal(h.calls.every(call=>call.method==='GET'),true);
});
test('removed cancelled items clear obsolete drafts but retain edits still in the review',async()=>{
let current=review([item(),item({item_id:'2:DISP_ROOM_NO',source_sequence:2,field:'DISP_ROOM_NO',can_be_empty:false})]);
const h=harness(()=>response(200,current));
+2 -2
View File
@@ -76,8 +76,8 @@ class TemporaryPostgres:
connection.close()
raise
def reset_database(self, *, schema_version=20):
if schema_version not in (19, 20):
def reset_database(self, *, schema_version=21):
if schema_version not in (19, 20, 21):
raise ValueError("unsupported fixture schema version")
# Only this owned cluster has passed data_directory/socket checks above.
with self.connect(database="postgres", autocommit=True) as connection:
+8 -6
View File
@@ -59,6 +59,7 @@ class CancelledIngestionContractTests(unittest.TestCase):
old_envelope = replace(envelope, processor_version="4.2.0")
old_payload = copy.deepcopy(payload)
old_payload["processor_version"] = "4.2.0"
old_payload["outcome_counts"].pop("excluded_pm", None)
with self.assertRaisesRegex(IngestionError, "outcome counts contract"):
_validate_structured_payload_v4(old_payload, old_envelope)
del old_payload["outcome_counts"]["excluded_cancelled"]
@@ -69,6 +70,7 @@ class CancelledIngestionContractTests(unittest.TestCase):
raw, store, envelope, payload = self.generated(Path(temporary))
old = copy.deepcopy(payload)
old["processor_version"] = "4.2.0"
old["outcome_counts"].pop("excluded_pm", None)
del old["outcome_counts"]["excluded_cancelled"]
with self.assertRaisesRegex(IngestionError, "structured outcome is invalid"):
_validate_structured_payload_v4(old, replace(envelope, processor_version="4.2.0"))
@@ -81,9 +83,9 @@ class CancelledIngestionContractTests(unittest.TestCase):
def test_finance_counts_preserve_old_payload_and_add_new_count(self):
counts = dict(retained=2, excluded_rate_code=1, duplicate=3,
validation_failed=4, price_unmatched=5)
self.assertEqual(_outcome_counts({"outcome_counts": counts}), (2, 1, 3, 4, 5, 0))
self.assertEqual(_outcome_counts({"outcome_counts": counts}), (2, 1, 3, 4, 5, 0, 0))
counts["excluded_cancelled"] = 6
self.assertEqual(_outcome_counts({"outcome_counts": counts}), (2, 1, 3, 4, 5, 6))
self.assertEqual(_outcome_counts({"outcome_counts": counts}), (2, 1, 3, 4, 5, 6, 0))
def test_cancelled_count_balances_pending_price_review(self):
with tempfile.TemporaryDirectory() as temporary:
@@ -152,17 +154,17 @@ class CancelledFinanceMigrationTests(unittest.TestCase):
connection.execute("RESET ROLE")
def test_empty_database_allows_guarded_rollback_and_reapply(self):
self.database.reset_database()
self.database.reset_database(schema_version=20)
repository = PostgresIngestionRepository(DatabaseConfig("owned-fixture"), connect=self.database.connect)
repository.assert_data_source_schema()
repository.assert_cancelled_scope_schema()
with self.database.connect(autocommit=True) as connection:
connection.execute((PROJECT / "database/020_daily_cancelled_exclusion.down.sql").read_text(), prepare=False)
with self.assertRaises(IngestionError) as missing:
repository.assert_data_source_schema()
repository.assert_cancelled_scope_schema()
self.assertEqual(missing.exception.code, "DATABASE_MIGRATION_MISSING")
with self.database.connect(autocommit=True) as connection:
connection.execute((PROJECT / "database/020_daily_cancelled_exclusion.sql").read_text(), prepare=False)
repository.assert_data_source_schema()
repository.assert_cancelled_scope_schema()
if __name__ == "__main__":
+1 -1
View File
@@ -157,7 +157,7 @@ class CancelledScopeTests(unittest.TestCase):
self.assertEqual((payload["source_rows"], payload["removed_by_rate_code"], payload["output_rows"]), (2, 0, 1))
self.assertEqual(payload["outcome_counts"]["excluded_cancelled"], 1)
self.assertEqual(set(payload["outcome_counts"]), core.FINAL_OUTCOMES)
self.assertEqual(payload["processor_version"], "4.3.0")
self.assertEqual(payload["processor_version"], "4.4.0")
self.assertEqual([row["outcome"] for row in payload["records"]], ["excluded_cancelled", "retained"])
self.assertEqual(payload["records"][0]["decision_codes"], ["RESERVATION_CANCELLED_EXCLUDED"])
self.assertEqual(payload["records"][0]["confirmation_no"], "SYNTHETIC-CONF-1")
+16
View File
@@ -41,6 +41,7 @@ class RuntimeTests(unittest.TestCase):
self.store = ManagedObjectStore(FilesystemObjectBackend(self.root / 'objects', create=True))
self.repository = InMemoryIngestionRepository()
self.repository.assert_cancelled_scope_schema = Mock()
self.repository.assert_pm_scope_schema = Mock()
self.processor = CountingProcessor(self.policy)
self.client = Mock()
with patch.object(processing_runtime, 'compose_object_store', return_value=
@@ -59,6 +60,21 @@ class RuntimeTests(unittest.TestCase):
def test_processing_composition_checks_finance_cancellation_schema(self):
self.repository.assert_cancelled_scope_schema.assert_called_once_with()
self.repository.assert_pm_scope_schema.assert_called_once_with()
def test_missing_pm_migration_closes_storage_and_disables_composition(self):
from arr_ingestion.contracts import IngestionError
self.repository.assert_pm_scope_schema.side_effect = IngestionError(
"DATABASE_MIGRATION_MISSING", "ARR processing requires migration 021")
client = Mock()
with patch.object(processing_runtime, 'compose_object_store', return_value=
processing_runtime.ObjectStoreRuntime(client, self.store)), \
patch.object(processing_runtime, 'PostgresIngestionRepository', return_value=self.repository), \
self.assertRaises(IngestionError) as error:
processing_runtime.compose_programmatic_processing(
project_root=Path(__file__).resolve().parents[1], connect=Mock())
self.assertEqual(error.exception.code, "DATABASE_MIGRATION_MISSING")
client.close.assert_called_once_with()
def test_missing_cancellation_migration_closes_storage_and_disables_composition(self):
from arr_ingestion.contracts import IngestionError
+1 -1
View File
@@ -533,7 +533,7 @@ class PostgresIngestionTests(unittest.TestCase):
and "INSERT INTO finance.daily_versions" in value
)
self.assertEqual(version_insert.count("%s"), 21)
self.assertEqual(version_insert.count("%s"), 22)
class Migration008ContractTests(unittest.TestCase):
+4 -3
View File
@@ -236,7 +236,7 @@ class ArrOperaDailyIngestTests(unittest.TestCase):
self.assertNotIn(forbidden, scripts)
self.assertEqual(core.RESULT_VERSION, "4.0")
self.assertEqual(core.STRUCTURED_RESULT_SCHEMA_VERSION, "4.0")
self.assertEqual(core.PROCESSOR_VERSION, "4.3.0")
self.assertEqual(core.PROCESSOR_VERSION, "4.4.0")
self.assertEqual(len(core.DAILY_HEADERS), 19)
self.assertEqual(len(core.RATE_WHITELIST), 20)
@@ -302,7 +302,7 @@ class ArrOperaDailyIngestTests(unittest.TestCase):
)
self.assertEqual(payload["result_schema_version"], "4.0")
self.assertEqual(payload["processor_version"], "4.3.0")
self.assertEqual(payload["processor_version"], "4.4.0")
self.assertEqual(payload["source_rows"], 5)
self.assertEqual(
payload["outcome_counts"],
@@ -310,6 +310,7 @@ class ArrOperaDailyIngestTests(unittest.TestCase):
"candidate": 0,
"duplicate": 1,
"excluded_cancelled": 0,
"excluded_pm": 0,
"excluded_rate_code": 1,
"price_unmatched": 0,
"retained": 3,
@@ -802,7 +803,7 @@ class ArrOperaDailyIngestTests(unittest.TestCase):
},
}
)
self.assertEqual(delivery.processor_version, "4.3.0")
self.assertEqual(delivery.processor_version, "4.4.0")
self.assertEqual(delivery.result_schema_version, "4.0")
self.assertEqual(delivery.business_date, date(2026, 7, 27))
+315
View File
@@ -0,0 +1,315 @@
"""Approved PM exclusion contracts and additive Finance upgrade; all inputs synthetic."""
from __future__ import annotations
import copy
from dataclasses import replace
import json
import os
from pathlib import Path
import tempfile
import unittest
from openpyxl import load_workbook
from arr_ingestion.contracts import DeliveryEnvelope, IngestionError
from arr_ingestion.postgres import DatabaseConfig, PostgresIngestionRepository, _outcome_counts
from arr_ingestion.validation import DeliveryValidator, _validate_structured_payload_v4
from arr_processing.policy import load_processor_policy
from tests.local_postgres import TemporaryPostgres
from tests import test_arr_direct_data as direct_helpers
from tests.test_arr_ingestion_validation import build_delivery, policy
from tests.test_arr_opera_daily_ingest import reservation, xml_document
PROJECT = Path(__file__).resolve().parents[1]
def pm_row(*, status=None):
value = reservation(1, rate_code="", rate_amount="0", company="").replace(
"<ROOM_CATEGORY_LABEL>SYNTHETIC ROOM TYPE</ROOM_CATEGORY_LABEL>",
"<ROOM_CATEGORY_LABEL> pm </ROOM_CATEGORY_LABEL>",
).replace("<DISP_ROOM_NO>SYNTHETIC-ROOM-1</DISP_ROOM_NO>", "<DISP_ROOM_NO/>")
if status:
value = value.replace("</G_RESERVATION>", f"<RESV_STATUS>{status}</RESV_STATUS></G_RESERVATION>")
return value
class PMIngestionContractTests(unittest.TestCase):
def generated(self, root, *rows):
raw, store, _ = build_delivery(xml_document(*rows), root)
envelope = DeliveryEnvelope.from_dict(json.loads(raw))
payload = json.loads(store.objects[envelope.artifacts["structured_result_json"].object_key])
return raw, store, envelope, payload
def test_pm_missing_required_fields_is_audited_before_validation_and_pricing(self):
with tempfile.TemporaryDirectory() as temporary:
raw, store, _, payload = self.generated(Path(temporary), pm_row(), reservation(2))
verified = DeliveryValidator(store, policy()).validate(raw)
self.assertEqual(verified.envelope.status, "success")
self.assertEqual(payload["outcome_counts"]["excluded_pm"], 1)
self.assertEqual(payload["output_rows"], 1)
row = payload["records"][0]
self.assertEqual(row["outcome"], "excluded_pm")
self.assertEqual(row["decision_codes"], ["ROOM_TYPE_PM_EXCLUDED"])
for field in ("real_price", "total_price", "kb_amount", "channel_key", "pricing_method"):
self.assertIsNone(row[field])
def test_pm_count_is_required_only_for_44_and_older_versions_stay_readable(self):
with tempfile.TemporaryDirectory() as temporary:
_, _, envelope, payload = self.generated(Path(temporary), reservation(1))
self.assertEqual(payload["processor_version"], "4.4.0")
self.assertEqual(payload["outcome_counts"]["excluded_pm"], 0)
for field in ("excluded_pm", "excluded_cancelled"):
missing = copy.deepcopy(payload)
del missing["outcome_counts"][field]
with self.assertRaisesRegex(IngestionError, "outcome counts contract"):
_validate_structured_payload_v4(missing, envelope)
for version in ("4.3.0", "4.2.0"):
older = copy.deepcopy(payload)
older["processor_version"] = version
with self.assertRaisesRegex(IngestionError, "outcome counts contract"):
_validate_structured_payload_v4(older, replace(envelope, processor_version=version))
del older["outcome_counts"]["excluded_pm"]
if version == "4.2.0":
del older["outcome_counts"]["excluded_cancelled"]
_validate_structured_payload_v4(older, replace(envelope, processor_version=version))
def test_pm_record_cannot_be_relabelled_or_given_price_facts(self):
with tempfile.TemporaryDirectory() as temporary:
_, _, envelope, payload = self.generated(Path(temporary), pm_row(), reservation(2))
changes = {
"room_category_label": "PM-SUITE",
"decision_codes": ["RATE_CODE_EXCLUDED"],
"real_price": 0,
"total_price": 0,
"kb_amount": 0,
"channel_key": "Group",
"pricing_method": "zero_price_exception",
}
for field, value in changes.items():
changed = copy.deepcopy(payload)
changed["records"][0][field] = value
with self.subTest(field=field), self.assertRaisesRegex(IngestionError, "PM exclusion"):
_validate_structured_payload_v4(changed, envelope)
older = copy.deepcopy(payload)
older["processor_version"] = "4.3.0"
del older["outcome_counts"]["excluded_pm"]
with self.assertRaisesRegex(IngestionError, "structured outcome is invalid"):
_validate_structured_payload_v4(older, replace(envelope, processor_version="4.3.0"))
changed = copy.deepcopy(payload)
changed["outcome_counts"]["excluded_pm"] = 0
with self.assertRaisesRegex(IngestionError, "outcomes do not reconcile"):
_validate_structured_payload_v4(changed, envelope)
def test_pm_balances_price_review_and_failure_and_cancelled_takes_precedence(self):
cases = (
(reservation(2, rate_amount="1800"), "review_required"),
(reservation(2, departure="2026-07-26"), "failed"),
)
for row, status in cases:
with self.subTest(status=status), tempfile.TemporaryDirectory() as temporary:
raw, store, _, payload = self.generated(Path(temporary), pm_row(), row)
self.assertEqual(DeliveryValidator(store, policy()).validate(raw).envelope.status, status)
self.assertEqual(payload["outcome_counts"]["excluded_pm"], 1)
with tempfile.TemporaryDirectory() as temporary:
raw, store, _, payload = self.generated(Path(temporary), pm_row(status="CXL"), reservation(2))
DeliveryValidator(store, policy()).validate(raw)
self.assertEqual(payload["outcome_counts"]["excluded_cancelled"], 1)
self.assertEqual(payload["outcome_counts"]["excluded_pm"], 0)
def test_finance_defaults_only_old_pm_counts_and_rejects_44_missing_keys(self):
counts = dict(retained=2, excluded_rate_code=1, duplicate=3,
validation_failed=4, price_unmatched=5)
self.assertEqual(_outcome_counts({"outcome_counts": counts}), (2, 1, 3, 4, 5, 0, 0))
counts["excluded_cancelled"] = 6
self.assertEqual(_outcome_counts({"processor_version": "4.3.0", "outcome_counts": counts}),
(2, 1, 3, 4, 5, 6, 0))
with self.assertRaises(IngestionError):
_outcome_counts({"processor_version": "4.4.0", "outcome_counts": counts})
counts["excluded_pm"] = 7
self.assertEqual(_outcome_counts({"processor_version": "4.4.0", "outcome_counts": counts}),
(2, 1, 3, 4, 5, 6, 7))
del counts["excluded_cancelled"]
with self.assertRaises(IngestionError):
_outcome_counts({"processor_version": "4.4.0", "outcome_counts": counts})
@unittest.skipUnless(os.environ.get("ARR_TEST_LOCAL_POSTGRES") == "1", "owned PostgreSQL opt-in required")
class PMFinanceMigrationTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.database = TemporaryPostgres().__enter__()
cls.addClassCleanup(cls.database.__exit__, None, None, None)
def test_additive_upgrade_preserves_cancelled_history_and_role_privileges(self):
import psycopg
self.database.reset_database(schema_version=20)
repository = PostgresIngestionRepository(DatabaseConfig("owned-fixture"), connect=self.database.connect)
repository.assert_cancelled_scope_schema()
with self.assertRaises(IngestionError) as missing:
repository.assert_pm_scope_schema()
self.assertEqual(missing.exception.code, "DATABASE_MIGRATION_MISSING")
with self.database.connect(autocommit=True) as connection:
artifact = connection.execute("""INSERT INTO ingestion.artifacts
(artifact_kind,storage_provider,bucket_alias,object_key,original_filename,sha256,byte_size)
VALUES ('opera_xml','local_fixture','fixture','old/source.xml','source.xml',%s,0) RETURNING id""",
("a" * 64,)).fetchone()[0]
output_ids = {}
for kind in ("daily_xlsx", "result_json", "structured_result_json"):
output_ids[kind] = connection.execute("""INSERT INTO ingestion.artifacts
(artifact_kind,storage_provider,bucket_alias,object_key,original_filename,sha256,byte_size)
VALUES (%s,'local_fixture','fixture',%s,%s,%s,0) RETURNING id""",
(kind, "old/" + kind, kind, "d" * 64)).fetchone()[0]
run = connection.execute("""INSERT INTO ingestion.processing_runs
(run_key,pipeline_type,source_artifact_id,result_artifact_id,run_status,result_delivery_mode,business_date,
delivered_processor_version,delivered_rule_set_sha256,result_schema_version,
delivery_sha256,validated_at,finished_at)
VALUES ('old-cancelled','opera_daily',%s,%s,'accepted','artifact_callback','2026-07-27',
'4.3.0',%s,'4.0',%s,now(),now()) RETURNING id""",
(artifact, output_ids["result_json"], "b" * 64, "c" * 64)).fetchone()[0]
version = connection.execute("""INSERT INTO finance.daily_versions
(business_date,version_no,processing_run_id,source_artifact_id,daily_report_artifact_id,
result_json_artifact_id,structured_result_artifact_id,version_status,
processor_version,rule_set_sha256,result_schema_version,result_sha256,source_rows,
retained_rows,excluded_rate_code_rows,duplicate_rows,validation_failed_rows,
price_unmatched_rows,excluded_cancelled_rows,validated_at,result_delivery_mode)
VALUES ('2026-07-27',1,%s,%s,%s,%s,%s,'validated','4.3.0',%s,'4.0',%s,
2,1,0,0,0,0,1,now(),'artifact_callback') RETURNING id""",
(run, artifact, output_ids["daily_xlsx"], output_ids["result_json"],
output_ids["structured_result_json"], "b" * 64, "c" * 64)).fetchone()[0]
connection.execute("""INSERT INTO finance.daily_records
(daily_version_id,source_sequence,source_location,outcome,decision_codes,
booking_source_match_status)
VALUES (%s,1,'reservation[1]','excluded_cancelled',
ARRAY['RESERVATION_CANCELLED_EXCLUDED'],'missing_group_code')""", (version,))
# Processor 4.3 included this PM row. A schema upgrade must preserve it.
connection.execute("""INSERT INTO finance.daily_records
(daily_version_id,source_sequence,source_location,outcome,decision_codes,block_code,
adults,children,company_name,company_key,confirmation_no,disp_room_no,
effective_rate_amount,full_name,no_of_rooms,products,rate_code,room_category_label,
arrival,departure,nights,real_price,total_price,channel_key,pricing_method,
booking_source_match_status)
VALUES (%s,2,'reservation[2]','retained',ARRAY['PRICE_REFERENCE_EXACT'],'',2,0,
'SYNTHETIC COMPANY','SYNTHETIC COMPANY','SYNTHETIC CONF','SYNTHETIC ROOM',
900,'SYNTHETIC GUEST',1,'SYNTHETIC','GRPA1','PM','2026-07-27','2026-07-28',
1,900,900,'Group','price_reference_exact','missing_group_code')""", (version,))
before = connection.execute("SELECT to_jsonb(v) FROM finance.daily_versions v WHERE id=%s", (version,)).fetchone()[0]
records_before = connection.execute("SELECT to_jsonb(r) FROM finance.daily_records r ORDER BY source_sequence").fetchall()
guards = connection.execute("SELECT tgname,pg_get_triggerdef(oid) FROM pg_trigger WHERE NOT tgisinternal ORDER BY tgname").fetchall()
connection.execute("CREATE ROLE arr_pm_fixture")
connection.execute("CREATE ROLE arr_pm_column_fixture")
connection.execute("GRANT USAGE ON SCHEMA finance TO arr_pm_fixture,arr_pm_column_fixture")
connection.execute("GRANT SELECT,INSERT ON finance.daily_versions TO arr_pm_fixture")
connection.execute("GRANT INSERT (excluded_cancelled_rows) ON finance.daily_versions TO arr_pm_column_fixture")
connection.execute((PROJECT / "database/021_daily_pm_exclusion.sql").read_text(), prepare=False)
repository.assert_data_source_schema()
after = connection.execute("SELECT to_jsonb(v) FROM finance.daily_versions v WHERE id=%s", (version,)).fetchone()[0]
self.assertEqual(after.pop("excluded_pm_rows"), 0)
self.assertEqual(after, before)
self.assertEqual(connection.execute("SELECT to_jsonb(r) FROM finance.daily_records r ORDER BY source_sequence").fetchall(), records_before)
self.assertEqual(connection.execute("SELECT tgname,pg_get_triggerdef(oid) FROM pg_trigger WHERE NOT tgisinternal ORDER BY tgname").fetchall(), guards)
self.assertEqual(connection.execute("SELECT has_column_privilege('arr_pm_fixture','finance.daily_versions','excluded_pm_rows','INSERT'),has_table_privilege('arr_pm_fixture','finance.daily_versions','UPDATE'),has_column_privilege('arr_pm_column_fixture','finance.daily_versions','excluded_pm_rows','INSERT')").fetchone(), (True, False, False))
for statement in (
"UPDATE finance.daily_versions SET excluded_pm_rows=-1 WHERE id=%s",
"UPDATE finance.daily_versions SET excluded_pm_rows=1 WHERE id=%s",
):
with self.assertRaises(psycopg.errors.CheckViolation):
connection.execute(statement, (version,))
connection.execute("UPDATE finance.daily_versions SET excluded_cancelled_rows=0,excluded_pm_rows=1 WHERE id=%s", (version,))
with self.assertRaisesRegex(psycopg.errors.RaiseException, "PM-exclusion facts exist"):
connection.execute((PROJECT / "database/021_daily_pm_exclusion.down.sql").read_text(), prepare=False)
connection.execute("ROLLBACK")
connection.execute("SET ROLE arr_pm_fixture")
with self.assertRaises(psycopg.errors.InsufficientPrivilege):
connection.execute("UPDATE finance.daily_versions SET excluded_pm_rows=0 WHERE id=%s", (version,))
connection.execute("RESET ROLE")
def test_empty_rollback_preserves_020_and_allows_reapply(self):
self.database.reset_database()
repository = PostgresIngestionRepository(DatabaseConfig("owned-fixture"), connect=self.database.connect)
repository.assert_data_source_schema()
with self.database.connect(autocommit=True) as connection:
connection.execute((PROJECT / "database/021_daily_pm_exclusion.down.sql").read_text(), prepare=False)
repository.assert_cancelled_scope_schema()
with self.assertRaises(IngestionError) as missing:
repository.assert_pm_scope_schema()
self.assertEqual(missing.exception.code, "DATABASE_MIGRATION_MISSING")
with self.database.connect(autocommit=True) as connection:
connection.execute((PROJECT / "database/021_daily_pm_exclusion.sql").read_text(), prepare=False)
repository.assert_data_source_schema()
@unittest.skipUnless(os.environ.get("ARR_TEST_LOCAL_POSTGRES") == "1", "owned PostgreSQL opt-in required")
class PMFinanceProcessingTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.database = TemporaryPostgres().__enter__()
cls.addClassCleanup(cls.database.__exit__, None, None, None)
cls.policy = load_processor_policy(PROJECT)
setUp = direct_helpers.DirectDataPostgresTests.setUp
configure = direct_helpers.DirectDataTests.configure
execute = direct_helpers.DirectDataTests.execute
sql = direct_helpers.DirectDataPostgresTests.sql
count = direct_helpers.DirectDataPostgresTests.count
monthly_worker = direct_helpers.DirectDataPostgresTests.monthly_worker
def make_first_pm(self):
row = self.transport.rows[0]
row["roomStay"]["roomType"] = "PM"
row["roomStay"]["currentRoomInfo"]["roomType"] = "PM"
for rate in row["roomStay"]["roomRates"]:
rate["roomType"] = "PM"
def test_direct_pm_is_audited_and_daily_monthly_exclude_it_without_refetch(self):
self.make_first_pm()
self.assertEqual(self.execute().status, "succeeded")
self.assertEqual(self.sql("SELECT result_schema_version,source_rows,retained_rows,excluded_pm_rows FROM finance.daily_versions"), [("5.0", 6, 5, 1)])
self.assertEqual(self.sql("SELECT outcome,decision_codes,real_price,channel_key FROM finance.daily_records WHERE source_sequence=1"), [("excluded_pm", ["ROOM_TYPE_PM_EXCLUDED"], None, None)])
self.assertEqual(self.count("finance.v_active_daily_facts"), 5)
self.assertEqual(self.monthly_worker().process_next().status, "published")
key, size = self.sql("SELECT a.object_key,a.byte_size FROM reporting.monthly_runs r JOIN ingestion.artifacts a ON a.id=r.workbook_artifact_id")[0]
output = self.files / "monthly.xlsx"
self.store.materialize(key, output, size)
workbook = load_workbook(output, data_only=False)
try:
self.assertEqual(sum(cell.data_type == "f" for sheet in workbook for row in sheet for cell in row), 5)
finally:
workbook.close()
calls = len(self.transport.calls)
self.assertEqual(self.execute().status, "succeeded")
self.assertEqual(len(self.transport.calls), calls)
self.assertEqual(self.count("finance.daily_versions"), 1)
def test_xml_pm_is_audited_without_required_field_or_price_review(self):
outcome = self.coordinator.submit("source.xml", xml_document(pm_row(), reservation(2)).encode())
self.assertEqual(outcome["status"], "succeeded")
self.assertEqual(self.sql("SELECT result_schema_version,source_rows,retained_rows,excluded_pm_rows FROM finance.daily_versions"), [("4.0", 2, 1, 1)])
self.assertEqual(self.sql("SELECT outcome FROM finance.daily_records ORDER BY source_sequence"), [("excluded_pm",), ("retained",)])
self.assertEqual(self.count("finance.v_active_daily_facts"), 1)
self.assertEqual(self.monthly_worker().process_next().status, "published")
def test_xml_failed_batch_keeps_pm_audit_without_monthly_commit(self):
outcome = self.coordinator.submit("source.xml", xml_document(pm_row(), reservation(2, departure="2026-07-26")).encode())
self.assertEqual(outcome["status"], "failed")
self.assertEqual(self.sql("SELECT version_status,source_rows,excluded_pm_rows,validation_failed_rows FROM finance.daily_versions"), [("rejected", 2, 1, 1)])
self.assertEqual(self.count("finance.current_daily_versions"), 0)
self.assertEqual(self.monthly_worker().process_next().status, "idle")
def test_database_pm_record_classification_guard_rejects_wrong_room_or_price(self):
import psycopg
self.make_first_pm()
self.assertEqual(self.execute().status, "succeeded")
with self.database.connect(autocommit=True) as connection:
for statement in (
"UPDATE finance.daily_records SET room_category_label='PM-SUITE' WHERE source_sequence=1",
"UPDATE finance.daily_records SET real_price=0 WHERE source_sequence=1",
"UPDATE finance.daily_records SET channel_key='Group' WHERE source_sequence=1",
"UPDATE finance.daily_records SET decision_codes=ARRAY['RATE_CODE_EXCLUDED'] WHERE source_sequence=1",
):
with self.subTest(statement=statement), self.assertRaises(psycopg.errors.CheckViolation):
connection.execute(statement)
if __name__ == "__main__":
unittest.main()
+98
View File
@@ -0,0 +1,98 @@
"""PM eligibility is shared with processing, before source-field completion."""
import json
from pathlib import Path
import tempfile
import unittest
from arr_processing.policy import load_processor_policy
from arr_web.arr_data_review import DataFieldReviews
from arr_web.contracts import PortalError
from tests.test_arr_data_review import source_document, raw, gap, CONTEXT, DAY, REQUEST, ACTOR, SOURCE_MANIFEST
class PMReviewTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory(prefix="arr-pm-review-")
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name) / "reviews"
self.policy = load_processor_policy(Path(__file__).resolve().parents[1])
self.service = DataFieldReviews(root=self.root, policy=self.policy, context=CONTEXT)
self.document = source_document(2)
def prepare(self):
return self.service.prepare(REQUEST, raw(self.document), SOURCE_MANIFEST, DAY)
def test_explicit_pm_needs_no_fields_and_does_not_hide_other_rows(self):
self.document["records"][0]["fields"]["ROOM_CATEGORY_LABEL"] = {"state": "available", "value": " pm "}
for field in ("DISP_ROOM_NO", "RATE_CODE", "BLOCK_CODE", "ADULTS"):
gap(self.document, field)
gap(self.document, "DISP_ROOM_NO", sequence=2)
review = self.prepare()
self.assertEqual([item["item_id"] for item in review["items"]], ["2:DISP_ROOM_NO"])
self.assertEqual(review["excluded_pm_count"], 1)
self.assertEqual(review["excluded_cancelled_count"], 0)
with self.assertRaisesRegex(PortalError, "PM"):
self.service.update(REQUEST, "1:BLOCK_CODE", review["revision"], "IGNORED", ACTOR)
def test_zero_price_or_nine_thousand_room_number_does_not_identify_pm(self):
for row in self.document["records"]:
row["fields"]["EFFECTIVE_RATE_AMOUNT"] = {"state": "available", "value": "0"}
row["fields"]["DISP_ROOM_NO"] = {"state": "available", "value": "9002"}
row["fields"]["ROOM_CATEGORY_LABEL"] = {"state": "available", "value": "PMX"}
gap(self.document, "ADULTS")
review = self.prepare()
self.assertEqual(review["excluded_pm_count"], 0)
self.assertIn("1:ADULTS", [item["item_id"] for item in review["items"]])
def test_cancelled_pm_is_counted_once_as_cancelled(self):
row = self.document["records"][0]
row["reservation_status"] = "Cancelled"
row["fields"]["ROOM_CATEGORY_LABEL"] = {"state": "available", "value": "PM"}
gap(self.document, "BLOCK_CODE", sequence=2)
review = self.prepare()
self.assertEqual((review["excluded_cancelled_count"], review["excluded_pm_count"]), (1, 0))
def test_prior_decisions_remain_audited_when_completed_room_type_is_pm(self):
gap(self.document, "BLOCK_CODE")
gap(self.document, "ROOM_CATEGORY_LABEL")
review = self.prepare()
original = self.service.original(REQUEST)
review = self.service.update(REQUEST, "1:BLOCK_CODE", review["revision"], "VERIFIED", ACTOR)
review = self.service.update(REQUEST, "1:ROOM_CATEGORY_LABEL", review["revision"], "PM", ACTOR)
self.assertEqual([item["item_id"] for item in review["items"]], ["1:ROOM_CATEGORY_LABEL"])
self.assertTrue(review["items"][0]["confirmed"])
self.assertEqual(review["excluded_pm_count"], 1)
self.assertTrue(review["can_finalize"])
with self.assertRaisesRegex(PortalError, "PM"):
self.service.update(REQUEST, "1:BLOCK_CODE", review["revision"], "CHANGED", ACTOR)
self.service.finalize(REQUEST, review["revision"], ACTOR)
derived = json.loads(self.service.payload(REQUEST)[0])
self.assertEqual(self.service.original(REQUEST), original)
changes = derived["manual_data_review"]["manifest"]["changes"]
self.assertEqual({item["field"]: item["value"] for item in changes},
{"BLOCK_CODE": "VERIFIED", "ROOM_CATEGORY_LABEL": "PM"})
def test_existing_room_type_normalization_is_shared_by_exclusion_count_and_edit_guard(self):
self.document["records"][0]["fields"]["ROOM_CATEGORY_LABEL"] = {"state": "available", "value": "PM😀"}
gap(self.document, "BLOCK_CODE")
gap(self.document, "DISP_ROOM_NO", sequence=2)
review = self.prepare()
self.assertEqual(review["excluded_pm_count"], 1)
self.assertEqual([item["item_id"] for item in review["items"]], ["2:DISP_ROOM_NO"])
with self.assertRaisesRegex(PortalError, "PM"):
self.service.update(REQUEST, "1:BLOCK_CODE", review["revision"], "IGNORED", ACTOR)
def test_manually_completed_room_type_can_be_corrected_before_finalization(self):
gap(self.document, "ROOM_CATEGORY_LABEL")
gap(self.document, "BLOCK_CODE")
review = self.prepare()
review = self.service.update(REQUEST, "1:ROOM_CATEGORY_LABEL", review["revision"], "PM", ACTOR)
self.assertEqual(review["pending_count"], 0)
review = self.service.update(REQUEST, "1:ROOM_CATEGORY_LABEL", review["revision"], "SU6", ACTOR)
self.assertEqual(review["excluded_pm_count"], 0)
self.assertEqual(review["pending_count"], 1)
self.assertIn("1:BLOCK_CODE", [item["item_id"] for item in review["items"]])
if __name__ == "__main__":
unittest.main()
+228
View File
@@ -0,0 +1,228 @@
"""Offline PM scope checks for both daily inputs and the frozen legacy replay."""
from __future__ import annotations
import argparse
import contextlib
import copy
import io
import json
from pathlib import Path
import tempfile
import unittest
import xml.etree.ElementTree as ET
from tests.test_arr_opera_daily_ingest import (
core, reservation, run_processor, validator, xml_document,
)
from tests.test_arr_data_review import source_document
def pm_xml_row(sequence, *, label="PM", room=None, status=None, incomplete=False,
rate_amount="900"):
node = ET.fromstring(reservation(sequence, room=room, rate_amount=rate_amount))
node.find("ROOM_CATEGORY_LABEL").text = label
if status is not None:
ET.SubElement(node, "SHORT_RESV_STATUS").text = status
if incomplete:
for field, value in (("RATE_CODE", ""), ("DISP_ROOM_NO", ""),
("COMPANY_NAME", ""), ("FULL_NAME", ""),
("ADULTS", "-1"), ("NO_OF_ROOMS", "0"),
("EFFECTIVE_RATE_AMOUNT", "INVALID"),
("TRUNC_END", "2026-07-26")):
node.find(field).text = value
return ET.tostring(node, encoding="unicode")
class PMScopeTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory(prefix="arr-pm-scope-")
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
def classify_xml(self, *rows, apply_scope=True):
path = self.root / "source.xml"
path.write_text(xml_document(*rows))
business_date, nodes = core.read_xml(path)
return core.classify_input_records(nodes, business_date, apply_scope=apply_scope)
def run_data(self, document, name="output"):
path = self.root / (name + ".json")
path.write_text(json.dumps(document))
output = self.root / name
args = argparse.Namespace(xml=None, data_json=str(path), output_dir=str(output),
result_json=str(output / "result.json"),
structured_result_json=str(output / "structured-result.json"))
before = path.read_bytes()
with contextlib.redirect_stdout(io.StringIO()):
exit_code = core.process(args)
self.assertEqual(path.read_bytes(), before)
return exit_code, json.loads((output / "structured-result.json").read_text())
def test_xml_pm_precedes_rate_and_required_fields_but_cancelled_takes_precedence(self):
rows, eligible, removed_rate, duplicates, errors = self.classify_xml(
pm_xml_row(1, label=" pm ", incomplete=True),
pm_xml_row(2, status="CXL", incomplete=True), reservation(3))
self.assertEqual(errors, [])
self.assertEqual((removed_rate, duplicates), (0, 0))
self.assertEqual([row["_SOURCE_INDEX"] for row in eligible], [3])
self.assertEqual([row["_OUTCOME"] for row in rows],
["excluded_pm", "excluded_cancelled", "pending"])
self.assertEqual(rows[0]["_DECISION_CODES"], ["ROOM_TYPE_PM_EXCLUDED"])
self.assertEqual(rows[1]["_DECISION_CODES"], ["RESERVATION_CANCELLED_EXCLUDED"])
self.assertEqual(rows[0]["ROOM_CATEGORY_LABEL"], "pm")
self.assertEqual(rows[0]["CONFIRMATION_NO"], "SYNTHETIC-CONF-1")
self.assertEqual(rows[0]["_SOURCE_LOCATION"], "reservation[1]")
def test_only_exact_explicit_pm_room_type_is_excluded(self):
for label in ("PM", "pm", " pM ", "\tPm\n"):
with self.subTest(label=label):
rows, _, _, _, errors = self.classify_xml(
pm_xml_row(1, label=label, incomplete=True), reservation(2))
self.assertEqual(errors, [])
self.assertTrue(core.is_pm_record(rows[0]))
for label in (None, "", "PM1", "PM ROOM", "P M", "SU6"):
with self.subTest(label=label):
rows, _, _, _, errors = self.classify_xml(
pm_xml_row(1, label=label, room="PM", rate_amount="0"), reservation(2))
self.assertEqual(errors, [])
self.assertFalse(core.is_pm_record(rows[0]))
self.assertEqual(rows[0]["_OUTCOME"], "pending")
def test_pm_does_not_consume_a_real_room_duplicate_key_or_create_price_issue(self):
rows, eligible, _, duplicates, errors = self.classify_xml(
pm_xml_row(1, room="SAME-ROOM", rate_amount="0"),
reservation(2, room="SAME-ROOM"), reservation(3, room="SAME-ROOM"))
self.assertEqual(errors, [])
self.assertEqual([row["_SOURCE_INDEX"] for row in eligible], [2])
self.assertEqual(duplicates, 1)
self.assertEqual([row["_OUTCOME"] for row in rows],
["excluded_pm", "pending", "duplicate"])
self.assertEqual(rows[2]["_DUPLICATE_OF_SOURCE_SEQUENCE"], 2)
prices = core.load_price_map(core.PRICE_REFERENCE)
self.assertEqual(core.apply_prices_classified(eligible, prices), [])
self.assertEqual(core.review_issues(eligible, prices), [])
def test_xml_success_and_independent_replay_count_pm_and_cancelled_separately(self):
code, source, output, _, payload = run_processor(xml_document(
pm_xml_row(1, label=" pm ", incomplete=True),
pm_xml_row(2, status="CXL", incomplete=True), reservation(3)), self.root)
self.assertEqual(code, 0, payload["errors"])
self.assertEqual(payload["processor_version"], "4.4.0")
self.assertEqual(payload["outcome_counts"]["excluded_pm"], 1)
self.assertEqual(payload["outcome_counts"]["excluded_cancelled"], 1)
self.assertEqual(sum(payload["outcome_counts"].values()), 3)
self.assertEqual((payload["source_rows"], payload["output_rows"]), (3, 1))
self.assertEqual(payload["artifacts"]["source_xml"]["sha256"], core.sha256_file(source))
self.assertEqual(payload["records"][0]["room_category_label"], "pm")
for field in ("real_price", "total_price", "kb_amount", "channel_key", "pricing_method"):
self.assertIsNone(payload["records"][0][field])
# A structurally valid cancellation reclassification cannot replace PM evidence.
forged = copy.deepcopy(payload)
forged["records"][0]["outcome"] = "excluded_cancelled"
forged["records"][0]["decision_codes"] = ["RESERVATION_CANCELLED_EXCLUDED"]
forged["outcome_counts"]["excluded_cancelled"] += 1
forged["outcome_counts"]["excluded_pm"] -= 1
core.validate_structured_completeness(forged)
structured = output / "structured-result.json"
structured.write_text(json.dumps(forged))
errors = validator.validate(argparse.Namespace(xml=str(source),
daily=str(next(output.glob("*.xlsx"))), result_json=str(output / "result.json"),
structured_result_json=str(structured), price_reference=str(core.PRICE_REFERENCE)))
self.assertIn("OUTPUT_STRUCTURED_OUTCOME_MISMATCH", {error.code for error in errors})
self.assertIn("OUTPUT_STRUCTURED_RECORD_MISMATCH", {error.code for error in errors})
def test_pm_self_validation_and_schemas_require_explicit_type_reason_and_null_facts(self):
code, _, _, _, payload = run_processor(xml_document(
pm_xml_row(1, label="pM"), reservation(2)), self.root)
self.assertEqual(code, 0, payload["errors"])
for field, value in (("room_category_label", "SU6"),
("decision_codes", ["RATE_CODE_NOT_WHITELISTED"]),
("real_price", 0), ("total_price", 0), ("kb_amount", 0),
("channel_key", "QBD"), ("pricing_method", "manual_review")):
with self.subTest(field=field):
forged = copy.deepcopy(payload)
forged["records"][0][field] = value
with self.assertRaises(core.ProcessingFailure):
core.validate_structured_completeness(forged)
forged = copy.deepcopy(payload)
del forged["outcome_counts"]["excluded_pm"]
with self.assertRaises(core.ProcessingFailure):
core.validate_structured_completeness(forged)
for name in ("structured-result.schema.json", "data-structured-result.schema.json"):
schema = json.loads((core.SKILL_ROOT / "references" / name).read_text())
self.assertEqual(schema["properties"]["processor_version"]["const"], "4.4.0")
self.assertIn("excluded_pm", schema["properties"]["outcome_counts"]["required"])
rule = next(item for item in schema["$defs"]["record"]["allOf"]
if item["if"]["properties"]["outcome"].get("const") == "excluded_pm")
self.assertEqual(rule["then"]["properties"]["decision_codes"]["const"],
["ROOM_TYPE_PM_EXCLUDED"])
self.assertEqual(rule["then"]["properties"]["real_price"]["type"], "null")
def test_xml_review_and_failure_keep_pm_outside_pricing_and_other_validation(self):
code, _, _, _, payload = run_processor(xml_document(
pm_xml_row(1, incomplete=True), reservation(2, rate_amount="8765")),
self.root / "review")
self.assertEqual(code, 0, payload["errors"])
self.assertEqual(payload["status"], "review_required")
self.assertEqual(payload["outcome_counts"]["excluded_pm"], 1)
self.assertEqual(payload["review_required_rows"], 1)
self.assertEqual(payload["review_issues"][0]["affected_records"], 1)
invalid = ET.fromstring(reservation(2))
invalid.find("DISP_ROOM_NO").text = ""
code, _, _, _, payload = run_processor(xml_document(
pm_xml_row(1, incomplete=True), ET.tostring(invalid, encoding="unicode")),
self.root / "failure")
self.assertNotEqual(code, 0)
self.assertEqual(payload["status"], "failed")
self.assertEqual(payload["outcome_counts"]["excluded_pm"], 1)
self.assertEqual(payload["outcome_counts"]["validation_failed"], 1)
def test_data_success_review_and_failure_keep_pm_outside_business_rules(self):
baseline = source_document(2)
for field in baseline["records"][0]["fields"]:
baseline["records"][0]["fields"][field] = {"state": "missing", "value": None}
baseline["records"][0]["fields"]["ROOM_CATEGORY_LABEL"] = {"state": "available", "value": " pm "}
baseline["input_complete"] = False
baseline["status"] = "collected_with_gaps"
code, payload = self.run_data(baseline)
self.assertEqual(code, 0, payload["errors"])
self.assertEqual(payload["outcome_counts"]["excluded_pm"], 1)
self.assertEqual(payload["output_rows"], 1)
self.assertEqual(payload["removed_by_rate_code"], 0)
self.assertEqual(payload["records"][0]["room_category_label"], "pm")
review = copy.deepcopy(baseline)
review["records"][1]["fields"]["EFFECTIVE_RATE_AMOUNT"]["value"] = "8765"
code, payload = self.run_data(review, "review")
self.assertEqual(code, 0, payload["errors"])
self.assertEqual(payload["status"], "review_required")
self.assertEqual(payload["outcome_counts"]["excluded_pm"], 1)
self.assertEqual(payload["review_required_rows"], 1)
failed = copy.deepcopy(baseline)
failed["records"][1]["fields"]["DISP_ROOM_NO"] = {"state": "missing", "value": None}
code, payload = self.run_data(failed, "failure")
self.assertNotEqual(code, 0)
self.assertEqual(payload["status"], "failed")
self.assertEqual(payload["outcome_counts"]["excluded_pm"], 1)
self.assertEqual(payload["outcome_counts"]["validation_failed"], 1)
def test_frozen_legacy_v3_replay_keeps_pm_and_never_projects_new_outcomes(self):
code, _, _, _, payload = run_processor(xml_document(
pm_xml_row(1), reservation(2)), self.root, legacy_v3_output=True)
self.assertEqual(code, 0, payload["errors"])
self.assertEqual(payload["result_schema_version"], "3.0")
self.assertEqual(payload["output_rows"], 2)
self.assertEqual(set(payload["outcome_counts"]), core.LEGACY_DIRECT_FINAL_OUTCOMES)
self.assertEqual([row["outcome"] for row in payload["records"]], ["retained", "retained"])
rows, _, _, _, _ = self.classify_xml(pm_xml_row(1), reservation(2))
for function in (core.build_legacy_direct_structured_result,
core.build_legacy_direct_failed_structured_result):
with self.subTest(function=function.__name__), self.assertRaises(core.ProcessingFailure):
if function is core.build_legacy_direct_structured_result:
function(None, rows[:1], [], self.root / "source.xml", None, None)
else:
function(None, rows[:1], self.root / "source.xml", None, None, [])
if __name__ == "__main__":
unittest.main()