172 lines
11 KiB
Python
172 lines
11 KiB
Python
"""Cancellation outcome contracts and additive Finance migration, synthetic only."""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
from dataclasses import replace
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
|
|
from arr_ingestion.contracts import DeliveryEnvelope, IngestionError
|
|
from arr_ingestion.postgres import DatabaseConfig, PostgresIngestionRepository, _outcome_counts
|
|
from arr_ingestion.validation import DeliveryValidator, _validate_structured_payload_v4
|
|
from tests.local_postgres import TemporaryPostgres
|
|
from tests.test_arr_ingestion_validation import build_delivery, policy
|
|
from tests.test_arr_opera_daily_ingest import reservation, xml_document
|
|
|
|
|
|
PROJECT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def cancelled_xml(*, rate_amount="900"):
|
|
cancelled = reservation(1).replace(
|
|
"<DISP_ROOM_NO>SYNTHETIC-ROOM-1</DISP_ROOM_NO>", "<DISP_ROOM_NO/>"
|
|
).replace("</G_RESERVATION>", "<RESV_STATUS>CANCELLED</RESV_STATUS></G_RESERVATION>")
|
|
return xml_document(cancelled, reservation(2, rate_amount=rate_amount))
|
|
|
|
|
|
class CancelledIngestionContractTests(unittest.TestCase):
|
|
def generated(self, root):
|
|
raw, store, _ = build_delivery(cancelled_xml(), root)
|
|
envelope = DeliveryEnvelope.from_dict(json.loads(raw))
|
|
payload = json.loads(store.objects[envelope.artifacts["structured_result_json"].object_key])
|
|
return raw, store, envelope, payload
|
|
|
|
def test_new_outcome_is_validated_and_replayed_without_room(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
raw, store, _, payload = self.generated(Path(temporary))
|
|
verified = DeliveryValidator(store, policy()).validate(raw)
|
|
self.assertEqual(verified.envelope.status, "success")
|
|
self.assertEqual(payload["outcome_counts"]["excluded_cancelled"], 1)
|
|
self.assertEqual(payload["output_rows"], 1)
|
|
self.assertEqual(payload["records"][0]["outcome"], "excluded_cancelled")
|
|
self.assertEqual(payload["records"][0]["disp_room_no"], "")
|
|
self.assertNotIn("removed_as_cancelled", payload)
|
|
|
|
def test_new_identity_requires_zero_count_and_old_identity_rejects_new_key(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
raw, store, _ = build_delivery(xml_document(reservation(1)), Path(temporary))
|
|
envelope = DeliveryEnvelope.from_dict(json.loads(raw))
|
|
payload = json.loads(store.objects[envelope.artifacts["structured_result_json"].object_key])
|
|
self.assertEqual(payload["outcome_counts"]["excluded_cancelled"], 0)
|
|
_validate_structured_payload_v4(payload, envelope)
|
|
missing = copy.deepcopy(payload)
|
|
del missing["outcome_counts"]["excluded_cancelled"]
|
|
with self.assertRaisesRegex(IngestionError, "outcome counts contract"):
|
|
_validate_structured_payload_v4(missing, envelope)
|
|
old_envelope = replace(envelope, processor_version="4.2.0")
|
|
old_payload = copy.deepcopy(payload)
|
|
old_payload["processor_version"] = "4.2.0"
|
|
old_payload["outcome_counts"].pop("excluded_pm", None)
|
|
with self.assertRaisesRegex(IngestionError, "outcome counts contract"):
|
|
_validate_structured_payload_v4(old_payload, old_envelope)
|
|
del old_payload["outcome_counts"]["excluded_cancelled"]
|
|
_validate_structured_payload_v4(old_payload, old_envelope)
|
|
|
|
def test_old_identity_cannot_disguise_cancelled_record_or_rule_hash(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
raw, store, envelope, payload = self.generated(Path(temporary))
|
|
old = copy.deepcopy(payload)
|
|
old["processor_version"] = "4.2.0"
|
|
old["outcome_counts"].pop("excluded_pm", None)
|
|
del old["outcome_counts"]["excluded_cancelled"]
|
|
with self.assertRaisesRegex(IngestionError, "structured outcome is invalid"):
|
|
_validate_structured_payload_v4(old, replace(envelope, processor_version="4.2.0"))
|
|
bad_hash = json.loads(raw)
|
|
bad_hash["rule_set_sha256"] = "0" * 64
|
|
with self.assertRaises(IngestionError) as caught:
|
|
DeliveryValidator(store, policy()).validate(json.dumps(bad_hash).encode())
|
|
self.assertEqual(caught.exception.code, "PROCESSOR_NOT_ALLOWED")
|
|
|
|
def test_finance_counts_preserve_old_payload_and_add_new_count(self):
|
|
counts = dict(retained=2, excluded_rate_code=1, duplicate=3,
|
|
validation_failed=4, price_unmatched=5)
|
|
self.assertEqual(_outcome_counts({"outcome_counts": counts}), (2, 1, 3, 4, 5, 0, 0))
|
|
counts["excluded_cancelled"] = 6
|
|
self.assertEqual(_outcome_counts({"outcome_counts": counts}), (2, 1, 3, 4, 5, 6, 0))
|
|
|
|
def test_cancelled_count_balances_pending_price_review(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
raw, store, _ = build_delivery(cancelled_xml(rate_amount="1800"), Path(temporary))
|
|
verified = DeliveryValidator(store, policy()).validate(raw)
|
|
self.assertEqual(verified.envelope.status, "review_required")
|
|
self.assertEqual(verified.structured_payload["outcome_counts"]["excluded_cancelled"], 1)
|
|
self.assertEqual(verified.structured_payload["review_required_rows"], 1)
|
|
|
|
|
|
@unittest.skipUnless(os.environ.get("ARR_TEST_LOCAL_POSTGRES") == "1", "owned PostgreSQL opt-in required")
|
|
class CancelledFinanceMigrationTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.database = TemporaryPostgres().__enter__()
|
|
cls.addClassCleanup(cls.database.__exit__, None, None, None)
|
|
|
|
def test_upgrade_preserves_history_permissions_guards_and_count_constraints(self):
|
|
import psycopg
|
|
self.database.reset_database(schema_version=19)
|
|
repository = PostgresIngestionRepository(DatabaseConfig("owned-fixture"), connect=self.database.connect)
|
|
with self.assertRaises(IngestionError) as missing:
|
|
repository.assert_cancelled_scope_schema()
|
|
self.assertEqual(missing.exception.code, "DATABASE_MIGRATION_MISSING")
|
|
with self.database.connect(autocommit=True) as connection:
|
|
artifact = connection.execute("""INSERT INTO ingestion.artifacts
|
|
(artifact_kind, storage_provider, bucket_alias, object_key, original_filename, sha256, byte_size)
|
|
VALUES ('opera_xml','local_fixture','fixture','old/source.xml','source.xml',%s,0) RETURNING id""",
|
|
("a" * 64,)).fetchone()[0]
|
|
run = connection.execute("""INSERT INTO ingestion.processing_runs
|
|
(run_key,pipeline_type,source_artifact_id,run_status,result_delivery_mode,business_date,
|
|
delivered_processor_version,delivered_rule_set_sha256,result_schema_version,
|
|
delivery_sha256,validated_at,finished_at)
|
|
VALUES ('old-success','opera_daily',%s,'accepted','direct_mcp','2026-07-27',
|
|
'3.0.0',%s,'3.0',%s,now(),now()) RETURNING id""",
|
|
(artifact, "b" * 64, "c" * 64)).fetchone()[0]
|
|
version = connection.execute("""INSERT INTO finance.daily_versions
|
|
(business_date,version_no,processing_run_id,source_artifact_id,version_status,
|
|
processor_version,rule_set_sha256,result_schema_version,result_sha256,source_rows,
|
|
retained_rows,excluded_rate_code_rows,duplicate_rows,validation_failed_rows,
|
|
price_unmatched_rows,validated_at,result_delivery_mode)
|
|
VALUES ('2026-07-27',1,%s,%s,'validated','3.0.0',%s,'3.0',%s,
|
|
1,0,1,0,0,0,now(),'direct_mcp') RETURNING id""",
|
|
(run, artifact, "b" * 64, "c" * 64)).fetchone()[0]
|
|
before = connection.execute("SELECT to_jsonb(v) FROM finance.daily_versions v WHERE id=%s", (version,)).fetchone()[0]
|
|
guards = connection.execute("SELECT tgname,pg_get_triggerdef(oid) FROM pg_trigger WHERE NOT tgisinternal ORDER BY tgname").fetchall()
|
|
connection.execute("CREATE ROLE arr_cancelled_fixture")
|
|
connection.execute("GRANT USAGE ON SCHEMA finance TO arr_cancelled_fixture")
|
|
connection.execute("GRANT SELECT, INSERT ON finance.daily_versions TO arr_cancelled_fixture")
|
|
connection.execute((PROJECT / "database/020_daily_cancelled_exclusion.sql").read_text(), prepare=False)
|
|
repository.assert_cancelled_scope_schema()
|
|
after = connection.execute("SELECT to_jsonb(v) FROM finance.daily_versions v WHERE id=%s", (version,)).fetchone()[0]
|
|
self.assertEqual(after.pop("excluded_cancelled_rows"), 0)
|
|
self.assertEqual(after, before)
|
|
self.assertEqual(connection.execute("SELECT tgname,pg_get_triggerdef(oid) FROM pg_trigger WHERE NOT tgisinternal ORDER BY tgname").fetchall(), guards)
|
|
self.assertEqual(connection.execute("SELECT has_column_privilege('arr_cancelled_fixture','finance.daily_versions','excluded_cancelled_rows','INSERT'),has_table_privilege('arr_cancelled_fixture','finance.daily_versions','UPDATE')").fetchone(), (True, False))
|
|
with self.assertRaises(psycopg.errors.CheckViolation):
|
|
connection.execute("UPDATE finance.daily_versions SET excluded_cancelled_rows=1 WHERE id=%s", (version,))
|
|
connection.execute("UPDATE finance.daily_versions SET excluded_rate_code_rows=0, excluded_cancelled_rows=1 WHERE id=%s", (version,))
|
|
with self.assertRaisesRegex(psycopg.errors.RaiseException, "cancellation-exclusion facts exist"):
|
|
connection.execute((PROJECT / "database/020_daily_cancelled_exclusion.down.sql").read_text(), prepare=False)
|
|
connection.execute("ROLLBACK")
|
|
connection.execute("SET ROLE arr_cancelled_fixture")
|
|
with self.assertRaises(psycopg.errors.InsufficientPrivilege):
|
|
connection.execute("UPDATE finance.daily_versions SET excluded_cancelled_rows=0 WHERE id=%s", (version,))
|
|
connection.execute("RESET ROLE")
|
|
|
|
def test_empty_database_allows_guarded_rollback_and_reapply(self):
|
|
self.database.reset_database(schema_version=20)
|
|
repository = PostgresIngestionRepository(DatabaseConfig("owned-fixture"), connect=self.database.connect)
|
|
repository.assert_cancelled_scope_schema()
|
|
with self.database.connect(autocommit=True) as connection:
|
|
connection.execute((PROJECT / "database/020_daily_cancelled_exclusion.down.sql").read_text(), prepare=False)
|
|
with self.assertRaises(IngestionError) as missing:
|
|
repository.assert_cancelled_scope_schema()
|
|
self.assertEqual(missing.exception.code, "DATABASE_MIGRATION_MISSING")
|
|
with self.database.connect(autocommit=True) as connection:
|
|
connection.execute((PROJECT / "database/020_daily_cancelled_exclusion.sql").read_text(), prepare=False)
|
|
repository.assert_cancelled_scope_schema()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|