228 lines
13 KiB
Python
228 lines
13 KiB
Python
"""Date-grouped portal reads against owned disposable PostgreSQL only."""
|
|
from datetime import date, timedelta
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
|
|
from arr_ingestion.postgres import DatabaseConfig as IngestionConfig, PostgresIngestionRepository
|
|
from arr_ingestion.service import IngestionService
|
|
from arr_ingestion.validation import DeliveryValidator
|
|
from arr_processing.local import LocalDailyProcessor
|
|
from arr_processing.policy import load_processor_policy
|
|
from arr_storage.filesystem import FilesystemObjectBackend
|
|
from arr_storage.store import ManagedObjectStore
|
|
from arr_web.programmatic import ProgrammaticUploadCoordinator
|
|
from arr_web.repository import PortalDataError, PostgresPortalRepository
|
|
from monthly_reports.publishing import AtomicReportPublisher, OpenpyxlWorkbookBuilder
|
|
from monthly_reports.repository import DatabaseConfig, PostgresReportRepository
|
|
from monthly_reports.service import MonthlyReportService
|
|
from monthly_reports.worker import MonthlyOutboxWorker, PostgresOutboxRepository
|
|
from tests.local_postgres import TemporaryPostgres
|
|
from tests.test_arr_opera_daily_ingest import reservation, xml_document
|
|
|
|
|
|
PROJECT = Path(__file__).resolve().parents[1]
|
|
DAY = date(2026, 7, 2)
|
|
|
|
|
|
@unittest.skipUnless(os.environ.get("ARR_TEST_LOCAL_POSTGRES") == "1", "owned disposable PostgreSQL opt-in required")
|
|
class DailyReportReadPostgresTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.database = TemporaryPostgres().__enter__()
|
|
cls.addClassCleanup(cls.database.__exit__, None, None, None)
|
|
cls.policy = load_processor_policy(PROJECT)
|
|
|
|
def setUp(self):
|
|
self.database.reset_database(schema_version=22)
|
|
temporary = tempfile.TemporaryDirectory(prefix="arr-daily-read-fixture-")
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name)
|
|
self.store = ManagedObjectStore(FilesystemObjectBackend(self.root / "objects", create=True))
|
|
self.ingestion = PostgresIngestionRepository(IngestionConfig("owned-fixture"), connect=self.database.connect)
|
|
service = IngestionService(DeliveryValidator(self.store, self.policy), self.ingestion)
|
|
self.uploads = ProgrammaticUploadCoordinator(self.store, self.ingestion, service, LocalDailyProcessor(self.policy),
|
|
self.policy.processor_version, self.policy.rule_set_sha256)
|
|
self.portal = PostgresPortalRepository("owned-fixture", connect=self.database.connect)
|
|
config = DatabaseConfig("owned-fixture")
|
|
reports = PostgresReportRepository(config, connect=self.database.connect)
|
|
output = self.root / "outputs/monthly"
|
|
monthly = MonthlyReportService(reports, OpenpyxlWorkbookBuilder(),
|
|
AtomicReportPublisher(self.root, output, object_store=self.store), output / ".staging")
|
|
self.worker = MonthlyOutboxWorker(PostgresOutboxRepository(config, connect=self.database.connect), reports, monthly)
|
|
|
|
def sql(self, query, parameters=None):
|
|
with self.database.connect() as connection:
|
|
cursor = connection.execute(query, parameters)
|
|
return cursor.fetchall() if cursor.description else []
|
|
|
|
def upload(self, label, day=DAY, **changes):
|
|
xml = xml_document(reservation(1, confirmation=f"SYN-{label}", room=f"SYN-ROOM-{label}", **changes))
|
|
xml = (xml.replace("2026-07-27", day.isoformat())
|
|
.replace("2026-07-28", (day + timedelta(days=1)).isoformat())
|
|
.replace("20260727", day.strftime("%Y%m%d"))
|
|
.replace("27-07-26", day.strftime("%d-%m-%y")))
|
|
return self.uploads.submit(f"synthetic-{label}.XML", xml.encode())
|
|
|
|
def accepted(self, label, day=DAY):
|
|
result = self.upload(label, day)
|
|
self.assertEqual(result["status"], "succeeded", result)
|
|
return result
|
|
|
|
def retire(self, uploaded):
|
|
return self.ingestion.retire_daily_job(uploaded["job_id"], date.fromisoformat(uploaded["business_date"]), "synthetic-operator")
|
|
|
|
def drain(self):
|
|
for _attempt in range(20):
|
|
result = self.worker.process_next()
|
|
if result.status == "idle":
|
|
return
|
|
self.assertIn(result.status, {"published", "withdrawn"}, result)
|
|
self.fail("synthetic monthly worker did not settle")
|
|
|
|
def assert_download_missing(self, callback):
|
|
with self.assertRaises(PortalDataError) as raised:
|
|
callback()
|
|
self.assertEqual(raised.exception.code, "DOWNLOAD_NOT_FOUND")
|
|
|
|
def test_one_date_group_keeps_current_success_separate_from_latest_pending_then_failed_replacement(self):
|
|
accepted = self.accepted("current")
|
|
pending = self.upload("pending", company="SYNTHETIC UNPRICED COMPANY")
|
|
self.assertEqual(pending["status"], "needs_review")
|
|
groups, total = self.portal.list_daily_reports("2026-07")
|
|
self.assertEqual((total, len(groups)), (1, 1))
|
|
group = groups[0]
|
|
self.assertEqual(group["business_date"], DAY.isoformat())
|
|
self.assertEqual(group["current_job"]["job_id"], accepted["job_id"])
|
|
self.assertTrue(group["current_job"]["is_current"])
|
|
self.assertEqual(group["latest_job"]["job_id"], pending["job_id"])
|
|
self.assertEqual(group["latest_job"]["status"], "needs_review")
|
|
self.assertFalse(group["latest_job"]["is_current"])
|
|
self.assertIsNone(group["latest_job"]["daily_report_sha256"])
|
|
self.assertEqual(group["history_count"], 1)
|
|
failed = self.upload("failed", departure="2026-07-01")
|
|
self.assertEqual(failed["status"], "failed")
|
|
selected = self.portal.get_daily_report(DAY.isoformat())
|
|
self.assertEqual(selected["current_job"]["job_id"], accepted["job_id"])
|
|
self.assertEqual((selected["latest_job"]["job_id"], selected["latest_job"]["status"]), (failed["job_id"], "failed"))
|
|
self.assertEqual(selected["history_count"], 2)
|
|
self.assertEqual(self.portal.resolve_daily_download(accepted["job_id"]).sha256,
|
|
selected["current_job"]["daily_report_sha256"])
|
|
self.assert_download_missing(lambda: self.portal.resolve_daily_download(pending["job_id"]))
|
|
|
|
def test_deleting_an_old_success_preserves_current_and_monthly_but_blocks_old_download(self):
|
|
old = self.accepted("old")
|
|
current = self.accepted("current")
|
|
self.drain()
|
|
monthly_before = self.portal.list_monthly_runs("2026-07")
|
|
result = self.retire(old)
|
|
self.assertFalse(result["current_removed"])
|
|
self.assertFalse(result["monthly_refresh_required"])
|
|
selected = self.portal.get_daily_report(DAY.isoformat())
|
|
self.assertEqual(selected["current_job"]["job_id"], current["job_id"])
|
|
self.assertEqual(selected["latest_job"]["job_id"], current["job_id"])
|
|
self.assertEqual(self.portal.list_monthly_runs("2026-07"), monthly_before)
|
|
history, total = self.portal.list_daily_report_history(DAY.isoformat())
|
|
self.assertEqual(total, 2)
|
|
deleted = next(job for job in history if job["job_id"] == old["job_id"])
|
|
self.assertEqual(deleted["status"], "deleted")
|
|
self.assertTrue(deleted["is_retired"])
|
|
self.assertFalse(deleted["is_current"])
|
|
self.assertFalse(deleted["can_delete"])
|
|
self.assertIsNone(deleted["daily_report_sha256"])
|
|
self.assert_download_missing(lambda: self.portal.resolve_daily_download(old["job_id"]))
|
|
self.assertIsNotNone(self.portal.resolve_daily_download(current["job_id"]))
|
|
|
|
def test_deleting_current_does_not_turn_old_history_into_an_effective_report(self):
|
|
old = self.accepted("old")
|
|
current = self.accepted("current")
|
|
self.retire(current)
|
|
selected = self.portal.get_daily_report(DAY.isoformat())
|
|
self.assertIsNone(selected["current_job"])
|
|
self.assertFalse(selected["latest_job"]["is_current"])
|
|
self.assertEqual(selected["latest_job"]["job_id"], old["job_id"])
|
|
history, total = self.portal.list_daily_report_history(DAY.isoformat())
|
|
self.assertEqual(total, 2)
|
|
self.assertFalse(any(job["is_current"] for job in history))
|
|
deleted = next(job for job in history if job["job_id"] == current["job_id"])
|
|
self.assertTrue(deleted["is_retired"])
|
|
self.assertFalse(deleted["can_delete"])
|
|
self.assertIsNone(deleted["daily_report_sha256"])
|
|
self.assertEqual(self.sql("SELECT count(*) FROM finance.current_daily_versions"), [(0,)])
|
|
self.assertEqual(self.portal.list_months(), [])
|
|
self.assert_download_missing(lambda: self.portal.resolve_daily_download(current["job_id"]))
|
|
|
|
def test_empty_month_hides_withdrawn_publications_counts_old_download_and_bi(self):
|
|
uploaded = self.accepted("only")
|
|
self.drain()
|
|
monthly, total = self.portal.list_monthly_runs("2026-07")
|
|
self.assertEqual(total, 1)
|
|
report_id = monthly[0]["report_id"]
|
|
self.assertIsNotNone(self.portal.resolve_monthly_download(report_id))
|
|
self.assertEqual(len(self.portal.list_months()), 1)
|
|
self.retire(uploaded)
|
|
# Visibility changes with retirement; the asynchronous ack is not needed
|
|
# before old artifacts and analytics disappear from the portal.
|
|
self.assertEqual(self.portal.list_monthly_runs("2026-07"), ([], 0))
|
|
history_months = self.portal.list_history_month_counts()
|
|
self.assertEqual(history_months[0]["monthly_count"], 0)
|
|
self.assertEqual(self.portal.list_months(), [])
|
|
self.assert_download_missing(lambda: self.portal.resolve_monthly_download(report_id))
|
|
with self.assertRaises(PortalDataError) as raised:
|
|
self.portal.read_dashboard("2026-07")
|
|
self.assertEqual(raised.exception.code, "ANALYTICS_MONTH_NOT_FOUND")
|
|
self.assertEqual(self.portal.list_daily_reports("2026-07"), ([], 0))
|
|
selected = self.portal.get_daily_report(DAY.isoformat())
|
|
self.assertIsNone(selected["current_job"])
|
|
self.assertIsNone(selected["latest_job"])
|
|
history, total = self.portal.list_daily_report_history(DAY.isoformat())
|
|
self.assertEqual((total, history[0]["status"]), (1, "deleted"))
|
|
self.drain()
|
|
self.assertEqual(self.portal.list_monthly_runs("2026-07"), ([], 0))
|
|
|
|
def test_empty_selected_day_and_history_return_date_context_without_a_fabricated_report(self):
|
|
selected = self.portal.get_daily_report("2026-07-19")
|
|
self.assertEqual(selected, {"business_date": "2026-07-19", "current_job": None, "latest_job": None, "history_count": 0})
|
|
self.assertEqual(self.portal.list_daily_report_history("2026-07-19"), ([], 0))
|
|
self.assertEqual(self.portal.list_daily_reports("2026-07"), ([], 0))
|
|
|
|
def test_selected_date_is_independent_of_latest_upload_and_date_pagination_counts_days(self):
|
|
first = self.accepted("first", date(2026, 7, 2))
|
|
second = self.accepted("second", date(2026, 7, 3))
|
|
self.accepted("first-replaced", date(2026, 7, 2))
|
|
groups, total = self.portal.list_daily_reports("2026-07", limit=1)
|
|
self.assertEqual(total, 2)
|
|
self.assertEqual(groups[0]["business_date"], "2026-07-03")
|
|
self.assertEqual(groups[0]["current_job"]["job_id"], second["job_id"])
|
|
selected = self.portal.get_daily_report("2026-07-03")
|
|
self.assertEqual(selected["current_job"]["job_id"], second["job_id"])
|
|
history, count = self.portal.list_daily_report_history("2026-07-02", limit=1, offset=1)
|
|
self.assertEqual(count, 2)
|
|
self.assertEqual(history[0]["job_id"], first["job_id"])
|
|
|
|
def test_date_views_work_under_a_select_only_role_that_cannot_retire_reports(self):
|
|
uploaded = self.accepted("read-role")
|
|
self.retire(uploaded)
|
|
self.sql("CREATE ROLE arr_read_lifecycle_fixture NOLOGIN")
|
|
self.sql("GRANT USAGE ON SCHEMA ingestion, finance, reporting TO arr_read_lifecycle_fixture")
|
|
self.sql("GRANT SELECT ON ALL TABLES IN SCHEMA ingestion, finance, reporting TO arr_read_lifecycle_fixture")
|
|
def read_connection(_dsn):
|
|
connection = self.database.connect()
|
|
connection.execute("SET ROLE arr_read_lifecycle_fixture")
|
|
connection.commit()
|
|
return connection
|
|
portal = PostgresPortalRepository("owned-select-only", connect=read_connection)
|
|
self.assertEqual(portal.list_daily_reports("2026-07"), ([], 0))
|
|
history, total = portal.list_daily_report_history(DAY.isoformat())
|
|
self.assertEqual((total, history[0]["status"]), (1, "deleted"))
|
|
self.assertIsNone(portal.get_daily_report(DAY.isoformat())["current_job"])
|
|
import psycopg
|
|
with read_connection(None) as connection:
|
|
with self.assertRaises(psycopg.errors.InsufficientPrivilege):
|
|
connection.execute("DELETE FROM finance.current_daily_versions")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|