"""Read-only PostgreSQL snapshot source for generated monthly reports.""" from __future__ import annotations import os import re import time import hashlib from dataclasses import dataclass from datetime import date from decimal import Decimal from pathlib import PurePosixPath from typing import Any, Callable, List, Optional, Protocol, Sequence, Tuple from monthly_reports.contracts import ( PROCESSOR_VERSION, RESULT_SCHEMA_VERSION, ChannelObservation, DailyVersionPin, ErrorCode, MonthlyFact, MonthlyReport, MonthlySnapshot, rule_set_sha256, ) from monthly_reports.core import month_bounds TARGET_DATABASE = "booking_test" DATABASE_ENV = "MONTHLY_REPORT_DATABASE_URL" FALLBACK_DATABASE_ENV = "ARR_DATABASE_URL" TRANSIENT_SQLSTATES = frozenset({"23505", "40001", "40P01", "55P03"}) MAX_TRANSACTION_ATTEMPTS = 4 SHA256_RE = re.compile(r"^[0-9a-f]{64}$") SOURCE_FACTS_SQL = """ SELECT facts.id, facts.daily_version_id, versions.business_date, facts.channel_key, facts.arrival, facts.departure, facts.nights, facts.adults, facts.children, facts.block_code, facts.no_of_rooms, facts.company_name, facts.confirmation_no, facts.disp_room_no, facts.effective_rate_amount, facts.full_name, facts.res_comment, facts.trace_text, facts.products, facts.rate_code, facts.room_category_label, facts.real_price, facts.total_price, facts.kb_amount FROM finance.v_active_daily_facts AS facts JOIN finance.daily_versions AS versions ON versions.id = facts.daily_version_id WHERE versions.business_date BETWEEN %s AND %s ORDER BY facts.id """.strip() CURRENT_DAILY_SQL = """ SELECT business_date, daily_version_id FROM finance.current_daily_versions WHERE business_date BETWEEN %s AND %s ORDER BY business_date """.strip() CHANNEL_OBSERVATIONS_SQL = """ SELECT current_version.business_date, current_version.daily_version_id, metrics.channel_key, metrics.channel_order, metrics.row_count FROM finance.current_daily_versions AS current_version JOIN finance.daily_channel_metrics AS metrics ON metrics.daily_version_id = current_version.daily_version_id WHERE current_version.business_date BETWEEN %s AND %s ORDER BY current_version.business_date, metrics.channel_order, metrics.channel_key """.strip() PREFERRED_CHANNELS_SQL = """ SELECT metrics.channel_key FROM finance.current_daily_versions AS current_version JOIN finance.daily_channel_metrics AS metrics ON metrics.daily_version_id = current_version.daily_version_id WHERE current_version.business_date BETWEEN %s AND %s GROUP BY metrics.channel_key ORDER BY min(current_version.business_date), min(metrics.channel_order), metrics.channel_key """.strip() class RepositoryError(RuntimeError): def __init__(self, code: str, safe_message: str): super().__init__(safe_message) self.code = code self.safe_message = safe_message @dataclass(frozen=True) class DatabaseConfig: dsn: str @classmethod def from_environment(cls) -> "DatabaseConfig": dsn = os.environ.get(DATABASE_ENV, "").strip() if not dsn: dsn = os.environ.get(FALLBACK_DATABASE_ENV, "").strip() if not dsn: raise RepositoryError( ErrorCode.REQUEST_INVALID, f"{DATABASE_ENV} or {FALLBACK_DATABASE_ENV} is required", ) return cls(dsn=dsn) @dataclass(frozen=True) class ReservedReport: report_version_id: int version_no: int period_start: Optional[date] = None as_of_date: Optional[date] = None daily_versions: Tuple[DailyVersionPin, ...] = tuple() @dataclass(frozen=True) class FileMetadata: file_kind: str original_filename: str storage_key: str sha256: str byte_size: int mime_type: str def validate(self) -> None: storage = PurePosixPath(self.storage_key) if ( self.file_kind not in {"monthly_xlsx", "result_json"} or not self.storage_key or storage.is_absolute() or ".." in storage.parts or "\\" in self.storage_key or "\x00" in self.storage_key or self.original_filename != PurePosixPath(self.original_filename).name or "/" in self.original_filename or not SHA256_RE.fullmatch(self.sha256) or self.byte_size <= 0 or not self.mime_type.strip() ): raise RepositoryError( ErrorCode.PUBLISH_FAILED, "generated artifact metadata is invalid", ) class ReportRepository(Protocol): def load_snapshot(self, year: int, month: int, as_of_date: date) -> MonthlySnapshot: ... def reserve_report(self, report: MonthlyReport) -> ReservedReport: ... def activate_report( self, reservation: ReservedReport, artifact: FileMetadata, result_json: FileMetadata, ) -> None: ... def mark_failed(self, reservation: ReservedReport, code: str, safe_message: str) -> None: ... def _default_connect(dsn: str) -> Any: try: import psycopg # type: ignore[import-not-found] except ImportError: raise RepositoryError( ErrorCode.INTERNAL_ERROR, "PostgreSQL driver is unavailable; install requirements-monthly-reports.txt", ) from None try: return psycopg.connect(dsn, autocommit=False) except Exception: raise RepositoryError( ErrorCode.DATABASE_FAILED, "database connection failed", ) from None def _decimal(value: Any) -> Decimal: return value if isinstance(value, Decimal) else Decimal(str(value)) def _is_transient_database_error(error: Exception) -> bool: sqlstate = getattr(error, "sqlstate", None) if not isinstance(sqlstate, str): sqlstate = getattr(getattr(error, "diag", None), "sqlstate", None) return isinstance(sqlstate, str) and sqlstate in TRANSIENT_SQLSTATES class PostgresReportRepository: """Loads one repeatable snapshot and atomically publishes its lineage.""" def __init__( self, config: DatabaseConfig, connect: Optional[Callable[[str], Any]] = None, ) -> None: self._config = config self._connect = connect or _default_connect def _open(self) -> Any: try: return self._connect(self._config.dsn) except RepositoryError: raise except Exception: raise RepositoryError( ErrorCode.DATABASE_FAILED, "database connection failed", ) from None @staticmethod def _assert_target(cursor: Any) -> None: cursor.execute("SELECT current_database()") row = cursor.fetchone() if not row or row[0] != TARGET_DATABASE: raise RepositoryError( ErrorCode.DATABASE_FAILED, "database target is not booking_test", ) @classmethod def _begin_serializable(cls, cursor: Any) -> None: cursor.execute( "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY" ) cursor.execute("SET LOCAL lock_timeout = '10s'") cursor.execute("SET LOCAL statement_timeout = '30s'") cls._assert_target(cursor) def _run_serializable( self, operation: Callable[[Any], Any], safe_message: str, ) -> Any: for attempt_index in range(MAX_TRANSACTION_ATTEMPTS): connection = self._open() retry = False try: with connection.transaction(): with connection.cursor() as cursor: self._begin_serializable(cursor) return operation(cursor) except RepositoryError: raise except Exception as error: retry = ( attempt_index + 1 < MAX_TRANSACTION_ATTEMPTS and _is_transient_database_error(error) ) if not retry: raise RepositoryError(ErrorCode.DATABASE_FAILED, safe_message) from None finally: connection.close() if retry: time.sleep(0.02 * (2**attempt_index)) raise RepositoryError(ErrorCode.DATABASE_FAILED, safe_message) def load_snapshot(self, year: int, month: int, as_of_date: date) -> MonthlySnapshot: month_start, _month_end = month_bounds(year, month) connection = self._open() try: with connection.transaction(): with connection.cursor() as cursor: cursor.execute( "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY" ) cursor.execute("SET LOCAL statement_timeout = '30s'") self._assert_target(cursor) cursor.execute(SOURCE_FACTS_SQL, (month_start, as_of_date)) fact_rows = list(cursor.fetchall()) cursor.execute(CURRENT_DAILY_SQL, (month_start, as_of_date)) pin_rows = list(cursor.fetchall()) cursor.execute(CHANNEL_OBSERVATIONS_SQL, (month_start, as_of_date)) observation_rows = list(cursor.fetchall()) cursor.execute(PREFERRED_CHANNELS_SQL, (month_start, _month_end)) preferred_rows = list(cursor.fetchall()) facts = tuple( MonthlyFact( daily_record_id=int(row[0]), daily_version_id=int(row[1]), business_date=row[2], channel_key=str(row[3] or ""), arrival=row[4], departure=row[5], nights=int(row[6]), adults=int(row[7]), children=int(row[8]), block_code=row[9], no_of_rooms=int(row[10]), company_name=str(row[11] or ""), confirmation_no=str(row[12] or ""), disp_room_no=str(row[13] or ""), effective_rate_amount=_decimal(row[14]), full_name=str(row[15] or ""), res_comment=row[16], trace_text=row[17], products=row[18], rate_code=str(row[19] or ""), room_category_label=row[20], real_price=_decimal(row[21]), total_price=_decimal(row[22]), kb_amount=None if row[23] is None else _decimal(row[23]), ) for row in fact_rows ) pins = tuple( DailyVersionPin(business_date=row[0], daily_version_id=int(row[1])) for row in pin_rows ) observations = tuple( ChannelObservation( business_date=row[0], daily_version_id=int(row[1]), worksheet=str(row[2] or ""), worksheet_order=None if row[3] is None else int(row[3]), row_count=int(row[4]), ) for row in observation_rows ) return MonthlySnapshot( facts=facts, daily_versions=pins, channel_observations=observations, preferred_channel_order=tuple(str(row[0] or "") for row in preferred_rows), ) except RepositoryError: raise except Exception: raise RepositoryError( ErrorCode.DATABASE_FAILED, "database snapshot query failed", ) from None finally: connection.close() @staticmethod def _lock_scope(cursor: Any, period_start: date, period_end: date) -> None: lock_key = f"monthly_channel:{period_start.isoformat()}:{period_end.isoformat()}:*" cursor.execute( "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (lock_key,), ) @staticmethod def _current_pins( cursor: Any, period_start: date, as_of_date: date, ) -> Tuple[DailyVersionPin, ...]: cursor.execute( CURRENT_DAILY_SQL, (period_start, as_of_date), ) return tuple( DailyVersionPin(business_date=row[0], daily_version_id=int(row[1])) for row in cursor.fetchall() ) @classmethod def _validate_pins_current(cls, cursor: Any, report: MonthlyReport) -> None: period_start, _period_end = month_bounds(report.report_year, report.report_month) if cls._current_pins(cursor, period_start, report.as_of_date) != report.daily_versions: raise RepositoryError( ErrorCode.SOURCE_SNAPSHOT_STALE, "daily source versions changed while the monthly report was being generated", ) def reserve_report(self, report: MonthlyReport) -> ReservedReport: period_start, period_end = month_bounds(report.report_year, report.report_month) def operation(cursor: Any) -> ReservedReport: self._lock_scope(cursor, period_start, period_end) self._validate_pins_current(cursor, report) identity = { "period_start": period_start.isoformat(), "period_end": period_end.isoformat(), "as_of_date": report.as_of_date.isoformat(), "processor_version": PROCESSOR_VERSION, "rule_set_sha256": rule_set_sha256(), "result_schema_version": RESULT_SCHEMA_VERSION, "daily_versions": [ [pin.business_date.isoformat(), pin.daily_version_id] for pin in report.daily_versions ], "channel_manifest": [list(item) for item in report.channel_manifest], } digest = hashlib.sha256( repr(sorted(identity.items())).encode("utf-8") ).hexdigest() generation_id = int(digest[:12], 16) return ReservedReport( generation_id, generation_id, period_start, report.as_of_date, report.daily_versions, ) return self._run_serializable( operation, "monthly report source reservation failed", ) def activate_report( self, reservation: ReservedReport, artifact: FileMetadata, result_json: FileMetadata, ) -> None: artifact.validate() result_json.validate() if reservation.period_start is None or reservation.as_of_date is None: raise RepositoryError( ErrorCode.PUBLISH_FAILED, "monthly source reservation is incomplete", ) def operation(cursor: Any) -> None: period_start, period_end = month_bounds( reservation.period_start.year, reservation.period_start.month, ) self._lock_scope(cursor, period_start, period_end) current = self._current_pins( cursor, period_start, reservation.as_of_date, ) if current != reservation.daily_versions: raise RepositoryError( ErrorCode.SOURCE_SNAPSHOT_STALE, "daily source versions changed before publication", ) self._run_serializable( operation, "monthly report source revalidation failed", ) def mark_failed( self, reservation: ReservedReport, code: str, safe_message: str, ) -> None: # Monthly rows and versions are intentionally not persisted. The # caller's privacy-minimized result JSON is the failure record. _ = (reservation, code, safe_message)