479 lines
16 KiB
Python
479 lines
16 KiB
Python
"""Read-only PostgreSQL snapshot source for generated monthly reports."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import time
|
|
import hashlib
|
|
from dataclasses import dataclass
|
|
from datetime import date
|
|
from decimal import Decimal
|
|
from pathlib import PurePosixPath
|
|
from typing import Any, Callable, List, Optional, Protocol, Sequence, Tuple
|
|
|
|
from monthly_reports.contracts import (
|
|
PROCESSOR_VERSION,
|
|
RESULT_SCHEMA_VERSION,
|
|
ChannelObservation,
|
|
DailyVersionPin,
|
|
ErrorCode,
|
|
MonthlyFact,
|
|
MonthlyReport,
|
|
MonthlySnapshot,
|
|
rule_set_sha256,
|
|
)
|
|
from monthly_reports.core import month_bounds
|
|
|
|
|
|
TARGET_DATABASE = "booking_test"
|
|
DATABASE_ENV = "MONTHLY_REPORT_DATABASE_URL"
|
|
FALLBACK_DATABASE_ENV = "ARR_DATABASE_URL"
|
|
TRANSIENT_SQLSTATES = frozenset({"23505", "40001", "40P01", "55P03"})
|
|
MAX_TRANSACTION_ATTEMPTS = 4
|
|
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
|
|
|
|
SOURCE_FACTS_SQL = """
|
|
SELECT
|
|
facts.id,
|
|
facts.daily_version_id,
|
|
versions.business_date,
|
|
facts.channel_key,
|
|
facts.arrival,
|
|
facts.departure,
|
|
facts.nights,
|
|
facts.adults,
|
|
facts.children,
|
|
facts.block_code,
|
|
facts.no_of_rooms,
|
|
facts.company_name,
|
|
facts.confirmation_no,
|
|
facts.disp_room_no,
|
|
facts.effective_rate_amount,
|
|
facts.full_name,
|
|
facts.res_comment,
|
|
facts.trace_text,
|
|
facts.products,
|
|
facts.rate_code,
|
|
facts.room_category_label,
|
|
facts.real_price,
|
|
facts.total_price,
|
|
facts.kb_amount
|
|
FROM finance.v_active_daily_facts AS facts
|
|
JOIN finance.daily_versions AS versions
|
|
ON versions.id = facts.daily_version_id
|
|
WHERE versions.business_date BETWEEN %s AND %s
|
|
ORDER BY facts.id
|
|
""".strip()
|
|
|
|
CURRENT_DAILY_SQL = """
|
|
SELECT business_date, daily_version_id
|
|
FROM finance.current_daily_versions
|
|
WHERE business_date BETWEEN %s AND %s
|
|
ORDER BY business_date
|
|
""".strip()
|
|
|
|
CHANNEL_OBSERVATIONS_SQL = """
|
|
SELECT
|
|
current_version.business_date,
|
|
current_version.daily_version_id,
|
|
metrics.channel_key,
|
|
metrics.channel_order,
|
|
metrics.row_count
|
|
FROM finance.current_daily_versions AS current_version
|
|
JOIN finance.daily_channel_metrics AS metrics
|
|
ON metrics.daily_version_id = current_version.daily_version_id
|
|
WHERE current_version.business_date BETWEEN %s AND %s
|
|
ORDER BY
|
|
current_version.business_date,
|
|
metrics.channel_order,
|
|
metrics.channel_key
|
|
""".strip()
|
|
|
|
PREFERRED_CHANNELS_SQL = """
|
|
SELECT metrics.channel_key
|
|
FROM finance.current_daily_versions AS current_version
|
|
JOIN finance.daily_channel_metrics AS metrics
|
|
ON metrics.daily_version_id = current_version.daily_version_id
|
|
WHERE current_version.business_date BETWEEN %s AND %s
|
|
GROUP BY metrics.channel_key
|
|
ORDER BY
|
|
min(current_version.business_date),
|
|
min(metrics.channel_order),
|
|
metrics.channel_key
|
|
""".strip()
|
|
|
|
|
|
class RepositoryError(RuntimeError):
|
|
def __init__(self, code: str, safe_message: str):
|
|
super().__init__(safe_message)
|
|
self.code = code
|
|
self.safe_message = safe_message
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DatabaseConfig:
|
|
dsn: str
|
|
|
|
@classmethod
|
|
def from_environment(cls) -> "DatabaseConfig":
|
|
dsn = os.environ.get(DATABASE_ENV, "").strip()
|
|
if not dsn:
|
|
dsn = os.environ.get(FALLBACK_DATABASE_ENV, "").strip()
|
|
if not dsn:
|
|
raise RepositoryError(
|
|
ErrorCode.REQUEST_INVALID,
|
|
f"{DATABASE_ENV} or {FALLBACK_DATABASE_ENV} is required",
|
|
)
|
|
return cls(dsn=dsn)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReservedReport:
|
|
report_version_id: int
|
|
version_no: int
|
|
period_start: Optional[date] = None
|
|
as_of_date: Optional[date] = None
|
|
daily_versions: Tuple[DailyVersionPin, ...] = tuple()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FileMetadata:
|
|
file_kind: str
|
|
original_filename: str
|
|
storage_key: str
|
|
sha256: str
|
|
byte_size: int
|
|
mime_type: str
|
|
|
|
def validate(self) -> None:
|
|
storage = PurePosixPath(self.storage_key)
|
|
if (
|
|
self.file_kind not in {"monthly_xlsx", "result_json"}
|
|
or not self.storage_key
|
|
or storage.is_absolute()
|
|
or ".." in storage.parts
|
|
or "\\" in self.storage_key
|
|
or "\x00" in self.storage_key
|
|
or self.original_filename != PurePosixPath(self.original_filename).name
|
|
or "/" in self.original_filename
|
|
or not SHA256_RE.fullmatch(self.sha256)
|
|
or self.byte_size <= 0
|
|
or not self.mime_type.strip()
|
|
):
|
|
raise RepositoryError(
|
|
ErrorCode.PUBLISH_FAILED,
|
|
"generated artifact metadata is invalid",
|
|
)
|
|
|
|
|
|
class ReportRepository(Protocol):
|
|
def load_snapshot(self, year: int, month: int, as_of_date: date) -> MonthlySnapshot:
|
|
...
|
|
|
|
def reserve_report(self, report: MonthlyReport) -> ReservedReport:
|
|
...
|
|
|
|
def activate_report(
|
|
self,
|
|
reservation: ReservedReport,
|
|
artifact: FileMetadata,
|
|
result_json: FileMetadata,
|
|
) -> None:
|
|
...
|
|
|
|
def mark_failed(self, reservation: ReservedReport, code: str, safe_message: str) -> None:
|
|
...
|
|
|
|
|
|
def _default_connect(dsn: str) -> Any:
|
|
try:
|
|
import psycopg # type: ignore[import-not-found]
|
|
except ImportError:
|
|
raise RepositoryError(
|
|
ErrorCode.INTERNAL_ERROR,
|
|
"PostgreSQL driver is unavailable; install requirements-monthly-reports.txt",
|
|
) from None
|
|
try:
|
|
return psycopg.connect(dsn, autocommit=False)
|
|
except Exception:
|
|
raise RepositoryError(
|
|
ErrorCode.DATABASE_FAILED,
|
|
"database connection failed",
|
|
) from None
|
|
|
|
|
|
def _decimal(value: Any) -> Decimal:
|
|
return value if isinstance(value, Decimal) else Decimal(str(value))
|
|
|
|
|
|
def _is_transient_database_error(error: Exception) -> bool:
|
|
sqlstate = getattr(error, "sqlstate", None)
|
|
if not isinstance(sqlstate, str):
|
|
sqlstate = getattr(getattr(error, "diag", None), "sqlstate", None)
|
|
return isinstance(sqlstate, str) and sqlstate in TRANSIENT_SQLSTATES
|
|
|
|
|
|
class PostgresReportRepository:
|
|
"""Loads one repeatable snapshot and atomically publishes its lineage."""
|
|
|
|
def __init__(
|
|
self,
|
|
config: DatabaseConfig,
|
|
connect: Optional[Callable[[str], Any]] = None,
|
|
) -> None:
|
|
self._config = config
|
|
self._connect = connect or _default_connect
|
|
|
|
def _open(self) -> Any:
|
|
try:
|
|
return self._connect(self._config.dsn)
|
|
except RepositoryError:
|
|
raise
|
|
except Exception:
|
|
raise RepositoryError(
|
|
ErrorCode.DATABASE_FAILED,
|
|
"database connection failed",
|
|
) from None
|
|
|
|
@staticmethod
|
|
def _assert_target(cursor: Any) -> None:
|
|
cursor.execute("SELECT current_database()")
|
|
row = cursor.fetchone()
|
|
if not row or row[0] != TARGET_DATABASE:
|
|
raise RepositoryError(
|
|
ErrorCode.DATABASE_FAILED,
|
|
"database target is not booking_test",
|
|
)
|
|
|
|
@classmethod
|
|
def _begin_serializable(cls, cursor: Any) -> None:
|
|
cursor.execute(
|
|
"SET TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY"
|
|
)
|
|
cursor.execute("SET LOCAL lock_timeout = '10s'")
|
|
cursor.execute("SET LOCAL statement_timeout = '30s'")
|
|
cls._assert_target(cursor)
|
|
|
|
def _run_serializable(
|
|
self,
|
|
operation: Callable[[Any], Any],
|
|
safe_message: str,
|
|
) -> Any:
|
|
for attempt_index in range(MAX_TRANSACTION_ATTEMPTS):
|
|
connection = self._open()
|
|
retry = False
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin_serializable(cursor)
|
|
return operation(cursor)
|
|
except RepositoryError:
|
|
raise
|
|
except Exception as error:
|
|
retry = (
|
|
attempt_index + 1 < MAX_TRANSACTION_ATTEMPTS
|
|
and _is_transient_database_error(error)
|
|
)
|
|
if not retry:
|
|
raise RepositoryError(ErrorCode.DATABASE_FAILED, safe_message) from None
|
|
finally:
|
|
connection.close()
|
|
if retry:
|
|
time.sleep(0.02 * (2**attempt_index))
|
|
raise RepositoryError(ErrorCode.DATABASE_FAILED, safe_message)
|
|
|
|
def load_snapshot(self, year: int, month: int, as_of_date: date) -> MonthlySnapshot:
|
|
month_start, _month_end = month_bounds(year, month)
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
cursor.execute(
|
|
"SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"
|
|
)
|
|
cursor.execute("SET LOCAL statement_timeout = '30s'")
|
|
self._assert_target(cursor)
|
|
cursor.execute(SOURCE_FACTS_SQL, (month_start, as_of_date))
|
|
fact_rows = list(cursor.fetchall())
|
|
cursor.execute(CURRENT_DAILY_SQL, (month_start, as_of_date))
|
|
pin_rows = list(cursor.fetchall())
|
|
cursor.execute(CHANNEL_OBSERVATIONS_SQL, (month_start, as_of_date))
|
|
observation_rows = list(cursor.fetchall())
|
|
cursor.execute(PREFERRED_CHANNELS_SQL, (month_start, _month_end))
|
|
preferred_rows = list(cursor.fetchall())
|
|
|
|
facts = tuple(
|
|
MonthlyFact(
|
|
daily_record_id=int(row[0]),
|
|
daily_version_id=int(row[1]),
|
|
business_date=row[2],
|
|
channel_key=str(row[3] or ""),
|
|
arrival=row[4],
|
|
departure=row[5],
|
|
nights=int(row[6]),
|
|
adults=int(row[7]),
|
|
children=int(row[8]),
|
|
block_code=row[9],
|
|
no_of_rooms=int(row[10]),
|
|
company_name=str(row[11] or ""),
|
|
confirmation_no=str(row[12] or ""),
|
|
disp_room_no=str(row[13] or ""),
|
|
effective_rate_amount=_decimal(row[14]),
|
|
full_name=str(row[15] or ""),
|
|
res_comment=row[16],
|
|
trace_text=row[17],
|
|
products=row[18],
|
|
rate_code=str(row[19] or ""),
|
|
room_category_label=row[20],
|
|
real_price=_decimal(row[21]),
|
|
total_price=_decimal(row[22]),
|
|
kb_amount=None if row[23] is None else _decimal(row[23]),
|
|
)
|
|
for row in fact_rows
|
|
)
|
|
pins = tuple(
|
|
DailyVersionPin(business_date=row[0], daily_version_id=int(row[1]))
|
|
for row in pin_rows
|
|
)
|
|
observations = tuple(
|
|
ChannelObservation(
|
|
business_date=row[0],
|
|
daily_version_id=int(row[1]),
|
|
worksheet=str(row[2] or ""),
|
|
worksheet_order=None if row[3] is None else int(row[3]),
|
|
row_count=int(row[4]),
|
|
)
|
|
for row in observation_rows
|
|
)
|
|
return MonthlySnapshot(
|
|
facts=facts,
|
|
daily_versions=pins,
|
|
channel_observations=observations,
|
|
preferred_channel_order=tuple(str(row[0] or "") for row in preferred_rows),
|
|
)
|
|
except RepositoryError:
|
|
raise
|
|
except Exception:
|
|
raise RepositoryError(
|
|
ErrorCode.DATABASE_FAILED,
|
|
"database snapshot query failed",
|
|
) from None
|
|
finally:
|
|
connection.close()
|
|
|
|
@staticmethod
|
|
def _lock_scope(cursor: Any, period_start: date, period_end: date) -> None:
|
|
lock_key = f"monthly_channel:{period_start.isoformat()}:{period_end.isoformat()}:*"
|
|
cursor.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))",
|
|
(lock_key,),
|
|
)
|
|
|
|
@staticmethod
|
|
def _current_pins(
|
|
cursor: Any,
|
|
period_start: date,
|
|
as_of_date: date,
|
|
) -> Tuple[DailyVersionPin, ...]:
|
|
cursor.execute(
|
|
CURRENT_DAILY_SQL,
|
|
(period_start, as_of_date),
|
|
)
|
|
return tuple(
|
|
DailyVersionPin(business_date=row[0], daily_version_id=int(row[1]))
|
|
for row in cursor.fetchall()
|
|
)
|
|
|
|
@classmethod
|
|
def _validate_pins_current(cls, cursor: Any, report: MonthlyReport) -> None:
|
|
period_start, _period_end = month_bounds(report.report_year, report.report_month)
|
|
if cls._current_pins(cursor, period_start, report.as_of_date) != report.daily_versions:
|
|
raise RepositoryError(
|
|
ErrorCode.SOURCE_SNAPSHOT_STALE,
|
|
"daily source versions changed while the monthly report was being generated",
|
|
)
|
|
|
|
def reserve_report(self, report: MonthlyReport) -> ReservedReport:
|
|
period_start, period_end = month_bounds(report.report_year, report.report_month)
|
|
|
|
def operation(cursor: Any) -> ReservedReport:
|
|
self._lock_scope(cursor, period_start, period_end)
|
|
self._validate_pins_current(cursor, report)
|
|
identity = {
|
|
"period_start": period_start.isoformat(),
|
|
"period_end": period_end.isoformat(),
|
|
"as_of_date": report.as_of_date.isoformat(),
|
|
"processor_version": PROCESSOR_VERSION,
|
|
"rule_set_sha256": rule_set_sha256(),
|
|
"result_schema_version": RESULT_SCHEMA_VERSION,
|
|
"daily_versions": [
|
|
[pin.business_date.isoformat(), pin.daily_version_id]
|
|
for pin in report.daily_versions
|
|
],
|
|
"channel_manifest": [list(item) for item in report.channel_manifest],
|
|
}
|
|
digest = hashlib.sha256(
|
|
repr(sorted(identity.items())).encode("utf-8")
|
|
).hexdigest()
|
|
generation_id = int(digest[:12], 16)
|
|
return ReservedReport(
|
|
generation_id,
|
|
generation_id,
|
|
period_start,
|
|
report.as_of_date,
|
|
report.daily_versions,
|
|
)
|
|
|
|
return self._run_serializable(
|
|
operation,
|
|
"monthly report source reservation failed",
|
|
)
|
|
|
|
def activate_report(
|
|
self,
|
|
reservation: ReservedReport,
|
|
artifact: FileMetadata,
|
|
result_json: FileMetadata,
|
|
) -> None:
|
|
artifact.validate()
|
|
result_json.validate()
|
|
if reservation.period_start is None or reservation.as_of_date is None:
|
|
raise RepositoryError(
|
|
ErrorCode.PUBLISH_FAILED,
|
|
"monthly source reservation is incomplete",
|
|
)
|
|
|
|
def operation(cursor: Any) -> None:
|
|
period_start, period_end = month_bounds(
|
|
reservation.period_start.year,
|
|
reservation.period_start.month,
|
|
)
|
|
self._lock_scope(cursor, period_start, period_end)
|
|
current = self._current_pins(
|
|
cursor,
|
|
period_start,
|
|
reservation.as_of_date,
|
|
)
|
|
if current != reservation.daily_versions:
|
|
raise RepositoryError(
|
|
ErrorCode.SOURCE_SNAPSHOT_STALE,
|
|
"daily source versions changed before publication",
|
|
)
|
|
|
|
self._run_serializable(
|
|
operation,
|
|
"monthly report source revalidation failed",
|
|
)
|
|
|
|
def mark_failed(
|
|
self,
|
|
reservation: ReservedReport,
|
|
code: str,
|
|
safe_message: str,
|
|
) -> None:
|
|
# Monthly rows and versions are intentionally not persisted. The
|
|
# caller's privacy-minimized result JSON is the failure record.
|
|
_ = (reservation, code, safe_message)
|