feat: prepare ARR for controlled public deployment
This commit is contained in:
Wyndham ARR
478
monthly_reports/repository.py
Normal file
478
monthly_reports/repository.py
Normal file
@@ -0,0 +1,478 @@
|
||||
"""Read-only PostgreSQL snapshot source for generated monthly reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any, Callable, List, Optional, Protocol, Sequence, Tuple
|
||||
|
||||
from monthly_reports.contracts import (
|
||||
PROCESSOR_VERSION,
|
||||
RESULT_SCHEMA_VERSION,
|
||||
ChannelObservation,
|
||||
DailyVersionPin,
|
||||
ErrorCode,
|
||||
MonthlyFact,
|
||||
MonthlyReport,
|
||||
MonthlySnapshot,
|
||||
rule_set_sha256,
|
||||
)
|
||||
from monthly_reports.core import month_bounds
|
||||
|
||||
|
||||
TARGET_DATABASE = "booking_test"
|
||||
DATABASE_ENV = "MONTHLY_REPORT_DATABASE_URL"
|
||||
FALLBACK_DATABASE_ENV = "ARR_DATABASE_URL"
|
||||
TRANSIENT_SQLSTATES = frozenset({"23505", "40001", "40P01", "55P03"})
|
||||
MAX_TRANSACTION_ATTEMPTS = 4
|
||||
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
SOURCE_FACTS_SQL = """
|
||||
SELECT
|
||||
facts.id,
|
||||
facts.daily_version_id,
|
||||
versions.business_date,
|
||||
facts.channel_key,
|
||||
facts.arrival,
|
||||
facts.departure,
|
||||
facts.nights,
|
||||
facts.adults,
|
||||
facts.children,
|
||||
facts.block_code,
|
||||
facts.no_of_rooms,
|
||||
facts.company_name,
|
||||
facts.confirmation_no,
|
||||
facts.disp_room_no,
|
||||
facts.effective_rate_amount,
|
||||
facts.full_name,
|
||||
facts.res_comment,
|
||||
facts.trace_text,
|
||||
facts.products,
|
||||
facts.rate_code,
|
||||
facts.room_category_label,
|
||||
facts.real_price,
|
||||
facts.total_price,
|
||||
facts.kb_amount
|
||||
FROM finance.v_active_daily_facts AS facts
|
||||
JOIN finance.daily_versions AS versions
|
||||
ON versions.id = facts.daily_version_id
|
||||
WHERE versions.business_date BETWEEN %s AND %s
|
||||
ORDER BY facts.id
|
||||
""".strip()
|
||||
|
||||
CURRENT_DAILY_SQL = """
|
||||
SELECT business_date, daily_version_id
|
||||
FROM finance.current_daily_versions
|
||||
WHERE business_date BETWEEN %s AND %s
|
||||
ORDER BY business_date
|
||||
""".strip()
|
||||
|
||||
CHANNEL_OBSERVATIONS_SQL = """
|
||||
SELECT
|
||||
current_version.business_date,
|
||||
current_version.daily_version_id,
|
||||
metrics.channel_key,
|
||||
metrics.channel_order,
|
||||
metrics.row_count
|
||||
FROM finance.current_daily_versions AS current_version
|
||||
JOIN finance.daily_channel_metrics AS metrics
|
||||
ON metrics.daily_version_id = current_version.daily_version_id
|
||||
WHERE current_version.business_date BETWEEN %s AND %s
|
||||
ORDER BY
|
||||
current_version.business_date,
|
||||
metrics.channel_order,
|
||||
metrics.channel_key
|
||||
""".strip()
|
||||
|
||||
PREFERRED_CHANNELS_SQL = """
|
||||
SELECT metrics.channel_key
|
||||
FROM finance.current_daily_versions AS current_version
|
||||
JOIN finance.daily_channel_metrics AS metrics
|
||||
ON metrics.daily_version_id = current_version.daily_version_id
|
||||
WHERE current_version.business_date BETWEEN %s AND %s
|
||||
GROUP BY metrics.channel_key
|
||||
ORDER BY
|
||||
min(current_version.business_date),
|
||||
min(metrics.channel_order),
|
||||
metrics.channel_key
|
||||
""".strip()
|
||||
|
||||
|
||||
class RepositoryError(RuntimeError):
|
||||
def __init__(self, code: str, safe_message: str):
|
||||
super().__init__(safe_message)
|
||||
self.code = code
|
||||
self.safe_message = safe_message
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatabaseConfig:
|
||||
dsn: str
|
||||
|
||||
@classmethod
|
||||
def from_environment(cls) -> "DatabaseConfig":
|
||||
dsn = os.environ.get(DATABASE_ENV, "").strip()
|
||||
if not dsn:
|
||||
dsn = os.environ.get(FALLBACK_DATABASE_ENV, "").strip()
|
||||
if not dsn:
|
||||
raise RepositoryError(
|
||||
ErrorCode.REQUEST_INVALID,
|
||||
f"{DATABASE_ENV} or {FALLBACK_DATABASE_ENV} is required",
|
||||
)
|
||||
return cls(dsn=dsn)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReservedReport:
|
||||
report_version_id: int
|
||||
version_no: int
|
||||
period_start: Optional[date] = None
|
||||
as_of_date: Optional[date] = None
|
||||
daily_versions: Tuple[DailyVersionPin, ...] = tuple()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FileMetadata:
|
||||
file_kind: str
|
||||
original_filename: str
|
||||
storage_key: str
|
||||
sha256: str
|
||||
byte_size: int
|
||||
mime_type: str
|
||||
|
||||
def validate(self) -> None:
|
||||
storage = PurePosixPath(self.storage_key)
|
||||
if (
|
||||
self.file_kind not in {"monthly_xlsx", "result_json"}
|
||||
or not self.storage_key
|
||||
or storage.is_absolute()
|
||||
or ".." in storage.parts
|
||||
or "\\" in self.storage_key
|
||||
or "\x00" in self.storage_key
|
||||
or self.original_filename != PurePosixPath(self.original_filename).name
|
||||
or "/" in self.original_filename
|
||||
or not SHA256_RE.fullmatch(self.sha256)
|
||||
or self.byte_size <= 0
|
||||
or not self.mime_type.strip()
|
||||
):
|
||||
raise RepositoryError(
|
||||
ErrorCode.PUBLISH_FAILED,
|
||||
"generated artifact metadata is invalid",
|
||||
)
|
||||
|
||||
|
||||
class ReportRepository(Protocol):
|
||||
def load_snapshot(self, year: int, month: int, as_of_date: date) -> MonthlySnapshot:
|
||||
...
|
||||
|
||||
def reserve_report(self, report: MonthlyReport) -> ReservedReport:
|
||||
...
|
||||
|
||||
def activate_report(
|
||||
self,
|
||||
reservation: ReservedReport,
|
||||
artifact: FileMetadata,
|
||||
result_json: FileMetadata,
|
||||
) -> None:
|
||||
...
|
||||
|
||||
def mark_failed(self, reservation: ReservedReport, code: str, safe_message: str) -> None:
|
||||
...
|
||||
|
||||
|
||||
def _default_connect(dsn: str) -> Any:
|
||||
try:
|
||||
import psycopg # type: ignore[import-not-found]
|
||||
except ImportError:
|
||||
raise RepositoryError(
|
||||
ErrorCode.INTERNAL_ERROR,
|
||||
"PostgreSQL driver is unavailable; install requirements-monthly-reports.txt",
|
||||
) from None
|
||||
try:
|
||||
return psycopg.connect(dsn, autocommit=False)
|
||||
except Exception:
|
||||
raise RepositoryError(
|
||||
ErrorCode.DATABASE_FAILED,
|
||||
"database connection failed",
|
||||
) from None
|
||||
|
||||
|
||||
def _decimal(value: Any) -> Decimal:
|
||||
return value if isinstance(value, Decimal) else Decimal(str(value))
|
||||
|
||||
|
||||
def _is_transient_database_error(error: Exception) -> bool:
|
||||
sqlstate = getattr(error, "sqlstate", None)
|
||||
if not isinstance(sqlstate, str):
|
||||
sqlstate = getattr(getattr(error, "diag", None), "sqlstate", None)
|
||||
return isinstance(sqlstate, str) and sqlstate in TRANSIENT_SQLSTATES
|
||||
|
||||
|
||||
class PostgresReportRepository:
|
||||
"""Loads one repeatable snapshot and atomically publishes its lineage."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: DatabaseConfig,
|
||||
connect: Optional[Callable[[str], Any]] = None,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._connect = connect or _default_connect
|
||||
|
||||
def _open(self) -> Any:
|
||||
try:
|
||||
return self._connect(self._config.dsn)
|
||||
except RepositoryError:
|
||||
raise
|
||||
except Exception:
|
||||
raise RepositoryError(
|
||||
ErrorCode.DATABASE_FAILED,
|
||||
"database connection failed",
|
||||
) from None
|
||||
|
||||
@staticmethod
|
||||
def _assert_target(cursor: Any) -> None:
|
||||
cursor.execute("SELECT current_database()")
|
||||
row = cursor.fetchone()
|
||||
if not row or row[0] != TARGET_DATABASE:
|
||||
raise RepositoryError(
|
||||
ErrorCode.DATABASE_FAILED,
|
||||
"database target is not booking_test",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _begin_serializable(cls, cursor: Any) -> None:
|
||||
cursor.execute(
|
||||
"SET TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY"
|
||||
)
|
||||
cursor.execute("SET LOCAL lock_timeout = '10s'")
|
||||
cursor.execute("SET LOCAL statement_timeout = '30s'")
|
||||
cls._assert_target(cursor)
|
||||
|
||||
def _run_serializable(
|
||||
self,
|
||||
operation: Callable[[Any], Any],
|
||||
safe_message: str,
|
||||
) -> Any:
|
||||
for attempt_index in range(MAX_TRANSACTION_ATTEMPTS):
|
||||
connection = self._open()
|
||||
retry = False
|
||||
try:
|
||||
with connection.transaction():
|
||||
with connection.cursor() as cursor:
|
||||
self._begin_serializable(cursor)
|
||||
return operation(cursor)
|
||||
except RepositoryError:
|
||||
raise
|
||||
except Exception as error:
|
||||
retry = (
|
||||
attempt_index + 1 < MAX_TRANSACTION_ATTEMPTS
|
||||
and _is_transient_database_error(error)
|
||||
)
|
||||
if not retry:
|
||||
raise RepositoryError(ErrorCode.DATABASE_FAILED, safe_message) from None
|
||||
finally:
|
||||
connection.close()
|
||||
if retry:
|
||||
time.sleep(0.02 * (2**attempt_index))
|
||||
raise RepositoryError(ErrorCode.DATABASE_FAILED, safe_message)
|
||||
|
||||
def load_snapshot(self, year: int, month: int, as_of_date: date) -> MonthlySnapshot:
|
||||
month_start, _month_end = month_bounds(year, month)
|
||||
connection = self._open()
|
||||
try:
|
||||
with connection.transaction():
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"
|
||||
)
|
||||
cursor.execute("SET LOCAL statement_timeout = '30s'")
|
||||
self._assert_target(cursor)
|
||||
cursor.execute(SOURCE_FACTS_SQL, (month_start, as_of_date))
|
||||
fact_rows = list(cursor.fetchall())
|
||||
cursor.execute(CURRENT_DAILY_SQL, (month_start, as_of_date))
|
||||
pin_rows = list(cursor.fetchall())
|
||||
cursor.execute(CHANNEL_OBSERVATIONS_SQL, (month_start, as_of_date))
|
||||
observation_rows = list(cursor.fetchall())
|
||||
cursor.execute(PREFERRED_CHANNELS_SQL, (month_start, _month_end))
|
||||
preferred_rows = list(cursor.fetchall())
|
||||
|
||||
facts = tuple(
|
||||
MonthlyFact(
|
||||
daily_record_id=int(row[0]),
|
||||
daily_version_id=int(row[1]),
|
||||
business_date=row[2],
|
||||
channel_key=str(row[3] or ""),
|
||||
arrival=row[4],
|
||||
departure=row[5],
|
||||
nights=int(row[6]),
|
||||
adults=int(row[7]),
|
||||
children=int(row[8]),
|
||||
block_code=row[9],
|
||||
no_of_rooms=int(row[10]),
|
||||
company_name=str(row[11] or ""),
|
||||
confirmation_no=str(row[12] or ""),
|
||||
disp_room_no=str(row[13] or ""),
|
||||
effective_rate_amount=_decimal(row[14]),
|
||||
full_name=str(row[15] or ""),
|
||||
res_comment=row[16],
|
||||
trace_text=row[17],
|
||||
products=row[18],
|
||||
rate_code=str(row[19] or ""),
|
||||
room_category_label=row[20],
|
||||
real_price=_decimal(row[21]),
|
||||
total_price=_decimal(row[22]),
|
||||
kb_amount=None if row[23] is None else _decimal(row[23]),
|
||||
)
|
||||
for row in fact_rows
|
||||
)
|
||||
pins = tuple(
|
||||
DailyVersionPin(business_date=row[0], daily_version_id=int(row[1]))
|
||||
for row in pin_rows
|
||||
)
|
||||
observations = tuple(
|
||||
ChannelObservation(
|
||||
business_date=row[0],
|
||||
daily_version_id=int(row[1]),
|
||||
worksheet=str(row[2] or ""),
|
||||
worksheet_order=None if row[3] is None else int(row[3]),
|
||||
row_count=int(row[4]),
|
||||
)
|
||||
for row in observation_rows
|
||||
)
|
||||
return MonthlySnapshot(
|
||||
facts=facts,
|
||||
daily_versions=pins,
|
||||
channel_observations=observations,
|
||||
preferred_channel_order=tuple(str(row[0] or "") for row in preferred_rows),
|
||||
)
|
||||
except RepositoryError:
|
||||
raise
|
||||
except Exception:
|
||||
raise RepositoryError(
|
||||
ErrorCode.DATABASE_FAILED,
|
||||
"database snapshot query failed",
|
||||
) from None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
@staticmethod
|
||||
def _lock_scope(cursor: Any, period_start: date, period_end: date) -> None:
|
||||
lock_key = f"monthly_channel:{period_start.isoformat()}:{period_end.isoformat()}:*"
|
||||
cursor.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))",
|
||||
(lock_key,),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _current_pins(
|
||||
cursor: Any,
|
||||
period_start: date,
|
||||
as_of_date: date,
|
||||
) -> Tuple[DailyVersionPin, ...]:
|
||||
cursor.execute(
|
||||
CURRENT_DAILY_SQL,
|
||||
(period_start, as_of_date),
|
||||
)
|
||||
return tuple(
|
||||
DailyVersionPin(business_date=row[0], daily_version_id=int(row[1]))
|
||||
for row in cursor.fetchall()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _validate_pins_current(cls, cursor: Any, report: MonthlyReport) -> None:
|
||||
period_start, _period_end = month_bounds(report.report_year, report.report_month)
|
||||
if cls._current_pins(cursor, period_start, report.as_of_date) != report.daily_versions:
|
||||
raise RepositoryError(
|
||||
ErrorCode.SOURCE_SNAPSHOT_STALE,
|
||||
"daily source versions changed while the monthly report was being generated",
|
||||
)
|
||||
|
||||
def reserve_report(self, report: MonthlyReport) -> ReservedReport:
|
||||
period_start, period_end = month_bounds(report.report_year, report.report_month)
|
||||
|
||||
def operation(cursor: Any) -> ReservedReport:
|
||||
self._lock_scope(cursor, period_start, period_end)
|
||||
self._validate_pins_current(cursor, report)
|
||||
identity = {
|
||||
"period_start": period_start.isoformat(),
|
||||
"period_end": period_end.isoformat(),
|
||||
"as_of_date": report.as_of_date.isoformat(),
|
||||
"processor_version": PROCESSOR_VERSION,
|
||||
"rule_set_sha256": rule_set_sha256(),
|
||||
"result_schema_version": RESULT_SCHEMA_VERSION,
|
||||
"daily_versions": [
|
||||
[pin.business_date.isoformat(), pin.daily_version_id]
|
||||
for pin in report.daily_versions
|
||||
],
|
||||
"channel_manifest": [list(item) for item in report.channel_manifest],
|
||||
}
|
||||
digest = hashlib.sha256(
|
||||
repr(sorted(identity.items())).encode("utf-8")
|
||||
).hexdigest()
|
||||
generation_id = int(digest[:12], 16)
|
||||
return ReservedReport(
|
||||
generation_id,
|
||||
generation_id,
|
||||
period_start,
|
||||
report.as_of_date,
|
||||
report.daily_versions,
|
||||
)
|
||||
|
||||
return self._run_serializable(
|
||||
operation,
|
||||
"monthly report source reservation failed",
|
||||
)
|
||||
|
||||
def activate_report(
|
||||
self,
|
||||
reservation: ReservedReport,
|
||||
artifact: FileMetadata,
|
||||
result_json: FileMetadata,
|
||||
) -> None:
|
||||
artifact.validate()
|
||||
result_json.validate()
|
||||
if reservation.period_start is None or reservation.as_of_date is None:
|
||||
raise RepositoryError(
|
||||
ErrorCode.PUBLISH_FAILED,
|
||||
"monthly source reservation is incomplete",
|
||||
)
|
||||
|
||||
def operation(cursor: Any) -> None:
|
||||
period_start, period_end = month_bounds(
|
||||
reservation.period_start.year,
|
||||
reservation.period_start.month,
|
||||
)
|
||||
self._lock_scope(cursor, period_start, period_end)
|
||||
current = self._current_pins(
|
||||
cursor,
|
||||
period_start,
|
||||
reservation.as_of_date,
|
||||
)
|
||||
if current != reservation.daily_versions:
|
||||
raise RepositoryError(
|
||||
ErrorCode.SOURCE_SNAPSHOT_STALE,
|
||||
"daily source versions changed before publication",
|
||||
)
|
||||
|
||||
self._run_serializable(
|
||||
operation,
|
||||
"monthly report source revalidation failed",
|
||||
)
|
||||
|
||||
def mark_failed(
|
||||
self,
|
||||
reservation: ReservedReport,
|
||||
code: str,
|
||||
safe_message: str,
|
||||
) -> None:
|
||||
# Monthly rows and versions are intentionally not persisted. The
|
||||
# caller's privacy-minimized result JSON is the failure record.
|
||||
_ = (reservation, code, safe_message)
|
||||
Reference in New Issue
Block a user