feat: prepare ARR for controlled public deployment
This commit is contained in:
Wyndham ARR
584
channel_analytics/postgres.py
Normal file
584
channel_analytics/postgres.py
Normal file
@@ -0,0 +1,584 @@
|
||||
"""Read-only PostgreSQL provider for live channel analytics and details."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar
|
||||
import hashlib
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from channel_analytics.contracts import (
|
||||
DETAIL_VERSION,
|
||||
UNLABELED_ROOM_TYPE,
|
||||
AggregatedRoomFact,
|
||||
AnalyticsError,
|
||||
build_dashboard,
|
||||
json_number,
|
||||
validate_month_key,
|
||||
)
|
||||
|
||||
|
||||
TARGET_DATABASE = "booking_test"
|
||||
DATABASE_ENV = "DASHBOARD_DATABASE_URL"
|
||||
FALLBACK_DATABASE_ENV = "ARR_DATABASE_URL"
|
||||
|
||||
|
||||
CURRENT_REPORT_SQL = """
|
||||
SELECT
|
||||
max(current_version.business_date) AS as_of_date,
|
||||
max(current_version.activated_at) AS updated_at,
|
||||
count(*) AS daily_version_count
|
||||
FROM finance.current_daily_versions AS current_version
|
||||
WHERE current_version.business_date BETWEEN %s AND %s
|
||||
""".strip()
|
||||
|
||||
REPORT_MANIFEST_SQL = """
|
||||
SELECT
|
||||
metrics.channel_key,
|
||||
sum(metrics.row_count) AS row_count
|
||||
FROM finance.current_daily_versions AS current_version
|
||||
JOIN finance.daily_channel_metrics AS metrics
|
||||
ON metrics.daily_version_id = current_version.daily_version_id
|
||||
WHERE current_version.business_date BETWEEN %s AND %s
|
||||
GROUP BY metrics.channel_key
|
||||
ORDER BY
|
||||
min(current_version.business_date),
|
||||
min(metrics.channel_order),
|
||||
metrics.channel_key
|
||||
""".strip()
|
||||
|
||||
REPORT_PIN_COUNT_SQL = """
|
||||
SELECT
|
||||
current_version.business_date,
|
||||
current_version.daily_version_id,
|
||||
source.sha256
|
||||
FROM finance.current_daily_versions AS current_version
|
||||
JOIN finance.daily_versions AS version
|
||||
ON version.id = current_version.daily_version_id
|
||||
JOIN ingestion.artifacts AS source
|
||||
ON source.id = version.source_artifact_id
|
||||
WHERE current_version.business_date BETWEEN %s AND %s
|
||||
ORDER BY current_version.business_date, current_version.daily_version_id
|
||||
""".strip()
|
||||
|
||||
ROOM_AGGREGATES_SQL = """
|
||||
SELECT
|
||||
facts.channel_key,
|
||||
coalesce(nullif(btrim(facts.room_category_label), ''), %s) AS room_type,
|
||||
count(*) AS reservation_rows,
|
||||
sum(facts.no_of_rooms) AS rooms_sold,
|
||||
sum(facts.no_of_rooms * facts.nights) AS room_nights,
|
||||
sum(facts.total_price) AS total_price
|
||||
FROM finance.v_active_daily_facts AS facts
|
||||
WHERE facts.business_date BETWEEN %s AND %s
|
||||
GROUP BY facts.channel_key, room_type
|
||||
ORDER BY facts.channel_key, room_type
|
||||
""".strip()
|
||||
|
||||
CHANNEL_DETAIL_SQL = """
|
||||
SELECT
|
||||
facts.arrival,
|
||||
facts.departure,
|
||||
facts.nights,
|
||||
facts.no_of_rooms,
|
||||
facts.company_name,
|
||||
facts.rate_code,
|
||||
coalesce(nullif(btrim(facts.room_category_label), ''), %s) AS room_type,
|
||||
facts.real_price,
|
||||
facts.total_price
|
||||
FROM finance.v_active_daily_facts AS facts
|
||||
WHERE facts.business_date BETWEEN %s AND %s
|
||||
AND facts.channel_key = %s
|
||||
ORDER BY facts.arrival, facts.id
|
||||
LIMIT %s OFFSET %s
|
||||
""".strip()
|
||||
|
||||
MONTHS_SQL = """
|
||||
SELECT DISTINCT date_trunc('month', business_date)::date AS period_start
|
||||
FROM finance.current_daily_versions
|
||||
ORDER BY period_start DESC
|
||||
""".strip()
|
||||
|
||||
|
||||
class AnalyticsRepositoryError(AnalyticsError):
|
||||
pass
|
||||
|
||||
|
||||
class AnalyticsMonthNotFound(AnalyticsRepositoryError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
"ANALYTICS_MONTH_NOT_FOUND",
|
||||
"current monthly database projection was not found",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatabaseConfig:
|
||||
dsn: str
|
||||
|
||||
@classmethod
|
||||
def from_environment(cls) -> "DatabaseConfig":
|
||||
dsn = os.environ.get(DATABASE_ENV, "").strip()
|
||||
if not dsn:
|
||||
dsn = os.environ.get(FALLBACK_DATABASE_ENV, "").strip()
|
||||
if not dsn:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_CONFIG_MISSING",
|
||||
f"{DATABASE_ENV} or {FALLBACK_DATABASE_ENV} is required",
|
||||
)
|
||||
return cls(dsn)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChannelManifestItem:
|
||||
worksheet: str
|
||||
worksheet_order: int
|
||||
row_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReportContext:
|
||||
period_start: date
|
||||
period_end: date
|
||||
as_of_date: date
|
||||
activated_at: datetime
|
||||
source_projection_sha256: str
|
||||
daily_version_count: int
|
||||
channels: Tuple[ChannelManifestItem, ...]
|
||||
|
||||
@property
|
||||
def month_key(self) -> str:
|
||||
return f"{self.period_start.year:04d}-{self.period_start.month:02d}"
|
||||
|
||||
@property
|
||||
def filename(self) -> str:
|
||||
return f"database-projection-{self.month_key}.json"
|
||||
|
||||
|
||||
def _default_connect(dsn: str) -> Any:
|
||||
try:
|
||||
import psycopg # type: ignore[import-not-found]
|
||||
except ImportError:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_DRIVER_UNAVAILABLE",
|
||||
"PostgreSQL driver is unavailable",
|
||||
) from None
|
||||
try:
|
||||
return psycopg.connect(dsn, autocommit=False)
|
||||
except Exception:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_UNAVAILABLE",
|
||||
"database connection failed",
|
||||
) from None
|
||||
|
||||
|
||||
def _integer(value: Any, field_name: str) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_RESULT_INVALID",
|
||||
f"database {field_name} is invalid",
|
||||
)
|
||||
try:
|
||||
number = Decimal(str(value))
|
||||
except (InvalidOperation, TypeError, ValueError):
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_RESULT_INVALID",
|
||||
f"database {field_name} is invalid",
|
||||
) from None
|
||||
if (
|
||||
not number.is_finite()
|
||||
or number < 0
|
||||
or number != number.to_integral_value()
|
||||
):
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_RESULT_INVALID",
|
||||
f"database {field_name} is invalid",
|
||||
)
|
||||
return int(number)
|
||||
|
||||
|
||||
def _decimal(value: Any, field_name: str) -> Decimal:
|
||||
try:
|
||||
number = Decimal(str(value))
|
||||
except (InvalidOperation, TypeError, ValueError):
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_RESULT_INVALID",
|
||||
f"database {field_name} is invalid",
|
||||
) from None
|
||||
if not number.is_finite() or number < 0:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_RESULT_INVALID",
|
||||
f"database {field_name} is invalid",
|
||||
)
|
||||
return number
|
||||
|
||||
|
||||
def _date(value: Any, field_name: str) -> date:
|
||||
if isinstance(value, datetime):
|
||||
value = value.date()
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
try:
|
||||
return date.fromisoformat(str(value))
|
||||
except ValueError:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_RESULT_INVALID",
|
||||
f"database {field_name} is invalid",
|
||||
) from None
|
||||
|
||||
|
||||
def _datetime(value: Any) -> datetime:
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
try:
|
||||
return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_RESULT_INVALID",
|
||||
"database activated_at is invalid",
|
||||
) from None
|
||||
|
||||
|
||||
def _projection_sha256(rows: List[Tuple[Any, ...]]) -> str:
|
||||
identity = "\n".join(
|
||||
f"{_date(row[0], 'business_date').isoformat()}:{_integer(row[1], 'daily_version_id')}:{str(row[2]).lower()}"
|
||||
for row in rows
|
||||
)
|
||||
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class PostgresAnalyticsRepository:
|
||||
def __init__(
|
||||
self,
|
||||
config: DatabaseConfig,
|
||||
connect: Optional[Callable[[str], Any]] = None,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._connect = connect or _default_connect
|
||||
|
||||
def _open(self) -> Any:
|
||||
try:
|
||||
return self._connect(self._config.dsn)
|
||||
except AnalyticsRepositoryError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_UNAVAILABLE",
|
||||
"database connection failed",
|
||||
) from None
|
||||
|
||||
@staticmethod
|
||||
def _begin(cursor: Any) -> None:
|
||||
cursor.execute(
|
||||
"SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"
|
||||
)
|
||||
cursor.execute("SET LOCAL statement_timeout = '10s'")
|
||||
cursor.execute("SET LOCAL lock_timeout = '3s'")
|
||||
cursor.execute(
|
||||
"SELECT current_database(), current_setting('transaction_read_only')"
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if not row or row[0] != TARGET_DATABASE or row[1] != "on":
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_TARGET_INVALID",
|
||||
"database target or read-only mode is invalid",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _period(month_key: str) -> Tuple[date, date]:
|
||||
year, month = validate_month_key(month_key)
|
||||
return (
|
||||
date(year, month, 1),
|
||||
date(year, month, calendar.monthrange(year, month)[1]),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _load_context(
|
||||
cursor: Any,
|
||||
period_start: date,
|
||||
period_end: date,
|
||||
) -> ReportContext:
|
||||
cursor.execute(CURRENT_REPORT_SQL, (period_start, period_end))
|
||||
row = cursor.fetchone()
|
||||
if not row or row[0] is None or _integer(row[2], "daily_version_count") == 0:
|
||||
raise AnalyticsMonthNotFound()
|
||||
as_of_date = _date(row[0], "as_of_date")
|
||||
activated_at = _datetime(row[1])
|
||||
daily_version_count = _integer(row[2], "daily_version_count")
|
||||
|
||||
cursor.execute(REPORT_PIN_COUNT_SQL, (period_start, period_end))
|
||||
identity_rows = list(cursor.fetchall())
|
||||
if len(identity_rows) != daily_version_count:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_RESULT_INVALID",
|
||||
"current daily version identity is incomplete",
|
||||
)
|
||||
for identity_row in identity_rows:
|
||||
digest = str(identity_row[2] or "").lower()
|
||||
if len(digest) != 64 or any(
|
||||
character not in "0123456789abcdef" for character in digest
|
||||
):
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_RESULT_INVALID",
|
||||
"current source artifact identity is invalid",
|
||||
)
|
||||
|
||||
cursor.execute(REPORT_MANIFEST_SQL, (period_start, period_end))
|
||||
manifest_rows = list(cursor.fetchall())
|
||||
channels = tuple(
|
||||
ChannelManifestItem(
|
||||
worksheet=str(item[0] or ""),
|
||||
worksheet_order=order,
|
||||
row_count=_integer(item[1], "row_count"),
|
||||
)
|
||||
for order, item in enumerate(manifest_rows, 1)
|
||||
)
|
||||
if (
|
||||
not channels
|
||||
or len({item.worksheet for item in channels}) != len(channels)
|
||||
or any(not item.worksheet.strip() for item in channels)
|
||||
):
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_CHANNEL_MANIFEST_INVALID",
|
||||
"monthly channel projection is invalid",
|
||||
)
|
||||
return ReportContext(
|
||||
period_start=period_start,
|
||||
period_end=period_end,
|
||||
as_of_date=as_of_date,
|
||||
activated_at=activated_at,
|
||||
source_projection_sha256=_projection_sha256(identity_rows),
|
||||
daily_version_count=daily_version_count,
|
||||
channels=channels,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _aggregates(
|
||||
cursor: Any,
|
||||
context: ReportContext,
|
||||
) -> Tuple[AggregatedRoomFact, ...]:
|
||||
cursor.execute(
|
||||
ROOM_AGGREGATES_SQL,
|
||||
(
|
||||
UNLABELED_ROOM_TYPE,
|
||||
context.period_start,
|
||||
context.period_end,
|
||||
),
|
||||
)
|
||||
facts = tuple(
|
||||
AggregatedRoomFact(
|
||||
worksheet=str(row[0] or ""),
|
||||
room_type=str(row[1] or UNLABELED_ROOM_TYPE),
|
||||
reservation_rows=_integer(row[2], "reservation_rows"),
|
||||
rooms_sold=_integer(row[3], "rooms_sold"),
|
||||
room_nights=_integer(row[4], "room_nights"),
|
||||
total_price=_decimal(row[5], "total_price"),
|
||||
)
|
||||
for row in cursor.fetchall()
|
||||
)
|
||||
actual_counts: Dict[str, int] = {}
|
||||
for fact in facts:
|
||||
actual_counts[fact.worksheet] = (
|
||||
actual_counts.get(fact.worksheet, 0)
|
||||
+ fact.reservation_rows
|
||||
)
|
||||
expected_names = {item.worksheet for item in context.channels}
|
||||
if (
|
||||
not set(actual_counts).issubset(expected_names)
|
||||
or any(
|
||||
item.row_count != actual_counts.get(item.worksheet, 0)
|
||||
for item in context.channels
|
||||
)
|
||||
):
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_CHANNEL_MANIFEST_INVALID",
|
||||
"monthly channel projection does not match active facts",
|
||||
)
|
||||
return facts
|
||||
|
||||
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
||||
period_start, period_end = self._period(month_key)
|
||||
connection = self._open()
|
||||
try:
|
||||
with connection.transaction():
|
||||
with connection.cursor() as cursor:
|
||||
self._begin(cursor)
|
||||
context = self._load_context(
|
||||
cursor,
|
||||
period_start,
|
||||
period_end,
|
||||
)
|
||||
facts = self._aggregates(cursor, context)
|
||||
return build_dashboard(
|
||||
month_key=context.month_key,
|
||||
updated_at=context.activated_at.isoformat(),
|
||||
max_arrival_date=context.as_of_date.isoformat(),
|
||||
source_monthly_sha256=context.source_projection_sha256,
|
||||
channel_names=(item.worksheet for item in context.channels),
|
||||
facts=facts,
|
||||
)
|
||||
except AnalyticsError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_QUERY_FAILED",
|
||||
"database analytics query failed",
|
||||
) from None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def read_channel_detail(
|
||||
self,
|
||||
month_key: str,
|
||||
worksheet: str,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> Dict[str, Any]:
|
||||
if (
|
||||
not isinstance(worksheet, str)
|
||||
or not worksheet.strip()
|
||||
or worksheet != worksheet.strip()
|
||||
or isinstance(limit, bool)
|
||||
or isinstance(offset, bool)
|
||||
or limit < 1
|
||||
or limit > 500
|
||||
or offset < 0
|
||||
):
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DETAIL_REQUEST_INVALID",
|
||||
"channel detail request is invalid",
|
||||
)
|
||||
period_start, period_end = self._period(month_key)
|
||||
connection = self._open()
|
||||
try:
|
||||
with connection.transaction():
|
||||
with connection.cursor() as cursor:
|
||||
self._begin(cursor)
|
||||
context = self._load_context(
|
||||
cursor,
|
||||
period_start,
|
||||
period_end,
|
||||
)
|
||||
manifest = {
|
||||
item.worksheet: item for item in context.channels
|
||||
}
|
||||
if worksheet not in manifest:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_CHANNEL_NOT_FOUND",
|
||||
"channel is not present in the current month",
|
||||
)
|
||||
cursor.execute(
|
||||
CHANNEL_DETAIL_SQL,
|
||||
(
|
||||
UNLABELED_ROOM_TYPE,
|
||||
context.period_start,
|
||||
context.period_end,
|
||||
worksheet,
|
||||
limit,
|
||||
offset,
|
||||
),
|
||||
)
|
||||
rows = [
|
||||
{
|
||||
"arrival": _date(row[0], "arrival").isoformat(),
|
||||
"departure": _date(
|
||||
row[1],
|
||||
"departure",
|
||||
).isoformat(),
|
||||
"nights": _integer(row[2], "nights"),
|
||||
"no_of_rooms": _integer(
|
||||
row[3],
|
||||
"no_of_rooms",
|
||||
),
|
||||
"company_name": str(row[4] or ""),
|
||||
"rate_code": str(row[5] or ""),
|
||||
"room_type": str(
|
||||
row[6] or UNLABELED_ROOM_TYPE
|
||||
),
|
||||
"real_price": json_number(
|
||||
_decimal(row[7], "real_price")
|
||||
),
|
||||
"total_price": json_number(
|
||||
_decimal(row[8], "total_price")
|
||||
),
|
||||
}
|
||||
for row in cursor.fetchall()
|
||||
]
|
||||
return {
|
||||
"version": DETAIL_VERSION,
|
||||
"month_key": context.month_key,
|
||||
"worksheet": worksheet,
|
||||
"updated_at": context.activated_at.isoformat(),
|
||||
"max_arrival_date": context.as_of_date.isoformat(),
|
||||
"source_monthly_sha256": (
|
||||
context.source_projection_sha256
|
||||
),
|
||||
"total_rows": manifest[worksheet].row_count,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"rows": rows,
|
||||
}
|
||||
except AnalyticsError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_QUERY_FAILED",
|
||||
"database channel detail query failed",
|
||||
) from None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def list_months(self) -> List[Dict[str, Any]]:
|
||||
connection = self._open()
|
||||
try:
|
||||
with connection.transaction():
|
||||
with connection.cursor() as cursor:
|
||||
self._begin(cursor)
|
||||
cursor.execute(MONTHS_SQL)
|
||||
period_rows = list(cursor.fetchall())
|
||||
contexts = [
|
||||
self._load_context(
|
||||
cursor,
|
||||
period_start,
|
||||
date(
|
||||
period_start.year,
|
||||
period_start.month,
|
||||
calendar.monthrange(
|
||||
period_start.year,
|
||||
period_start.month,
|
||||
)[1],
|
||||
),
|
||||
)
|
||||
for period_start in (
|
||||
_date(row[0], "period_start")
|
||||
for row in period_rows
|
||||
)
|
||||
]
|
||||
return [
|
||||
{
|
||||
"month_key": context.month_key,
|
||||
"max_arrival_date": context.as_of_date.isoformat(),
|
||||
"updated_at": context.activated_at.isoformat(),
|
||||
"filename": context.filename,
|
||||
"source_monthly_sha256": (
|
||||
context.source_projection_sha256
|
||||
),
|
||||
"channel_count": len(context.channels),
|
||||
"row_count": sum(
|
||||
item.row_count for item in context.channels
|
||||
),
|
||||
}
|
||||
for context in contexts
|
||||
]
|
||||
except AnalyticsError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AnalyticsRepositoryError(
|
||||
"ANALYTICS_DATABASE_QUERY_FAILED",
|
||||
"database monthly list query failed",
|
||||
) from None
|
||||
finally:
|
||||
connection.close()
|
||||
Reference in New Issue
Block a user