585 lines
20 KiB
Python
585 lines
20 KiB
Python
"""Read-only PostgreSQL provider for live channel analytics and details."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import calendar
|
|
import hashlib
|
|
import os
|
|
from dataclasses import dataclass
|
|
from datetime import date, datetime
|
|
from decimal import Decimal, InvalidOperation
|
|
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
|
|
from channel_analytics.contracts import (
|
|
DETAIL_VERSION,
|
|
UNLABELED_ROOM_TYPE,
|
|
AggregatedRoomFact,
|
|
AnalyticsError,
|
|
build_dashboard,
|
|
json_number,
|
|
validate_month_key,
|
|
)
|
|
|
|
|
|
TARGET_DATABASE = "booking_test"
|
|
DATABASE_ENV = "DASHBOARD_DATABASE_URL"
|
|
FALLBACK_DATABASE_ENV = "ARR_DATABASE_URL"
|
|
|
|
|
|
CURRENT_REPORT_SQL = """
|
|
SELECT
|
|
max(current_version.business_date) AS as_of_date,
|
|
max(current_version.activated_at) AS updated_at,
|
|
count(*) AS daily_version_count
|
|
FROM finance.current_daily_versions AS current_version
|
|
WHERE current_version.business_date BETWEEN %s AND %s
|
|
""".strip()
|
|
|
|
REPORT_MANIFEST_SQL = """
|
|
SELECT
|
|
metrics.channel_key,
|
|
sum(metrics.row_count) AS row_count
|
|
FROM finance.current_daily_versions AS current_version
|
|
JOIN finance.daily_channel_metrics AS metrics
|
|
ON metrics.daily_version_id = current_version.daily_version_id
|
|
WHERE current_version.business_date BETWEEN %s AND %s
|
|
GROUP BY metrics.channel_key
|
|
ORDER BY
|
|
min(current_version.business_date),
|
|
min(metrics.channel_order),
|
|
metrics.channel_key
|
|
""".strip()
|
|
|
|
REPORT_PIN_COUNT_SQL = """
|
|
SELECT
|
|
current_version.business_date,
|
|
current_version.daily_version_id,
|
|
source.sha256
|
|
FROM finance.current_daily_versions AS current_version
|
|
JOIN finance.daily_versions AS version
|
|
ON version.id = current_version.daily_version_id
|
|
JOIN ingestion.artifacts AS source
|
|
ON source.id = version.source_artifact_id
|
|
WHERE current_version.business_date BETWEEN %s AND %s
|
|
ORDER BY current_version.business_date, current_version.daily_version_id
|
|
""".strip()
|
|
|
|
ROOM_AGGREGATES_SQL = """
|
|
SELECT
|
|
facts.channel_key,
|
|
coalesce(nullif(btrim(facts.room_category_label), ''), %s) AS room_type,
|
|
count(*) AS reservation_rows,
|
|
sum(facts.no_of_rooms) AS rooms_sold,
|
|
sum(facts.no_of_rooms * facts.nights) AS room_nights,
|
|
sum(facts.total_price) AS total_price
|
|
FROM finance.v_active_daily_facts AS facts
|
|
WHERE facts.business_date BETWEEN %s AND %s
|
|
GROUP BY facts.channel_key, room_type
|
|
ORDER BY facts.channel_key, room_type
|
|
""".strip()
|
|
|
|
CHANNEL_DETAIL_SQL = """
|
|
SELECT
|
|
facts.arrival,
|
|
facts.departure,
|
|
facts.nights,
|
|
facts.no_of_rooms,
|
|
facts.company_name,
|
|
facts.rate_code,
|
|
coalesce(nullif(btrim(facts.room_category_label), ''), %s) AS room_type,
|
|
facts.real_price,
|
|
facts.total_price
|
|
FROM finance.v_active_daily_facts AS facts
|
|
WHERE facts.business_date BETWEEN %s AND %s
|
|
AND facts.channel_key = %s
|
|
ORDER BY facts.arrival, facts.id
|
|
LIMIT %s OFFSET %s
|
|
""".strip()
|
|
|
|
MONTHS_SQL = """
|
|
SELECT DISTINCT date_trunc('month', business_date)::date AS period_start
|
|
FROM finance.current_daily_versions
|
|
ORDER BY period_start DESC
|
|
""".strip()
|
|
|
|
|
|
class AnalyticsRepositoryError(AnalyticsError):
|
|
pass
|
|
|
|
|
|
class AnalyticsMonthNotFound(AnalyticsRepositoryError):
|
|
def __init__(self) -> None:
|
|
super().__init__(
|
|
"ANALYTICS_MONTH_NOT_FOUND",
|
|
"current monthly database projection was not found",
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DatabaseConfig:
|
|
dsn: str
|
|
|
|
@classmethod
|
|
def from_environment(cls) -> "DatabaseConfig":
|
|
dsn = os.environ.get(DATABASE_ENV, "").strip()
|
|
if not dsn:
|
|
dsn = os.environ.get(FALLBACK_DATABASE_ENV, "").strip()
|
|
if not dsn:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_CONFIG_MISSING",
|
|
f"{DATABASE_ENV} or {FALLBACK_DATABASE_ENV} is required",
|
|
)
|
|
return cls(dsn)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ChannelManifestItem:
|
|
worksheet: str
|
|
worksheet_order: int
|
|
row_count: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReportContext:
|
|
period_start: date
|
|
period_end: date
|
|
as_of_date: date
|
|
activated_at: datetime
|
|
source_projection_sha256: str
|
|
daily_version_count: int
|
|
channels: Tuple[ChannelManifestItem, ...]
|
|
|
|
@property
|
|
def month_key(self) -> str:
|
|
return f"{self.period_start.year:04d}-{self.period_start.month:02d}"
|
|
|
|
@property
|
|
def filename(self) -> str:
|
|
return f"database-projection-{self.month_key}.json"
|
|
|
|
|
|
def _default_connect(dsn: str) -> Any:
|
|
try:
|
|
import psycopg # type: ignore[import-not-found]
|
|
except ImportError:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_DRIVER_UNAVAILABLE",
|
|
"PostgreSQL driver is unavailable",
|
|
) from None
|
|
try:
|
|
return psycopg.connect(dsn, autocommit=False)
|
|
except Exception:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_UNAVAILABLE",
|
|
"database connection failed",
|
|
) from None
|
|
|
|
|
|
def _integer(value: Any, field_name: str) -> int:
|
|
if isinstance(value, bool):
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_RESULT_INVALID",
|
|
f"database {field_name} is invalid",
|
|
)
|
|
try:
|
|
number = Decimal(str(value))
|
|
except (InvalidOperation, TypeError, ValueError):
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_RESULT_INVALID",
|
|
f"database {field_name} is invalid",
|
|
) from None
|
|
if (
|
|
not number.is_finite()
|
|
or number < 0
|
|
or number != number.to_integral_value()
|
|
):
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_RESULT_INVALID",
|
|
f"database {field_name} is invalid",
|
|
)
|
|
return int(number)
|
|
|
|
|
|
def _decimal(value: Any, field_name: str) -> Decimal:
|
|
try:
|
|
number = Decimal(str(value))
|
|
except (InvalidOperation, TypeError, ValueError):
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_RESULT_INVALID",
|
|
f"database {field_name} is invalid",
|
|
) from None
|
|
if not number.is_finite() or number < 0:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_RESULT_INVALID",
|
|
f"database {field_name} is invalid",
|
|
)
|
|
return number
|
|
|
|
|
|
def _date(value: Any, field_name: str) -> date:
|
|
if isinstance(value, datetime):
|
|
value = value.date()
|
|
if isinstance(value, date):
|
|
return value
|
|
try:
|
|
return date.fromisoformat(str(value))
|
|
except ValueError:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_RESULT_INVALID",
|
|
f"database {field_name} is invalid",
|
|
) from None
|
|
|
|
|
|
def _datetime(value: Any) -> datetime:
|
|
if isinstance(value, datetime):
|
|
return value
|
|
try:
|
|
return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
except ValueError:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_RESULT_INVALID",
|
|
"database activated_at is invalid",
|
|
) from None
|
|
|
|
|
|
def _projection_sha256(rows: List[Tuple[Any, ...]]) -> str:
|
|
identity = "\n".join(
|
|
f"{_date(row[0], 'business_date').isoformat()}:{_integer(row[1], 'daily_version_id')}:{str(row[2]).lower()}"
|
|
for row in rows
|
|
)
|
|
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
|
|
|
|
|
|
class PostgresAnalyticsRepository:
|
|
def __init__(
|
|
self,
|
|
config: DatabaseConfig,
|
|
connect: Optional[Callable[[str], Any]] = None,
|
|
) -> None:
|
|
self._config = config
|
|
self._connect = connect or _default_connect
|
|
|
|
def _open(self) -> Any:
|
|
try:
|
|
return self._connect(self._config.dsn)
|
|
except AnalyticsRepositoryError:
|
|
raise
|
|
except Exception:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_UNAVAILABLE",
|
|
"database connection failed",
|
|
) from None
|
|
|
|
@staticmethod
|
|
def _begin(cursor: Any) -> None:
|
|
cursor.execute(
|
|
"SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"
|
|
)
|
|
cursor.execute("SET LOCAL statement_timeout = '10s'")
|
|
cursor.execute("SET LOCAL lock_timeout = '3s'")
|
|
cursor.execute(
|
|
"SELECT current_database(), current_setting('transaction_read_only')"
|
|
)
|
|
row = cursor.fetchone()
|
|
if not row or row[0] != TARGET_DATABASE or row[1] != "on":
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_TARGET_INVALID",
|
|
"database target or read-only mode is invalid",
|
|
)
|
|
|
|
@staticmethod
|
|
def _period(month_key: str) -> Tuple[date, date]:
|
|
year, month = validate_month_key(month_key)
|
|
return (
|
|
date(year, month, 1),
|
|
date(year, month, calendar.monthrange(year, month)[1]),
|
|
)
|
|
|
|
@staticmethod
|
|
def _load_context(
|
|
cursor: Any,
|
|
period_start: date,
|
|
period_end: date,
|
|
) -> ReportContext:
|
|
cursor.execute(CURRENT_REPORT_SQL, (period_start, period_end))
|
|
row = cursor.fetchone()
|
|
if not row or row[0] is None or _integer(row[2], "daily_version_count") == 0:
|
|
raise AnalyticsMonthNotFound()
|
|
as_of_date = _date(row[0], "as_of_date")
|
|
activated_at = _datetime(row[1])
|
|
daily_version_count = _integer(row[2], "daily_version_count")
|
|
|
|
cursor.execute(REPORT_PIN_COUNT_SQL, (period_start, period_end))
|
|
identity_rows = list(cursor.fetchall())
|
|
if len(identity_rows) != daily_version_count:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_RESULT_INVALID",
|
|
"current daily version identity is incomplete",
|
|
)
|
|
for identity_row in identity_rows:
|
|
digest = str(identity_row[2] or "").lower()
|
|
if len(digest) != 64 or any(
|
|
character not in "0123456789abcdef" for character in digest
|
|
):
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_RESULT_INVALID",
|
|
"current source artifact identity is invalid",
|
|
)
|
|
|
|
cursor.execute(REPORT_MANIFEST_SQL, (period_start, period_end))
|
|
manifest_rows = list(cursor.fetchall())
|
|
channels = tuple(
|
|
ChannelManifestItem(
|
|
worksheet=str(item[0] or ""),
|
|
worksheet_order=order,
|
|
row_count=_integer(item[1], "row_count"),
|
|
)
|
|
for order, item in enumerate(manifest_rows, 1)
|
|
)
|
|
if (
|
|
not channels
|
|
or len({item.worksheet for item in channels}) != len(channels)
|
|
or any(not item.worksheet.strip() for item in channels)
|
|
):
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_CHANNEL_MANIFEST_INVALID",
|
|
"monthly channel projection is invalid",
|
|
)
|
|
return ReportContext(
|
|
period_start=period_start,
|
|
period_end=period_end,
|
|
as_of_date=as_of_date,
|
|
activated_at=activated_at,
|
|
source_projection_sha256=_projection_sha256(identity_rows),
|
|
daily_version_count=daily_version_count,
|
|
channels=channels,
|
|
)
|
|
|
|
@staticmethod
|
|
def _aggregates(
|
|
cursor: Any,
|
|
context: ReportContext,
|
|
) -> Tuple[AggregatedRoomFact, ...]:
|
|
cursor.execute(
|
|
ROOM_AGGREGATES_SQL,
|
|
(
|
|
UNLABELED_ROOM_TYPE,
|
|
context.period_start,
|
|
context.period_end,
|
|
),
|
|
)
|
|
facts = tuple(
|
|
AggregatedRoomFact(
|
|
worksheet=str(row[0] or ""),
|
|
room_type=str(row[1] or UNLABELED_ROOM_TYPE),
|
|
reservation_rows=_integer(row[2], "reservation_rows"),
|
|
rooms_sold=_integer(row[3], "rooms_sold"),
|
|
room_nights=_integer(row[4], "room_nights"),
|
|
total_price=_decimal(row[5], "total_price"),
|
|
)
|
|
for row in cursor.fetchall()
|
|
)
|
|
actual_counts: Dict[str, int] = {}
|
|
for fact in facts:
|
|
actual_counts[fact.worksheet] = (
|
|
actual_counts.get(fact.worksheet, 0)
|
|
+ fact.reservation_rows
|
|
)
|
|
expected_names = {item.worksheet for item in context.channels}
|
|
if (
|
|
not set(actual_counts).issubset(expected_names)
|
|
or any(
|
|
item.row_count != actual_counts.get(item.worksheet, 0)
|
|
for item in context.channels
|
|
)
|
|
):
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_CHANNEL_MANIFEST_INVALID",
|
|
"monthly channel projection does not match active facts",
|
|
)
|
|
return facts
|
|
|
|
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
|
period_start, period_end = self._period(month_key)
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
context = self._load_context(
|
|
cursor,
|
|
period_start,
|
|
period_end,
|
|
)
|
|
facts = self._aggregates(cursor, context)
|
|
return build_dashboard(
|
|
month_key=context.month_key,
|
|
updated_at=context.activated_at.isoformat(),
|
|
max_arrival_date=context.as_of_date.isoformat(),
|
|
source_monthly_sha256=context.source_projection_sha256,
|
|
channel_names=(item.worksheet for item in context.channels),
|
|
facts=facts,
|
|
)
|
|
except AnalyticsError:
|
|
raise
|
|
except Exception:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_QUERY_FAILED",
|
|
"database analytics query failed",
|
|
) from None
|
|
finally:
|
|
connection.close()
|
|
|
|
def read_channel_detail(
|
|
self,
|
|
month_key: str,
|
|
worksheet: str,
|
|
limit: int = 100,
|
|
offset: int = 0,
|
|
) -> Dict[str, Any]:
|
|
if (
|
|
not isinstance(worksheet, str)
|
|
or not worksheet.strip()
|
|
or worksheet != worksheet.strip()
|
|
or isinstance(limit, bool)
|
|
or isinstance(offset, bool)
|
|
or limit < 1
|
|
or limit > 500
|
|
or offset < 0
|
|
):
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DETAIL_REQUEST_INVALID",
|
|
"channel detail request is invalid",
|
|
)
|
|
period_start, period_end = self._period(month_key)
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
context = self._load_context(
|
|
cursor,
|
|
period_start,
|
|
period_end,
|
|
)
|
|
manifest = {
|
|
item.worksheet: item for item in context.channels
|
|
}
|
|
if worksheet not in manifest:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_CHANNEL_NOT_FOUND",
|
|
"channel is not present in the current month",
|
|
)
|
|
cursor.execute(
|
|
CHANNEL_DETAIL_SQL,
|
|
(
|
|
UNLABELED_ROOM_TYPE,
|
|
context.period_start,
|
|
context.period_end,
|
|
worksheet,
|
|
limit,
|
|
offset,
|
|
),
|
|
)
|
|
rows = [
|
|
{
|
|
"arrival": _date(row[0], "arrival").isoformat(),
|
|
"departure": _date(
|
|
row[1],
|
|
"departure",
|
|
).isoformat(),
|
|
"nights": _integer(row[2], "nights"),
|
|
"no_of_rooms": _integer(
|
|
row[3],
|
|
"no_of_rooms",
|
|
),
|
|
"company_name": str(row[4] or ""),
|
|
"rate_code": str(row[5] or ""),
|
|
"room_type": str(
|
|
row[6] or UNLABELED_ROOM_TYPE
|
|
),
|
|
"real_price": json_number(
|
|
_decimal(row[7], "real_price")
|
|
),
|
|
"total_price": json_number(
|
|
_decimal(row[8], "total_price")
|
|
),
|
|
}
|
|
for row in cursor.fetchall()
|
|
]
|
|
return {
|
|
"version": DETAIL_VERSION,
|
|
"month_key": context.month_key,
|
|
"worksheet": worksheet,
|
|
"updated_at": context.activated_at.isoformat(),
|
|
"max_arrival_date": context.as_of_date.isoformat(),
|
|
"source_monthly_sha256": (
|
|
context.source_projection_sha256
|
|
),
|
|
"total_rows": manifest[worksheet].row_count,
|
|
"limit": limit,
|
|
"offset": offset,
|
|
"rows": rows,
|
|
}
|
|
except AnalyticsError:
|
|
raise
|
|
except Exception:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_QUERY_FAILED",
|
|
"database channel detail query failed",
|
|
) from None
|
|
finally:
|
|
connection.close()
|
|
|
|
def list_months(self) -> List[Dict[str, Any]]:
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
cursor.execute(MONTHS_SQL)
|
|
period_rows = list(cursor.fetchall())
|
|
contexts = [
|
|
self._load_context(
|
|
cursor,
|
|
period_start,
|
|
date(
|
|
period_start.year,
|
|
period_start.month,
|
|
calendar.monthrange(
|
|
period_start.year,
|
|
period_start.month,
|
|
)[1],
|
|
),
|
|
)
|
|
for period_start in (
|
|
_date(row[0], "period_start")
|
|
for row in period_rows
|
|
)
|
|
]
|
|
return [
|
|
{
|
|
"month_key": context.month_key,
|
|
"max_arrival_date": context.as_of_date.isoformat(),
|
|
"updated_at": context.activated_at.isoformat(),
|
|
"filename": context.filename,
|
|
"source_monthly_sha256": (
|
|
context.source_projection_sha256
|
|
),
|
|
"channel_count": len(context.channels),
|
|
"row_count": sum(
|
|
item.row_count for item in context.channels
|
|
),
|
|
}
|
|
for context in contexts
|
|
]
|
|
except AnalyticsError:
|
|
raise
|
|
except Exception:
|
|
raise AnalyticsRepositoryError(
|
|
"ANALYTICS_DATABASE_QUERY_FAILED",
|
|
"database monthly list query failed",
|
|
) from None
|
|
finally:
|
|
connection.close()
|