413 lines
14 KiB
Python
413 lines
14 KiB
Python
"""Read-only portal repositories for daily history, monthly lineage, and BI."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass
|
|
from datetime import date, datetime
|
|
from decimal import Decimal
|
|
from typing import Any, Callable, Dict, List, Optional, Protocol
|
|
|
|
from channel_analytics.postgres import (
|
|
DatabaseConfig as AnalyticsDatabaseConfig,
|
|
PostgresAnalyticsRepository,
|
|
)
|
|
from arr_web.downloads import ArtifactDescriptor
|
|
|
|
|
|
TARGET_DATABASE = "booking_test"
|
|
|
|
|
|
class PortalDataError(RuntimeError):
|
|
def __init__(self, code: str, safe_message: str):
|
|
super().__init__(safe_message)
|
|
self.code = code
|
|
self.safe_message = safe_message
|
|
|
|
|
|
class PortalRepository(Protocol):
|
|
def list_jobs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
|
...
|
|
|
|
def list_monthly_runs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
|
...
|
|
|
|
def list_months(self) -> List[Dict[str, Any]]:
|
|
...
|
|
|
|
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
|
...
|
|
|
|
def read_channel_detail(
|
|
self,
|
|
month_key: str,
|
|
worksheet: str,
|
|
limit: int,
|
|
offset: int,
|
|
) -> Dict[str, Any]:
|
|
...
|
|
|
|
def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor:
|
|
...
|
|
|
|
def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor:
|
|
...
|
|
|
|
|
|
class UnavailablePortalRepository:
|
|
"""Fail-closed provider used when the ARR read role is not configured."""
|
|
|
|
@staticmethod
|
|
def _raise() -> None:
|
|
raise PortalDataError("DATABASE_UNAVAILABLE", "数据库读取服务暂不可用")
|
|
|
|
def list_jobs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
|
self._raise()
|
|
|
|
def list_monthly_runs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
|
self._raise()
|
|
|
|
def list_months(self) -> List[Dict[str, Any]]:
|
|
self._raise()
|
|
|
|
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
|
self._raise()
|
|
|
|
def read_channel_detail(
|
|
self,
|
|
month_key: str,
|
|
worksheet: str,
|
|
limit: int,
|
|
offset: int,
|
|
) -> Dict[str, Any]:
|
|
self._raise()
|
|
|
|
def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor:
|
|
self._raise()
|
|
|
|
def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor:
|
|
self._raise()
|
|
|
|
|
|
def _default_connect(dsn: str) -> Any:
|
|
try:
|
|
import psycopg # type: ignore[import-not-found]
|
|
except ImportError:
|
|
raise PortalDataError("DATABASE_DRIVER_UNAVAILABLE", "数据库驱动不可用") from None
|
|
try:
|
|
return psycopg.connect(dsn, autocommit=False)
|
|
except Exception:
|
|
raise PortalDataError("DATABASE_UNAVAILABLE", "数据库连接失败") from None
|
|
|
|
|
|
def _iso(value: Any) -> Optional[str]:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, (date, datetime)):
|
|
return value.isoformat()
|
|
return str(value)
|
|
|
|
|
|
def _number(value: Any) -> int | float:
|
|
number = value if isinstance(value, Decimal) else Decimal(str(value or 0))
|
|
return int(number) if number == number.to_integral_value() else float(number)
|
|
|
|
|
|
JOBS_SQL = """
|
|
SELECT
|
|
run.run_key,
|
|
CASE run.run_status
|
|
WHEN 'accepted' THEN 'succeeded'
|
|
WHEN 'rejected' THEN 'failed'
|
|
ELSE run.run_status
|
|
END AS public_status,
|
|
run.business_date,
|
|
run.failure_code,
|
|
run.created_at,
|
|
run.updated_at,
|
|
run.finished_at,
|
|
source.original_filename,
|
|
source.byte_size,
|
|
version.retained_rows,
|
|
version.version_no,
|
|
daily_artifact.sha256,
|
|
coalesce(room_totals.no_of_rooms, 0) AS no_of_rooms
|
|
FROM ingestion.processing_runs AS run
|
|
JOIN ingestion.artifacts AS source
|
|
ON source.id = run.source_artifact_id
|
|
LEFT JOIN finance.daily_versions AS version
|
|
ON version.processing_run_id = run.id
|
|
LEFT JOIN ingestion.artifacts AS daily_artifact
|
|
ON daily_artifact.id = version.daily_report_artifact_id
|
|
LEFT JOIN LATERAL (
|
|
SELECT coalesce(sum(record.no_of_rooms), 0) AS no_of_rooms
|
|
FROM finance.daily_records AS record
|
|
WHERE record.daily_version_id = version.id
|
|
AND record.outcome = 'retained'
|
|
) AS room_totals ON true
|
|
WHERE run.pipeline_type = 'opera_daily'
|
|
AND (
|
|
(run.business_date >= %s AND run.business_date < %s)
|
|
OR (
|
|
run.business_date IS NULL
|
|
AND run.created_at >= %s
|
|
AND run.created_at < %s
|
|
)
|
|
)
|
|
ORDER BY run.created_at DESC, run.id DESC
|
|
LIMIT %s
|
|
""".strip()
|
|
|
|
|
|
MONTHLY_RUNS_SQL = """
|
|
WITH selected_versions AS (
|
|
SELECT
|
|
current_version.business_date,
|
|
current_version.daily_version_id,
|
|
current_version.activated_at
|
|
FROM finance.current_daily_versions AS current_version
|
|
WHERE current_version.business_date >= %s
|
|
AND current_version.business_date < %s
|
|
),
|
|
selected_facts AS (
|
|
SELECT fact.id, fact.channel_key
|
|
FROM finance.v_active_daily_facts AS fact
|
|
WHERE fact.business_date >= %s
|
|
AND fact.business_date < %s
|
|
)
|
|
SELECT
|
|
NULL::bigint AS report_id,
|
|
min(selected_versions.business_date)::date AS period_start,
|
|
max(selected_versions.business_date)::date AS as_of_date,
|
|
1::integer AS version_no,
|
|
'source_ready'::text AS report_status,
|
|
true AS is_current,
|
|
min(selected_versions.activated_at) AS created_at,
|
|
max(selected_versions.activated_at) AS updated_at,
|
|
NULL::text AS failure_code,
|
|
NULL::text AS original_filename,
|
|
NULL::text AS artifact_sha256,
|
|
(SELECT count(DISTINCT channel_key) FROM selected_facts) AS channel_count,
|
|
(SELECT count(*) FROM selected_facts) AS row_count
|
|
FROM selected_versions
|
|
HAVING count(*) > 0
|
|
LIMIT %s
|
|
""".strip()
|
|
|
|
|
|
DAILY_DOWNLOAD_SQL = """
|
|
SELECT
|
|
artifact.artifact_kind,
|
|
artifact.original_filename,
|
|
artifact.object_key,
|
|
artifact.sha256,
|
|
artifact.byte_size,
|
|
artifact.mime_type
|
|
FROM ingestion.processing_runs AS run
|
|
JOIN finance.daily_versions AS version
|
|
ON version.processing_run_id = run.id
|
|
JOIN ingestion.artifacts AS artifact
|
|
ON artifact.id = version.daily_report_artifact_id
|
|
WHERE run.run_key = %s
|
|
AND run.run_status = 'accepted'
|
|
AND version.version_status IN ('active', 'superseded')
|
|
""".strip()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PostgresPortalRepository:
|
|
"""One read-only boundary; never exposes guest fields or object keys."""
|
|
|
|
dsn: str
|
|
connect: Callable[[str], Any] = _default_connect
|
|
|
|
def __post_init__(self) -> None:
|
|
object.__setattr__(
|
|
self,
|
|
"_analytics",
|
|
PostgresAnalyticsRepository(
|
|
AnalyticsDatabaseConfig(self.dsn),
|
|
connect=self.connect,
|
|
),
|
|
)
|
|
|
|
@classmethod
|
|
def from_environment(cls) -> "PostgresPortalRepository":
|
|
dsn = (
|
|
os.environ.get("DASHBOARD_DATABASE_URL", "").strip()
|
|
or os.environ.get("ARR_DATABASE_URL", "").strip()
|
|
)
|
|
if not dsn:
|
|
raise PortalDataError("DATABASE_CONFIG_MISSING", "数据库读取配置缺失")
|
|
return cls(dsn)
|
|
|
|
def _open(self) -> Any:
|
|
try:
|
|
return self.connect(self.dsn)
|
|
except PortalDataError:
|
|
raise
|
|
except Exception:
|
|
raise PortalDataError("DATABASE_UNAVAILABLE", "数据库连接失败") from None
|
|
|
|
@staticmethod
|
|
def _begin(cursor: Any) -> None:
|
|
cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
|
|
cursor.execute("SET LOCAL statement_timeout = '10s'")
|
|
cursor.execute("SET LOCAL lock_timeout = '3s'")
|
|
cursor.execute("SELECT current_database(), current_setting('transaction_read_only')")
|
|
row = cursor.fetchone()
|
|
if not row or row[0] != TARGET_DATABASE or row[1] != "on":
|
|
raise PortalDataError("DATABASE_TARGET_INVALID", "数据库读取门禁未通过")
|
|
|
|
@staticmethod
|
|
def _month_bounds(month_key: str) -> tuple[date, date]:
|
|
year, month = (int(item) for item in month_key.split("-"))
|
|
start = date(year, month, 1)
|
|
end = date(year + (month == 12), 1 if month == 12 else month + 1, 1)
|
|
return start, end
|
|
|
|
def list_jobs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
|
start, end = self._month_bounds(month_key)
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
cursor.execute(JOBS_SQL, (start, end, start, end, limit))
|
|
rows = list(cursor.fetchall())
|
|
return [
|
|
{
|
|
"job_id": str(row[0]),
|
|
"status": str(row[1]),
|
|
"arrival_date": _iso(row[2]),
|
|
"failure_code": str(row[3]) if row[3] else None,
|
|
"created_at": _iso(row[4]),
|
|
"updated_at": _iso(row[5]),
|
|
"finished_at": _iso(row[6]),
|
|
"filename": str(row[7]),
|
|
"byte_size": int(row[8]),
|
|
"output_rows": int(row[9] or 0),
|
|
"version_no": int(row[10]) if row[10] is not None else None,
|
|
"daily_report_sha256": str(row[11]) if row[11] else None,
|
|
"no_of_rooms": int(row[12] or 0),
|
|
}
|
|
for row in rows
|
|
]
|
|
except PortalDataError:
|
|
raise
|
|
except Exception:
|
|
raise PortalDataError("DATABASE_QUERY_FAILED", "日报记录查询失败") from None
|
|
finally:
|
|
connection.close()
|
|
|
|
def list_monthly_runs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
|
start, _end = self._month_bounds(month_key)
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
_start, end = self._month_bounds(month_key)
|
|
cursor.execute(MONTHLY_RUNS_SQL, (start, end, start, end, limit))
|
|
rows = list(cursor.fetchall())
|
|
return [
|
|
{
|
|
"report_id": int(row[0]) if row[0] is not None else None,
|
|
"month_key": start.strftime("%Y-%m"),
|
|
"as_of_date": _iso(row[2]),
|
|
"version_no": int(row[3]),
|
|
"status": str(row[4]),
|
|
"is_current": bool(row[5]),
|
|
"created_at": _iso(row[6]),
|
|
"updated_at": _iso(row[7]),
|
|
"failure_code": str(row[8]) if row[8] else None,
|
|
"filename": str(row[9]) if row[9] else None,
|
|
"artifact_sha256": str(row[10]) if row[10] else None,
|
|
"channel_count": int(row[11]),
|
|
"row_count": int(row[12]),
|
|
}
|
|
for row in rows
|
|
]
|
|
except PortalDataError:
|
|
raise
|
|
except Exception:
|
|
raise PortalDataError("DATABASE_QUERY_FAILED", "月报记录查询失败") from None
|
|
finally:
|
|
connection.close()
|
|
|
|
def list_months(self) -> List[Dict[str, Any]]:
|
|
try:
|
|
return self._analytics.list_months()
|
|
except Exception as error:
|
|
code = getattr(error, "code", "DATABASE_QUERY_FAILED")
|
|
raise PortalDataError(code, "月报月份查询失败") from None
|
|
|
|
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
|
try:
|
|
return self._analytics.read_dashboard(month_key)
|
|
except Exception as error:
|
|
code = getattr(error, "code", "DATABASE_QUERY_FAILED")
|
|
message = "该月份暂无已发布月报" if code == "ANALYTICS_MONTH_NOT_FOUND" else "看板查询失败"
|
|
raise PortalDataError(code, message) from None
|
|
|
|
def read_channel_detail(
|
|
self,
|
|
month_key: str,
|
|
worksheet: str,
|
|
limit: int,
|
|
offset: int,
|
|
) -> Dict[str, Any]:
|
|
try:
|
|
return self._analytics.read_channel_detail(
|
|
month_key,
|
|
worksheet,
|
|
limit=limit,
|
|
offset=offset,
|
|
)
|
|
except Exception as error:
|
|
code = getattr(error, "code", "DATABASE_QUERY_FAILED")
|
|
raise PortalDataError(code, "渠道明细查询失败") from None
|
|
|
|
def _resolve_download(self, statement: str, value: object) -> ArtifactDescriptor:
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
cursor.execute(statement, (value,))
|
|
rows = list(cursor.fetchall())
|
|
if len(rows) != 1:
|
|
raise PortalDataError("DOWNLOAD_NOT_FOUND", "文件不存在或尚未生成")
|
|
row = rows[0]
|
|
descriptor = ArtifactDescriptor(
|
|
file_kind=str(row[0] or ""),
|
|
original_filename=str(row[1] or ""),
|
|
storage_key=str(row[2] or ""),
|
|
sha256=str(row[3] or "").lower(),
|
|
byte_size=int(row[4]),
|
|
mime_type=str(row[5] or "application/octet-stream"),
|
|
)
|
|
descriptor.validate()
|
|
return descriptor
|
|
except PortalDataError:
|
|
raise
|
|
except Exception as error:
|
|
if hasattr(error, "code") and str(getattr(error, "code")).startswith("DOWNLOAD_"):
|
|
raise PortalDataError(str(getattr(error, "code")), str(error)) from None
|
|
raise PortalDataError("DATABASE_QUERY_FAILED", "文件身份查询失败") from None
|
|
finally:
|
|
connection.close()
|
|
|
|
def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor:
|
|
if not isinstance(job_id, str) or not 1 <= len(job_id) <= 128:
|
|
raise PortalDataError("DOWNLOAD_REQUEST_INVALID", "下载请求无效")
|
|
return self._resolve_download(DAILY_DOWNLOAD_SQL, job_id)
|
|
|
|
def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor:
|
|
if isinstance(report_id, bool) or not isinstance(report_id, int) or report_id < 1:
|
|
raise PortalDataError("DOWNLOAD_REQUEST_INVALID", "下载请求无效")
|
|
raise PortalDataError(
|
|
"DOWNLOAD_NOT_FOUND",
|
|
"月报行与版本不在数据库重复保存；请从本次生成结果下载",
|
|
)
|