feat: prepare ARR for controlled public deployment
This commit is contained in:
Wyndham ARR
412
arr_web/repository.py
Normal file
412
arr_web/repository.py
Normal file
@@ -0,0 +1,412 @@
|
||||
"""Read-only portal repositories for daily history, monthly lineage, and BI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any, Callable, Dict, List, Optional, Protocol
|
||||
|
||||
from channel_analytics.postgres import (
|
||||
DatabaseConfig as AnalyticsDatabaseConfig,
|
||||
PostgresAnalyticsRepository,
|
||||
)
|
||||
from arr_web.downloads import ArtifactDescriptor
|
||||
|
||||
|
||||
TARGET_DATABASE = "booking_test"
|
||||
|
||||
|
||||
class PortalDataError(RuntimeError):
|
||||
def __init__(self, code: str, safe_message: str):
|
||||
super().__init__(safe_message)
|
||||
self.code = code
|
||||
self.safe_message = safe_message
|
||||
|
||||
|
||||
class PortalRepository(Protocol):
|
||||
def list_jobs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
...
|
||||
|
||||
def list_monthly_runs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
...
|
||||
|
||||
def list_months(self) -> List[Dict[str, Any]]:
|
||||
...
|
||||
|
||||
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
||||
...
|
||||
|
||||
def read_channel_detail(
|
||||
self,
|
||||
month_key: str,
|
||||
worksheet: str,
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> Dict[str, Any]:
|
||||
...
|
||||
|
||||
def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor:
|
||||
...
|
||||
|
||||
def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor:
|
||||
...
|
||||
|
||||
|
||||
class UnavailablePortalRepository:
|
||||
"""Fail-closed provider used when the ARR read role is not configured."""
|
||||
|
||||
@staticmethod
|
||||
def _raise() -> None:
|
||||
raise PortalDataError("DATABASE_UNAVAILABLE", "数据库读取服务暂不可用")
|
||||
|
||||
def list_jobs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
self._raise()
|
||||
|
||||
def list_monthly_runs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
self._raise()
|
||||
|
||||
def list_months(self) -> List[Dict[str, Any]]:
|
||||
self._raise()
|
||||
|
||||
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
||||
self._raise()
|
||||
|
||||
def read_channel_detail(
|
||||
self,
|
||||
month_key: str,
|
||||
worksheet: str,
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> Dict[str, Any]:
|
||||
self._raise()
|
||||
|
||||
def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor:
|
||||
self._raise()
|
||||
|
||||
def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor:
|
||||
self._raise()
|
||||
|
||||
|
||||
def _default_connect(dsn: str) -> Any:
|
||||
try:
|
||||
import psycopg # type: ignore[import-not-found]
|
||||
except ImportError:
|
||||
raise PortalDataError("DATABASE_DRIVER_UNAVAILABLE", "数据库驱动不可用") from None
|
||||
try:
|
||||
return psycopg.connect(dsn, autocommit=False)
|
||||
except Exception:
|
||||
raise PortalDataError("DATABASE_UNAVAILABLE", "数据库连接失败") from None
|
||||
|
||||
|
||||
def _iso(value: Any) -> Optional[str]:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (date, datetime)):
|
||||
return value.isoformat()
|
||||
return str(value)
|
||||
|
||||
|
||||
def _number(value: Any) -> int | float:
|
||||
number = value if isinstance(value, Decimal) else Decimal(str(value or 0))
|
||||
return int(number) if number == number.to_integral_value() else float(number)
|
||||
|
||||
|
||||
JOBS_SQL = """
|
||||
SELECT
|
||||
run.run_key,
|
||||
CASE run.run_status
|
||||
WHEN 'accepted' THEN 'succeeded'
|
||||
WHEN 'rejected' THEN 'failed'
|
||||
ELSE run.run_status
|
||||
END AS public_status,
|
||||
run.business_date,
|
||||
run.failure_code,
|
||||
run.created_at,
|
||||
run.updated_at,
|
||||
run.finished_at,
|
||||
source.original_filename,
|
||||
source.byte_size,
|
||||
version.retained_rows,
|
||||
version.version_no,
|
||||
daily_artifact.sha256,
|
||||
coalesce(room_totals.no_of_rooms, 0) AS no_of_rooms
|
||||
FROM ingestion.processing_runs AS run
|
||||
JOIN ingestion.artifacts AS source
|
||||
ON source.id = run.source_artifact_id
|
||||
LEFT JOIN finance.daily_versions AS version
|
||||
ON version.processing_run_id = run.id
|
||||
LEFT JOIN ingestion.artifacts AS daily_artifact
|
||||
ON daily_artifact.id = version.daily_report_artifact_id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT coalesce(sum(record.no_of_rooms), 0) AS no_of_rooms
|
||||
FROM finance.daily_records AS record
|
||||
WHERE record.daily_version_id = version.id
|
||||
AND record.outcome = 'retained'
|
||||
) AS room_totals ON true
|
||||
WHERE run.pipeline_type = 'opera_daily'
|
||||
AND (
|
||||
(run.business_date >= %s AND run.business_date < %s)
|
||||
OR (
|
||||
run.business_date IS NULL
|
||||
AND run.created_at >= %s
|
||||
AND run.created_at < %s
|
||||
)
|
||||
)
|
||||
ORDER BY run.created_at DESC, run.id DESC
|
||||
LIMIT %s
|
||||
""".strip()
|
||||
|
||||
|
||||
MONTHLY_RUNS_SQL = """
|
||||
WITH selected_versions AS (
|
||||
SELECT
|
||||
current_version.business_date,
|
||||
current_version.daily_version_id,
|
||||
current_version.activated_at
|
||||
FROM finance.current_daily_versions AS current_version
|
||||
WHERE current_version.business_date >= %s
|
||||
AND current_version.business_date < %s
|
||||
),
|
||||
selected_facts AS (
|
||||
SELECT fact.id, fact.channel_key
|
||||
FROM finance.v_active_daily_facts AS fact
|
||||
WHERE fact.business_date >= %s
|
||||
AND fact.business_date < %s
|
||||
)
|
||||
SELECT
|
||||
NULL::bigint AS report_id,
|
||||
min(selected_versions.business_date)::date AS period_start,
|
||||
max(selected_versions.business_date)::date AS as_of_date,
|
||||
1::integer AS version_no,
|
||||
'source_ready'::text AS report_status,
|
||||
true AS is_current,
|
||||
min(selected_versions.activated_at) AS created_at,
|
||||
max(selected_versions.activated_at) AS updated_at,
|
||||
NULL::text AS failure_code,
|
||||
NULL::text AS original_filename,
|
||||
NULL::text AS artifact_sha256,
|
||||
(SELECT count(DISTINCT channel_key) FROM selected_facts) AS channel_count,
|
||||
(SELECT count(*) FROM selected_facts) AS row_count
|
||||
FROM selected_versions
|
||||
HAVING count(*) > 0
|
||||
LIMIT %s
|
||||
""".strip()
|
||||
|
||||
|
||||
DAILY_DOWNLOAD_SQL = """
|
||||
SELECT
|
||||
artifact.artifact_kind,
|
||||
artifact.original_filename,
|
||||
artifact.object_key,
|
||||
artifact.sha256,
|
||||
artifact.byte_size,
|
||||
artifact.mime_type
|
||||
FROM ingestion.processing_runs AS run
|
||||
JOIN finance.daily_versions AS version
|
||||
ON version.processing_run_id = run.id
|
||||
JOIN ingestion.artifacts AS artifact
|
||||
ON artifact.id = version.daily_report_artifact_id
|
||||
WHERE run.run_key = %s
|
||||
AND run.run_status = 'accepted'
|
||||
AND version.version_status IN ('active', 'superseded')
|
||||
""".strip()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PostgresPortalRepository:
|
||||
"""One read-only boundary; never exposes guest fields or object keys."""
|
||||
|
||||
dsn: str
|
||||
connect: Callable[[str], Any] = _default_connect
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"_analytics",
|
||||
PostgresAnalyticsRepository(
|
||||
AnalyticsDatabaseConfig(self.dsn),
|
||||
connect=self.connect,
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_environment(cls) -> "PostgresPortalRepository":
|
||||
dsn = (
|
||||
os.environ.get("DASHBOARD_DATABASE_URL", "").strip()
|
||||
or os.environ.get("ARR_DATABASE_URL", "").strip()
|
||||
)
|
||||
if not dsn:
|
||||
raise PortalDataError("DATABASE_CONFIG_MISSING", "数据库读取配置缺失")
|
||||
return cls(dsn)
|
||||
|
||||
def _open(self) -> Any:
|
||||
try:
|
||||
return self.connect(self.dsn)
|
||||
except PortalDataError:
|
||||
raise
|
||||
except Exception:
|
||||
raise PortalDataError("DATABASE_UNAVAILABLE", "数据库连接失败") from None
|
||||
|
||||
@staticmethod
|
||||
def _begin(cursor: Any) -> None:
|
||||
cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
|
||||
cursor.execute("SET LOCAL statement_timeout = '10s'")
|
||||
cursor.execute("SET LOCAL lock_timeout = '3s'")
|
||||
cursor.execute("SELECT current_database(), current_setting('transaction_read_only')")
|
||||
row = cursor.fetchone()
|
||||
if not row or row[0] != TARGET_DATABASE or row[1] != "on":
|
||||
raise PortalDataError("DATABASE_TARGET_INVALID", "数据库读取门禁未通过")
|
||||
|
||||
@staticmethod
|
||||
def _month_bounds(month_key: str) -> tuple[date, date]:
|
||||
year, month = (int(item) for item in month_key.split("-"))
|
||||
start = date(year, month, 1)
|
||||
end = date(year + (month == 12), 1 if month == 12 else month + 1, 1)
|
||||
return start, end
|
||||
|
||||
def list_jobs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
start, end = self._month_bounds(month_key)
|
||||
connection = self._open()
|
||||
try:
|
||||
with connection.transaction():
|
||||
with connection.cursor() as cursor:
|
||||
self._begin(cursor)
|
||||
cursor.execute(JOBS_SQL, (start, end, start, end, limit))
|
||||
rows = list(cursor.fetchall())
|
||||
return [
|
||||
{
|
||||
"job_id": str(row[0]),
|
||||
"status": str(row[1]),
|
||||
"arrival_date": _iso(row[2]),
|
||||
"failure_code": str(row[3]) if row[3] else None,
|
||||
"created_at": _iso(row[4]),
|
||||
"updated_at": _iso(row[5]),
|
||||
"finished_at": _iso(row[6]),
|
||||
"filename": str(row[7]),
|
||||
"byte_size": int(row[8]),
|
||||
"output_rows": int(row[9] or 0),
|
||||
"version_no": int(row[10]) if row[10] is not None else None,
|
||||
"daily_report_sha256": str(row[11]) if row[11] else None,
|
||||
"no_of_rooms": int(row[12] or 0),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
except PortalDataError:
|
||||
raise
|
||||
except Exception:
|
||||
raise PortalDataError("DATABASE_QUERY_FAILED", "日报记录查询失败") from None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def list_monthly_runs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
start, _end = self._month_bounds(month_key)
|
||||
connection = self._open()
|
||||
try:
|
||||
with connection.transaction():
|
||||
with connection.cursor() as cursor:
|
||||
self._begin(cursor)
|
||||
_start, end = self._month_bounds(month_key)
|
||||
cursor.execute(MONTHLY_RUNS_SQL, (start, end, start, end, limit))
|
||||
rows = list(cursor.fetchall())
|
||||
return [
|
||||
{
|
||||
"report_id": int(row[0]) if row[0] is not None else None,
|
||||
"month_key": start.strftime("%Y-%m"),
|
||||
"as_of_date": _iso(row[2]),
|
||||
"version_no": int(row[3]),
|
||||
"status": str(row[4]),
|
||||
"is_current": bool(row[5]),
|
||||
"created_at": _iso(row[6]),
|
||||
"updated_at": _iso(row[7]),
|
||||
"failure_code": str(row[8]) if row[8] else None,
|
||||
"filename": str(row[9]) if row[9] else None,
|
||||
"artifact_sha256": str(row[10]) if row[10] else None,
|
||||
"channel_count": int(row[11]),
|
||||
"row_count": int(row[12]),
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
except PortalDataError:
|
||||
raise
|
||||
except Exception:
|
||||
raise PortalDataError("DATABASE_QUERY_FAILED", "月报记录查询失败") from None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def list_months(self) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
return self._analytics.list_months()
|
||||
except Exception as error:
|
||||
code = getattr(error, "code", "DATABASE_QUERY_FAILED")
|
||||
raise PortalDataError(code, "月报月份查询失败") from None
|
||||
|
||||
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
||||
try:
|
||||
return self._analytics.read_dashboard(month_key)
|
||||
except Exception as error:
|
||||
code = getattr(error, "code", "DATABASE_QUERY_FAILED")
|
||||
message = "该月份暂无已发布月报" if code == "ANALYTICS_MONTH_NOT_FOUND" else "看板查询失败"
|
||||
raise PortalDataError(code, message) from None
|
||||
|
||||
def read_channel_detail(
|
||||
self,
|
||||
month_key: str,
|
||||
worksheet: str,
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> Dict[str, Any]:
|
||||
try:
|
||||
return self._analytics.read_channel_detail(
|
||||
month_key,
|
||||
worksheet,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
except Exception as error:
|
||||
code = getattr(error, "code", "DATABASE_QUERY_FAILED")
|
||||
raise PortalDataError(code, "渠道明细查询失败") from None
|
||||
|
||||
def _resolve_download(self, statement: str, value: object) -> ArtifactDescriptor:
|
||||
connection = self._open()
|
||||
try:
|
||||
with connection.transaction():
|
||||
with connection.cursor() as cursor:
|
||||
self._begin(cursor)
|
||||
cursor.execute(statement, (value,))
|
||||
rows = list(cursor.fetchall())
|
||||
if len(rows) != 1:
|
||||
raise PortalDataError("DOWNLOAD_NOT_FOUND", "文件不存在或尚未生成")
|
||||
row = rows[0]
|
||||
descriptor = ArtifactDescriptor(
|
||||
file_kind=str(row[0] or ""),
|
||||
original_filename=str(row[1] or ""),
|
||||
storage_key=str(row[2] or ""),
|
||||
sha256=str(row[3] or "").lower(),
|
||||
byte_size=int(row[4]),
|
||||
mime_type=str(row[5] or "application/octet-stream"),
|
||||
)
|
||||
descriptor.validate()
|
||||
return descriptor
|
||||
except PortalDataError:
|
||||
raise
|
||||
except Exception as error:
|
||||
if hasattr(error, "code") and str(getattr(error, "code")).startswith("DOWNLOAD_"):
|
||||
raise PortalDataError(str(getattr(error, "code")), str(error)) from None
|
||||
raise PortalDataError("DATABASE_QUERY_FAILED", "文件身份查询失败") from None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor:
|
||||
if not isinstance(job_id, str) or not 1 <= len(job_id) <= 128:
|
||||
raise PortalDataError("DOWNLOAD_REQUEST_INVALID", "下载请求无效")
|
||||
return self._resolve_download(DAILY_DOWNLOAD_SQL, job_id)
|
||||
|
||||
def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor:
|
||||
if isinstance(report_id, bool) or not isinstance(report_id, int) or report_id < 1:
|
||||
raise PortalDataError("DOWNLOAD_REQUEST_INVALID", "下载请求无效")
|
||||
raise PortalDataError(
|
||||
"DOWNLOAD_NOT_FOUND",
|
||||
"月报行与版本不在数据库重复保存；请从本次生成结果下载",
|
||||
)
|
||||
Reference in New Issue
Block a user