"""Read-only portal repositories for daily history, monthly lineage, and BI.""" from __future__ import annotations import os from dataclasses import dataclass from datetime import date, datetime from decimal import Decimal from typing import Any, Callable, Dict, List, Optional, Protocol from channel_analytics.postgres import ( DatabaseConfig as AnalyticsDatabaseConfig, PostgresAnalyticsRepository, ) from arr_web.downloads import ArtifactDescriptor from arr_web.job_trace import build_job_trace TARGET_DATABASE = "booking_test" class PortalDataError(RuntimeError): def __init__(self, code: str, safe_message: str): super().__init__(safe_message) self.code = code self.safe_message = safe_message class PortalRepository(Protocol): def list_jobs( self, month_key: str, limit: int = 50, offset: int = 0, ) -> tuple[List[Dict[str, Any]], int]: ... def get_job_trace(self, job_id: str) -> Dict[str, Any]: ... def list_monthly_runs( self, month_key: str, limit: int = 50, offset: int = 0, ) -> tuple[List[Dict[str, Any]], int]: ... def list_history_month_counts(self) -> List[Dict[str, Any]]: ... def list_months(self) -> List[Dict[str, Any]]: ... def read_dashboard(self, month_key: str) -> Dict[str, Any]: ... def read_channel_detail( self, month_key: str, worksheet: str, limit: int, offset: int, ) -> Dict[str, Any]: ... def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor: ... def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor: ... class UnavailablePortalRepository: """Fail-closed provider used when the ARR read role is not configured.""" @staticmethod def _raise() -> None: raise PortalDataError("DATABASE_UNAVAILABLE", "数据库读取服务暂不可用") def list_jobs( self, month_key: str, limit: int = 50, offset: int = 0, ) -> tuple[List[Dict[str, Any]], int]: self._raise() def get_job_trace(self, job_id: str) -> Dict[str, Any]: self._raise() def list_monthly_runs( self, month_key: str, limit: int = 50, offset: int = 0, ) -> tuple[List[Dict[str, Any]], int]: self._raise() def list_history_month_counts(self) -> List[Dict[str, Any]]: self._raise() def list_months(self) -> List[Dict[str, Any]]: self._raise() def read_dashboard(self, month_key: str) -> Dict[str, Any]: self._raise() def read_channel_detail( self, month_key: str, worksheet: str, limit: int, offset: int, ) -> Dict[str, Any]: self._raise() def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor: self._raise() def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor: self._raise() def _default_connect(dsn: str) -> Any: try: import psycopg # type: ignore[import-not-found] except ImportError: raise PortalDataError("DATABASE_DRIVER_UNAVAILABLE", "数据库驱动不可用") from None try: return psycopg.connect(dsn, autocommit=False) except Exception: raise PortalDataError("DATABASE_UNAVAILABLE", "数据库连接失败") from None def _iso(value: Any) -> Optional[str]: if value is None: return None if isinstance(value, (date, datetime)): return value.isoformat() return str(value) def _number(value: Any) -> int | float: number = value if isinstance(value, Decimal) else Decimal(str(value or 0)) return int(number) if number == number.to_integral_value() else float(number) JOBS_SQL = """ SELECT run.run_key, CASE run.run_status WHEN 'accepted' THEN 'succeeded' WHEN 'rejected' THEN 'failed' ELSE run.run_status END AS public_status, run.business_date, run.failure_code, run.created_at, run.updated_at, run.finished_at, run.uploaded_filename, source.byte_size, version.retained_rows, version.version_no, daily_artifact.sha256, coalesce(room_totals.no_of_rooms, 0) AS no_of_rooms FROM ingestion.processing_runs AS run JOIN ingestion.artifacts AS source ON source.id = run.source_artifact_id LEFT JOIN finance.daily_versions AS version ON version.processing_run_id = run.id LEFT JOIN ingestion.artifacts AS daily_artifact ON daily_artifact.id = version.daily_report_artifact_id LEFT JOIN LATERAL ( SELECT coalesce(sum(record.no_of_rooms), 0) AS no_of_rooms FROM finance.daily_records AS record WHERE record.daily_version_id = version.id AND record.outcome = 'retained' ) AS room_totals ON true WHERE run.pipeline_type = 'opera_daily' AND ( (run.business_date >= %s AND run.business_date < %s) OR ( run.business_date IS NULL AND run.created_at >= %s AND run.created_at < %s ) ) ORDER BY run.created_at DESC, run.id DESC LIMIT %s OFFSET %s """.strip() JOBS_COUNT_SQL = """ SELECT count(*) FROM ingestion.processing_runs AS run WHERE run.pipeline_type = 'opera_daily' AND ( (run.business_date >= %s AND run.business_date < %s) OR ( run.business_date IS NULL AND run.created_at >= %s AND run.created_at < %s ) ) """.strip() JOB_TRACE_RUN_SQL = """ SELECT run.id, run.run_key, run.run_status, CASE run.run_status WHEN 'accepted' THEN 'succeeded' WHEN 'rejected' THEN 'failed' ELSE run.run_status END AS public_status, run.result_delivery_mode, run.business_date, run.failure_code, run.failure_message, run.created_at, run.updated_at, run.validated_at, run.finished_at, run.uploaded_filename, source.byte_size, source.sha256, run.requested_processor_version, run.delivered_processor_version, run.requested_rule_set_sha256, run.delivered_rule_set_sha256, run.result_schema_version FROM ingestion.processing_runs AS run JOIN ingestion.artifacts AS source ON source.id = run.source_artifact_id WHERE run.pipeline_type = 'opera_daily' AND run.run_key = %s """.strip() JOB_TRACE_ATTEMPTS_SQL = """ SELECT attempt.id, attempt.attempt_no, attempt.attempt_status, attempt.remote_run_id, attempt.failure_code, attempt.failure_message, attempt.created_at, attempt.started_at, attempt.finished_at FROM ingestion.processing_attempts AS attempt WHERE attempt.processing_run_id = %s ORDER BY attempt.attempt_no, attempt.id """.strip() JOB_TRACE_DELIVERIES_SQL = """ SELECT delivery.delivery_key, attempt.attempt_no, delivery.delivery_status, delivery.result_status, delivery.processor_version, delivery.result_schema_version, delivery.business_date, delivery.daily_version_id, delivery.failure_code, delivery.failure_message, delivery.received_at, delivery.validated_at, delivery.committed_at FROM ingestion.processing_deliveries AS delivery LEFT JOIN ingestion.processing_attempts AS attempt ON attempt.id = delivery.attempt_id AND attempt.processing_run_id = delivery.processing_run_id WHERE delivery.processing_run_id = %s ORDER BY delivery.received_at, delivery.id """.strip() JOB_TRACE_SUBMISSIONS_SQL = """ SELECT submission.submission_key, attempt.attempt_no, submission.submission_status, submission.contract_version, submission.business_date, submission.processor_version, submission.result_schema_version, submission.record_count, submission.daily_version_id, submission.failure_code, submission.failure_message, submission.created_at, submission.validation_started_at, submission.finished_at FROM ingestion.result_submissions AS submission JOIN ingestion.processing_attempts AS attempt ON attempt.id = submission.attempt_id AND attempt.processing_run_id = submission.processing_run_id WHERE submission.processing_run_id = %s ORDER BY submission.created_at, submission.id """.strip() JOB_TRACE_VERSIONS_SQL = """ SELECT version.id, version.business_date, version.version_no, version.version_status, version.result_delivery_mode, version.source_rows, version.retained_rows, version.excluded_rate_code_rows, version.duplicate_rows, version.validation_failed_rows, version.price_unmatched_rows, version.failure_code, version.failure_message, version.created_at, version.validated_at, version.activated_at, version.superseded_at FROM finance.daily_versions AS version WHERE version.processing_run_id = %s ORDER BY COALESCE(version.version_no, 0), version.id """.strip() JOB_TRACE_OUTBOX_SQL = """ SELECT event.event_key, event.event_type, event.publish_status, event.publish_attempts, event.available_at, event.created_at, event.published_at, event.last_error_code FROM ingestion.outbox_events AS event WHERE event.aggregate_type = 'processing_run' AND event.aggregate_id = %s ORDER BY event.created_at, event.id """.strip() MONTHLY_RUNS_SQL = """ SELECT run.id AS report_id, run.period_start, run.as_of_date AS max_arrival_date, run.version_no, run.report_status, run.report_status = 'active' AS is_current, run.created_at, run.updated_at, run.failure_code, artifact.original_filename, artifact.sha256, run.channel_count, run.row_count FROM reporting.monthly_runs AS run LEFT JOIN ingestion.artifacts AS artifact ON artifact.id = run.workbook_artifact_id WHERE run.period_start = %s ORDER BY run.version_no DESC, run.id DESC LIMIT %s OFFSET %s """.strip() MONTHLY_RUNS_COUNT_SQL = """ SELECT count(*) FROM reporting.monthly_runs AS run WHERE run.period_start = %s """.strip() HISTORY_MONTH_COUNTS_SQL = """ WITH daily AS ( SELECT CASE WHEN run.business_date IS NOT NULL THEN date_trunc('month', run.business_date)::date ELSE date_trunc( 'month', run.created_at AT TIME ZONE 'Asia/Bangkok' )::date END AS period_start, count(*) AS record_count FROM ingestion.processing_runs AS run WHERE run.pipeline_type = 'opera_daily' GROUP BY 1 ), monthly AS ( SELECT run.period_start, count(*) AS record_count FROM reporting.monthly_runs AS run GROUP BY run.period_start ) SELECT coalesce(daily.period_start, monthly.period_start) AS period_start, coalesce(daily.record_count, 0) AS daily_count, coalesce(monthly.record_count, 0) AS monthly_count FROM daily FULL OUTER JOIN monthly USING (period_start) ORDER BY period_start DESC """.strip() DAILY_DOWNLOAD_SQL = """ SELECT artifact.artifact_kind, artifact.original_filename, artifact.object_key, artifact.sha256, artifact.byte_size, artifact.mime_type FROM ingestion.processing_runs AS run JOIN finance.daily_versions AS version ON version.processing_run_id = run.id JOIN ingestion.artifacts AS artifact ON artifact.id = version.daily_report_artifact_id WHERE run.run_key = %s AND run.run_status = 'accepted' AND version.version_status IN ('active', 'superseded') """.strip() MONTHLY_DOWNLOAD_SQL = """ SELECT artifact.artifact_kind, artifact.original_filename, artifact.object_key, artifact.sha256, artifact.byte_size, artifact.mime_type FROM reporting.monthly_runs AS run JOIN ingestion.artifacts AS artifact ON artifact.id = run.workbook_artifact_id WHERE run.id = %s AND run.report_status IN ('active', 'superseded') AND artifact.artifact_kind = 'monthly_xlsx' AND artifact.storage_provider = 'local' """.strip() @dataclass(frozen=True) class PostgresPortalRepository: """One read-only boundary; never exposes guest fields or object keys.""" dsn: str connect: Callable[[str], Any] = _default_connect def __post_init__(self) -> None: object.__setattr__( self, "_analytics", PostgresAnalyticsRepository( AnalyticsDatabaseConfig(self.dsn), connect=self.connect, ), ) @classmethod def from_environment(cls) -> "PostgresPortalRepository": dsn = ( os.environ.get("DASHBOARD_DATABASE_URL", "").strip() or os.environ.get("ARR_DATABASE_URL", "").strip() ) if not dsn: raise PortalDataError("DATABASE_CONFIG_MISSING", "数据库读取配置缺失") return cls(dsn) def _open(self) -> Any: try: return self.connect(self.dsn) except PortalDataError: raise except Exception: raise PortalDataError("DATABASE_UNAVAILABLE", "数据库连接失败") from None @staticmethod def _begin(cursor: Any) -> None: cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") cursor.execute("SET LOCAL statement_timeout = '10s'") cursor.execute("SET LOCAL lock_timeout = '3s'") cursor.execute("SELECT current_database(), current_setting('transaction_read_only')") row = cursor.fetchone() if not row or row[0] != TARGET_DATABASE or row[1] != "on": raise PortalDataError("DATABASE_TARGET_INVALID", "数据库读取门禁未通过") @staticmethod def _month_bounds(month_key: str) -> tuple[date, date]: year, month = (int(item) for item in month_key.split("-")) start = date(year, month, 1) end = date(year + (month == 12), 1 if month == 12 else month + 1, 1) return start, end def list_jobs( self, month_key: str, limit: int = 50, offset: int = 0, ) -> tuple[List[Dict[str, Any]], int]: start, end = self._month_bounds(month_key) connection = self._open() try: with connection.transaction(): with connection.cursor() as cursor: self._begin(cursor) cursor.execute(JOBS_COUNT_SQL, (start, end, start, end)) count_row = cursor.fetchone() total = int(count_row[0] or 0) if count_row else 0 cursor.execute( JOBS_SQL, (start, end, start, end, limit, offset), ) rows = list(cursor.fetchall()) return [ { "job_id": str(row[0]), "status": str(row[1]), "arrival_date": _iso(row[2]), "failure_code": str(row[3]) if row[3] else None, "created_at": _iso(row[4]), "updated_at": _iso(row[5]), "finished_at": _iso(row[6]), "filename": str(row[7]) if row[7] else None, "byte_size": int(row[8]), "output_rows": int(row[9] or 0), "version_no": int(row[10]) if row[10] is not None else None, "daily_report_sha256": str(row[11]) if row[11] else None, "no_of_rooms": int(row[12] or 0), } for row in rows ], total except PortalDataError: raise except Exception: raise PortalDataError("DATABASE_QUERY_FAILED", "日报记录查询失败") from None finally: connection.close() def get_job_trace(self, job_id: str) -> Dict[str, Any]: connection = self._open() try: with connection.transaction(): with connection.cursor() as cursor: self._begin(cursor) cursor.execute(JOB_TRACE_RUN_SQL, (job_id,)) row = cursor.fetchone() if row is None: raise PortalDataError("JOB_NOT_FOUND", "任务不存在或已过保留期") run = { "id": int(row[0]), "job_id": str(row[1]), "run_status": str(row[2]), "public_status": str(row[3]), "delivery_mode": str(row[4]), "business_date": row[5], "failure_code": str(row[6]) if row[6] else None, "failure_message": str(row[7]) if row[7] else None, "created_at": row[8], "updated_at": row[9], "validated_at": row[10], "finished_at": row[11], "filename": str(row[12]) if row[12] else None, "byte_size": int(row[13]), "source_sha256": str(row[14]), "requested_processor_version": ( str(row[15]) if row[15] else None ), "delivered_processor_version": ( str(row[16]) if row[16] else None ), "requested_rule_set_sha256": ( str(row[17]) if row[17] else None ), "delivered_rule_set_sha256": ( str(row[18]) if row[18] else None ), "result_schema_version": str(row[19]) if row[19] else None, } run_id = int(row[0]) cursor.execute(JOB_TRACE_ATTEMPTS_SQL, (run_id,)) attempts = [ { "attempt_id": int(item[0]), "attempt_no": int(item[1]), "attempt_status": str(item[2]), "remote_run_id": str(item[3]) if item[3] else None, "failure_code": str(item[4]) if item[4] else None, "failure_message": str(item[5]) if item[5] else None, "created_at": item[6], "started_at": item[7], "finished_at": item[8], } for item in cursor.fetchall() ] cursor.execute(JOB_TRACE_DELIVERIES_SQL, (run_id,)) deliveries = [ { "delivery_key": str(item[0]), "attempt_no": int(item[1]) if item[1] is not None else None, "delivery_status": str(item[2]), "result_status": str(item[3]), "processor_version": str(item[4]), "result_schema_version": str(item[5]), "business_date": item[6], "daily_version_id": ( int(item[7]) if item[7] is not None else None ), "failure_code": str(item[8]) if item[8] else None, "failure_message": str(item[9]) if item[9] else None, "received_at": item[10], "validated_at": item[11], "committed_at": item[12], } for item in cursor.fetchall() ] cursor.execute(JOB_TRACE_SUBMISSIONS_SQL, (run_id,)) submissions = [ { "submission_key": str(item[0]), "attempt_no": int(item[1]), "submission_status": str(item[2]), "contract_version": str(item[3]), "business_date": item[4], "processor_version": str(item[5]), "result_schema_version": str(item[6]), "record_count": int(item[7]), "daily_version_id": ( int(item[8]) if item[8] is not None else None ), "failure_code": str(item[9]) if item[9] else None, "failure_message": str(item[10]) if item[10] else None, "created_at": item[11], "validation_started_at": item[12], "finished_at": item[13], } for item in cursor.fetchall() ] cursor.execute(JOB_TRACE_VERSIONS_SQL, (run_id,)) versions = [ { "daily_version_id": int(item[0]), "business_date": item[1], "version_no": ( int(item[2]) if item[2] is not None else None ), "version_status": str(item[3]), "result_delivery_mode": str(item[4]), "source_rows": int(item[5]), "retained_rows": int(item[6]), "excluded_rate_code_rows": int(item[7]), "duplicate_rows": int(item[8]), "validation_failed_rows": int(item[9]), "price_unmatched_rows": int(item[10]), "failure_code": str(item[11]) if item[11] else None, "failure_message": str(item[12]) if item[12] else None, "created_at": item[13], "validated_at": item[14], "activated_at": item[15], "superseded_at": item[16], } for item in cursor.fetchall() ] cursor.execute(JOB_TRACE_OUTBOX_SQL, (run_id,)) outbox_events = [ { "event_key": str(item[0]), "event_type": str(item[1]), "publish_status": str(item[2]), "publish_attempts": int(item[3]), "available_at": item[4], "created_at": item[5], "published_at": item[6], "last_error_code": str(item[7]) if item[7] else None, } for item in cursor.fetchall() ] return build_job_trace( run, attempts=attempts, deliveries=deliveries, submissions=submissions, versions=versions, outbox_events=outbox_events, ) except PortalDataError: raise except Exception: raise PortalDataError("DATABASE_QUERY_FAILED", "任务日志查询失败") from None finally: connection.close() def list_monthly_runs( self, month_key: str, limit: int = 50, offset: int = 0, ) -> tuple[List[Dict[str, Any]], int]: start, _end = self._month_bounds(month_key) connection = self._open() try: with connection.transaction(): with connection.cursor() as cursor: self._begin(cursor) cursor.execute(MONTHLY_RUNS_COUNT_SQL, (start,)) count_row = cursor.fetchone() total = int(count_row[0] or 0) if count_row else 0 cursor.execute(MONTHLY_RUNS_SQL, (start, limit, offset)) rows = list(cursor.fetchall()) return [ { "report_id": int(row[0]) if row[0] is not None else None, "month_key": start.strftime("%Y-%m"), "max_arrival_date": _iso(row[2]), "as_of_date": _iso(row[2]), "version_no": int(row[3]), "status": str(row[4]), "is_current": bool(row[5]), "created_at": _iso(row[6]), "updated_at": _iso(row[7]), "failure_code": str(row[8]) if row[8] else None, "filename": str(row[9]) if row[9] else None, "artifact_sha256": str(row[10]) if row[10] else None, "channel_count": int(row[11]), "row_count": int(row[12]), } for row in rows ], total except PortalDataError: raise except Exception: raise PortalDataError("DATABASE_QUERY_FAILED", "月报记录查询失败") from None finally: connection.close() def list_history_month_counts(self) -> List[Dict[str, Any]]: connection = self._open() try: with connection.transaction(): with connection.cursor() as cursor: self._begin(cursor) cursor.execute(HISTORY_MONTH_COUNTS_SQL) rows = list(cursor.fetchall()) return [ { "month_key": row[0].strftime("%Y-%m"), "daily_count": int(row[1] or 0), "monthly_count": int(row[2] or 0), } for row in rows ] except PortalDataError: raise except Exception: raise PortalDataError( "DATABASE_QUERY_FAILED", "历史月份查询失败", ) from None finally: connection.close() def list_months(self) -> List[Dict[str, Any]]: try: return self._analytics.list_months() except Exception as error: code = getattr(error, "code", "DATABASE_QUERY_FAILED") raise PortalDataError(code, "月报月份查询失败") from None def read_dashboard(self, month_key: str) -> Dict[str, Any]: try: return self._analytics.read_dashboard(month_key) except Exception as error: code = getattr(error, "code", "DATABASE_QUERY_FAILED") message = "该月份暂无已发布月报" if code == "ANALYTICS_MONTH_NOT_FOUND" else "看板查询失败" raise PortalDataError(code, message) from None def read_channel_detail( self, month_key: str, worksheet: str, limit: int, offset: int, ) -> Dict[str, Any]: try: return self._analytics.read_channel_detail( month_key, worksheet, limit=limit, offset=offset, ) except Exception as error: code = getattr(error, "code", "DATABASE_QUERY_FAILED") raise PortalDataError(code, "渠道明细查询失败") from None def _resolve_download(self, statement: str, value: object) -> ArtifactDescriptor: connection = self._open() try: with connection.transaction(): with connection.cursor() as cursor: self._begin(cursor) cursor.execute(statement, (value,)) rows = list(cursor.fetchall()) if len(rows) != 1: raise PortalDataError("DOWNLOAD_NOT_FOUND", "文件不存在或尚未生成") row = rows[0] descriptor = ArtifactDescriptor( file_kind=str(row[0] or ""), original_filename=str(row[1] or ""), storage_key=str(row[2] or ""), sha256=str(row[3] or "").lower(), byte_size=int(row[4]), mime_type=str(row[5] or "application/octet-stream"), ) descriptor.validate() return descriptor except PortalDataError: raise except Exception as error: if hasattr(error, "code") and str(getattr(error, "code")).startswith("DOWNLOAD_"): raise PortalDataError(str(getattr(error, "code")), str(error)) from None raise PortalDataError("DATABASE_QUERY_FAILED", "文件身份查询失败") from None finally: connection.close() def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor: if not isinstance(job_id, str) or not 1 <= len(job_id) <= 128: raise PortalDataError("DOWNLOAD_REQUEST_INVALID", "下载请求无效") return self._resolve_download(DAILY_DOWNLOAD_SQL, job_id) def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor: if isinstance(report_id, bool) or not isinstance(report_id, int) or report_id < 1: raise PortalDataError("DOWNLOAD_REQUEST_INVALID", "下载请求无效") return self._resolve_download(MONTHLY_DOWNLOAD_SQL, report_id)