846 lines
29 KiB
Python
846 lines
29 KiB
Python
"""Read-only portal repositories for daily history, monthly lineage, and BI."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass
|
|
from datetime import date, datetime
|
|
from decimal import Decimal
|
|
from typing import Any, Callable, Dict, List, Optional, Protocol
|
|
|
|
from channel_analytics.postgres import (
|
|
DatabaseConfig as AnalyticsDatabaseConfig,
|
|
PostgresAnalyticsRepository,
|
|
)
|
|
from arr_web.downloads import ArtifactDescriptor
|
|
from arr_web.job_trace import build_job_trace
|
|
|
|
|
|
TARGET_DATABASE = "booking_test"
|
|
|
|
|
|
class PortalDataError(RuntimeError):
|
|
def __init__(self, code: str, safe_message: str):
|
|
super().__init__(safe_message)
|
|
self.code = code
|
|
self.safe_message = safe_message
|
|
|
|
|
|
class PortalRepository(Protocol):
|
|
def list_jobs(
|
|
self,
|
|
month_key: str,
|
|
limit: int = 50,
|
|
offset: int = 0,
|
|
) -> tuple[List[Dict[str, Any]], int]:
|
|
...
|
|
|
|
def get_job_trace(self, job_id: str) -> Dict[str, Any]:
|
|
...
|
|
|
|
def list_monthly_runs(
|
|
self,
|
|
month_key: str,
|
|
limit: int = 50,
|
|
offset: int = 0,
|
|
) -> tuple[List[Dict[str, Any]], int]:
|
|
...
|
|
|
|
def list_history_month_counts(self) -> List[Dict[str, Any]]:
|
|
...
|
|
|
|
def list_months(self) -> List[Dict[str, Any]]:
|
|
...
|
|
|
|
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
|
...
|
|
|
|
def read_channel_detail(
|
|
self,
|
|
month_key: str,
|
|
worksheet: str,
|
|
limit: int,
|
|
offset: int,
|
|
) -> Dict[str, Any]:
|
|
...
|
|
|
|
def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor:
|
|
...
|
|
|
|
def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor:
|
|
...
|
|
|
|
|
|
class UnavailablePortalRepository:
|
|
"""Fail-closed provider used when the ARR read role is not configured."""
|
|
|
|
@staticmethod
|
|
def _raise() -> None:
|
|
raise PortalDataError("DATABASE_UNAVAILABLE", "数据库读取服务暂不可用")
|
|
|
|
def list_jobs(
|
|
self,
|
|
month_key: str,
|
|
limit: int = 50,
|
|
offset: int = 0,
|
|
) -> tuple[List[Dict[str, Any]], int]:
|
|
self._raise()
|
|
|
|
def get_job_trace(self, job_id: str) -> Dict[str, Any]:
|
|
self._raise()
|
|
|
|
def list_monthly_runs(
|
|
self,
|
|
month_key: str,
|
|
limit: int = 50,
|
|
offset: int = 0,
|
|
) -> tuple[List[Dict[str, Any]], int]:
|
|
self._raise()
|
|
|
|
def list_history_month_counts(self) -> List[Dict[str, Any]]:
|
|
self._raise()
|
|
|
|
def list_months(self) -> List[Dict[str, Any]]:
|
|
self._raise()
|
|
|
|
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
|
self._raise()
|
|
|
|
def read_channel_detail(
|
|
self,
|
|
month_key: str,
|
|
worksheet: str,
|
|
limit: int,
|
|
offset: int,
|
|
) -> Dict[str, Any]:
|
|
self._raise()
|
|
|
|
def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor:
|
|
self._raise()
|
|
|
|
def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor:
|
|
self._raise()
|
|
|
|
|
|
def _default_connect(dsn: str) -> Any:
|
|
try:
|
|
import psycopg # type: ignore[import-not-found]
|
|
except ImportError:
|
|
raise PortalDataError("DATABASE_DRIVER_UNAVAILABLE", "数据库驱动不可用") from None
|
|
try:
|
|
return psycopg.connect(dsn, autocommit=False)
|
|
except Exception:
|
|
raise PortalDataError("DATABASE_UNAVAILABLE", "数据库连接失败") from None
|
|
|
|
|
|
def _iso(value: Any) -> Optional[str]:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, (date, datetime)):
|
|
return value.isoformat()
|
|
return str(value)
|
|
|
|
|
|
def _number(value: Any) -> int | float:
|
|
number = value if isinstance(value, Decimal) else Decimal(str(value or 0))
|
|
return int(number) if number == number.to_integral_value() else float(number)
|
|
|
|
|
|
JOBS_SQL = """
|
|
SELECT
|
|
run.run_key,
|
|
CASE run.run_status
|
|
WHEN 'accepted' THEN 'succeeded'
|
|
WHEN 'rejected' THEN 'failed'
|
|
ELSE run.run_status
|
|
END AS public_status,
|
|
run.business_date,
|
|
run.failure_code,
|
|
run.created_at,
|
|
run.updated_at,
|
|
run.finished_at,
|
|
run.uploaded_filename,
|
|
source.byte_size,
|
|
version.retained_rows,
|
|
version.version_no,
|
|
daily_artifact.sha256,
|
|
coalesce(room_totals.no_of_rooms, 0) AS no_of_rooms
|
|
FROM ingestion.processing_runs AS run
|
|
JOIN ingestion.artifacts AS source
|
|
ON source.id = run.source_artifact_id
|
|
LEFT JOIN finance.daily_versions AS version
|
|
ON version.processing_run_id = run.id
|
|
LEFT JOIN ingestion.artifacts AS daily_artifact
|
|
ON daily_artifact.id = version.daily_report_artifact_id
|
|
LEFT JOIN LATERAL (
|
|
SELECT coalesce(sum(record.no_of_rooms), 0) AS no_of_rooms
|
|
FROM finance.daily_records AS record
|
|
WHERE record.daily_version_id = version.id
|
|
AND record.outcome = 'retained'
|
|
) AS room_totals ON true
|
|
WHERE run.pipeline_type = 'opera_daily'
|
|
AND (
|
|
(run.business_date >= %s AND run.business_date < %s)
|
|
OR (
|
|
run.business_date IS NULL
|
|
AND run.created_at >= %s
|
|
AND run.created_at < %s
|
|
)
|
|
)
|
|
ORDER BY run.created_at DESC, run.id DESC
|
|
LIMIT %s OFFSET %s
|
|
""".strip()
|
|
|
|
|
|
JOBS_COUNT_SQL = """
|
|
SELECT count(*)
|
|
FROM ingestion.processing_runs AS run
|
|
WHERE run.pipeline_type = 'opera_daily'
|
|
AND (
|
|
(run.business_date >= %s AND run.business_date < %s)
|
|
OR (
|
|
run.business_date IS NULL
|
|
AND run.created_at >= %s
|
|
AND run.created_at < %s
|
|
)
|
|
)
|
|
""".strip()
|
|
|
|
|
|
JOB_TRACE_RUN_SQL = """
|
|
SELECT
|
|
run.id,
|
|
run.run_key,
|
|
run.run_status,
|
|
CASE run.run_status
|
|
WHEN 'accepted' THEN 'succeeded'
|
|
WHEN 'rejected' THEN 'failed'
|
|
ELSE run.run_status
|
|
END AS public_status,
|
|
run.result_delivery_mode,
|
|
run.business_date,
|
|
run.failure_code,
|
|
run.failure_message,
|
|
run.created_at,
|
|
run.updated_at,
|
|
run.validated_at,
|
|
run.finished_at,
|
|
run.uploaded_filename,
|
|
source.byte_size,
|
|
source.sha256,
|
|
run.requested_processor_version,
|
|
run.delivered_processor_version,
|
|
run.requested_rule_set_sha256,
|
|
run.delivered_rule_set_sha256,
|
|
run.result_schema_version
|
|
FROM ingestion.processing_runs AS run
|
|
JOIN ingestion.artifacts AS source
|
|
ON source.id = run.source_artifact_id
|
|
WHERE run.pipeline_type = 'opera_daily'
|
|
AND run.run_key = %s
|
|
""".strip()
|
|
|
|
|
|
JOB_TRACE_ATTEMPTS_SQL = """
|
|
SELECT
|
|
attempt.id,
|
|
attempt.attempt_no,
|
|
attempt.attempt_status,
|
|
attempt.remote_run_id,
|
|
attempt.failure_code,
|
|
attempt.failure_message,
|
|
attempt.created_at,
|
|
attempt.started_at,
|
|
attempt.finished_at
|
|
FROM ingestion.processing_attempts AS attempt
|
|
WHERE attempt.processing_run_id = %s
|
|
ORDER BY attempt.attempt_no, attempt.id
|
|
""".strip()
|
|
|
|
|
|
JOB_TRACE_DELIVERIES_SQL = """
|
|
SELECT
|
|
delivery.delivery_key,
|
|
attempt.attempt_no,
|
|
delivery.delivery_status,
|
|
delivery.result_status,
|
|
delivery.processor_version,
|
|
delivery.result_schema_version,
|
|
delivery.business_date,
|
|
delivery.daily_version_id,
|
|
delivery.failure_code,
|
|
delivery.failure_message,
|
|
delivery.received_at,
|
|
delivery.validated_at,
|
|
delivery.committed_at
|
|
FROM ingestion.processing_deliveries AS delivery
|
|
LEFT JOIN ingestion.processing_attempts AS attempt
|
|
ON attempt.id = delivery.attempt_id
|
|
AND attempt.processing_run_id = delivery.processing_run_id
|
|
WHERE delivery.processing_run_id = %s
|
|
ORDER BY delivery.received_at, delivery.id
|
|
""".strip()
|
|
|
|
|
|
JOB_TRACE_SUBMISSIONS_SQL = """
|
|
SELECT
|
|
submission.submission_key,
|
|
attempt.attempt_no,
|
|
submission.submission_status,
|
|
submission.contract_version,
|
|
submission.business_date,
|
|
submission.processor_version,
|
|
submission.result_schema_version,
|
|
submission.record_count,
|
|
submission.daily_version_id,
|
|
submission.failure_code,
|
|
submission.failure_message,
|
|
submission.created_at,
|
|
submission.validation_started_at,
|
|
submission.finished_at
|
|
FROM ingestion.result_submissions AS submission
|
|
JOIN ingestion.processing_attempts AS attempt
|
|
ON attempt.id = submission.attempt_id
|
|
AND attempt.processing_run_id = submission.processing_run_id
|
|
WHERE submission.processing_run_id = %s
|
|
ORDER BY submission.created_at, submission.id
|
|
""".strip()
|
|
|
|
|
|
JOB_TRACE_VERSIONS_SQL = """
|
|
SELECT
|
|
version.id,
|
|
version.business_date,
|
|
version.version_no,
|
|
version.version_status,
|
|
version.result_delivery_mode,
|
|
version.source_rows,
|
|
version.retained_rows,
|
|
version.excluded_rate_code_rows,
|
|
version.duplicate_rows,
|
|
version.validation_failed_rows,
|
|
version.price_unmatched_rows,
|
|
version.failure_code,
|
|
version.failure_message,
|
|
version.created_at,
|
|
version.validated_at,
|
|
version.activated_at,
|
|
version.superseded_at
|
|
FROM finance.daily_versions AS version
|
|
WHERE version.processing_run_id = %s
|
|
ORDER BY COALESCE(version.version_no, 0), version.id
|
|
""".strip()
|
|
|
|
|
|
JOB_TRACE_OUTBOX_SQL = """
|
|
SELECT
|
|
event.event_key,
|
|
event.event_type,
|
|
event.publish_status,
|
|
event.publish_attempts,
|
|
event.available_at,
|
|
event.created_at,
|
|
event.published_at,
|
|
event.last_error_code
|
|
FROM ingestion.outbox_events AS event
|
|
WHERE event.aggregate_type = 'processing_run'
|
|
AND event.aggregate_id = %s
|
|
ORDER BY event.created_at, event.id
|
|
""".strip()
|
|
|
|
|
|
MONTHLY_RUNS_SQL = """
|
|
SELECT
|
|
run.id AS report_id,
|
|
run.period_start,
|
|
run.as_of_date AS max_arrival_date,
|
|
run.version_no,
|
|
run.report_status,
|
|
run.report_status = 'active' AS is_current,
|
|
run.created_at,
|
|
run.updated_at,
|
|
run.failure_code,
|
|
artifact.original_filename,
|
|
artifact.sha256,
|
|
run.channel_count,
|
|
run.row_count
|
|
FROM reporting.monthly_runs AS run
|
|
LEFT JOIN ingestion.artifacts AS artifact
|
|
ON artifact.id = run.workbook_artifact_id
|
|
WHERE run.period_start = %s
|
|
ORDER BY run.version_no DESC, run.id DESC
|
|
LIMIT %s OFFSET %s
|
|
""".strip()
|
|
|
|
|
|
MONTHLY_RUNS_COUNT_SQL = """
|
|
SELECT count(*)
|
|
FROM reporting.monthly_runs AS run
|
|
WHERE run.period_start = %s
|
|
""".strip()
|
|
|
|
|
|
HISTORY_MONTH_COUNTS_SQL = """
|
|
WITH daily AS (
|
|
SELECT
|
|
CASE
|
|
WHEN run.business_date IS NOT NULL
|
|
THEN date_trunc('month', run.business_date)::date
|
|
ELSE date_trunc(
|
|
'month',
|
|
run.created_at AT TIME ZONE 'Asia/Bangkok'
|
|
)::date
|
|
END AS period_start,
|
|
count(*) AS record_count
|
|
FROM ingestion.processing_runs AS run
|
|
WHERE run.pipeline_type = 'opera_daily'
|
|
GROUP BY 1
|
|
),
|
|
monthly AS (
|
|
SELECT run.period_start, count(*) AS record_count
|
|
FROM reporting.monthly_runs AS run
|
|
GROUP BY run.period_start
|
|
)
|
|
SELECT
|
|
coalesce(daily.period_start, monthly.period_start) AS period_start,
|
|
coalesce(daily.record_count, 0) AS daily_count,
|
|
coalesce(monthly.record_count, 0) AS monthly_count
|
|
FROM daily
|
|
FULL OUTER JOIN monthly USING (period_start)
|
|
ORDER BY period_start DESC
|
|
""".strip()
|
|
|
|
|
|
DAILY_DOWNLOAD_SQL = """
|
|
SELECT
|
|
artifact.artifact_kind,
|
|
artifact.original_filename,
|
|
artifact.object_key,
|
|
artifact.sha256,
|
|
artifact.byte_size,
|
|
artifact.mime_type
|
|
FROM ingestion.processing_runs AS run
|
|
JOIN finance.daily_versions AS version
|
|
ON version.processing_run_id = run.id
|
|
JOIN ingestion.artifacts AS artifact
|
|
ON artifact.id = version.daily_report_artifact_id
|
|
WHERE run.run_key = %s
|
|
AND run.run_status = 'accepted'
|
|
AND version.version_status IN ('active', 'superseded')
|
|
""".strip()
|
|
|
|
MONTHLY_DOWNLOAD_SQL = """
|
|
SELECT
|
|
artifact.artifact_kind,
|
|
artifact.original_filename,
|
|
artifact.object_key,
|
|
artifact.sha256,
|
|
artifact.byte_size,
|
|
artifact.mime_type
|
|
FROM reporting.monthly_runs AS run
|
|
JOIN ingestion.artifacts AS artifact
|
|
ON artifact.id = run.workbook_artifact_id
|
|
WHERE run.id = %s
|
|
AND run.report_status IN ('active', 'superseded')
|
|
AND artifact.artifact_kind = 'monthly_xlsx'
|
|
AND artifact.storage_provider = 'local'
|
|
""".strip()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PostgresPortalRepository:
|
|
"""One read-only boundary; never exposes guest fields or object keys."""
|
|
|
|
dsn: str
|
|
connect: Callable[[str], Any] = _default_connect
|
|
|
|
def __post_init__(self) -> None:
|
|
object.__setattr__(
|
|
self,
|
|
"_analytics",
|
|
PostgresAnalyticsRepository(
|
|
AnalyticsDatabaseConfig(self.dsn),
|
|
connect=self.connect,
|
|
),
|
|
)
|
|
|
|
@classmethod
|
|
def from_environment(cls) -> "PostgresPortalRepository":
|
|
dsn = (
|
|
os.environ.get("DASHBOARD_DATABASE_URL", "").strip()
|
|
or os.environ.get("ARR_DATABASE_URL", "").strip()
|
|
)
|
|
if not dsn:
|
|
raise PortalDataError("DATABASE_CONFIG_MISSING", "数据库读取配置缺失")
|
|
return cls(dsn)
|
|
|
|
def _open(self) -> Any:
|
|
try:
|
|
return self.connect(self.dsn)
|
|
except PortalDataError:
|
|
raise
|
|
except Exception:
|
|
raise PortalDataError("DATABASE_UNAVAILABLE", "数据库连接失败") from None
|
|
|
|
@staticmethod
|
|
def _begin(cursor: Any) -> None:
|
|
cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
|
|
cursor.execute("SET LOCAL statement_timeout = '10s'")
|
|
cursor.execute("SET LOCAL lock_timeout = '3s'")
|
|
cursor.execute("SELECT current_database(), current_setting('transaction_read_only')")
|
|
row = cursor.fetchone()
|
|
if not row or row[0] != TARGET_DATABASE or row[1] != "on":
|
|
raise PortalDataError("DATABASE_TARGET_INVALID", "数据库读取门禁未通过")
|
|
|
|
@staticmethod
|
|
def _month_bounds(month_key: str) -> tuple[date, date]:
|
|
year, month = (int(item) for item in month_key.split("-"))
|
|
start = date(year, month, 1)
|
|
end = date(year + (month == 12), 1 if month == 12 else month + 1, 1)
|
|
return start, end
|
|
|
|
def list_jobs(
|
|
self,
|
|
month_key: str,
|
|
limit: int = 50,
|
|
offset: int = 0,
|
|
) -> tuple[List[Dict[str, Any]], int]:
|
|
start, end = self._month_bounds(month_key)
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
cursor.execute(JOBS_COUNT_SQL, (start, end, start, end))
|
|
count_row = cursor.fetchone()
|
|
total = int(count_row[0] or 0) if count_row else 0
|
|
cursor.execute(
|
|
JOBS_SQL,
|
|
(start, end, start, end, limit, offset),
|
|
)
|
|
rows = list(cursor.fetchall())
|
|
return [
|
|
{
|
|
"job_id": str(row[0]),
|
|
"status": str(row[1]),
|
|
"arrival_date": _iso(row[2]),
|
|
"failure_code": str(row[3]) if row[3] else None,
|
|
"created_at": _iso(row[4]),
|
|
"updated_at": _iso(row[5]),
|
|
"finished_at": _iso(row[6]),
|
|
"filename": str(row[7]) if row[7] else None,
|
|
"byte_size": int(row[8]),
|
|
"output_rows": int(row[9] or 0),
|
|
"version_no": int(row[10]) if row[10] is not None else None,
|
|
"daily_report_sha256": str(row[11]) if row[11] else None,
|
|
"no_of_rooms": int(row[12] or 0),
|
|
}
|
|
for row in rows
|
|
], total
|
|
except PortalDataError:
|
|
raise
|
|
except Exception:
|
|
raise PortalDataError("DATABASE_QUERY_FAILED", "日报记录查询失败") from None
|
|
finally:
|
|
connection.close()
|
|
|
|
def get_job_trace(self, job_id: str) -> Dict[str, Any]:
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
cursor.execute(JOB_TRACE_RUN_SQL, (job_id,))
|
|
row = cursor.fetchone()
|
|
if row is None:
|
|
raise PortalDataError("JOB_NOT_FOUND", "任务不存在或已过保留期")
|
|
run = {
|
|
"id": int(row[0]),
|
|
"job_id": str(row[1]),
|
|
"run_status": str(row[2]),
|
|
"public_status": str(row[3]),
|
|
"delivery_mode": str(row[4]),
|
|
"business_date": row[5],
|
|
"failure_code": str(row[6]) if row[6] else None,
|
|
"failure_message": str(row[7]) if row[7] else None,
|
|
"created_at": row[8],
|
|
"updated_at": row[9],
|
|
"validated_at": row[10],
|
|
"finished_at": row[11],
|
|
"filename": str(row[12]) if row[12] else None,
|
|
"byte_size": int(row[13]),
|
|
"source_sha256": str(row[14]),
|
|
"requested_processor_version": (
|
|
str(row[15]) if row[15] else None
|
|
),
|
|
"delivered_processor_version": (
|
|
str(row[16]) if row[16] else None
|
|
),
|
|
"requested_rule_set_sha256": (
|
|
str(row[17]) if row[17] else None
|
|
),
|
|
"delivered_rule_set_sha256": (
|
|
str(row[18]) if row[18] else None
|
|
),
|
|
"result_schema_version": str(row[19]) if row[19] else None,
|
|
}
|
|
run_id = int(row[0])
|
|
|
|
cursor.execute(JOB_TRACE_ATTEMPTS_SQL, (run_id,))
|
|
attempts = [
|
|
{
|
|
"attempt_id": int(item[0]),
|
|
"attempt_no": int(item[1]),
|
|
"attempt_status": str(item[2]),
|
|
"remote_run_id": str(item[3]) if item[3] else None,
|
|
"failure_code": str(item[4]) if item[4] else None,
|
|
"failure_message": str(item[5]) if item[5] else None,
|
|
"created_at": item[6],
|
|
"started_at": item[7],
|
|
"finished_at": item[8],
|
|
}
|
|
for item in cursor.fetchall()
|
|
]
|
|
|
|
cursor.execute(JOB_TRACE_DELIVERIES_SQL, (run_id,))
|
|
deliveries = [
|
|
{
|
|
"delivery_key": str(item[0]),
|
|
"attempt_no": int(item[1]) if item[1] is not None else None,
|
|
"delivery_status": str(item[2]),
|
|
"result_status": str(item[3]),
|
|
"processor_version": str(item[4]),
|
|
"result_schema_version": str(item[5]),
|
|
"business_date": item[6],
|
|
"daily_version_id": (
|
|
int(item[7]) if item[7] is not None else None
|
|
),
|
|
"failure_code": str(item[8]) if item[8] else None,
|
|
"failure_message": str(item[9]) if item[9] else None,
|
|
"received_at": item[10],
|
|
"validated_at": item[11],
|
|
"committed_at": item[12],
|
|
}
|
|
for item in cursor.fetchall()
|
|
]
|
|
|
|
cursor.execute(JOB_TRACE_SUBMISSIONS_SQL, (run_id,))
|
|
submissions = [
|
|
{
|
|
"submission_key": str(item[0]),
|
|
"attempt_no": int(item[1]),
|
|
"submission_status": str(item[2]),
|
|
"contract_version": str(item[3]),
|
|
"business_date": item[4],
|
|
"processor_version": str(item[5]),
|
|
"result_schema_version": str(item[6]),
|
|
"record_count": int(item[7]),
|
|
"daily_version_id": (
|
|
int(item[8]) if item[8] is not None else None
|
|
),
|
|
"failure_code": str(item[9]) if item[9] else None,
|
|
"failure_message": str(item[10]) if item[10] else None,
|
|
"created_at": item[11],
|
|
"validation_started_at": item[12],
|
|
"finished_at": item[13],
|
|
}
|
|
for item in cursor.fetchall()
|
|
]
|
|
|
|
cursor.execute(JOB_TRACE_VERSIONS_SQL, (run_id,))
|
|
versions = [
|
|
{
|
|
"daily_version_id": int(item[0]),
|
|
"business_date": item[1],
|
|
"version_no": (
|
|
int(item[2]) if item[2] is not None else None
|
|
),
|
|
"version_status": str(item[3]),
|
|
"result_delivery_mode": str(item[4]),
|
|
"source_rows": int(item[5]),
|
|
"retained_rows": int(item[6]),
|
|
"excluded_rate_code_rows": int(item[7]),
|
|
"duplicate_rows": int(item[8]),
|
|
"validation_failed_rows": int(item[9]),
|
|
"price_unmatched_rows": int(item[10]),
|
|
"failure_code": str(item[11]) if item[11] else None,
|
|
"failure_message": str(item[12]) if item[12] else None,
|
|
"created_at": item[13],
|
|
"validated_at": item[14],
|
|
"activated_at": item[15],
|
|
"superseded_at": item[16],
|
|
}
|
|
for item in cursor.fetchall()
|
|
]
|
|
|
|
cursor.execute(JOB_TRACE_OUTBOX_SQL, (run_id,))
|
|
outbox_events = [
|
|
{
|
|
"event_key": str(item[0]),
|
|
"event_type": str(item[1]),
|
|
"publish_status": str(item[2]),
|
|
"publish_attempts": int(item[3]),
|
|
"available_at": item[4],
|
|
"created_at": item[5],
|
|
"published_at": item[6],
|
|
"last_error_code": str(item[7]) if item[7] else None,
|
|
}
|
|
for item in cursor.fetchall()
|
|
]
|
|
return build_job_trace(
|
|
run,
|
|
attempts=attempts,
|
|
deliveries=deliveries,
|
|
submissions=submissions,
|
|
versions=versions,
|
|
outbox_events=outbox_events,
|
|
)
|
|
except PortalDataError:
|
|
raise
|
|
except Exception:
|
|
raise PortalDataError("DATABASE_QUERY_FAILED", "任务日志查询失败") from None
|
|
finally:
|
|
connection.close()
|
|
|
|
def list_monthly_runs(
|
|
self,
|
|
month_key: str,
|
|
limit: int = 50,
|
|
offset: int = 0,
|
|
) -> tuple[List[Dict[str, Any]], int]:
|
|
start, _end = self._month_bounds(month_key)
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
cursor.execute(MONTHLY_RUNS_COUNT_SQL, (start,))
|
|
count_row = cursor.fetchone()
|
|
total = int(count_row[0] or 0) if count_row else 0
|
|
cursor.execute(MONTHLY_RUNS_SQL, (start, limit, offset))
|
|
rows = list(cursor.fetchall())
|
|
return [
|
|
{
|
|
"report_id": int(row[0]) if row[0] is not None else None,
|
|
"month_key": start.strftime("%Y-%m"),
|
|
"max_arrival_date": _iso(row[2]),
|
|
"as_of_date": _iso(row[2]),
|
|
"version_no": int(row[3]),
|
|
"status": str(row[4]),
|
|
"is_current": bool(row[5]),
|
|
"created_at": _iso(row[6]),
|
|
"updated_at": _iso(row[7]),
|
|
"failure_code": str(row[8]) if row[8] else None,
|
|
"filename": str(row[9]) if row[9] else None,
|
|
"artifact_sha256": str(row[10]) if row[10] else None,
|
|
"channel_count": int(row[11]),
|
|
"row_count": int(row[12]),
|
|
}
|
|
for row in rows
|
|
], total
|
|
except PortalDataError:
|
|
raise
|
|
except Exception:
|
|
raise PortalDataError("DATABASE_QUERY_FAILED", "月报记录查询失败") from None
|
|
finally:
|
|
connection.close()
|
|
|
|
def list_history_month_counts(self) -> List[Dict[str, Any]]:
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
cursor.execute(HISTORY_MONTH_COUNTS_SQL)
|
|
rows = list(cursor.fetchall())
|
|
return [
|
|
{
|
|
"month_key": row[0].strftime("%Y-%m"),
|
|
"daily_count": int(row[1] or 0),
|
|
"monthly_count": int(row[2] or 0),
|
|
}
|
|
for row in rows
|
|
]
|
|
except PortalDataError:
|
|
raise
|
|
except Exception:
|
|
raise PortalDataError(
|
|
"DATABASE_QUERY_FAILED",
|
|
"历史月份查询失败",
|
|
) from None
|
|
finally:
|
|
connection.close()
|
|
|
|
def list_months(self) -> List[Dict[str, Any]]:
|
|
try:
|
|
return self._analytics.list_months()
|
|
except Exception as error:
|
|
code = getattr(error, "code", "DATABASE_QUERY_FAILED")
|
|
raise PortalDataError(code, "月报月份查询失败") from None
|
|
|
|
def read_dashboard(self, month_key: str) -> Dict[str, Any]:
|
|
try:
|
|
return self._analytics.read_dashboard(month_key)
|
|
except Exception as error:
|
|
code = getattr(error, "code", "DATABASE_QUERY_FAILED")
|
|
message = "该月份暂无已发布月报" if code == "ANALYTICS_MONTH_NOT_FOUND" else "看板查询失败"
|
|
raise PortalDataError(code, message) from None
|
|
|
|
def read_channel_detail(
|
|
self,
|
|
month_key: str,
|
|
worksheet: str,
|
|
limit: int,
|
|
offset: int,
|
|
) -> Dict[str, Any]:
|
|
try:
|
|
return self._analytics.read_channel_detail(
|
|
month_key,
|
|
worksheet,
|
|
limit=limit,
|
|
offset=offset,
|
|
)
|
|
except Exception as error:
|
|
code = getattr(error, "code", "DATABASE_QUERY_FAILED")
|
|
raise PortalDataError(code, "渠道明细查询失败") from None
|
|
|
|
def _resolve_download(self, statement: str, value: object) -> ArtifactDescriptor:
|
|
connection = self._open()
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
cursor.execute(statement, (value,))
|
|
rows = list(cursor.fetchall())
|
|
if len(rows) != 1:
|
|
raise PortalDataError("DOWNLOAD_NOT_FOUND", "文件不存在或尚未生成")
|
|
row = rows[0]
|
|
descriptor = ArtifactDescriptor(
|
|
file_kind=str(row[0] or ""),
|
|
original_filename=str(row[1] or ""),
|
|
storage_key=str(row[2] or ""),
|
|
sha256=str(row[3] or "").lower(),
|
|
byte_size=int(row[4]),
|
|
mime_type=str(row[5] or "application/octet-stream"),
|
|
)
|
|
descriptor.validate()
|
|
return descriptor
|
|
except PortalDataError:
|
|
raise
|
|
except Exception as error:
|
|
if hasattr(error, "code") and str(getattr(error, "code")).startswith("DOWNLOAD_"):
|
|
raise PortalDataError(str(getattr(error, "code")), str(error)) from None
|
|
raise PortalDataError("DATABASE_QUERY_FAILED", "文件身份查询失败") from None
|
|
finally:
|
|
connection.close()
|
|
|
|
def resolve_daily_download(self, job_id: str) -> ArtifactDescriptor:
|
|
if not isinstance(job_id, str) or not 1 <= len(job_id) <= 128:
|
|
raise PortalDataError("DOWNLOAD_REQUEST_INVALID", "下载请求无效")
|
|
return self._resolve_download(DAILY_DOWNLOAD_SQL, job_id)
|
|
|
|
def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor:
|
|
if isinstance(report_id, bool) or not isinstance(report_id, int) or report_id < 1:
|
|
raise PortalDataError("DOWNLOAD_REQUEST_INVALID", "下载请求无效")
|
|
return self._resolve_download(MONTHLY_DOWNLOAD_SQL, report_id)
|