shiyuyun/wyndham-ARR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: sync latest ARR implementation

Browse Source
This commit is contained in:
Wyndham ARR
2026-07-31 15:11:42 +08:00
parent d6f8a747fa
commit bf7939dd1a
185 changed files with 17527 additions and 2260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

468
arr_web/repository.py
View File

@@ -13,6 +13,7 @@ from channel_analytics.postgres import (
PostgresAnalyticsRepository,
)
from arr_web.downloads import ArtifactDescriptor
from arr_web.job_trace import build_job_trace
TARGET_DATABASE = "booking_test"
@@ -26,10 +27,23 @@ class PortalDataError(RuntimeError):
class PortalRepository(Protocol):
def list_jobs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
def list_jobs(
self,
month_key: str,
limit: int = 50,
offset: int = 0,
) -> tuple[List[Dict[str, Any]], int]:
...
def list_monthly_runs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
def get_job_trace(self, job_id: str) -> Dict[str, Any]:
...
def list_monthly_runs(
self,
month_key: str,
limit: int = 50,
offset: int = 0,
) -> tuple[List[Dict[str, Any]], int]:
...
def list_months(self) -> List[Dict[str, Any]]:
@@ -61,10 +75,23 @@ class UnavailablePortalRepository:
def _raise() -> None:
raise PortalDataError("DATABASE_UNAVAILABLE", "数据库读取服务暂不可用")
def list_jobs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
def list_jobs(
self,
month_key: str,
limit: int = 50,
offset: int = 0,
) -> tuple[List[Dict[str, Any]], int]:
self._raise()
def list_monthly_runs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
def get_job_trace(self, job_id: str) -> Dict[str, Any]:
self._raise()
def list_monthly_runs(
self,
month_key: str,
limit: int = 50,
offset: int = 0,
) -> tuple[List[Dict[str, Any]], int]:
self._raise()
def list_months(self) -> List[Dict[str, Any]]:
@@ -126,7 +153,7 @@ SELECT
run.created_at,
run.updated_at,
run.finished_at,
source.original_filename,
run.uploaded_filename,
source.byte_size,
version.retained_rows,
version.version_no,
@@ -155,43 +182,195 @@ WHERE run.pipeline_type = 'opera_daily'
)
)
ORDER BY run.created_at DESC, run.id DESC
LIMIT %s
LIMIT %s OFFSET %s
""".strip()
JOBS_COUNT_SQL = """
SELECT count(*)
FROM ingestion.processing_runs AS run
WHERE run.pipeline_type = 'opera_daily'
AND (
(run.business_date >= %s AND run.business_date < %s)
OR (
run.business_date IS NULL
AND run.created_at >= %s
AND run.created_at < %s
)
)
""".strip()
JOB_TRACE_RUN_SQL = """
SELECT
run.id,
run.run_key,
run.run_status,
CASE run.run_status
WHEN 'accepted' THEN 'succeeded'
WHEN 'rejected' THEN 'failed'
ELSE run.run_status
END AS public_status,
run.result_delivery_mode,
run.business_date,
run.failure_code,
run.failure_message,
run.created_at,
run.updated_at,
run.validated_at,
run.finished_at,
run.uploaded_filename,
source.byte_size,
source.sha256,
run.requested_processor_version,
run.delivered_processor_version,
run.requested_rule_set_sha256,
run.delivered_rule_set_sha256,
run.result_schema_version
FROM ingestion.processing_runs AS run
JOIN ingestion.artifacts AS source
ON source.id = run.source_artifact_id
WHERE run.pipeline_type = 'opera_daily'
AND run.run_key = %s
""".strip()
JOB_TRACE_ATTEMPTS_SQL = """
SELECT
attempt.id,
attempt.attempt_no,
attempt.attempt_status,
attempt.remote_run_id,
attempt.failure_code,
attempt.failure_message,
attempt.created_at,
attempt.started_at,
attempt.finished_at
FROM ingestion.processing_attempts AS attempt
WHERE attempt.processing_run_id = %s
ORDER BY attempt.attempt_no, attempt.id
""".strip()
JOB_TRACE_DELIVERIES_SQL = """
SELECT
delivery.delivery_key,
attempt.attempt_no,
delivery.delivery_status,
delivery.result_status,
delivery.processor_version,
delivery.result_schema_version,
delivery.business_date,
delivery.daily_version_id,
delivery.failure_code,
delivery.failure_message,
delivery.received_at,
delivery.validated_at,
delivery.committed_at
FROM ingestion.processing_deliveries AS delivery
LEFT JOIN ingestion.processing_attempts AS attempt
ON attempt.id = delivery.attempt_id
AND attempt.processing_run_id = delivery.processing_run_id
WHERE delivery.processing_run_id = %s
ORDER BY delivery.received_at, delivery.id
""".strip()
JOB_TRACE_SUBMISSIONS_SQL = """
SELECT
submission.submission_key,
attempt.attempt_no,
submission.submission_status,
submission.contract_version,
submission.business_date,
submission.processor_version,
submission.result_schema_version,
submission.record_count,
submission.daily_version_id,
submission.failure_code,
submission.failure_message,
submission.created_at,
submission.validation_started_at,
submission.finished_at
FROM ingestion.result_submissions AS submission
JOIN ingestion.processing_attempts AS attempt
ON attempt.id = submission.attempt_id
AND attempt.processing_run_id = submission.processing_run_id
WHERE submission.processing_run_id = %s
ORDER BY submission.created_at, submission.id
""".strip()
JOB_TRACE_VERSIONS_SQL = """
SELECT
version.id,
version.business_date,
version.version_no,
version.version_status,
version.result_delivery_mode,
version.source_rows,
version.retained_rows,
version.excluded_rate_code_rows,
version.duplicate_rows,
version.validation_failed_rows,
version.price_unmatched_rows,
version.failure_code,
version.failure_message,
version.created_at,
version.validated_at,
version.activated_at,
version.superseded_at
FROM finance.daily_versions AS version
WHERE version.processing_run_id = %s
ORDER BY COALESCE(version.version_no, 0), version.id
""".strip()
JOB_TRACE_OUTBOX_SQL = """
SELECT
event.event_key,
event.event_type,
event.publish_status,
event.publish_attempts,
event.available_at,
event.created_at,
event.published_at,
event.last_error_code
FROM ingestion.outbox_events AS event
WHERE event.aggregate_type = 'processing_run'
AND event.aggregate_id = %s
ORDER BY event.created_at, event.id
""".strip()
MONTHLY_RUNS_SQL = """
WITH selected_versions AS (
SELECT
current_version.business_date,
current_version.daily_version_id,
current_version.activated_at
FROM finance.current_daily_versions AS current_version
WHERE current_version.business_date >= %s
AND current_version.business_date < %s
),
selected_facts AS (
SELECT fact.id, fact.channel_key
FROM finance.v_active_daily_facts AS fact
WHERE fact.business_date >= %s
AND fact.business_date < %s
)
SELECT
NULL::bigint AS report_id,
min(selected_versions.business_date)::date AS period_start,
max(selected_versions.business_date)::date AS as_of_date,
1::integer AS version_no,
'source_ready'::text AS report_status,
true AS is_current,
min(selected_versions.activated_at) AS created_at,
max(selected_versions.activated_at) AS updated_at,
NULL::text AS failure_code,
NULL::text AS original_filename,
NULL::text AS artifact_sha256,
(SELECT count(DISTINCT channel_key) FROM selected_facts) AS channel_count,
(SELECT count(*) FROM selected_facts) AS row_count
FROM selected_versions
HAVING count(*) > 0
LIMIT %s
run.id AS report_id,
run.period_start,
run.as_of_date AS max_arrival_date,
run.version_no,
run.report_status,
run.report_status = 'active' AS is_current,
run.created_at,
run.updated_at,
run.failure_code,
artifact.original_filename,
artifact.sha256,
run.channel_count,
run.row_count
FROM reporting.monthly_runs AS run
LEFT JOIN ingestion.artifacts AS artifact
ON artifact.id = run.workbook_artifact_id
WHERE run.period_start = %s
ORDER BY run.version_no DESC, run.id DESC
LIMIT %s OFFSET %s
""".strip()
MONTHLY_RUNS_COUNT_SQL = """
SELECT count(*)
FROM reporting.monthly_runs AS run
WHERE run.period_start = %s
""".strip()
@@ -213,6 +392,23 @@ WHERE run.run_key = %s
AND version.version_status IN ('active', 'superseded')
""".strip()
MONTHLY_DOWNLOAD_SQL = """
SELECT
artifact.artifact_kind,
artifact.original_filename,
artifact.object_key,
artifact.sha256,
artifact.byte_size,
artifact.mime_type
FROM reporting.monthly_runs AS run
JOIN ingestion.artifacts AS artifact
ON artifact.id = run.workbook_artifact_id
WHERE run.id = %s
AND run.report_status IN ('active', 'superseded')
AND artifact.artifact_kind = 'monthly_xlsx'
AND artifact.storage_provider = 'local'
""".strip()
@dataclass(frozen=True)
class PostgresPortalRepository:
@@ -266,14 +462,25 @@ class PostgresPortalRepository:
end = date(year + (month == 12), 1 if month == 12 else month + 1, 1)
return start, end
def list_jobs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
def list_jobs(
self,
month_key: str,
limit: int = 50,
offset: int = 0,
) -> tuple[List[Dict[str, Any]], int]:
start, end = self._month_bounds(month_key)
connection = self._open()
try:
with connection.transaction():
with connection.cursor() as cursor:
self._begin(cursor)
cursor.execute(JOBS_SQL, (start, end, start, end, limit))
cursor.execute(JOBS_COUNT_SQL, (start, end, start, end))
count_row = cursor.fetchone()
total = int(count_row[0] or 0) if count_row else 0
cursor.execute(
JOBS_SQL,
(start, end, start, end, limit, offset),
)
rows = list(cursor.fetchall())
return [
{
@@ -284,7 +491,7 @@ class PostgresPortalRepository:
"created_at": _iso(row[4]),
"updated_at": _iso(row[5]),
"finished_at": _iso(row[6]),
"filename": str(row[7]),
"filename": str(row[7]) if row[7] else None,
"byte_size": int(row[8]),
"output_rows": int(row[9] or 0),
"version_no": int(row[10]) if row[10] is not None else None,
@@ -292,7 +499,7 @@ class PostgresPortalRepository:
"no_of_rooms": int(row[12] or 0),
}
for row in rows
]
], total
except PortalDataError:
raise
except Exception:
@@ -300,20 +507,186 @@ class PostgresPortalRepository:
finally:
connection.close()
def list_monthly_runs(self, month_key: str, limit: int = 100) -> List[Dict[str, Any]]:
def get_job_trace(self, job_id: str) -> Dict[str, Any]:
connection = self._open()
try:
with connection.transaction():
with connection.cursor() as cursor:
self._begin(cursor)
cursor.execute(JOB_TRACE_RUN_SQL, (job_id,))
row = cursor.fetchone()
if row is None:
raise PortalDataError("JOB_NOT_FOUND", "任务不存在或已过保留期")
run = {
"id": int(row[0]),
"job_id": str(row[1]),
"run_status": str(row[2]),
"public_status": str(row[3]),
"delivery_mode": str(row[4]),
"business_date": row[5],
"failure_code": str(row[6]) if row[6] else None,
"failure_message": str(row[7]) if row[7] else None,
"created_at": row[8],
"updated_at": row[9],
"validated_at": row[10],
"finished_at": row[11],
"filename": str(row[12]) if row[12] else None,
"byte_size": int(row[13]),
"source_sha256": str(row[14]),
"requested_processor_version": (
str(row[15]) if row[15] else None
),
"delivered_processor_version": (
str(row[16]) if row[16] else None
),
"requested_rule_set_sha256": (
str(row[17]) if row[17] else None
),
"delivered_rule_set_sha256": (
str(row[18]) if row[18] else None
),
"result_schema_version": str(row[19]) if row[19] else None,
}
run_id = int(row[0])
cursor.execute(JOB_TRACE_ATTEMPTS_SQL, (run_id,))
attempts = [
{
"attempt_id": int(item[0]),
"attempt_no": int(item[1]),
"attempt_status": str(item[2]),
"remote_run_id": str(item[3]) if item[3] else None,
"failure_code": str(item[4]) if item[4] else None,
"failure_message": str(item[5]) if item[5] else None,
"created_at": item[6],
"started_at": item[7],
"finished_at": item[8],
}
for item in cursor.fetchall()
]
cursor.execute(JOB_TRACE_DELIVERIES_SQL, (run_id,))
deliveries = [
{
"delivery_key": str(item[0]),
"attempt_no": int(item[1]) if item[1] is not None else None,
"delivery_status": str(item[2]),
"result_status": str(item[3]),
"processor_version": str(item[4]),
"result_schema_version": str(item[5]),
"business_date": item[6],
"daily_version_id": (
int(item[7]) if item[7] is not None else None
),
"failure_code": str(item[8]) if item[8] else None,
"failure_message": str(item[9]) if item[9] else None,
"received_at": item[10],
"validated_at": item[11],
"committed_at": item[12],
}
for item in cursor.fetchall()
]
cursor.execute(JOB_TRACE_SUBMISSIONS_SQL, (run_id,))
submissions = [
{
"submission_key": str(item[0]),
"attempt_no": int(item[1]),
"submission_status": str(item[2]),
"contract_version": str(item[3]),
"business_date": item[4],
"processor_version": str(item[5]),
"result_schema_version": str(item[6]),
"record_count": int(item[7]),
"daily_version_id": (
int(item[8]) if item[8] is not None else None
),
"failure_code": str(item[9]) if item[9] else None,
"failure_message": str(item[10]) if item[10] else None,
"created_at": item[11],
"validation_started_at": item[12],
"finished_at": item[13],
}
for item in cursor.fetchall()
]
cursor.execute(JOB_TRACE_VERSIONS_SQL, (run_id,))
versions = [
{
"daily_version_id": int(item[0]),
"business_date": item[1],
"version_no": (
int(item[2]) if item[2] is not None else None
),
"version_status": str(item[3]),
"result_delivery_mode": str(item[4]),
"source_rows": int(item[5]),
"retained_rows": int(item[6]),
"excluded_rate_code_rows": int(item[7]),
"duplicate_rows": int(item[8]),
"validation_failed_rows": int(item[9]),
"price_unmatched_rows": int(item[10]),
"failure_code": str(item[11]) if item[11] else None,
"failure_message": str(item[12]) if item[12] else None,
"created_at": item[13],
"validated_at": item[14],
"activated_at": item[15],
"superseded_at": item[16],
}
for item in cursor.fetchall()
]
cursor.execute(JOB_TRACE_OUTBOX_SQL, (run_id,))
outbox_events = [
{
"event_key": str(item[0]),
"event_type": str(item[1]),
"publish_status": str(item[2]),
"publish_attempts": int(item[3]),
"available_at": item[4],
"created_at": item[5],
"published_at": item[6],
"last_error_code": str(item[7]) if item[7] else None,
}
for item in cursor.fetchall()
]
return build_job_trace(
run,
attempts=attempts,
deliveries=deliveries,
submissions=submissions,
versions=versions,
outbox_events=outbox_events,
)
except PortalDataError:
raise
except Exception:
raise PortalDataError("DATABASE_QUERY_FAILED", "任务日志查询失败") from None
finally:
connection.close()
def list_monthly_runs(
self,
month_key: str,
limit: int = 50,
offset: int = 0,
) -> tuple[List[Dict[str, Any]], int]:
start, _end = self._month_bounds(month_key)
connection = self._open()
try:
with connection.transaction():
with connection.cursor() as cursor:
self._begin(cursor)
_start, end = self._month_bounds(month_key)
cursor.execute(MONTHLY_RUNS_SQL, (start, end, start, end, limit))
cursor.execute(MONTHLY_RUNS_COUNT_SQL, (start,))
count_row = cursor.fetchone()
total = int(count_row[0] or 0) if count_row else 0
cursor.execute(MONTHLY_RUNS_SQL, (start, limit, offset))
rows = list(cursor.fetchall())
return [
{
"report_id": int(row[0]) if row[0] is not None else None,
"month_key": start.strftime("%Y-%m"),
"max_arrival_date": _iso(row[2]),
"as_of_date": _iso(row[2]),
"version_no": int(row[3]),
"status": str(row[4]),
@@ -327,7 +700,7 @@ class PostgresPortalRepository:
"row_count": int(row[12]),
}
for row in rows
]
], total
except PortalDataError:
raise
except Exception:
@@ -406,7 +779,4 @@ class PostgresPortalRepository:
def resolve_monthly_download(self, report_id: int) -> ArtifactDescriptor:
if isinstance(report_id, bool) or not isinstance(report_id, int) or report_id < 1:
raise PortalDataError("DOWNLOAD_REQUEST_INVALID", "下载请求无效")
raise PortalDataError(
"DOWNLOAD_NOT_FOUND",
"月报行与版本不在数据库重复保存；请从本次生成结果下载",
)
return self._resolve_download(MONTHLY_DOWNLOAD_SQL, report_id)