shiyuyun/wyndham-ARR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: sync latest ARR implementation

Browse Source
This commit is contained in:
Wyndham ARR
2026-07-31 15:11:42 +08:00
parent d6f8a747fa
commit bf7939dd1a
185 changed files with 17527 additions and 2260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

483
monthly_reports/repository.py
View File

@@ -1,4 +1,4 @@
"""Read-only PostgreSQL snapshot source for generated monthly reports."""
"""PostgreSQL snapshot, publication metadata, and ARRIVAL-derived report scope."""
from __future__ import annotations
@@ -6,8 +6,9 @@ import os
import re
import time
import hashlib
import json
from dataclasses import dataclass
from datetime import date
from datetime import date, timedelta
from decimal import Decimal
from pathlib import PurePosixPath
from typing import Any, Callable, List, Optional, Protocol, Sequence, Tuple
@@ -32,6 +33,9 @@ FALLBACK_DATABASE_ENV = "ARR_DATABASE_URL"
TRANSIENT_SQLSTATES = frozenset({"23505", "40001", "40P01", "55P03"})
MAX_TRANSACTION_ATTEMPTS = 4
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
SAFE_CODE_RE = re.compile(r"^[A-Z][A-Z0-9_]{0,63}$")
LOCAL_STORAGE_PROVIDER = "local"
LOCAL_BUCKET_ALIAS = "arr-project-root"
SOURCE_FACTS_SQL = """
@@ -63,7 +67,7 @@ SELECT
FROM finance.v_active_daily_facts AS facts
JOIN finance.daily_versions AS versions
ON versions.id = facts.daily_version_id
WHERE versions.business_date BETWEEN %s AND %s
WHERE facts.arrival BETWEEN %s AND %s
ORDER BY facts.id
""".strip()
@@ -104,6 +108,25 @@ ORDER BY
metrics.channel_key
""".strip()
TRIGGER_ARRIVAL_SCOPE_SQL = """
SELECT min(record.arrival), max(record.arrival)
FROM finance.daily_versions AS version
JOIN ingestion.processing_runs AS run
ON run.id = version.processing_run_id
JOIN finance.daily_records AS record
ON record.daily_version_id = version.id
WHERE version.id = %s
AND run.run_status = 'accepted'
AND record.outcome = 'retained'
""".strip()
MONTH_MAX_ARRIVAL_SQL = """
SELECT max(fact.arrival)
FROM finance.v_active_daily_facts AS fact
WHERE fact.arrival >= %s
AND fact.arrival < %s
""".strip()
class RepositoryError(RuntimeError):
def __init__(self, code: str, safe_message: str):
@@ -129,15 +152,6 @@ class DatabaseConfig:
return cls(dsn=dsn)
@dataclass(frozen=True)
class ReservedReport:
report_version_id: int
version_no: int
period_start: Optional[date] = None
as_of_date: Optional[date] = None
daily_versions: Tuple[DailyVersionPin, ...] = tuple()
@dataclass(frozen=True)
class FileMetadata:
file_kind: str
@@ -168,6 +182,32 @@ class FileMetadata:
)
@dataclass(frozen=True)
class ReservedReport:
report_version_id: int
version_no: int
period_start: Optional[date] = None
as_of_date: Optional[date] = None
daily_versions: Tuple[DailyVersionPin, ...] = tuple()
source_snapshot_sha256: Optional[str] = None
report_status: str = "reserved"
published_artifact: Optional[FileMetadata] = None
@property
def already_published(self) -> bool:
return (
self.report_status in {"active", "superseded"}
and self.published_artifact is not None
)
@dataclass(frozen=True)
class DerivedMonthlyRequest:
report_year: int
report_month: int
as_of_date: date
class ReportRepository(Protocol):
def load_snapshot(self, year: int, month: int, as_of_date: date) -> MonthlySnapshot:
...
@@ -180,6 +220,7 @@ class ReportRepository(Protocol):
reservation: ReservedReport,
artifact: FileMetadata,
result_json: FileMetadata,
semantic_sha256: str,
) -> None:
...
@@ -248,9 +289,11 @@ class PostgresReportRepository:
)
@classmethod
def _begin_serializable(cls, cursor: Any) -> None:
def _begin_serializable(cls, cursor: Any, *, read_only: bool = False) -> None:
cursor.execute(
"SET TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY"
if read_only
else "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"
)
cursor.execute("SET LOCAL lock_timeout = '10s'")
cursor.execute("SET LOCAL statement_timeout = '30s'")
@@ -260,6 +303,8 @@ class PostgresReportRepository:
self,
operation: Callable[[Any], Any],
safe_message: str,
*,
read_only: bool = False,
) -> Any:
for attempt_index in range(MAX_TRANSACTION_ATTEMPTS):
connection = self._open()
@@ -267,7 +312,7 @@ class PostgresReportRepository:
try:
with connection.transaction():
with connection.cursor() as cursor:
self._begin_serializable(cursor)
self._begin_serializable(cursor, read_only=read_only)
return operation(cursor)
except RepositoryError:
raise
@@ -363,6 +408,54 @@ class PostgresReportRepository:
finally:
connection.close()
def derive_monthly_request(
self,
daily_version_id: int,
) -> Optional[DerivedMonthlyRequest]:
if isinstance(daily_version_id, bool) or daily_version_id < 1:
raise RepositoryError(
ErrorCode.REQUEST_INVALID,
"daily version identity is invalid",
)
def operation(cursor: Any) -> Optional[DerivedMonthlyRequest]:
cursor.execute(TRIGGER_ARRIVAL_SCOPE_SQL, (daily_version_id,))
row = cursor.fetchone()
if not row or row[0] is None or row[1] is None:
return None
first_arrival, last_arrival = row[0], row[1]
if (
first_arrival.year != last_arrival.year
or first_arrival.month != last_arrival.month
):
raise RepositoryError(
ErrorCode.SOURCE_INVALID,
"one daily version spans multiple ARRIVAL months",
)
period_start, period_end = month_bounds(
first_arrival.year,
first_arrival.month,
)
cursor.execute(
MONTH_MAX_ARRIVAL_SQL,
(period_start, period_end + timedelta(days=1)),
)
latest = cursor.fetchone()
as_of_date = latest[0] if latest else None
if as_of_date is None:
return None
return DerivedMonthlyRequest(
report_year=as_of_date.year,
report_month=as_of_date.month,
as_of_date=as_of_date,
)
return self._run_serializable(
operation,
"monthly ARRIVAL scope query failed",
read_only=True,
)
@staticmethod
def _lock_scope(cursor: Any, period_start: date, period_end: date) -> None:
lock_key = f"monthly_channel:{period_start.isoformat()}:{period_end.isoformat()}:*"
@@ -395,35 +488,170 @@ class PostgresReportRepository:
"daily source versions changed while the monthly report was being generated",
)
@staticmethod
def _source_snapshot_sha256(report: MonthlyReport) -> str:
identity = {
"period_start": date(
report.report_year,
report.report_month,
1,
).isoformat(),
"as_of_date": report.as_of_date.isoformat(),
"processor_version": PROCESSOR_VERSION,
"rule_set_sha256": rule_set_sha256(),
"result_schema_version": RESULT_SCHEMA_VERSION,
"daily_versions": [
[pin.business_date.isoformat(), pin.daily_version_id]
for pin in report.daily_versions
],
"channel_manifest": [list(item) for item in report.channel_manifest],
}
return hashlib.sha256(
json.dumps(
identity,
ensure_ascii=False,
separators=(",", ":"),
sort_keys=True,
).encode("utf-8")
).hexdigest()
@staticmethod
def _published_artifact(row: Sequence[Any]) -> Optional[FileMetadata]:
if row[3] is None:
return None
metadata = FileMetadata(
file_kind=str(row[3]),
original_filename=str(row[4]),
storage_key=str(row[5]),
sha256=str(row[6]),
byte_size=int(row[7]),
mime_type=str(row[8] or "application/octet-stream"),
)
metadata.validate()
return metadata
def reserve_report(self, report: MonthlyReport) -> ReservedReport:
period_start, period_end = month_bounds(report.report_year, report.report_month)
snapshot_sha256 = self._source_snapshot_sha256(report)
def operation(cursor: Any) -> ReservedReport:
self._lock_scope(cursor, period_start, period_end)
self._validate_pins_current(cursor, report)
identity = {
"period_start": period_start.isoformat(),
"period_end": period_end.isoformat(),
"as_of_date": report.as_of_date.isoformat(),
"processor_version": PROCESSOR_VERSION,
"rule_set_sha256": rule_set_sha256(),
"result_schema_version": RESULT_SCHEMA_VERSION,
"daily_versions": [
[pin.business_date.isoformat(), pin.daily_version_id]
for pin in report.daily_versions
],
"channel_manifest": [list(item) for item in report.channel_manifest],
}
digest = hashlib.sha256(
repr(sorted(identity.items())).encode("utf-8")
).hexdigest()
generation_id = int(digest[:12], 16)
cursor.execute(
"""
SELECT
run.id,
run.version_no,
run.report_status,
artifact.artifact_kind,
artifact.original_filename,
artifact.object_key,
artifact.sha256,
artifact.byte_size,
artifact.mime_type
FROM reporting.monthly_runs AS run
LEFT JOIN ingestion.artifacts AS artifact
ON artifact.id = run.workbook_artifact_id
WHERE run.period_start = %s
AND run.source_snapshot_sha256 = %s
FOR UPDATE OF run
""",
(period_start, snapshot_sha256),
)
existing = cursor.fetchone()
if existing:
status = str(existing[2])
if status == "failed":
cursor.execute(
"""
UPDATE reporting.monthly_runs
SET report_status = 'reserved',
failure_code = NULL,
failure_message = NULL,
failed_at = NULL,
updated_at = now()
WHERE id = %s
""",
(int(existing[0]),),
)
status = "reserved"
return ReservedReport(
report_version_id=int(existing[0]),
version_no=int(existing[1]),
period_start=period_start,
as_of_date=report.as_of_date,
daily_versions=report.daily_versions,
source_snapshot_sha256=snapshot_sha256,
report_status=status,
published_artifact=self._published_artifact(existing),
)
cursor.execute(
"""
SELECT COALESCE(max(version_no), 0) + 1
FROM reporting.monthly_runs
WHERE period_start = %s
""",
(period_start,),
)
version_no = int(cursor.fetchone()[0])
cursor.execute(
"""
INSERT INTO reporting.monthly_runs (
period_start,
as_of_date,
version_no,
source_snapshot_sha256,
report_status,
processor_version,
rule_set_sha256,
result_schema_version,
row_count,
channel_count
)
VALUES (%s, %s, %s, %s, 'reserved', %s, %s, %s, %s, %s)
RETURNING id
""",
(
period_start,
report.as_of_date,
version_no,
snapshot_sha256,
PROCESSOR_VERSION,
rule_set_sha256(),
RESULT_SCHEMA_VERSION,
report.row_count,
len(report.channel_manifest),
),
)
report_id = int(cursor.fetchone()[0])
for pin in report.daily_versions:
cursor.execute(
"""
INSERT INTO reporting.monthly_run_daily_versions (
report_id, business_date, daily_version_id
)
VALUES (%s, %s, %s)
""",
(report_id, pin.business_date, pin.daily_version_id),
)
for worksheet, worksheet_order, row_count in report.channel_manifest:
cursor.execute(
"""
INSERT INTO reporting.monthly_channel_manifest (
report_id, worksheet, worksheet_order, row_count
)
VALUES (%s, %s, %s, %s)
""",
(report_id, worksheet, worksheet_order, row_count),
)
return ReservedReport(
generation_id,
generation_id,
period_start,
report.as_of_date,
report.daily_versions,
report_version_id=report_id,
version_no=version_no,
period_start=period_start,
as_of_date=report.as_of_date,
daily_versions=report.daily_versions,
source_snapshot_sha256=snapshot_sha256,
)
return self._run_serializable(
@@ -431,14 +659,82 @@ class PostgresReportRepository:
"monthly report source reservation failed",
)
@staticmethod
def _ensure_local_artifact(cursor: Any, metadata: FileMetadata) -> int:
cursor.execute(
"""
SELECT
id,
artifact_kind,
original_filename,
sha256,
byte_size,
mime_type
FROM ingestion.artifacts
WHERE storage_provider = %s
AND bucket_alias = %s
AND object_key = %s
AND object_version_id IS NULL
FOR SHARE
""",
(LOCAL_STORAGE_PROVIDER, LOCAL_BUCKET_ALIAS, metadata.storage_key),
)
row = cursor.fetchone()
if row:
if (
str(row[1]) != metadata.file_kind
or str(row[2]) != metadata.original_filename
or str(row[3]) != metadata.sha256
or int(row[4]) != metadata.byte_size
or str(row[5] or "") != metadata.mime_type
):
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"stored monthly artifact identity conflicts",
)
return int(row[0])
cursor.execute(
"""
INSERT INTO ingestion.artifacts (
artifact_kind,
storage_provider,
bucket_alias,
object_key,
original_filename,
sha256,
byte_size,
mime_type
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id
""",
(
metadata.file_kind,
LOCAL_STORAGE_PROVIDER,
LOCAL_BUCKET_ALIAS,
metadata.storage_key,
metadata.original_filename,
metadata.sha256,
metadata.byte_size,
metadata.mime_type,
),
)
return int(cursor.fetchone()[0])
def activate_report(
self,
reservation: ReservedReport,
artifact: FileMetadata,
result_json: FileMetadata,
semantic_sha256: str,
) -> None:
artifact.validate()
result_json.validate()
if not SHA256_RE.fullmatch(semantic_sha256):
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"monthly semantic identity is invalid",
)
if reservation.period_start is None or reservation.as_of_date is None:
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
@@ -461,6 +757,95 @@ class PostgresReportRepository:
ErrorCode.SOURCE_SNAPSHOT_STALE,
"daily source versions changed before publication",
)
cursor.execute(
"""
SELECT report_status, source_snapshot_sha256
FROM reporting.monthly_runs
WHERE id = %s
AND period_start = %s
AND version_no = %s
FOR UPDATE
""",
(
reservation.report_version_id,
period_start,
reservation.version_no,
),
)
run = cursor.fetchone()
if (
not run
or reservation.source_snapshot_sha256 is None
or str(run[1]) != reservation.source_snapshot_sha256
):
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"monthly report reservation identity changed",
)
workbook_artifact_id = self._ensure_local_artifact(cursor, artifact)
result_artifact_id = self._ensure_local_artifact(cursor, result_json)
if str(run[0]) in {"active", "superseded"}:
cursor.execute(
"""
SELECT workbook_artifact_id, result_artifact_id, semantic_sha256
FROM reporting.monthly_runs
WHERE id = %s
""",
(reservation.report_version_id,),
)
published = cursor.fetchone()
if (
not published
or int(published[0]) != workbook_artifact_id
or int(published[1]) != result_artifact_id
or str(published[2]) != semantic_sha256
):
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"published monthly report identity conflicts",
)
return
if str(run[0]) != "reserved":
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"monthly report reservation is not publishable",
)
cursor.execute(
"""
UPDATE reporting.monthly_runs
SET report_status = 'superseded',
superseded_at = now(),
updated_at = now()
WHERE period_start = %s
AND report_status = 'active'
AND id <> %s
""",
(period_start, reservation.report_version_id),
)
cursor.execute(
"""
UPDATE reporting.monthly_runs
SET report_status = 'active',
workbook_artifact_id = %s,
result_artifact_id = %s,
semantic_sha256 = %s,
published_at = now(),
updated_at = now()
WHERE id = %s
AND report_status = 'reserved'
""",
(
workbook_artifact_id,
result_artifact_id,
semantic_sha256,
reservation.report_version_id,
),
)
if cursor.rowcount != 1:
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"monthly report activation did not complete",
)
self._run_serializable(
operation,
@@ -473,6 +858,26 @@ class PostgresReportRepository:
code: str,
safe_message: str,
) -> None:
# Monthly rows and versions are intentionally not persisted. The
# caller's privacy-minimized result JSON is the failure record.
_ = (reservation, code, safe_message)
if not SAFE_CODE_RE.fullmatch(code):
code = ErrorCode.INTERNAL_ERROR
message = str(safe_message).strip()[:500] or "monthly report processing failed"
def operation(cursor: Any) -> None:
cursor.execute(
"""
UPDATE reporting.monthly_runs
SET report_status = 'failed',
failure_code = %s,
failure_message = %s,
failed_at = now(),
updated_at = now()
WHERE id = %s
AND report_status = 'reserved'
""",
(code, message, reservation.report_version_id),
)
self._run_serializable(
operation,
"monthly report failure state could not be recorded",
)