shiyuyun/wyndham-ARR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: sync latest ARR implementation

Browse Source
This commit is contained in:
Wyndham ARR
2026-07-31 15:11:42 +08:00
parent d6f8a747fa
commit bf7939dd1a
185 changed files with 17527 additions and 2260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
monthly_reports/README.md
View File

@@ -2,17 +2,27 @@
`monthly_reports` is an ordinary deterministic program. It does not call an Agent/Skill and does not read a prior monthly workbook.
Flow:
Automatic flow:
1. Open one `REPEATABLE READ READ ONLY` snapshot of `finance.v_active_daily_facts`, current daily-version pins and current channel metrics.
2. Build the channel worksheets and the 20-column monthly rows from those facts.
3. Reserve a deterministic generation identity and recheck that all daily pins are still current.
4. Generate a typed, formula-free XLSX; reopen and validate it.
5. Recheck current pins before publishing the file to the controlled output/OSS layer.
1. A dedicated worker leases one successful `arr.daily_version_committed` outbox event with `FOR UPDATE SKIP LOCKED`.
2. It looks up that version's retained `ARRIVAL` values in PostgreSQL. The affected month and report watermark come from those facts; the XML filename and wall clock are ignored.
3. It opens a repeatable-read snapshot of current Finance facts and daily-version pins for that month. `as_of_date` is the greatest `ARRIVAL` actually included in the snapshot.
4. It reserves an idempotent publication identity in `reporting.monthly_runs` with immutable daily lineage and channel manifest rows.
5. It generates the XLSX/result JSON, reopens the workbook, validates all values and formulas, and rechecks that the Finance pins are still current.
6. It registers both local artifacts and atomically activates the report. Only then is the outbox event marked `published`; transient failures are retried and exhausted events become `dead`.
Monthly rows and report versions are intentionally not persisted in PostgreSQL. The database remains the source of truth; generated XLSX/result files can be retained as artifacts outside the business fact model.
PostgreSQL stores publication metadata, version identity, lineage and artifact identities—not duplicate monthly business or guest rows. The portal lists `active`/`superseded` runs and downloads only registered artifacts after path, size and SHA-256 checks.
Run:
Run the dedicated worker:
```bash
python3 -m monthly_reports.worker \
--db-config /absolute/path/to/booking-test-db.env \
--node-binary /absolute/path/to/node \
--artifact-tool-module /absolute/path/to/artifact_tool.mjs
```
The CLI generation command remains a controlled recovery/diagnostic entrypoint:
```bash
python3 -m monthly_reports generate \
@@ -24,4 +34,4 @@ python3 -m monthly_reports generate \
The program reads `MONTHLY_REPORT_DATABASE_URL`, falling back to `ARR_DATABASE_URL`. The configured database must be `booking_test`; SuperAgent must never receive the DSN.
`TOTAL PRICE` is copied from the validated daily fact. `Booking Room` enrichment is a separate booking lookup and must not be used to recalculate price, dates or actual room count.
Every XLSX data-row `TOTAL PRICE` cell contains the exact row-relative formula `=R[row]*C[row]*G[row]`, meaning `REAL PRICE × NIGHTS × NO_OF_ROOMS`. The stored Finance `total_price` remains an independent audit expectation. `Booking Room` enrichment must not be used to recalculate price, dates or actual room count.

2
monthly_reports/contracts.py
View File

@@ -219,7 +219,7 @@ def rule_set_sha256() -> str:
"lexical_fallback",
],
"filename": "各渠道情况-YYYY年MM月-更新至M.D.xlsx",
"formula_policy": "no_formulas",
"formula_policy": "total_price_equals_real_price_times_nights_times_rooms_v1",
}
return hashlib.sha256(
json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8")

5
monthly_reports/core.py
View File

@@ -245,6 +245,11 @@ def build_monthly_report(
raise MonthlyReportError(ErrorCode.SOURCE_INVALID, "monthly source rows are duplicated")
for fact in facts:
_validate_fact(fact, start, as_of_date)
if facts and max(fact.arrival for fact in facts) != as_of_date:
raise MonthlyReportError(
ErrorCode.SOURCE_INVALID,
"monthly as-of date must equal the greatest included ARRIVAL",
)
pins = _validate_pins(facts, snapshot.daily_versions, start, as_of_date)
_validate_observations(facts, snapshot.channel_observations, pins)
ordered_channels = channel_order(snapshot)

9
monthly_reports/publishing.py
View File

@@ -158,7 +158,7 @@ class ArtifactToolBuilder:
or summary.get("as_of_date") != report.as_of_date.isoformat()
or summary.get("sheet_names") != expected_names
or summary.get("row_counts") != expected_rows
or summary.get("formula_count") != 0
or summary.get("formula_count") != report.row_count
or summary.get("preview_count") != len(report.channels)
or not isinstance(summary.get("semantic_sha256"), str)
or len(summary["semantic_sha256"]) != 64
@@ -356,7 +356,12 @@ class AtomicReportPublisher:
byte_size=result_path.stat().st_size,
mime_type="application/json",
)
repository.activate_report(reservation, artifact, result_json)
repository.activate_report(
reservation,
artifact,
result_json,
str(built.summary.get("semantic_sha256", "")),
)
return PublicationOutcome(
latest_path=latest_path,
latest_result_path=latest_result_path,

483
monthly_reports/repository.py
View File

@@ -1,4 +1,4 @@
"""Read-only PostgreSQL snapshot source for generated monthly reports."""
"""PostgreSQL snapshot, publication metadata, and ARRIVAL-derived report scope."""
from __future__ import annotations
@@ -6,8 +6,9 @@ import os
import re
import time
import hashlib
import json
from dataclasses import dataclass
from datetime import date
from datetime import date, timedelta
from decimal import Decimal
from pathlib import PurePosixPath
from typing import Any, Callable, List, Optional, Protocol, Sequence, Tuple
@@ -32,6 +33,9 @@ FALLBACK_DATABASE_ENV = "ARR_DATABASE_URL"
TRANSIENT_SQLSTATES = frozenset({"23505", "40001", "40P01", "55P03"})
MAX_TRANSACTION_ATTEMPTS = 4
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
SAFE_CODE_RE = re.compile(r"^[A-Z][A-Z0-9_]{0,63}$")
LOCAL_STORAGE_PROVIDER = "local"
LOCAL_BUCKET_ALIAS = "arr-project-root"
SOURCE_FACTS_SQL = """
@@ -63,7 +67,7 @@ SELECT
FROM finance.v_active_daily_facts AS facts
JOIN finance.daily_versions AS versions
ON versions.id = facts.daily_version_id
WHERE versions.business_date BETWEEN %s AND %s
WHERE facts.arrival BETWEEN %s AND %s
ORDER BY facts.id
""".strip()
@@ -104,6 +108,25 @@ ORDER BY
metrics.channel_key
""".strip()
TRIGGER_ARRIVAL_SCOPE_SQL = """
SELECT min(record.arrival), max(record.arrival)
FROM finance.daily_versions AS version
JOIN ingestion.processing_runs AS run
ON run.id = version.processing_run_id
JOIN finance.daily_records AS record
ON record.daily_version_id = version.id
WHERE version.id = %s
AND run.run_status = 'accepted'
AND record.outcome = 'retained'
""".strip()
MONTH_MAX_ARRIVAL_SQL = """
SELECT max(fact.arrival)
FROM finance.v_active_daily_facts AS fact
WHERE fact.arrival >= %s
AND fact.arrival < %s
""".strip()
class RepositoryError(RuntimeError):
def __init__(self, code: str, safe_message: str):
@@ -129,15 +152,6 @@ class DatabaseConfig:
return cls(dsn=dsn)
@dataclass(frozen=True)
class ReservedReport:
report_version_id: int
version_no: int
period_start: Optional[date] = None
as_of_date: Optional[date] = None
daily_versions: Tuple[DailyVersionPin, ...] = tuple()
@dataclass(frozen=True)
class FileMetadata:
file_kind: str
@@ -168,6 +182,32 @@ class FileMetadata:
)
@dataclass(frozen=True)
class ReservedReport:
report_version_id: int
version_no: int
period_start: Optional[date] = None
as_of_date: Optional[date] = None
daily_versions: Tuple[DailyVersionPin, ...] = tuple()
source_snapshot_sha256: Optional[str] = None
report_status: str = "reserved"
published_artifact: Optional[FileMetadata] = None
@property
def already_published(self) -> bool:
return (
self.report_status in {"active", "superseded"}
and self.published_artifact is not None
)
@dataclass(frozen=True)
class DerivedMonthlyRequest:
report_year: int
report_month: int
as_of_date: date
class ReportRepository(Protocol):
def load_snapshot(self, year: int, month: int, as_of_date: date) -> MonthlySnapshot:
...
@@ -180,6 +220,7 @@ class ReportRepository(Protocol):
reservation: ReservedReport,
artifact: FileMetadata,
result_json: FileMetadata,
semantic_sha256: str,
) -> None:
...
@@ -248,9 +289,11 @@ class PostgresReportRepository:
)
@classmethod
def _begin_serializable(cls, cursor: Any) -> None:
def _begin_serializable(cls, cursor: Any, *, read_only: bool = False) -> None:
cursor.execute(
"SET TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY"
if read_only
else "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"
)
cursor.execute("SET LOCAL lock_timeout = '10s'")
cursor.execute("SET LOCAL statement_timeout = '30s'")
@@ -260,6 +303,8 @@ class PostgresReportRepository:
self,
operation: Callable[[Any], Any],
safe_message: str,
*,
read_only: bool = False,
) -> Any:
for attempt_index in range(MAX_TRANSACTION_ATTEMPTS):
connection = self._open()
@@ -267,7 +312,7 @@ class PostgresReportRepository:
try:
with connection.transaction():
with connection.cursor() as cursor:
self._begin_serializable(cursor)
self._begin_serializable(cursor, read_only=read_only)
return operation(cursor)
except RepositoryError:
raise
@@ -363,6 +408,54 @@ class PostgresReportRepository:
finally:
connection.close()
def derive_monthly_request(
self,
daily_version_id: int,
) -> Optional[DerivedMonthlyRequest]:
if isinstance(daily_version_id, bool) or daily_version_id < 1:
raise RepositoryError(
ErrorCode.REQUEST_INVALID,
"daily version identity is invalid",
)
def operation(cursor: Any) -> Optional[DerivedMonthlyRequest]:
cursor.execute(TRIGGER_ARRIVAL_SCOPE_SQL, (daily_version_id,))
row = cursor.fetchone()
if not row or row[0] is None or row[1] is None:
return None
first_arrival, last_arrival = row[0], row[1]
if (
first_arrival.year != last_arrival.year
or first_arrival.month != last_arrival.month
):
raise RepositoryError(
ErrorCode.SOURCE_INVALID,
"one daily version spans multiple ARRIVAL months",
)
period_start, period_end = month_bounds(
first_arrival.year,
first_arrival.month,
)
cursor.execute(
MONTH_MAX_ARRIVAL_SQL,
(period_start, period_end + timedelta(days=1)),
)
latest = cursor.fetchone()
as_of_date = latest[0] if latest else None
if as_of_date is None:
return None
return DerivedMonthlyRequest(
report_year=as_of_date.year,
report_month=as_of_date.month,
as_of_date=as_of_date,
)
return self._run_serializable(
operation,
"monthly ARRIVAL scope query failed",
read_only=True,
)
@staticmethod
def _lock_scope(cursor: Any, period_start: date, period_end: date) -> None:
lock_key = f"monthly_channel:{period_start.isoformat()}:{period_end.isoformat()}:*"
@@ -395,35 +488,170 @@ class PostgresReportRepository:
"daily source versions changed while the monthly report was being generated",
)
@staticmethod
def _source_snapshot_sha256(report: MonthlyReport) -> str:
identity = {
"period_start": date(
report.report_year,
report.report_month,
1,
).isoformat(),
"as_of_date": report.as_of_date.isoformat(),
"processor_version": PROCESSOR_VERSION,
"rule_set_sha256": rule_set_sha256(),
"result_schema_version": RESULT_SCHEMA_VERSION,
"daily_versions": [
[pin.business_date.isoformat(), pin.daily_version_id]
for pin in report.daily_versions
],
"channel_manifest": [list(item) for item in report.channel_manifest],
}
return hashlib.sha256(
json.dumps(
identity,
ensure_ascii=False,
separators=(",", ":"),
sort_keys=True,
).encode("utf-8")
).hexdigest()
@staticmethod
def _published_artifact(row: Sequence[Any]) -> Optional[FileMetadata]:
if row[3] is None:
return None
metadata = FileMetadata(
file_kind=str(row[3]),
original_filename=str(row[4]),
storage_key=str(row[5]),
sha256=str(row[6]),
byte_size=int(row[7]),
mime_type=str(row[8] or "application/octet-stream"),
)
metadata.validate()
return metadata
def reserve_report(self, report: MonthlyReport) -> ReservedReport:
period_start, period_end = month_bounds(report.report_year, report.report_month)
snapshot_sha256 = self._source_snapshot_sha256(report)
def operation(cursor: Any) -> ReservedReport:
self._lock_scope(cursor, period_start, period_end)
self._validate_pins_current(cursor, report)
identity = {
"period_start": period_start.isoformat(),
"period_end": period_end.isoformat(),
"as_of_date": report.as_of_date.isoformat(),
"processor_version": PROCESSOR_VERSION,
"rule_set_sha256": rule_set_sha256(),
"result_schema_version": RESULT_SCHEMA_VERSION,
"daily_versions": [
[pin.business_date.isoformat(), pin.daily_version_id]
for pin in report.daily_versions
],
"channel_manifest": [list(item) for item in report.channel_manifest],
}
digest = hashlib.sha256(
repr(sorted(identity.items())).encode("utf-8")
).hexdigest()
generation_id = int(digest[:12], 16)
cursor.execute(
"""
SELECT
run.id,
run.version_no,
run.report_status,
artifact.artifact_kind,
artifact.original_filename,
artifact.object_key,
artifact.sha256,
artifact.byte_size,
artifact.mime_type
FROM reporting.monthly_runs AS run
LEFT JOIN ingestion.artifacts AS artifact
ON artifact.id = run.workbook_artifact_id
WHERE run.period_start = %s
AND run.source_snapshot_sha256 = %s
FOR UPDATE OF run
""",
(period_start, snapshot_sha256),
)
existing = cursor.fetchone()
if existing:
status = str(existing[2])
if status == "failed":
cursor.execute(
"""
UPDATE reporting.monthly_runs
SET report_status = 'reserved',
failure_code = NULL,
failure_message = NULL,
failed_at = NULL,
updated_at = now()
WHERE id = %s
""",
(int(existing[0]),),
)
status = "reserved"
return ReservedReport(
report_version_id=int(existing[0]),
version_no=int(existing[1]),
period_start=period_start,
as_of_date=report.as_of_date,
daily_versions=report.daily_versions,
source_snapshot_sha256=snapshot_sha256,
report_status=status,
published_artifact=self._published_artifact(existing),
)
cursor.execute(
"""
SELECT COALESCE(max(version_no), 0) + 1
FROM reporting.monthly_runs
WHERE period_start = %s
""",
(period_start,),
)
version_no = int(cursor.fetchone()[0])
cursor.execute(
"""
INSERT INTO reporting.monthly_runs (
period_start,
as_of_date,
version_no,
source_snapshot_sha256,
report_status,
processor_version,
rule_set_sha256,
result_schema_version,
row_count,
channel_count
)
VALUES (%s, %s, %s, %s, 'reserved', %s, %s, %s, %s, %s)
RETURNING id
""",
(
period_start,
report.as_of_date,
version_no,
snapshot_sha256,
PROCESSOR_VERSION,
rule_set_sha256(),
RESULT_SCHEMA_VERSION,
report.row_count,
len(report.channel_manifest),
),
)
report_id = int(cursor.fetchone()[0])
for pin in report.daily_versions:
cursor.execute(
"""
INSERT INTO reporting.monthly_run_daily_versions (
report_id, business_date, daily_version_id
)
VALUES (%s, %s, %s)
""",
(report_id, pin.business_date, pin.daily_version_id),
)
for worksheet, worksheet_order, row_count in report.channel_manifest:
cursor.execute(
"""
INSERT INTO reporting.monthly_channel_manifest (
report_id, worksheet, worksheet_order, row_count
)
VALUES (%s, %s, %s, %s)
""",
(report_id, worksheet, worksheet_order, row_count),
)
return ReservedReport(
generation_id,
generation_id,
period_start,
report.as_of_date,
report.daily_versions,
report_version_id=report_id,
version_no=version_no,
period_start=period_start,
as_of_date=report.as_of_date,
daily_versions=report.daily_versions,
source_snapshot_sha256=snapshot_sha256,
)
return self._run_serializable(
@@ -431,14 +659,82 @@ class PostgresReportRepository:
"monthly report source reservation failed",
)
@staticmethod
def _ensure_local_artifact(cursor: Any, metadata: FileMetadata) -> int:
cursor.execute(
"""
SELECT
id,
artifact_kind,
original_filename,
sha256,
byte_size,
mime_type
FROM ingestion.artifacts
WHERE storage_provider = %s
AND bucket_alias = %s
AND object_key = %s
AND object_version_id IS NULL
FOR SHARE
""",
(LOCAL_STORAGE_PROVIDER, LOCAL_BUCKET_ALIAS, metadata.storage_key),
)
row = cursor.fetchone()
if row:
if (
str(row[1]) != metadata.file_kind
or str(row[2]) != metadata.original_filename
or str(row[3]) != metadata.sha256
or int(row[4]) != metadata.byte_size
or str(row[5] or "") != metadata.mime_type
):
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"stored monthly artifact identity conflicts",
)
return int(row[0])
cursor.execute(
"""
INSERT INTO ingestion.artifacts (
artifact_kind,
storage_provider,
bucket_alias,
object_key,
original_filename,
sha256,
byte_size,
mime_type
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id
""",
(
metadata.file_kind,
LOCAL_STORAGE_PROVIDER,
LOCAL_BUCKET_ALIAS,
metadata.storage_key,
metadata.original_filename,
metadata.sha256,
metadata.byte_size,
metadata.mime_type,
),
)
return int(cursor.fetchone()[0])
def activate_report(
self,
reservation: ReservedReport,
artifact: FileMetadata,
result_json: FileMetadata,
semantic_sha256: str,
) -> None:
artifact.validate()
result_json.validate()
if not SHA256_RE.fullmatch(semantic_sha256):
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"monthly semantic identity is invalid",
)
if reservation.period_start is None or reservation.as_of_date is None:
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
@@ -461,6 +757,95 @@ class PostgresReportRepository:
ErrorCode.SOURCE_SNAPSHOT_STALE,
"daily source versions changed before publication",
)
cursor.execute(
"""
SELECT report_status, source_snapshot_sha256
FROM reporting.monthly_runs
WHERE id = %s
AND period_start = %s
AND version_no = %s
FOR UPDATE
""",
(
reservation.report_version_id,
period_start,
reservation.version_no,
),
)
run = cursor.fetchone()
if (
not run
or reservation.source_snapshot_sha256 is None
or str(run[1]) != reservation.source_snapshot_sha256
):
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"monthly report reservation identity changed",
)
workbook_artifact_id = self._ensure_local_artifact(cursor, artifact)
result_artifact_id = self._ensure_local_artifact(cursor, result_json)
if str(run[0]) in {"active", "superseded"}:
cursor.execute(
"""
SELECT workbook_artifact_id, result_artifact_id, semantic_sha256
FROM reporting.monthly_runs
WHERE id = %s
""",
(reservation.report_version_id,),
)
published = cursor.fetchone()
if (
not published
or int(published[0]) != workbook_artifact_id
or int(published[1]) != result_artifact_id
or str(published[2]) != semantic_sha256
):
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"published monthly report identity conflicts",
)
return
if str(run[0]) != "reserved":
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"monthly report reservation is not publishable",
)
cursor.execute(
"""
UPDATE reporting.monthly_runs
SET report_status = 'superseded',
superseded_at = now(),
updated_at = now()
WHERE period_start = %s
AND report_status = 'active'
AND id <> %s
""",
(period_start, reservation.report_version_id),
)
cursor.execute(
"""
UPDATE reporting.monthly_runs
SET report_status = 'active',
workbook_artifact_id = %s,
result_artifact_id = %s,
semantic_sha256 = %s,
published_at = now(),
updated_at = now()
WHERE id = %s
AND report_status = 'reserved'
""",
(
workbook_artifact_id,
result_artifact_id,
semantic_sha256,
reservation.report_version_id,
),
)
if cursor.rowcount != 1:
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"monthly report activation did not complete",
)
self._run_serializable(
operation,
@@ -473,6 +858,26 @@ class PostgresReportRepository:
code: str,
safe_message: str,
) -> None:
# Monthly rows and versions are intentionally not persisted. The
# caller's privacy-minimized result JSON is the failure record.
_ = (reservation, code, safe_message)
if not SAFE_CODE_RE.fullmatch(code):
code = ErrorCode.INTERNAL_ERROR
message = str(safe_message).strip()[:500] or "monthly report processing failed"
def operation(cursor: Any) -> None:
cursor.execute(
"""
UPDATE reporting.monthly_runs
SET report_status = 'failed',
failure_code = %s,
failure_message = %s,
failed_at = now(),
updated_at = now()
WHERE id = %s
AND report_status = 'reserved'
""",
(code, message, reservation.report_version_id),
)
self._run_serializable(
operation,
"monthly report failure state could not be recorded",
)

20
monthly_reports/service.py
View File

@@ -164,6 +164,26 @@ class MonthlyReportService:
reservation: Optional[ReservedReport] = None
try:
reservation = self._repository.reserve_report(report)
if reservation.already_published:
artifact = reservation.published_artifact
if artifact is None:
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
"published monthly artifact identity is missing",
)
return RunResult(
request=request,
status="success",
row_count=report.row_count,
channel_manifest=report.channel_manifest,
report_version_id=reservation.report_version_id,
version_no=reservation.version_no,
artifact={
"filename": artifact.original_filename,
"storage_key": artifact.storage_key,
"sha256": artifact.sha256,
},
)
with private_staging_directory(self._staging_root) as temp_dir:
work_dir = Path(temp_dir)
built = self._builder.build(report, work_dir)

447
monthly_reports/worker.py Normal file
View File

@@ -0,0 +1,447 @@
"""Dedicated reliable outbox worker for automatic monthly publication."""
from __future__ import annotations
import argparse
import json
import signal
import sys
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Mapping, Optional, Protocol, Sequence
from arr_database import controlled_connect
from monthly_reports.contracts import ErrorCode
from monthly_reports.publishing import ArtifactToolBuilder, AtomicReportPublisher
from monthly_reports.repository import (
DatabaseConfig,
DerivedMonthlyRequest,
PostgresReportRepository,
RepositoryError,
)
from monthly_reports.service import MonthlyReportService, RunRequest, RunResult
PROJECT_ROOT = Path(__file__).resolve().parents[1]
EVENT_TYPE = "arr.daily_version_committed"
DEFAULT_LEASE_SECONDS = 300
DEFAULT_MAX_ATTEMPTS = 8
CLAIM_EVENT_SQL = """
WITH candidate AS (
SELECT event.id
FROM ingestion.outbox_events AS event
WHERE event.event_type = %s
AND event.aggregate_type = 'processing_run'
AND event.available_at <= now()
AND event.publish_status IN ('pending', 'publishing')
ORDER BY event.created_at, event.id
FOR UPDATE SKIP LOCKED
LIMIT 1
)
UPDATE ingestion.outbox_events AS event
SET publish_status = 'publishing',
publish_attempts = event.publish_attempts + 1,
available_at = now() + (%s * interval '1 second'),
last_error_code = NULL
FROM candidate
WHERE event.id = candidate.id
RETURNING
event.id,
event.event_key,
event.payload,
event.publish_attempts
""".strip()
PUBLISH_EVENT_SQL = """
UPDATE ingestion.outbox_events AS event
SET publish_status = 'published',
published_at = now(),
available_at = now(),
last_error_code = NULL
WHERE event.id = %s
AND event.publish_status = 'publishing'
AND EXISTS (
SELECT 1
FROM reporting.monthly_runs AS run
JOIN ingestion.artifacts AS workbook
ON workbook.id = run.workbook_artifact_id
JOIN ingestion.artifacts AS result
ON result.id = run.result_artifact_id
WHERE run.id = %s
AND run.report_status IN ('active', 'superseded')
AND workbook.artifact_kind = 'monthly_xlsx'
AND result.artifact_kind = 'result_json'
)
""".strip()
FAIL_EVENT_SQL = """
UPDATE ingestion.outbox_events
SET publish_status = %s,
available_at = CASE
WHEN %s = 'pending' THEN now() + (%s * interval '1 second')
ELSE now()
END,
published_at = NULL,
last_error_code = %s
WHERE id = %s
AND publish_status = 'publishing'
""".strip()
class WorkerError(RuntimeError):
def __init__(self, code: str, *, retryable: bool):
super().__init__(code)
self.code = code
self.retryable = retryable
@dataclass(frozen=True)
class OutboxEvent:
event_id: int
event_key: str
payload: Mapping[str, Any]
publish_attempts: int
@dataclass(frozen=True)
class WorkerOutcome:
status: str
event_id: Optional[int] = None
report_id: Optional[int] = None
error_code: Optional[str] = None
def to_dict(self) -> dict[str, Any]:
payload: dict[str, Any] = {"status": self.status}
if self.event_id is not None:
payload["event_id"] = self.event_id
if self.report_id is not None:
payload["report_id"] = self.report_id
if self.error_code is not None:
payload["error_code"] = self.error_code
return payload
class OutboxRepository(Protocol):
def claim_next(self) -> Optional[OutboxEvent]:
...
def mark_published(self, event_id: int, report_id: int) -> None:
...
def mark_failed(self, event: OutboxEvent, code: str, *, retryable: bool) -> str:
...
class MonthlyRequestRepository(Protocol):
def derive_monthly_request(
self,
daily_version_id: int,
) -> Optional[DerivedMonthlyRequest]:
...
def _default_connect(dsn: str) -> Any:
try:
import psycopg # type: ignore[import-not-found]
except ImportError:
raise WorkerError("MONTHLY_WORKER_DATABASE_DRIVER_UNAVAILABLE", retryable=False) from None
try:
return psycopg.connect(dsn, autocommit=False)
except Exception:
raise WorkerError("MONTHLY_WORKER_DATABASE_UNAVAILABLE", retryable=True) from None
class PostgresOutboxRepository:
def __init__(
self,
config: DatabaseConfig,
*,
connect: Optional[Callable[[str], Any]] = None,
lease_seconds: int = DEFAULT_LEASE_SECONDS,
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
) -> None:
if lease_seconds < 30 or lease_seconds > 3600:
raise ValueError("worker lease must be between 30 and 3600 seconds")
if max_attempts < 1 or max_attempts > 100:
raise ValueError("worker max attempts is invalid")
self._config = config
self._connect = connect or _default_connect
self._lease_seconds = lease_seconds
self._max_attempts = max_attempts
def _open(self) -> Any:
try:
return self._connect(self._config.dsn)
except WorkerError:
raise
except Exception:
raise WorkerError("MONTHLY_WORKER_DATABASE_UNAVAILABLE", retryable=True) from None
@staticmethod
def _begin(cursor: Any) -> None:
cursor.execute("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
cursor.execute("SET LOCAL lock_timeout = '10s'")
cursor.execute("SET LOCAL statement_timeout = '30s'")
cursor.execute("SELECT current_database()")
row = cursor.fetchone()
if not row or row[0] != "booking_test":
raise WorkerError("MONTHLY_WORKER_DATABASE_TARGET_INVALID", retryable=False)
def claim_next(self) -> Optional[OutboxEvent]:
connection = self._open()
try:
with connection.transaction():
with connection.cursor() as cursor:
self._begin(cursor)
cursor.execute(CLAIM_EVENT_SQL, (EVENT_TYPE, self._lease_seconds))
row = cursor.fetchone()
if not row:
return None
payload = row[2]
if isinstance(payload, str):
payload = json.loads(payload)
if not isinstance(payload, Mapping):
payload = {}
return OutboxEvent(
event_id=int(row[0]),
event_key=str(row[1]),
payload=payload,
publish_attempts=int(row[3]),
)
except WorkerError:
raise
except Exception:
raise WorkerError("MONTHLY_WORKER_CLAIM_FAILED", retryable=True) from None
finally:
connection.close()
def mark_published(self, event_id: int, report_id: int) -> None:
connection = self._open()
try:
with connection.transaction():
with connection.cursor() as cursor:
self._begin(cursor)
cursor.execute(PUBLISH_EVENT_SQL, (event_id, report_id))
if cursor.rowcount != 1:
raise WorkerError("MONTHLY_WORKER_PUBLISH_ACK_FAILED", retryable=True)
except WorkerError:
raise
except Exception:
raise WorkerError("MONTHLY_WORKER_PUBLISH_ACK_FAILED", retryable=True) from None
finally:
connection.close()
def mark_failed(self, event: OutboxEvent, code: str, *, retryable: bool) -> str:
terminal = not retryable or event.publish_attempts >= self._max_attempts
status = "dead" if terminal else "pending"
delay_seconds = min(300, 2 ** min(event.publish_attempts, 8))
safe_code = code if code and len(code) <= 128 else "MONTHLY_WORKER_FAILED"
connection = self._open()
try:
with connection.transaction():
with connection.cursor() as cursor:
self._begin(cursor)
cursor.execute(
FAIL_EVENT_SQL,
(
status,
status,
delay_seconds,
safe_code,
event.event_id,
),
)
if cursor.rowcount != 1:
raise WorkerError("MONTHLY_WORKER_FAILURE_ACK_FAILED", retryable=True)
return status
except WorkerError:
raise
except Exception:
raise WorkerError("MONTHLY_WORKER_FAILURE_ACK_FAILED", retryable=True) from None
finally:
connection.close()
class MonthlyOutboxWorker:
def __init__(
self,
outbox: OutboxRepository,
requests: MonthlyRequestRepository,
service: MonthlyReportService,
) -> None:
self._outbox = outbox
self._requests = requests
self._service = service
@staticmethod
def _daily_version_id(event: OutboxEvent) -> int:
value = event.payload.get("daily_version_id")
if isinstance(value, bool):
raise WorkerError("MONTHLY_WORKER_EVENT_INVALID", retryable=False)
try:
daily_version_id = int(value)
except (TypeError, ValueError):
raise WorkerError("MONTHLY_WORKER_EVENT_INVALID", retryable=False) from None
if daily_version_id < 1:
raise WorkerError("MONTHLY_WORKER_EVENT_INVALID", retryable=False)
return daily_version_id
@staticmethod
def _result_failure(result: RunResult) -> WorkerError:
code = result.error_code or "MONTHLY_WORKER_REPORT_FAILED"
retryable = code not in {
ErrorCode.REQUEST_INVALID,
ErrorCode.SOURCE_INVALID,
}
return WorkerError(code, retryable=retryable)
def process_next(self) -> WorkerOutcome:
event = self._outbox.claim_next()
if event is None:
return WorkerOutcome(status="idle")
try:
daily_version_id = self._daily_version_id(event)
derived = self._requests.derive_monthly_request(daily_version_id)
if derived is None:
raise WorkerError("MONTHLY_WORKER_ARRIVAL_SCOPE_EMPTY", retryable=False)
result = self._service.run(
RunRequest(
derived.report_year,
derived.report_month,
derived.as_of_date,
)
)
if result.status != "success":
raise self._result_failure(result)
if (
result.report_version_id is None
or result.artifact is None
or not result.artifact.get("sha256")
):
raise WorkerError("MONTHLY_WORKER_REPORT_RECEIPT_INVALID", retryable=True)
self._outbox.mark_published(event.event_id, result.report_version_id)
return WorkerOutcome(
status="published",
event_id=event.event_id,
report_id=result.report_version_id,
)
except WorkerError as error:
status = self._outbox.mark_failed(
event,
error.code,
retryable=error.retryable,
)
return WorkerOutcome(
status=status,
event_id=event.event_id,
error_code=error.code,
)
except RepositoryError as error:
status = self._outbox.mark_failed(event, error.code, retryable=True)
return WorkerOutcome(
status=status,
event_id=event.event_id,
error_code=error.code,
)
except Exception:
code = "MONTHLY_WORKER_INTERNAL_ERROR"
status = self._outbox.mark_failed(event, code, retryable=True)
return WorkerOutcome(
status=status,
event_id=event.event_id,
error_code=code,
)
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="arr-monthly-worker")
parser.add_argument("--db-config", type=Path)
parser.add_argument("--driver-path", type=Path)
parser.add_argument("--node-binary", type=Path)
parser.add_argument("--artifact-tool-module", type=Path)
parser.add_argument("--output-root", type=Path)
parser.add_argument("--poll-seconds", type=float, default=2.0)
parser.add_argument("--lease-seconds", type=int, default=DEFAULT_LEASE_SECONDS)
parser.add_argument("--max-attempts", type=int, default=DEFAULT_MAX_ATTEMPTS)
parser.add_argument("--once", action="store_true")
return parser
def _runtime(args: argparse.Namespace) -> MonthlyOutboxWorker:
connect: Optional[Callable[[str], Any]] = None
if args.db_config:
connect = controlled_connect(args.db_config, args.driver_path)
config = DatabaseConfig("controlled")
else:
config = DatabaseConfig.from_environment()
repository = PostgresReportRepository(config, connect=connect)
outbox = PostgresOutboxRepository(
config,
connect=connect,
lease_seconds=args.lease_seconds,
max_attempts=args.max_attempts,
)
output_root = (args.output_root or PROJECT_ROOT / "outputs" / "monthly_reports").resolve()
try:
output_root.relative_to(PROJECT_ROOT)
except ValueError:
raise ValueError("worker output root must be inside the project") from None
builder = ArtifactToolBuilder(
PROJECT_ROOT / "monthly_reports" / "xlsx" / "build_workbook.mjs",
node_binary=str(args.node_binary) if args.node_binary else None,
artifact_tool_module=args.artifact_tool_module,
)
service = MonthlyReportService(
repository,
builder,
AtomicReportPublisher(PROJECT_ROOT, output_root),
output_root / ".staging",
)
return MonthlyOutboxWorker(outbox, repository, service)
def _emit(outcome: WorkerOutcome) -> None:
sys.stdout.write(json.dumps(outcome.to_dict(), sort_keys=True) + "\n")
sys.stdout.flush()
def main(argv: Optional[Sequence[str]] = None) -> int:
args = _parser().parse_args(argv)
if args.poll_seconds < 0.1 or args.poll_seconds > 60:
raise SystemExit("poll interval must be between 0.1 and 60 seconds")
worker = _runtime(args)
if args.once:
try:
outcome = worker.process_next()
except WorkerError as error:
outcome = WorkerOutcome(status="worker_error", error_code=error.code)
_emit(outcome)
return 0 if outcome.status in {"idle", "published"} else 2
stopping = False
def stop(_signum: int, _frame: object) -> None:
nonlocal stopping
stopping = True
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
while not stopping:
try:
outcome = worker.process_next()
except WorkerError as error:
outcome = WorkerOutcome(status="worker_error", error_code=error.code)
if outcome.status != "idle":
_emit(outcome)
if outcome.status in {"idle", "worker_error"} and not stopping:
time.sleep(args.poll_seconds)
return 0
if __name__ == "__main__":
raise SystemExit(main())

34
monthly_reports/xlsx/build_workbook.mjs
View File

@@ -242,6 +242,11 @@ function writeSheet(workbook, channel) {
const lastRow = matrix.length;
const used = sheet.getRange(`A1:${lastColumn}${lastRow}`);
used.values = matrix;
if (channel.rows.length > 0) {
sheet.getRange(`S2:S${lastRow}`).formulas = channel.rows.map((_, index) => [
`=R${index + 2}*C${index + 2}*G${index + 2}`,
]);
}
used.format.font = { name: "Arial", size: 10, color: BODY_FONT };
used.format.verticalAlignment = "center";
used.format.borders = {
@@ -328,22 +333,41 @@ async function validateWorkbook(workbook, payload, stage) {
}
});
});
const formulas = used?.formulas ?? [];
formulas.forEach((formulaRow, rowIndex) => {
formulaRow.forEach((formulaValue, columnIndex) => {
const formula = String(formulaValue ?? "").trim();
if (!formula) return;
formulaCount += 1;
const expectedFormula = `=R${rowIndex + 1}*C${rowIndex + 1}*G${rowIndex + 1}`;
if (
rowIndex < 1
|| columnIndex !== channel.headers.indexOf("TOTAL PRICE")
|| formula !== expectedFormula
) {
fail(`${stage} workbook contains an unauthorized formula`);
}
});
});
const inspected = await workbook.inspect({
kind: "formula",
sheetId: channel.worksheet,
range: `A1:${lastColumn}${expectedRows}`,
maxChars: 6000,
options: { maxResults: 100 },
options: { maxResults: Math.max(channel.rows.length + 10, 100) },
});
const inspectionText = String(inspected.ndjson ?? "");
formulaCount += inspectionText
.split("\n")
.filter((line) => line.includes('"kind":"formula"')).length;
if (/#REF!|#DIV\/0!|#VALUE!|#NAME\?|#N\/A/.test(inspectionText)) {
fail(`${stage} workbook contains a formula error`);
}
}
if (formulaCount !== 0) fail(`${stage} workbook must contain no formulas`);
const expectedFormulaCount = payload.channels.reduce(
(total, channel) => total + channel.rows.length,
0,
);
if (formulaCount !== expectedFormulaCount) {
fail(`${stage} workbook TOTAL PRICE formulas are incomplete`);
}
return formulaCount;
}