shiyuyun/wyndham-ARR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: move report artifacts to OSS storage

Browse Source
This commit is contained in:
Wyndham ARR
2026-08-04 12:37:26 +08:00
parent 727643f1f2
commit a2b86cdf10
45 changed files with 1288 additions and 1417 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
monthly_reports/README.md
View File

@@ -8,8 +8,8 @@ Automatic flow:
2. It looks up that version's retained `ARRIVAL` values in PostgreSQL. The affected month and report watermark come from those facts; the XML filename and wall clock are ignored.
3. It opens a repeatable-read snapshot of current Finance facts and daily-version pins for that month. `as_of_date` is the greatest `ARRIVAL` actually included in the snapshot.
4. It reserves an idempotent publication identity in `reporting.monthly_runs` with immutable daily lineage and channel manifest rows.
5. It generates the XLSX/result JSON, reopens the workbook, validates all values and formulas, and rechecks that the Finance pins are still current.
6. It registers both local artifacts and atomically activates the report. Only then is the outbox event marked `published`; transient failures are retried and exhausted events become `dead`.
5. It generates the XLSX/result JSON with Python/openpyxl, reopens the workbook, validates all values and formulas, and rechecks that the Finance pins are still current.
6. It uploads both artifacts to the existing private OSS through the immutable object-store adapter and atomically activates the report. Only then is the outbox event marked `published`; transient failures are retried and exhausted events become `dead`.
PostgreSQL stores publication metadata, version identity, lineage and artifact identities—not duplicate monthly business or guest rows. The portal lists `active`/`superseded` runs and downloads only registered artifacts after path, size and SHA-256 checks.
@@ -18,8 +18,7 @@ Run the dedicated worker:
```bash
python3 -m monthly_reports.worker \
--db-config /absolute/path/to/booking-test-db.env \
--node-binary /absolute/path/to/node \
--artifact-tool-module /absolute/path/to/artifact_tool.mjs
--output-root /app/outputs/monthly_reports
```
The CLI generation command remains a controlled recovery/diagnostic entrypoint:
@@ -28,10 +27,9 @@ The CLI generation command remains a controlled recovery/diagnostic entrypoint:
python3 -m monthly_reports generate \
--month 2026-07 \
--as-of 2026-07-31 \
--node-binary /absolute/path/to/node \
--artifact-tool-module /absolute/path/to/artifact_tool.mjs
--output-root /app/outputs/monthly_reports
```
The program reads `MONTHLY_REPORT_DATABASE_URL`, falling back to `ARR_DATABASE_URL`. The configured database must be `booking_test`; SuperAgent must never receive the DSN.
The program reads `MONTHLY_REPORT_DATABASE_URL`, falling back to `ARR_DATABASE_URL`, and uses the existing `ARR_OSS_REGION`, `ARR_OSS_BUCKET` and optional `ARR_OSS_ENDPOINT` settings. The configured database must be `booking_test`; SuperAgent must never receive the DSN. `output-root` is only staging/legacy-cache space; new downloads use the OSS identity.
Every XLSX data-row `TOTAL PRICE` cell contains the exact row-relative formula `=R[row]*C[row]*G[row]`, meaning `REAL PRICE × NIGHTS × NO_OF_ROOMS`. The stored Finance `total_price` remains an independent audit expectation. `Booking Room` enrichment must not be used to recalculate price, dates or actual room count.

43
monthly_reports/cli.py
View File

@@ -11,7 +11,8 @@ from pathlib import Path
from typing import Optional, Sequence, Tuple
from monthly_reports.contracts import ErrorCode, RESULT_SCHEMA_VERSION
from monthly_reports.publishing import ArtifactToolBuilder, AtomicReportPublisher
from arr_web.processing_runtime import compose_object_store
from monthly_reports.publishing import OpenpyxlWorkbookBuilder, AtomicReportPublisher
from monthly_reports.repository import DatabaseConfig, PostgresReportRepository, RepositoryError
from monthly_reports.service import MonthlyReportService, RunRequest, write_run_result
@@ -56,14 +57,6 @@ def _parser() -> SafeArgumentParser:
generate.add_argument("--month", required=True, help="report month in YYYY-MM")
generate.add_argument("--as-of", required=True, help="latest included business date")
generate.add_argument("--output-root", help="controlled output root inside the project")
generate.add_argument(
"--node-binary",
help="Node.js executable; defaults to MONTHLY_REPORT_NODE_BINARY or PATH",
)
generate.add_argument(
"--artifact-tool-module",
help="absolute path to artifact_tool.mjs when package resolution is unavailable",
)
return parser
@@ -96,23 +89,21 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
output_root = PROJECT_ROOT / output_root
output_root = output_root.resolve()
repository = PostgresReportRepository(DatabaseConfig.from_environment())
builder = ArtifactToolBuilder(
PROJECT_ROOT / "monthly_reports" / "xlsx" / "build_workbook.mjs",
node_binary=args.node_binary,
artifact_tool_module=(
Path(args.artifact_tool_module).expanduser()
if args.artifact_tool_module
else None
),
)
publisher = AtomicReportPublisher(PROJECT_ROOT, output_root)
service = MonthlyReportService(
repository,
builder,
publisher,
output_root / ".staging",
)
result = service.run(request)
storage_runtime = compose_object_store()
try:
service = MonthlyReportService(
repository,
OpenpyxlWorkbookBuilder(),
AtomicReportPublisher(
PROJECT_ROOT,
output_root,
object_store=storage_runtime.object_store,
),
output_root / ".staging",
)
result = service.run(request)
finally:
storage_runtime.close()
result_path = (
output_root
/ f"{year:04d}"

324
monthly_reports/publishing.py
View File

@@ -1,4 +1,4 @@
"""Artifact-tool adapter and recoverable atomic monthly-report publication."""
"""Python/openpyxl monthly workbook building and recoverable publication."""
from __future__ import annotations
@@ -6,19 +6,20 @@ import hashlib
import json
import os
import shutil
import subprocess
import tempfile
import uuid
from dataclasses import dataclass
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from pathlib import Path
from typing import Any, Dict, Mapping, Optional
from monthly_reports.contracts import (
RESULT_SCHEMA_VERSION,
XLSX_MIME,
ErrorCode,
MonthlyReport,
)
from openpyxl import Workbook, load_workbook
from openpyxl.styles import Alignment, Font, PatternFill
from openpyxl.utils import get_column_letter
from arr_storage.store import ManagedObjectStore
from monthly_reports.contracts import KB_HEADER, RESULT_SCHEMA_VERSION, XLSX_MIME, ErrorCode, MonthlyReport
from monthly_reports.repository import (
FileMetadata,
ReportRepository,
@@ -84,94 +85,178 @@ def _private_json(path: Path, payload: Mapping[str, Any]) -> None:
raise
class ArtifactToolBuilder:
def __init__(
self,
builder_script: Path,
node_binary: Optional[str] = None,
artifact_tool_module: Optional[Path] = None,
timeout_seconds: int = 180,
) -> None:
configured = (node_binary or os.environ.get("MONTHLY_REPORT_NODE_BINARY", "")).strip()
self._node_binary = configured or shutil.which("node") or ""
self._builder_script = builder_script.resolve()
self._artifact_tool_module = (
artifact_tool_module.resolve() if artifact_tool_module else None
)
self._timeout_seconds = timeout_seconds
class OpenpyxlWorkbookBuilder:
"""Build and re-open a validated monthly workbook without external runtimes."""
def build(self, report: MonthlyReport, work_dir: Path) -> BuiltWorkbook:
if not self._node_binary or not self._builder_script.is_file():
raise BuildError(
ErrorCode.OUTPUT_VALIDATION_FAILED,
"the monthly XLSX builder runtime is unavailable",
)
work_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(work_dir, 0o700)
payload_path = work_dir / "workbook-payload.json"
output_path = work_dir / report.filename
preview_dir = work_dir / "previews"
summary_path = work_dir / "workbook-summary.json"
_private_json(payload_path, report.to_workbook_payload())
environment = os.environ.copy()
if self._artifact_tool_module is not None:
environment["MONTHLY_REPORT_ARTIFACT_TOOL_MODULE"] = str(
self._artifact_tool_module
_INTEGER_HEADERS = frozenset({"NIGHTS", "ADULTS", "CHILDREN", "NO_OF_ROOMS"})
_DECIMAL_HEADERS = frozenset({"RATE_AMOUNT", "REAL PRICE", "TOTAL PRICE", KB_HEADER})
def __init__(self, *_legacy_args: Any, **_legacy_options: Any) -> None:
# The ignored arguments keep older local wrappers import-compatible while
# making the production builder entirely Python/openpyxl based.
_ = (_legacy_args, _legacy_options)
@staticmethod
def _safe_text(value: Any) -> str:
text = "" if value is None else str(value)
return "'" + text if text[:1] in {"=", "+", "-", "@"} else text
@staticmethod
def _date_key(value: Any) -> str:
if isinstance(value, datetime):
return value.date().isoformat()
if isinstance(value, date):
return value.isoformat()
if isinstance(value, str) and len(value) >= 10:
return value[:10]
return ""
@staticmethod
def _semantic_sha256(payload: Mapping[str, Any]) -> str:
encoded = json.dumps(
{"channels": payload["channels"]},
ensure_ascii=False,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
@classmethod
def _cell_value(cls, header: str, value: Any) -> Any:
if header in {"ARRIVAL", "DEPARTURE"}:
return date.fromisoformat(str(value))
if header in cls._INTEGER_HEADERS:
return int(value)
if header in cls._DECIMAL_HEADERS:
return float(Decimal(str(value)))
return cls._safe_text(value)
@staticmethod
def _style_sheet(worksheet: Any, headers: list[str]) -> None:
worksheet.freeze_panes = "A2"
worksheet.auto_filter.ref = f"A1:{get_column_letter(len(headers))}{max(1, worksheet.max_row)}"
for column, header in enumerate(headers, start=1):
cell = worksheet.cell(1, column)
cell.font = Font(name="Arial", size=10, bold=True, color="000000")
cell.fill = PatternFill("solid", fgColor="D9EAF7")
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
worksheet.column_dimensions[get_column_letter(column)].width = min(
34, max(12, len(header) + 2)
)
worksheet.row_dimensions[1].height = 28
for row in worksheet.iter_rows(min_row=2):
for cell in row:
cell.font = Font(name="Arial", size=10, color="222222")
cell.alignment = Alignment(vertical="center", wrap_text=True)
worksheet.row_dimensions[row[0].row].height = 24
for column in ("A", "B"):
for cell in worksheet[column][1:]:
cell.number_format = "yyyy-mm-dd"
@classmethod
def _write_sheet(cls, worksheet: Any, channel: Mapping[str, Any]) -> int:
headers = [str(header) for header in channel["headers"]]
worksheet.append(headers)
total_column = headers.index("TOTAL PRICE") + 1
for row in channel["rows"]:
row_index = worksheet.max_row + 1
values = [
cls._cell_value(header, row.get(header, ""))
for header in headers
]
values[total_column - 1] = f"=R{row_index}*C{row_index}*G{row_index}"
worksheet.append(values)
cls._style_sheet(worksheet, headers)
return len(channel["rows"])
@classmethod
def _validate_workbook(
cls,
output_path: Path,
report: MonthlyReport,
payload: Mapping[str, Any],
) -> int:
workbook = load_workbook(output_path, data_only=False, read_only=True)
try:
completed = subprocess.run(
[
self._node_binary,
str(self._builder_script),
str(payload_path),
str(output_path),
str(preview_dir),
str(summary_path),
],
cwd=self._builder_script.parent,
capture_output=True,
text=True,
timeout=self._timeout_seconds,
env=environment,
check=False,
)
except (OSError, subprocess.TimeoutExpired):
raise BuildError(
ErrorCode.OUTPUT_VALIDATION_FAILED,
"the monthly XLSX builder did not complete",
) from None
if completed.returncode != 0:
raise BuildError(
ErrorCode.OUTPUT_VALIDATION_FAILED,
"the monthly XLSX builder rejected the generated payload",
)
try:
summary = json.loads(summary_path.read_text(encoding="utf-8"))
expected_names = [channel.worksheet for channel in report.channels]
expected_rows = [len(channel.rows) for channel in report.channels]
if (
summary.get("status") != "success"
or summary.get("schema_version") != RESULT_SCHEMA_VERSION
or summary.get("filename") != report.filename
or summary.get("report_year") != report.report_year
or summary.get("report_month") != report.report_month
or summary.get("as_of_date") != report.as_of_date.isoformat()
or summary.get("sheet_names") != expected_names
or summary.get("row_counts") != expected_rows
or summary.get("formula_count") != report.row_count
or summary.get("preview_count") != len(report.channels)
or not isinstance(summary.get("semantic_sha256"), str)
or len(summary["semantic_sha256"]) != 64
or not output_path.is_file()
or output_path.stat().st_size <= 0
):
raise ValueError("builder summary mismatch")
except (OSError, ValueError, TypeError, json.JSONDecodeError, KeyError):
if workbook.sheetnames != expected_names:
raise ValueError("worksheet names do not match")
formula_count = 0
for channel, expected_row_count in zip(report.channels, expected_rows):
worksheet = workbook[channel.worksheet]
headers = list(channel.headers)
if worksheet.max_row != expected_row_count + 1 or worksheet.max_column != len(headers):
raise ValueError("worksheet dimensions do not match")
if [worksheet.cell(1, index).value for index in range(1, len(headers) + 1)] != headers:
raise ValueError("worksheet headers do not match")
for row_index, expected_row in enumerate(channel.rows, start=2):
payload_row = expected_row.to_payload(channel.worksheet == "DY-AI-Easy-KB")
for column, header in enumerate(headers, start=1):
cell = worksheet.cell(row_index, column)
if header == "TOTAL PRICE":
expected_formula = f"=R{row_index}*C{row_index}*G{row_index}"
if cell.value != expected_formula or cell.data_type != "f":
raise ValueError("total price formula does not match")
formula_count += 1
continue
actual = cell.value
expected = payload_row[header]
if header in {"ARRIVAL", "DEPARTURE"}:
matches = cls._date_key(actual) == str(expected)
elif header in cls._INTEGER_HEADERS:
matches = actual == int(expected)
elif header in cls._DECIMAL_HEADERS:
matches = Decimal(str(actual)) == Decimal(str(expected))
else:
matches = ("" if actual is None else actual) == cls._safe_text(expected)
if not matches:
raise ValueError("worksheet values do not match")
if formula_count != report.row_count:
raise ValueError("formula count does not match")
if cls._semantic_sha256(payload) != cls._semantic_sha256(report.to_workbook_payload()):
raise ValueError("semantic source differs")
return formula_count
finally:
workbook.close()
def build(self, report: MonthlyReport, work_dir: Path) -> BuiltWorkbook:
work_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(work_dir, 0o700)
output_path = work_dir / report.filename
summary_path = work_dir / "workbook-summary.json"
payload = report.to_workbook_payload()
try:
workbook = Workbook()
for index, channel in enumerate(payload["channels"]):
worksheet = workbook.active if index == 0 else workbook.create_sheet()
worksheet.title = str(channel["worksheet"])
self._write_sheet(worksheet, channel)
workbook.save(output_path)
workbook.close()
os.chmod(output_path, 0o600)
formula_count = self._validate_workbook(output_path, report, payload)
expected_names = [channel.worksheet for channel in report.channels]
expected_rows = [len(channel.rows) for channel in report.channels]
summary = {
"status": "success",
"schema_version": payload["schema_version"],
"report_year": report.report_year,
"report_month": report.report_month,
"as_of_date": report.as_of_date.isoformat(),
"filename": report.filename,
"sheet_names": expected_names,
"row_counts": expected_rows,
"formula_count": formula_count,
"semantic_sha256": self._semantic_sha256(payload),
"preview_count": len(expected_names),
"byte_size": output_path.stat().st_size,
}
_private_json(summary_path, summary)
except (OSError, ValueError, TypeError, InvalidOperation, KeyError):
raise BuildError(
ErrorCode.OUTPUT_VALIDATION_FAILED,
"the monthly XLSX builder result could not be validated",
) from None
os.chmod(output_path, 0o600)
return BuiltWorkbook(
path=output_path,
sha256=sha256_file(output_path),
@@ -179,11 +264,19 @@ class ArtifactToolBuilder:
summary=summary,
)
class AtomicReportPublisher:
def __init__(self, project_root: Path, output_root: Path) -> None:
def __init__(
self,
project_root: Path,
output_root: Path,
*,
object_store: Optional[ManagedObjectStore] = None,
bucket_alias: str = "arr-private",
) -> None:
self._project_root = project_root.resolve()
self._output_root = output_root.resolve()
self._object_store = object_store
self._bucket_alias = bucket_alias
try:
self._output_root.relative_to(self._project_root)
except ValueError:
@@ -250,6 +343,8 @@ class AtomicReportPublisher:
artifact_storage_key: str,
artifact_sha256: str,
semantic_sha256: str,
storage_provider: str,
bucket_alias: str,
) -> Dict[str, Any]:
return {
"schema_version": RESULT_SCHEMA_VERSION,
@@ -264,6 +359,8 @@ class AtomicReportPublisher:
"storage_key": artifact_storage_key,
"sha256": artifact_sha256,
"semantic_sha256": semantic_sha256,
"storage_provider": storage_provider,
"bucket_alias": bucket_alias,
},
"row_count": report.row_count,
"channel_manifest": [
@@ -312,13 +409,30 @@ class AtomicReportPublisher:
os.chmod(result_backup, 0o600)
archive_created = self._install_once(built.path, archive_path, built.sha256)
storage_provider = "local"
bucket_alias = "arr-project-root"
artifact_storage_key = self._storage_key(archive_path)
if self._object_store is not None:
uploaded_artifact = self._object_store.upload_committed(
job_id=f"monthly-report-{reservation.report_version_id}",
attempt_no=1,
role="monthly_report",
source=archive_path,
original_filename=report.filename,
expected_sha256=built.sha256,
expected_byte_size=built.byte_size,
)
storage_provider = "oss"
bucket_alias = self._bucket_alias
artifact_storage_key = uploaded_artifact.object_key
result_payload = self._success_result(
report,
reservation,
artifact_storage_key,
built.sha256,
str(built.summary.get("semantic_sha256", "")),
storage_provider,
bucket_alias,
)
if result_path.exists():
expected = json.dumps(
@@ -355,6 +469,38 @@ class AtomicReportPublisher:
sha256=sha256_file(result_path),
byte_size=result_path.stat().st_size,
mime_type="application/json",
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
if self._object_store is not None:
uploaded_result = self._object_store.upload_committed(
job_id=f"monthly-report-{reservation.report_version_id}",
attempt_no=1,
role="result_json",
source=result_path,
original_filename=result_path.name,
expected_sha256=result_json.sha256,
expected_byte_size=result_json.byte_size,
)
result_json = FileMetadata(
file_kind="result_json",
original_filename=result_path.name,
storage_key=uploaded_result.object_key,
sha256=uploaded_result.sha256,
byte_size=uploaded_result.byte_size,
mime_type=uploaded_result.mime_type,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
artifact = FileMetadata(
file_kind="monthly_xlsx",
original_filename=report.filename,
storage_key=artifact_storage_key,
sha256=built.sha256,
byte_size=built.byte_size,
mime_type=XLSX_MIME,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
repository.activate_report(
reservation,

44
monthly_reports/repository.py
View File

@@ -36,6 +36,8 @@ SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
SAFE_CODE_RE = re.compile(r"^[A-Z][A-Z0-9_]{0,63}$")
LOCAL_STORAGE_PROVIDER = "local"
LOCAL_BUCKET_ALIAS = "arr-project-root"
OSS_STORAGE_PROVIDER = "oss"
OSS_BUCKET_ALIAS = "arr-private"
SOURCE_FACTS_SQL = """
@@ -160,6 +162,8 @@ class FileMetadata:
sha256: str
byte_size: int
mime_type: str
storage_provider: str = LOCAL_STORAGE_PROVIDER
bucket_alias: str = LOCAL_BUCKET_ALIAS
def validate(self) -> None:
storage = PurePosixPath(self.storage_key)
@@ -175,6 +179,12 @@ class FileMetadata:
or not SHA256_RE.fullmatch(self.sha256)
or self.byte_size <= 0
or not self.mime_type.strip()
or not isinstance(self.storage_provider, str)
or self.storage_provider not in {"local", "local_fixture", "oss", "s3"}
or not isinstance(self.bucket_alias, str)
or not self.bucket_alias.strip()
or "/" in self.bucket_alias
or "\\" in self.bucket_alias
):
raise RepositoryError(
ErrorCode.PUBLISH_FAILED,
@@ -519,13 +529,23 @@ class PostgresReportRepository:
def _published_artifact(row: Sequence[Any]) -> Optional[FileMetadata]:
if row[3] is None:
return None
if len(row) >= 11:
provider = str(row[4] or LOCAL_STORAGE_PROVIDER)
bucket_alias = str(row[5] or LOCAL_BUCKET_ALIAS)
filename_index, key_index, hash_index, size_index, mime_index = 6, 7, 8, 9, 10
else:
provider = LOCAL_STORAGE_PROVIDER
bucket_alias = LOCAL_BUCKET_ALIAS
filename_index, key_index, hash_index, size_index, mime_index = 4, 5, 6, 7, 8
metadata = FileMetadata(
file_kind=str(row[3]),
original_filename=str(row[4]),
storage_key=str(row[5]),
sha256=str(row[6]),
byte_size=int(row[7]),
mime_type=str(row[8] or "application/octet-stream"),
original_filename=str(row[filename_index]),
storage_key=str(row[key_index]),
sha256=str(row[hash_index]),
byte_size=int(row[size_index]),
mime_type=str(row[mime_index] or "application/octet-stream"),
storage_provider=provider,
bucket_alias=bucket_alias,
)
metadata.validate()
return metadata
@@ -544,6 +564,8 @@ class PostgresReportRepository:
run.version_no,
run.report_status,
artifact.artifact_kind,
artifact.storage_provider,
artifact.bucket_alias,
artifact.original_filename,
artifact.object_key,
artifact.sha256,
@@ -660,7 +682,7 @@ class PostgresReportRepository:
)
@staticmethod
def _ensure_local_artifact(cursor: Any, metadata: FileMetadata) -> int:
def _ensure_artifact(cursor: Any, metadata: FileMetadata) -> int:
cursor.execute(
"""
SELECT
@@ -677,7 +699,7 @@ class PostgresReportRepository:
AND object_version_id IS NULL
FOR SHARE
""",
(LOCAL_STORAGE_PROVIDER, LOCAL_BUCKET_ALIAS, metadata.storage_key),
(metadata.storage_provider, metadata.bucket_alias, metadata.storage_key),
)
row = cursor.fetchone()
if row:
@@ -710,8 +732,8 @@ class PostgresReportRepository:
""",
(
metadata.file_kind,
LOCAL_STORAGE_PROVIDER,
LOCAL_BUCKET_ALIAS,
metadata.storage_provider,
metadata.bucket_alias,
metadata.storage_key,
metadata.original_filename,
metadata.sha256,
@@ -782,8 +804,8 @@ class PostgresReportRepository:
ErrorCode.PUBLISH_FAILED,
"monthly report reservation identity changed",
)
workbook_artifact_id = self._ensure_local_artifact(cursor, artifact)
result_artifact_id = self._ensure_local_artifact(cursor, result_json)
workbook_artifact_id = self._ensure_artifact(cursor, artifact)
result_artifact_id = self._ensure_artifact(cursor, result_json)
if str(run[0]) in {"active", "superseded"}:
cursor.execute(
"""

98
monthly_reports/worker.py
View File

@@ -13,7 +13,8 @@ from typing import Any, Callable, Mapping, Optional, Protocol, Sequence
from arr_database import controlled_connect
from monthly_reports.contracts import ErrorCode
from monthly_reports.publishing import ArtifactToolBuilder, AtomicReportPublisher
from arr_web.processing_runtime import ObjectStoreRuntime, compose_object_store
from monthly_reports.publishing import OpenpyxlWorkbookBuilder, AtomicReportPublisher
from monthly_reports.repository import (
DatabaseConfig,
DerivedMonthlyRequest,
@@ -273,10 +274,17 @@ class MonthlyOutboxWorker:
outbox: OutboxRepository,
requests: MonthlyRequestRepository,
service: MonthlyReportService,
storage_runtime: Optional[ObjectStoreRuntime] = None,
) -> None:
self._outbox = outbox
self._requests = requests
self._service = service
self._storage_runtime = storage_runtime
def close(self) -> None:
if self._storage_runtime is not None:
self._storage_runtime.close()
self._storage_runtime = None
@staticmethod
def _daily_version_id(event: OutboxEvent) -> int:
@@ -362,8 +370,6 @@ def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="arr-monthly-worker")
parser.add_argument("--db-config", type=Path)
parser.add_argument("--driver-path", type=Path)
parser.add_argument("--node-binary", type=Path)
parser.add_argument("--artifact-tool-module", type=Path)
parser.add_argument("--output-root", type=Path)
parser.add_argument("--poll-seconds", type=float, default=2.0)
parser.add_argument("--lease-seconds", type=int, default=DEFAULT_LEASE_SECONDS)
@@ -391,18 +397,27 @@ def _runtime(args: argparse.Namespace) -> MonthlyOutboxWorker:
output_root.relative_to(PROJECT_ROOT)
except ValueError:
raise ValueError("worker output root must be inside the project") from None
builder = ArtifactToolBuilder(
PROJECT_ROOT / "monthly_reports" / "xlsx" / "build_workbook.mjs",
node_binary=str(args.node_binary) if args.node_binary else None,
artifact_tool_module=args.artifact_tool_module,
)
service = MonthlyReportService(
repository,
builder,
AtomicReportPublisher(PROJECT_ROOT, output_root),
output_root / ".staging",
)
return MonthlyOutboxWorker(outbox, repository, service)
storage_runtime = compose_object_store()
try:
service = MonthlyReportService(
repository,
OpenpyxlWorkbookBuilder(),
AtomicReportPublisher(
PROJECT_ROOT,
output_root,
object_store=storage_runtime.object_store,
),
output_root / ".staging",
)
return MonthlyOutboxWorker(
outbox,
repository,
service,
storage_runtime=storage_runtime,
)
except Exception:
storage_runtime.close()
raise
def _emit(outcome: WorkerOutcome) -> None:
@@ -415,32 +430,35 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
if args.poll_seconds < 0.1 or args.poll_seconds > 60:
raise SystemExit("poll interval must be between 0.1 and 60 seconds")
worker = _runtime(args)
if args.once:
try:
outcome = worker.process_next()
except WorkerError as error:
outcome = WorkerOutcome(status="worker_error", error_code=error.code)
_emit(outcome)
return 0 if outcome.status in {"idle", "published"} else 2
stopping = False
def stop(_signum: int, _frame: object) -> None:
nonlocal stopping
stopping = True
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
while not stopping:
try:
outcome = worker.process_next()
except WorkerError as error:
outcome = WorkerOutcome(status="worker_error", error_code=error.code)
if outcome.status != "idle":
try:
if args.once:
try:
outcome = worker.process_next()
except WorkerError as error:
outcome = WorkerOutcome(status="worker_error", error_code=error.code)
_emit(outcome)
if outcome.status in {"idle", "worker_error"} and not stopping:
time.sleep(args.poll_seconds)
return 0
return 0 if outcome.status in {"idle", "published"} else 2
stopping = False
def stop(_signum: int, _frame: object) -> None:
nonlocal stopping
stopping = True
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
while not stopping:
try:
outcome = worker.process_next()
except WorkerError as error:
outcome = WorkerOutcome(status="worker_error", error_code=error.code)
if outcome.status != "idle":
_emit(outcome)
if outcome.status in {"idle", "worker_error"} and not stopping:
time.sleep(args.poll_seconds)
return 0
finally:
worker.close()
if __name__ == "__main__":

464
monthly_reports/xlsx/build_workbook.mjs
View File

@@ -1,464 +0,0 @@
import crypto from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import process from "node:process";
import { pathToFileURL } from "node:url";
let FileBlob;
let SpreadsheetFile;
let Workbook;
const STANDARD_HEADERS = [
"ARRIVAL",
"DEPARTURE",
"NIGHTS",
"ADULTS",
"CHILDREN",
"BLOCK_CODE",
"NO_OF_ROOMS",
"COMPANY_NAME",
"CONFIRMATION_NO",
"DISP_ROOM_NO",
"RATE_AMOUNT",
"FULL_NAME",
"RES_COMMENT",
"TRACE_TEXT",
"PRODUCTS",
"RATE_CODE",
"ROOM_CATEGORY_LABEL",
"REAL PRICE",
"TOTAL PRICE",
];
const KB_HEADER = "KB（100/晚/间）";
const KB_SHEET = "DY-AI-Easy-KB";
const DATE_FIELDS = new Set(["ARRIVAL", "DEPARTURE"]);
const INTEGER_FIELDS = new Set(["NIGHTS", "ADULTS", "CHILDREN", "NO_OF_ROOMS"]);
const DECIMAL_FIELDS = new Set(["RATE_AMOUNT", "REAL PRICE", "TOTAL PRICE", KB_HEADER]);
const TEXT_FIELDS = new Set([
"BLOCK_CODE",
"COMPANY_NAME",
"CONFIRMATION_NO",
"DISP_ROOM_NO",
"FULL_NAME",
"RES_COMMENT",
"TRACE_TEXT",
"PRODUCTS",
"RATE_CODE",
"ROOM_CATEGORY_LABEL",
]);
const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
const DECIMAL_PATTERN = /^(0|[1-9]\d*)(?:\.(\d{1,2}))?$/;
const INVALID_SHEET_CHARS = /[:\\/?*\[\]]/;
const HEADER_FILL = "#FFFFFF";
const BODY_FONT = "#222222";
const BORDER_COLOR = "#BFBFBF";
const MAX_PREVIEW_ROWS = 25;
function fail(message) {
throw new Error(message);
}
async function loadArtifactTool() {
const configured = String(process.env.MONTHLY_REPORT_ARTIFACT_TOOL_MODULE ?? "").trim();
const module = configured
? await import(pathToFileURL(path.resolve(configured)).href)
: await import("@oai/artifact-tool");
({ FileBlob, SpreadsheetFile, Workbook } = module);
if (!FileBlob || !SpreadsheetFile || !Workbook) fail("artifact-tool module is invalid");
}
function safeText(value) {
const text = String(value ?? "");
return /^[=+\-@]/.test(text) ? `'${text}` : text;
}
function excelDate(value) {
if (!DATE_PATTERN.test(String(value ?? ""))) fail("invalid date value");
const [year, month, day] = value.split("-").map(Number);
const parsed = new Date(Date.UTC(year, month - 1, day));
if (
parsed.getUTCFullYear() !== year
|| parsed.getUTCMonth() !== month - 1
|| parsed.getUTCDate() !== day
) {
fail("invalid date value");
}
return parsed;
}
function dateKey(value) {
if (value instanceof Date && !Number.isNaN(value.getTime())) {
return value.toISOString().slice(0, 10);
}
if (typeof value === "number" && Number.isFinite(value)) {
const epoch = Date.UTC(1899, 11, 30);
return new Date(epoch + Math.floor(value) * 86400000).toISOString().slice(0, 10);
}
if (typeof value === "string" && DATE_PATTERN.test(value.slice(0, 10))) {
return value.slice(0, 10);
}
return "";
}
function decimalNumber(value) {
const text = String(value ?? "");
const match = DECIMAL_PATTERN.exec(text);
if (!match) fail("invalid decimal value");
const number = Number(text);
if (
!Number.isFinite(number)
|| number < 0
|| !Number.isSafeInteger(Math.round(number * 100))
) {
fail("unsafe decimal value");
}
return number;
}
function columnName(index) {
let value = index;
let result = "";
while (value > 0) {
value -= 1;
result = String.fromCharCode(65 + (value % 26)) + result;
value = Math.floor(value / 26);
}
return result;
}
function validatePayload(payload) {
if (!payload || payload.schema_version !== "1.0") fail("invalid payload schema");
if (
!Number.isInteger(payload.report_year)
|| !Number.isInteger(payload.report_month)
|| payload.report_month < 1
|| payload.report_month > 12
|| !DATE_PATTERN.test(String(payload.as_of_date ?? ""))
|| typeof payload.filename !== "string"
|| !payload.filename.endsWith(".xlsx")
|| !Array.isArray(payload.channels)
|| payload.channels.length < 5
) {
fail("invalid monthly report contract");
}
const names = new Set();
payload.channels.forEach((channel, channelIndex) => {
const expectedHeaders = channel.worksheet === KB_SHEET
? [...STANDARD_HEADERS, KB_HEADER]
: STANDARD_HEADERS;
if (
typeof channel.worksheet !== "string"
|| channel.worksheet.length < 1
|| channel.worksheet.length > 31
|| channel.worksheet.trim() !== channel.worksheet
|| INVALID_SHEET_CHARS.test(channel.worksheet)
|| names.has(channel.worksheet)
|| channel.worksheet_order !== channelIndex + 1
|| !Array.isArray(channel.headers)
|| JSON.stringify(channel.headers) !== JSON.stringify(expectedHeaders)
|| !Array.isArray(channel.rows)
) {
fail("invalid channel worksheet contract");
}
names.add(channel.worksheet);
for (const row of channel.rows) {
if (!row || Object.keys(row).length !== expectedHeaders.length) {
fail("invalid monthly row contract");
}
for (const header of expectedHeaders) {
if (!Object.hasOwn(row, header)) fail("monthly row field is missing");
const value = row[header];
if (DATE_FIELDS.has(header)) {
excelDate(value);
} else if (INTEGER_FIELDS.has(header)) {
if (!Number.isInteger(value) || value < 0) fail("invalid integer value");
} else if (DECIMAL_FIELDS.has(header)) {
decimalNumber(value);
} else if (TEXT_FIELDS.has(header)) {
if (typeof value !== "string") fail("invalid text value");
} else {
fail("unknown monthly row field");
}
}
}
});
}
function rowValues(channel, row) {
return channel.headers.map((header) => {
const value = row[header];
if (DATE_FIELDS.has(header)) return excelDate(value);
if (INTEGER_FIELDS.has(header)) return value;
if (DECIMAL_FIELDS.has(header)) return decimalNumber(value);
return safeText(value);
});
}
function setColumnWidths(sheet, headers) {
const widths = {
ARRIVAL: 13,
DEPARTURE: 13,
NIGHTS: 9,
ADULTS: 9,
CHILDREN: 9,
BLOCK_CODE: 18,
NO_OF_ROOMS: 13,
COMPANY_NAME: 28,
CONFIRMATION_NO: 20,
DISP_ROOM_NO: 18,
RATE_AMOUNT: 14,
FULL_NAME: 22,
RES_COMMENT: 28,
TRACE_TEXT: 26,
PRODUCTS: 24,
RATE_CODE: 17,
ROOM_CATEGORY_LABEL: 22,
"REAL PRICE": 14,
"TOTAL PRICE": 16,
[KB_HEADER]: 18,
};
headers.forEach((header, index) => {
sheet.getRange(`${columnName(index + 1)}1`).format.columnWidth = widths[header] ?? 13;
});
}
function writeSheet(workbook, channel) {
const sheet = workbook.worksheets.add(channel.worksheet);
const matrix = [channel.headers, ...channel.rows.map((row) => rowValues(channel, row))];
const lastColumn = columnName(channel.headers.length);
const lastRow = matrix.length;
const used = sheet.getRange(`A1:${lastColumn}${lastRow}`);
used.values = matrix;
if (channel.rows.length > 0) {
sheet.getRange(`S2:S${lastRow}`).formulas = channel.rows.map((_, index) => [
`=R${index + 2}*C${index + 2}*G${index + 2}`,
]);
}
used.format.font = { name: "Arial", size: 10, color: BODY_FONT };
used.format.verticalAlignment = "center";
used.format.borders = {
top: { style: "thin", color: BORDER_COLOR },
bottom: { style: "thin", color: BORDER_COLOR },
left: { style: "thin", color: BORDER_COLOR },
right: { style: "thin", color: BORDER_COLOR },
};
setColumnWidths(sheet, channel.headers);
const header = sheet.getRange(`A1:${lastColumn}1`);
header.format.fill = HEADER_FILL;
header.format.font = { name: "Arial", size: 10, bold: true, color: "#000000" };
header.format.horizontalAlignment = "center";
header.format.rowHeight = 24;
header.format.wrapText = true;
sheet.freezePanes.freezeRows(1);
if (channel.rows.length > 0) {
const data = sheet.getRange(`A2:${lastColumn}${lastRow}`);
data.format.rowHeight = 22;
sheet.getRange(`A2:B${lastRow}`).setNumberFormat("dd-mmm-yy");
sheet.getRange(`C2:E${lastRow}`).setNumberFormat("0");
sheet.getRange(`G2:G${lastRow}`).setNumberFormat("0");
sheet.getRange(`K2:K${lastRow}`).setNumberFormat("0.##");
sheet.getRange(`R2:S${lastRow}`).setNumberFormat("0.##");
if (channel.worksheet === KB_SHEET) {
sheet.getRange(`T2:T${lastRow}`).setNumberFormat("0.##");
}
sheet.getRange(`A2:K${lastRow}`).format.horizontalAlignment = "center";
sheet.getRange(`L2:${lastColumn}${lastRow}`).format.horizontalAlignment = "left";
sheet.getRange(`F2:${lastColumn}${lastRow}`).format.wrapText = true;
}
sheet.showGridLines = true;
}
function expectedCell(header, value) {
if (DATE_FIELDS.has(header)) return { kind: "date", value: String(value) };
if (INTEGER_FIELDS.has(header)) return { kind: "number", value };
if (DECIMAL_FIELDS.has(header)) return { kind: "number", value: decimalNumber(value) };
// A leading apostrophe is the Excel formula-escape marker. The cell API
// exposes the displayed text after consuming that marker.
return { kind: "text", value: String(value ?? "") };
}
async function validateWorkbook(workbook, payload, stage) {
const names = workbook.worksheets.items.map((sheet) => sheet.name);
const expectedNames = payload.channels.map((channel) => channel.worksheet);
if (JSON.stringify(names) !== JSON.stringify(expectedNames)) {
fail(`${stage} worksheet names do not match`);
}
let formulaCount = 0;
for (const channel of payload.channels) {
const sheet = workbook.worksheets.getItem(channel.worksheet);
const expectedRows = channel.rows.length + 1;
const expectedColumns = channel.headers.length;
const lastColumn = columnName(expectedColumns);
const used = sheet.getUsedRange();
const values = used?.values ?? [];
if (
values.length !== expectedRows
|| (values[0]?.length ?? 0) !== expectedColumns
|| !channel.headers.every((header, index) => values[0][index] === header)
) {
fail(`${stage} worksheet dimensions or headers do not match`);
}
channel.rows.forEach((row, rowIndex) => {
const actual = values[rowIndex + 1] ?? [];
channel.headers.forEach((header, columnIndex) => {
const expected = expectedCell(header, row[header]);
const actualValue = actual[columnIndex];
if (
(expected.kind === "date" && dateKey(actualValue) !== expected.value)
|| (expected.kind === "number" && Number(actualValue) !== expected.value)
|| (expected.kind === "text" && String(actualValue ?? "") !== expected.value)
) {
fail(
`${stage} worksheet values do not match at ${channel.worksheet}!`
+ `${columnName(columnIndex + 1)}${rowIndex + 2}`,
);
}
});
});
const formulas = used?.formulas ?? [];
formulas.forEach((formulaRow, rowIndex) => {
formulaRow.forEach((formulaValue, columnIndex) => {
const formula = String(formulaValue ?? "").trim();
if (!formula) return;
formulaCount += 1;
const expectedFormula = `=R${rowIndex + 1}*C${rowIndex + 1}*G${rowIndex + 1}`;
if (
rowIndex < 1
|| columnIndex !== channel.headers.indexOf("TOTAL PRICE")
|| formula !== expectedFormula
) {
fail(`${stage} workbook contains an unauthorized formula`);
}
});
});
const inspected = await workbook.inspect({
kind: "formula",
sheetId: channel.worksheet,
range: `A1:${lastColumn}${expectedRows}`,
maxChars: 6000,
options: { maxResults: Math.max(channel.rows.length + 10, 100) },
});
const inspectionText = String(inspected.ndjson ?? "");
if (/#REF!|#DIV\/0!|#VALUE!|#NAME\?|#N\/A/.test(inspectionText)) {
fail(`${stage} workbook contains a formula error`);
}
}
const expectedFormulaCount = payload.channels.reduce(
(total, channel) => total + channel.rows.length,
0,
);
if (formulaCount !== expectedFormulaCount) {
fail(`${stage} workbook TOTAL PRICE formulas are incomplete`);
}
return formulaCount;
}
async function main() {
const [inputPath, outputPath, previewDir, summaryPath] = process.argv.slice(2);
if (!inputPath || !outputPath || !previewDir || !summaryPath) {
fail("usage: build_workbook.mjs INPUT_JSON OUTPUT_XLSX PREVIEW_DIR SUMMARY_JSON");
}
await loadArtifactTool();
const payload = JSON.parse(await fs.readFile(inputPath, "utf8"));
validatePayload(payload);
await fs.mkdir(path.dirname(outputPath), { recursive: true });
await fs.mkdir(previewDir, { recursive: true });
await fs.mkdir(path.dirname(summaryPath), { recursive: true });
const workbook = Workbook.create();
payload.channels.forEach((channel) => writeSheet(workbook, channel));
await validateWorkbook(workbook, payload, "pre-export");
const xlsx = await SpreadsheetFile.exportXlsx(workbook);
await xlsx.save(outputPath);
await fs.chmod(outputPath, 0o600);
const reopened = await SpreadsheetFile.importXlsx(await FileBlob.load(outputPath));
const formulaCount = await validateWorkbook(reopened, payload, "post-export");
const previews = [];
for (let index = 0; index < payload.channels.length; index += 1) {
const channel = payload.channels[index];
const lastColumn = columnName(channel.headers.length);
const previewRows = Math.min(channel.rows.length + 1, MAX_PREVIEW_ROWS);
const preview = await reopened.render({
sheetName: channel.worksheet,
range: `A1:${lastColumn}${previewRows}`,
autoCrop: "all",
scale: 1.2,
format: "png",
});
const previewPath = path.join(
previewDir,
`sheet-${String(index + 1).padStart(2, "0")}.png`,
);
await fs.writeFile(previewPath, new Uint8Array(await preview.arrayBuffer()), {
mode: 0o600,
});
previews.push(previewPath);
}
const stat = await fs.stat(outputPath);
const semanticSha256 = crypto
.createHash("sha256")
.update(JSON.stringify({ channels: payload.channels }), "utf8")
.digest("hex");
const summary = {
status: "success",
schema_version: payload.schema_version,
filename: payload.filename,
report_year: payload.report_year,
report_month: payload.report_month,
as_of_date: payload.as_of_date,
sheet_names: payload.channels.map((channel) => channel.worksheet),
row_counts: payload.channels.map((channel) => channel.rows.length),
formula_count: formulaCount,
semantic_sha256: semanticSha256,
preview_count: previews.length,
preview_rows: payload.channels.map((channel) => Math.min(channel.rows.length + 1, MAX_PREVIEW_ROWS)),
byte_size: stat.size,
};
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, {
encoding: "utf8",
mode: 0o600,
});
await fs.rm(`${outputPath}.inspect.ndjson`, { force: true });
process.stdout.write(`${JSON.stringify(summary)}\n`);
}
try {
await main();
} catch (_error) {
const outputPath = process.argv[3];
if (outputPath) {
await fs.rm(`${outputPath}.inspect.ndjson`, { force: true }).catch(() => undefined);
}
if (String(process.env.MONTHLY_REPORT_DEBUG ?? "") === "1") {
process.stderr.write(`${String(_error?.stack ?? _error)}\n`);
}
process.stderr.write(
`${JSON.stringify({
status: "failed",
code: "MONTHLY_REPORT_OUTPUT_VALIDATION_FAILED",
})}\n`,
);
process.exitCode = 4;
}

5
monthly_reports/xlsx/package.json
View File

@@ -1,5 +0,0 @@
{
"name": "arr-monthly-report-xlsx",
"private": true,
"type": "module"
}