shiyuyun/wyndham-ARR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: move report artifacts to OSS storage

Browse Source
This commit is contained in:
Wyndham ARR
2026-08-04 12:37:26 +08:00
parent 727643f1f2
commit a2b86cdf10
45 changed files with 1288 additions and 1417 deletions
Download Patch File Download Diff File Expand all files Collapse all files

324
monthly_reports/publishing.py
View File

@@ -1,4 +1,4 @@
"""Artifact-tool adapter and recoverable atomic monthly-report publication."""
"""Python/openpyxl monthly workbook building and recoverable publication."""
from __future__ import annotations
@@ -6,19 +6,20 @@ import hashlib
import json
import os
import shutil
import subprocess
import tempfile
import uuid
from dataclasses import dataclass
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from pathlib import Path
from typing import Any, Dict, Mapping, Optional
from monthly_reports.contracts import (
RESULT_SCHEMA_VERSION,
XLSX_MIME,
ErrorCode,
MonthlyReport,
)
from openpyxl import Workbook, load_workbook
from openpyxl.styles import Alignment, Font, PatternFill
from openpyxl.utils import get_column_letter
from arr_storage.store import ManagedObjectStore
from monthly_reports.contracts import KB_HEADER, RESULT_SCHEMA_VERSION, XLSX_MIME, ErrorCode, MonthlyReport
from monthly_reports.repository import (
FileMetadata,
ReportRepository,
@@ -84,94 +85,178 @@ def _private_json(path: Path, payload: Mapping[str, Any]) -> None:
raise
class ArtifactToolBuilder:
def __init__(
self,
builder_script: Path,
node_binary: Optional[str] = None,
artifact_tool_module: Optional[Path] = None,
timeout_seconds: int = 180,
) -> None:
configured = (node_binary or os.environ.get("MONTHLY_REPORT_NODE_BINARY", "")).strip()
self._node_binary = configured or shutil.which("node") or ""
self._builder_script = builder_script.resolve()
self._artifact_tool_module = (
artifact_tool_module.resolve() if artifact_tool_module else None
)
self._timeout_seconds = timeout_seconds
class OpenpyxlWorkbookBuilder:
"""Build and re-open a validated monthly workbook without external runtimes."""
def build(self, report: MonthlyReport, work_dir: Path) -> BuiltWorkbook:
if not self._node_binary or not self._builder_script.is_file():
raise BuildError(
ErrorCode.OUTPUT_VALIDATION_FAILED,
"the monthly XLSX builder runtime is unavailable",
)
work_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(work_dir, 0o700)
payload_path = work_dir / "workbook-payload.json"
output_path = work_dir / report.filename
preview_dir = work_dir / "previews"
summary_path = work_dir / "workbook-summary.json"
_private_json(payload_path, report.to_workbook_payload())
environment = os.environ.copy()
if self._artifact_tool_module is not None:
environment["MONTHLY_REPORT_ARTIFACT_TOOL_MODULE"] = str(
self._artifact_tool_module
_INTEGER_HEADERS = frozenset({"NIGHTS", "ADULTS", "CHILDREN", "NO_OF_ROOMS"})
_DECIMAL_HEADERS = frozenset({"RATE_AMOUNT", "REAL PRICE", "TOTAL PRICE", KB_HEADER})
def __init__(self, *_legacy_args: Any, **_legacy_options: Any) -> None:
# The ignored arguments keep older local wrappers import-compatible while
# making the production builder entirely Python/openpyxl based.
_ = (_legacy_args, _legacy_options)
@staticmethod
def _safe_text(value: Any) -> str:
text = "" if value is None else str(value)
return "'" + text if text[:1] in {"=", "+", "-", "@"} else text
@staticmethod
def _date_key(value: Any) -> str:
if isinstance(value, datetime):
return value.date().isoformat()
if isinstance(value, date):
return value.isoformat()
if isinstance(value, str) and len(value) >= 10:
return value[:10]
return ""
@staticmethod
def _semantic_sha256(payload: Mapping[str, Any]) -> str:
encoded = json.dumps(
{"channels": payload["channels"]},
ensure_ascii=False,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
@classmethod
def _cell_value(cls, header: str, value: Any) -> Any:
if header in {"ARRIVAL", "DEPARTURE"}:
return date.fromisoformat(str(value))
if header in cls._INTEGER_HEADERS:
return int(value)
if header in cls._DECIMAL_HEADERS:
return float(Decimal(str(value)))
return cls._safe_text(value)
@staticmethod
def _style_sheet(worksheet: Any, headers: list[str]) -> None:
worksheet.freeze_panes = "A2"
worksheet.auto_filter.ref = f"A1:{get_column_letter(len(headers))}{max(1, worksheet.max_row)}"
for column, header in enumerate(headers, start=1):
cell = worksheet.cell(1, column)
cell.font = Font(name="Arial", size=10, bold=True, color="000000")
cell.fill = PatternFill("solid", fgColor="D9EAF7")
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
worksheet.column_dimensions[get_column_letter(column)].width = min(
34, max(12, len(header) + 2)
)
worksheet.row_dimensions[1].height = 28
for row in worksheet.iter_rows(min_row=2):
for cell in row:
cell.font = Font(name="Arial", size=10, color="222222")
cell.alignment = Alignment(vertical="center", wrap_text=True)
worksheet.row_dimensions[row[0].row].height = 24
for column in ("A", "B"):
for cell in worksheet[column][1:]:
cell.number_format = "yyyy-mm-dd"
@classmethod
def _write_sheet(cls, worksheet: Any, channel: Mapping[str, Any]) -> int:
headers = [str(header) for header in channel["headers"]]
worksheet.append(headers)
total_column = headers.index("TOTAL PRICE") + 1
for row in channel["rows"]:
row_index = worksheet.max_row + 1
values = [
cls._cell_value(header, row.get(header, ""))
for header in headers
]
values[total_column - 1] = f"=R{row_index}*C{row_index}*G{row_index}"
worksheet.append(values)
cls._style_sheet(worksheet, headers)
return len(channel["rows"])
@classmethod
def _validate_workbook(
cls,
output_path: Path,
report: MonthlyReport,
payload: Mapping[str, Any],
) -> int:
workbook = load_workbook(output_path, data_only=False, read_only=True)
try:
completed = subprocess.run(
[
self._node_binary,
str(self._builder_script),
str(payload_path),
str(output_path),
str(preview_dir),
str(summary_path),
],
cwd=self._builder_script.parent,
capture_output=True,
text=True,
timeout=self._timeout_seconds,
env=environment,
check=False,
)
except (OSError, subprocess.TimeoutExpired):
raise BuildError(
ErrorCode.OUTPUT_VALIDATION_FAILED,
"the monthly XLSX builder did not complete",
) from None
if completed.returncode != 0:
raise BuildError(
ErrorCode.OUTPUT_VALIDATION_FAILED,
"the monthly XLSX builder rejected the generated payload",
)
try:
summary = json.loads(summary_path.read_text(encoding="utf-8"))
expected_names = [channel.worksheet for channel in report.channels]
expected_rows = [len(channel.rows) for channel in report.channels]
if (
summary.get("status") != "success"
or summary.get("schema_version") != RESULT_SCHEMA_VERSION
or summary.get("filename") != report.filename
or summary.get("report_year") != report.report_year
or summary.get("report_month") != report.report_month
or summary.get("as_of_date") != report.as_of_date.isoformat()
or summary.get("sheet_names") != expected_names
or summary.get("row_counts") != expected_rows
or summary.get("formula_count") != report.row_count
or summary.get("preview_count") != len(report.channels)
or not isinstance(summary.get("semantic_sha256"), str)
or len(summary["semantic_sha256"]) != 64
or not output_path.is_file()
or output_path.stat().st_size <= 0
):
raise ValueError("builder summary mismatch")
except (OSError, ValueError, TypeError, json.JSONDecodeError, KeyError):
if workbook.sheetnames != expected_names:
raise ValueError("worksheet names do not match")
formula_count = 0
for channel, expected_row_count in zip(report.channels, expected_rows):
worksheet = workbook[channel.worksheet]
headers = list(channel.headers)
if worksheet.max_row != expected_row_count + 1 or worksheet.max_column != len(headers):
raise ValueError("worksheet dimensions do not match")
if [worksheet.cell(1, index).value for index in range(1, len(headers) + 1)] != headers:
raise ValueError("worksheet headers do not match")
for row_index, expected_row in enumerate(channel.rows, start=2):
payload_row = expected_row.to_payload(channel.worksheet == "DY-AI-Easy-KB")
for column, header in enumerate(headers, start=1):
cell = worksheet.cell(row_index, column)
if header == "TOTAL PRICE":
expected_formula = f"=R{row_index}*C{row_index}*G{row_index}"
if cell.value != expected_formula or cell.data_type != "f":
raise ValueError("total price formula does not match")
formula_count += 1
continue
actual = cell.value
expected = payload_row[header]
if header in {"ARRIVAL", "DEPARTURE"}:
matches = cls._date_key(actual) == str(expected)
elif header in cls._INTEGER_HEADERS:
matches = actual == int(expected)
elif header in cls._DECIMAL_HEADERS:
matches = Decimal(str(actual)) == Decimal(str(expected))
else:
matches = ("" if actual is None else actual) == cls._safe_text(expected)
if not matches:
raise ValueError("worksheet values do not match")
if formula_count != report.row_count:
raise ValueError("formula count does not match")
if cls._semantic_sha256(payload) != cls._semantic_sha256(report.to_workbook_payload()):
raise ValueError("semantic source differs")
return formula_count
finally:
workbook.close()
def build(self, report: MonthlyReport, work_dir: Path) -> BuiltWorkbook:
work_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(work_dir, 0o700)
output_path = work_dir / report.filename
summary_path = work_dir / "workbook-summary.json"
payload = report.to_workbook_payload()
try:
workbook = Workbook()
for index, channel in enumerate(payload["channels"]):
worksheet = workbook.active if index == 0 else workbook.create_sheet()
worksheet.title = str(channel["worksheet"])
self._write_sheet(worksheet, channel)
workbook.save(output_path)
workbook.close()
os.chmod(output_path, 0o600)
formula_count = self._validate_workbook(output_path, report, payload)
expected_names = [channel.worksheet for channel in report.channels]
expected_rows = [len(channel.rows) for channel in report.channels]
summary = {
"status": "success",
"schema_version": payload["schema_version"],
"report_year": report.report_year,
"report_month": report.report_month,
"as_of_date": report.as_of_date.isoformat(),
"filename": report.filename,
"sheet_names": expected_names,
"row_counts": expected_rows,
"formula_count": formula_count,
"semantic_sha256": self._semantic_sha256(payload),
"preview_count": len(expected_names),
"byte_size": output_path.stat().st_size,
}
_private_json(summary_path, summary)
except (OSError, ValueError, TypeError, InvalidOperation, KeyError):
raise BuildError(
ErrorCode.OUTPUT_VALIDATION_FAILED,
"the monthly XLSX builder result could not be validated",
) from None
os.chmod(output_path, 0o600)
return BuiltWorkbook(
path=output_path,
sha256=sha256_file(output_path),
@@ -179,11 +264,19 @@ class ArtifactToolBuilder:
summary=summary,
)
class AtomicReportPublisher:
def __init__(self, project_root: Path, output_root: Path) -> None:
def __init__(
self,
project_root: Path,
output_root: Path,
*,
object_store: Optional[ManagedObjectStore] = None,
bucket_alias: str = "arr-private",
) -> None:
self._project_root = project_root.resolve()
self._output_root = output_root.resolve()
self._object_store = object_store
self._bucket_alias = bucket_alias
try:
self._output_root.relative_to(self._project_root)
except ValueError:
@@ -250,6 +343,8 @@ class AtomicReportPublisher:
artifact_storage_key: str,
artifact_sha256: str,
semantic_sha256: str,
storage_provider: str,
bucket_alias: str,
) -> Dict[str, Any]:
return {
"schema_version": RESULT_SCHEMA_VERSION,
@@ -264,6 +359,8 @@ class AtomicReportPublisher:
"storage_key": artifact_storage_key,
"sha256": artifact_sha256,
"semantic_sha256": semantic_sha256,
"storage_provider": storage_provider,
"bucket_alias": bucket_alias,
},
"row_count": report.row_count,
"channel_manifest": [
@@ -312,13 +409,30 @@ class AtomicReportPublisher:
os.chmod(result_backup, 0o600)
archive_created = self._install_once(built.path, archive_path, built.sha256)
storage_provider = "local"
bucket_alias = "arr-project-root"
artifact_storage_key = self._storage_key(archive_path)
if self._object_store is not None:
uploaded_artifact = self._object_store.upload_committed(
job_id=f"monthly-report-{reservation.report_version_id}",
attempt_no=1,
role="monthly_report",
source=archive_path,
original_filename=report.filename,
expected_sha256=built.sha256,
expected_byte_size=built.byte_size,
)
storage_provider = "oss"
bucket_alias = self._bucket_alias
artifact_storage_key = uploaded_artifact.object_key
result_payload = self._success_result(
report,
reservation,
artifact_storage_key,
built.sha256,
str(built.summary.get("semantic_sha256", "")),
storage_provider,
bucket_alias,
)
if result_path.exists():
expected = json.dumps(
@@ -355,6 +469,38 @@ class AtomicReportPublisher:
sha256=sha256_file(result_path),
byte_size=result_path.stat().st_size,
mime_type="application/json",
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
if self._object_store is not None:
uploaded_result = self._object_store.upload_committed(
job_id=f"monthly-report-{reservation.report_version_id}",
attempt_no=1,
role="result_json",
source=result_path,
original_filename=result_path.name,
expected_sha256=result_json.sha256,
expected_byte_size=result_json.byte_size,
)
result_json = FileMetadata(
file_kind="result_json",
original_filename=result_path.name,
storage_key=uploaded_result.object_key,
sha256=uploaded_result.sha256,
byte_size=uploaded_result.byte_size,
mime_type=uploaded_result.mime_type,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
artifact = FileMetadata(
file_kind="monthly_xlsx",
original_filename=report.filename,
storage_key=artifact_storage_key,
sha256=built.sha256,
byte_size=built.byte_size,
mime_type=XLSX_MIME,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
repository.activate_report(
reservation,