feat: prepare ARR for controlled public deployment
This commit is contained in:
Wyndham ARR
401
monthly_reports/publishing.py
Normal file
401
monthly_reports/publishing.py
Normal file
@@ -0,0 +1,401 @@
|
||||
"""Artifact-tool adapter and recoverable atomic monthly-report publication."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Mapping, Optional
|
||||
|
||||
from monthly_reports.contracts import (
|
||||
RESULT_SCHEMA_VERSION,
|
||||
XLSX_MIME,
|
||||
ErrorCode,
|
||||
MonthlyReport,
|
||||
)
|
||||
from monthly_reports.repository import (
|
||||
FileMetadata,
|
||||
ReportRepository,
|
||||
RepositoryError,
|
||||
ReservedReport,
|
||||
)
|
||||
|
||||
|
||||
class BuildError(RuntimeError):
|
||||
def __init__(self, code: str, safe_message: str):
|
||||
super().__init__(safe_message)
|
||||
self.code = code
|
||||
self.safe_message = safe_message
|
||||
|
||||
|
||||
class PublicationError(RuntimeError):
|
||||
def __init__(self, code: str, safe_message: str):
|
||||
super().__init__(safe_message)
|
||||
self.code = code
|
||||
self.safe_message = safe_message
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuiltWorkbook:
|
||||
path: Path
|
||||
sha256: str
|
||||
byte_size: int
|
||||
summary: Mapping[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublicationOutcome:
|
||||
latest_path: Path
|
||||
latest_result_path: Path
|
||||
archive_path: Path
|
||||
result_path: Path
|
||||
artifact: FileMetadata
|
||||
result_json: FileMetadata
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _private_json(path: Path, payload: Mapping[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, sort_keys=True, indent=2)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
except Exception:
|
||||
try:
|
||||
os.close(descriptor)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
class ArtifactToolBuilder:
|
||||
def __init__(
|
||||
self,
|
||||
builder_script: Path,
|
||||
node_binary: Optional[str] = None,
|
||||
artifact_tool_module: Optional[Path] = None,
|
||||
timeout_seconds: int = 180,
|
||||
) -> None:
|
||||
configured = (node_binary or os.environ.get("MONTHLY_REPORT_NODE_BINARY", "")).strip()
|
||||
self._node_binary = configured or shutil.which("node") or ""
|
||||
self._builder_script = builder_script.resolve()
|
||||
self._artifact_tool_module = (
|
||||
artifact_tool_module.resolve() if artifact_tool_module else None
|
||||
)
|
||||
self._timeout_seconds = timeout_seconds
|
||||
|
||||
def build(self, report: MonthlyReport, work_dir: Path) -> BuiltWorkbook:
|
||||
if not self._node_binary or not self._builder_script.is_file():
|
||||
raise BuildError(
|
||||
ErrorCode.OUTPUT_VALIDATION_FAILED,
|
||||
"the monthly XLSX builder runtime is unavailable",
|
||||
)
|
||||
work_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
os.chmod(work_dir, 0o700)
|
||||
payload_path = work_dir / "workbook-payload.json"
|
||||
output_path = work_dir / report.filename
|
||||
preview_dir = work_dir / "previews"
|
||||
summary_path = work_dir / "workbook-summary.json"
|
||||
_private_json(payload_path, report.to_workbook_payload())
|
||||
environment = os.environ.copy()
|
||||
if self._artifact_tool_module is not None:
|
||||
environment["MONTHLY_REPORT_ARTIFACT_TOOL_MODULE"] = str(
|
||||
self._artifact_tool_module
|
||||
)
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
self._node_binary,
|
||||
str(self._builder_script),
|
||||
str(payload_path),
|
||||
str(output_path),
|
||||
str(preview_dir),
|
||||
str(summary_path),
|
||||
],
|
||||
cwd=self._builder_script.parent,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self._timeout_seconds,
|
||||
env=environment,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
raise BuildError(
|
||||
ErrorCode.OUTPUT_VALIDATION_FAILED,
|
||||
"the monthly XLSX builder did not complete",
|
||||
) from None
|
||||
if completed.returncode != 0:
|
||||
raise BuildError(
|
||||
ErrorCode.OUTPUT_VALIDATION_FAILED,
|
||||
"the monthly XLSX builder rejected the generated payload",
|
||||
)
|
||||
try:
|
||||
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
expected_names = [channel.worksheet for channel in report.channels]
|
||||
expected_rows = [len(channel.rows) for channel in report.channels]
|
||||
if (
|
||||
summary.get("status") != "success"
|
||||
or summary.get("schema_version") != RESULT_SCHEMA_VERSION
|
||||
or summary.get("filename") != report.filename
|
||||
or summary.get("report_year") != report.report_year
|
||||
or summary.get("report_month") != report.report_month
|
||||
or summary.get("as_of_date") != report.as_of_date.isoformat()
|
||||
or summary.get("sheet_names") != expected_names
|
||||
or summary.get("row_counts") != expected_rows
|
||||
or summary.get("formula_count") != 0
|
||||
or summary.get("preview_count") != len(report.channels)
|
||||
or not isinstance(summary.get("semantic_sha256"), str)
|
||||
or len(summary["semantic_sha256"]) != 64
|
||||
or not output_path.is_file()
|
||||
or output_path.stat().st_size <= 0
|
||||
):
|
||||
raise ValueError("builder summary mismatch")
|
||||
except (OSError, ValueError, TypeError, json.JSONDecodeError, KeyError):
|
||||
raise BuildError(
|
||||
ErrorCode.OUTPUT_VALIDATION_FAILED,
|
||||
"the monthly XLSX builder result could not be validated",
|
||||
) from None
|
||||
os.chmod(output_path, 0o600)
|
||||
return BuiltWorkbook(
|
||||
path=output_path,
|
||||
sha256=sha256_file(output_path),
|
||||
byte_size=output_path.stat().st_size,
|
||||
summary=summary,
|
||||
)
|
||||
|
||||
|
||||
class AtomicReportPublisher:
|
||||
def __init__(self, project_root: Path, output_root: Path) -> None:
|
||||
self._project_root = project_root.resolve()
|
||||
self._output_root = output_root.resolve()
|
||||
try:
|
||||
self._output_root.relative_to(self._project_root)
|
||||
except ValueError:
|
||||
raise ValueError("output_root must be inside project_root") from None
|
||||
|
||||
@staticmethod
|
||||
def _fsync_parent(path: Path) -> None:
|
||||
descriptor = os.open(path.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
@classmethod
|
||||
def _atomic_copy(cls, source: Path, target: Path) -> None:
|
||||
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
temporary = target.parent / f".{target.name}.tmp-{uuid.uuid4().hex}"
|
||||
try:
|
||||
shutil.copyfile(source, temporary)
|
||||
os.chmod(temporary, 0o600)
|
||||
with temporary.open("rb") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, target)
|
||||
cls._fsync_parent(target)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
@classmethod
|
||||
def _atomic_json(cls, target: Path, payload: Mapping[str, Any]) -> None:
|
||||
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
temporary = target.parent / f".{target.name}.tmp-{uuid.uuid4().hex}"
|
||||
try:
|
||||
_private_json(temporary, payload)
|
||||
os.replace(temporary, target)
|
||||
cls._fsync_parent(target)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
@classmethod
|
||||
def _install_once(cls, source: Path, target: Path, expected_sha256: str) -> bool:
|
||||
if target.exists():
|
||||
if sha256_file(target) != expected_sha256:
|
||||
raise PublicationError(
|
||||
ErrorCode.PUBLISH_FAILED,
|
||||
"an immutable monthly archive path contains different content",
|
||||
)
|
||||
return False
|
||||
cls._atomic_copy(source, target)
|
||||
return True
|
||||
|
||||
def _storage_key(self, path: Path) -> str:
|
||||
try:
|
||||
return path.resolve().relative_to(self._project_root).as_posix()
|
||||
except ValueError:
|
||||
raise PublicationError(
|
||||
ErrorCode.PUBLISH_FAILED,
|
||||
"a monthly artifact escaped the controlled project root",
|
||||
) from None
|
||||
|
||||
@staticmethod
|
||||
def _success_result(
|
||||
report: MonthlyReport,
|
||||
reservation: ReservedReport,
|
||||
artifact_storage_key: str,
|
||||
artifact_sha256: str,
|
||||
semantic_sha256: str,
|
||||
) -> Dict[str, Any]:
|
||||
return {
|
||||
"schema_version": RESULT_SCHEMA_VERSION,
|
||||
"status": "success",
|
||||
"report_year": report.report_year,
|
||||
"report_month": report.report_month,
|
||||
"as_of_date": report.as_of_date.isoformat(),
|
||||
"report_version_id": reservation.report_version_id,
|
||||
"version_no": reservation.version_no,
|
||||
"artifact": {
|
||||
"filename": report.filename,
|
||||
"storage_key": artifact_storage_key,
|
||||
"sha256": artifact_sha256,
|
||||
"semantic_sha256": semantic_sha256,
|
||||
},
|
||||
"row_count": report.row_count,
|
||||
"channel_manifest": [
|
||||
{
|
||||
"worksheet": worksheet,
|
||||
"worksheet_order": order,
|
||||
"row_count": row_count,
|
||||
}
|
||||
for worksheet, order, row_count in report.channel_manifest
|
||||
],
|
||||
"daily_version_count": len(report.daily_versions),
|
||||
"warnings": [],
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
def publish(
|
||||
self,
|
||||
report: MonthlyReport,
|
||||
reservation: ReservedReport,
|
||||
built: BuiltWorkbook,
|
||||
repository: ReportRepository,
|
||||
work_dir: Path,
|
||||
) -> PublicationOutcome:
|
||||
month_root = self._output_root / f"{report.report_year:04d}" / f"{report.report_month:02d}"
|
||||
version_root = month_root / "archive" / f"v{reservation.version_no:04d}"
|
||||
archive_path = version_root / report.filename
|
||||
result_path = version_root / "result.json"
|
||||
latest_path = month_root / "latest.xlsx"
|
||||
latest_result_path = month_root / "latest.result.json"
|
||||
workbook_backup = work_dir / "previous-latest.xlsx"
|
||||
result_backup = work_dir / "previous-latest.result.json"
|
||||
latest_existed = latest_path.is_file()
|
||||
latest_result_existed = latest_result_path.is_file()
|
||||
archive_created = False
|
||||
result_created = False
|
||||
latest_replaced = False
|
||||
latest_result_replaced = False
|
||||
|
||||
try:
|
||||
month_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if latest_existed:
|
||||
shutil.copyfile(latest_path, workbook_backup)
|
||||
os.chmod(workbook_backup, 0o600)
|
||||
if latest_result_existed:
|
||||
shutil.copyfile(latest_result_path, result_backup)
|
||||
os.chmod(result_backup, 0o600)
|
||||
|
||||
archive_created = self._install_once(built.path, archive_path, built.sha256)
|
||||
artifact_storage_key = self._storage_key(archive_path)
|
||||
result_payload = self._success_result(
|
||||
report,
|
||||
reservation,
|
||||
artifact_storage_key,
|
||||
built.sha256,
|
||||
str(built.summary.get("semantic_sha256", "")),
|
||||
)
|
||||
if result_path.exists():
|
||||
expected = json.dumps(
|
||||
result_payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
indent=2,
|
||||
) + "\n"
|
||||
if result_path.read_text(encoding="utf-8") != expected:
|
||||
raise PublicationError(
|
||||
ErrorCode.PUBLISH_FAILED,
|
||||
"an immutable monthly result path contains different content",
|
||||
)
|
||||
else:
|
||||
self._atomic_json(result_path, result_payload)
|
||||
result_created = True
|
||||
|
||||
self._atomic_copy(built.path, latest_path)
|
||||
latest_replaced = True
|
||||
self._atomic_json(latest_result_path, result_payload)
|
||||
latest_result_replaced = True
|
||||
artifact = FileMetadata(
|
||||
file_kind="monthly_xlsx",
|
||||
original_filename=report.filename,
|
||||
storage_key=artifact_storage_key,
|
||||
sha256=built.sha256,
|
||||
byte_size=built.byte_size,
|
||||
mime_type=XLSX_MIME,
|
||||
)
|
||||
result_json = FileMetadata(
|
||||
file_kind="result_json",
|
||||
original_filename=result_path.name,
|
||||
storage_key=self._storage_key(result_path),
|
||||
sha256=sha256_file(result_path),
|
||||
byte_size=result_path.stat().st_size,
|
||||
mime_type="application/json",
|
||||
)
|
||||
repository.activate_report(reservation, artifact, result_json)
|
||||
return PublicationOutcome(
|
||||
latest_path=latest_path,
|
||||
latest_result_path=latest_result_path,
|
||||
archive_path=archive_path,
|
||||
result_path=result_path,
|
||||
artifact=artifact,
|
||||
result_json=result_json,
|
||||
)
|
||||
except Exception as error:
|
||||
if latest_result_replaced:
|
||||
if latest_result_existed and result_backup.is_file():
|
||||
self._atomic_copy(result_backup, latest_result_path)
|
||||
else:
|
||||
latest_result_path.unlink(missing_ok=True)
|
||||
if latest_replaced:
|
||||
if latest_existed and workbook_backup.is_file():
|
||||
self._atomic_copy(workbook_backup, latest_path)
|
||||
else:
|
||||
latest_path.unlink(missing_ok=True)
|
||||
if result_created:
|
||||
result_path.unlink(missing_ok=True)
|
||||
if archive_created:
|
||||
archive_path.unlink(missing_ok=True)
|
||||
code = getattr(error, "code", ErrorCode.PUBLISH_FAILED)
|
||||
safe_message = getattr(error, "safe_message", "monthly report publication failed")
|
||||
try:
|
||||
repository.mark_failed(reservation, code, safe_message)
|
||||
except Exception:
|
||||
pass
|
||||
if isinstance(error, PublicationError):
|
||||
raise
|
||||
if isinstance(error, RepositoryError):
|
||||
raise PublicationError(error.code, error.safe_message) from None
|
||||
raise PublicationError(
|
||||
ErrorCode.PUBLISH_FAILED,
|
||||
"monthly report publication failed",
|
||||
) from None
|
||||
|
||||
|
||||
def private_staging_directory(parent: Path):
|
||||
parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
return tempfile.TemporaryDirectory(prefix="monthly-report-", dir=parent)
|
||||
Reference in New Issue
Block a user