407 lines
15 KiB
Python
407 lines
15 KiB
Python
"""Artifact-tool adapter and recoverable atomic monthly-report publication."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Mapping, Optional
|
|
|
|
from monthly_reports.contracts import (
|
|
RESULT_SCHEMA_VERSION,
|
|
XLSX_MIME,
|
|
ErrorCode,
|
|
MonthlyReport,
|
|
)
|
|
from monthly_reports.repository import (
|
|
FileMetadata,
|
|
ReportRepository,
|
|
RepositoryError,
|
|
ReservedReport,
|
|
)
|
|
|
|
|
|
class BuildError(RuntimeError):
|
|
def __init__(self, code: str, safe_message: str):
|
|
super().__init__(safe_message)
|
|
self.code = code
|
|
self.safe_message = safe_message
|
|
|
|
|
|
class PublicationError(RuntimeError):
|
|
def __init__(self, code: str, safe_message: str):
|
|
super().__init__(safe_message)
|
|
self.code = code
|
|
self.safe_message = safe_message
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BuiltWorkbook:
|
|
path: Path
|
|
sha256: str
|
|
byte_size: int
|
|
summary: Mapping[str, Any]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PublicationOutcome:
|
|
latest_path: Path
|
|
latest_result_path: Path
|
|
archive_path: Path
|
|
result_path: Path
|
|
artifact: FileMetadata
|
|
result_json: FileMetadata
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _private_json(path: Path, payload: Mapping[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
try:
|
|
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
|
json.dump(payload, handle, ensure_ascii=False, sort_keys=True, indent=2)
|
|
handle.write("\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
except Exception:
|
|
try:
|
|
os.close(descriptor)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
|
|
|
|
class ArtifactToolBuilder:
|
|
def __init__(
|
|
self,
|
|
builder_script: Path,
|
|
node_binary: Optional[str] = None,
|
|
artifact_tool_module: Optional[Path] = None,
|
|
timeout_seconds: int = 180,
|
|
) -> None:
|
|
configured = (node_binary or os.environ.get("MONTHLY_REPORT_NODE_BINARY", "")).strip()
|
|
self._node_binary = configured or shutil.which("node") or ""
|
|
self._builder_script = builder_script.resolve()
|
|
self._artifact_tool_module = (
|
|
artifact_tool_module.resolve() if artifact_tool_module else None
|
|
)
|
|
self._timeout_seconds = timeout_seconds
|
|
|
|
def build(self, report: MonthlyReport, work_dir: Path) -> BuiltWorkbook:
|
|
if not self._node_binary or not self._builder_script.is_file():
|
|
raise BuildError(
|
|
ErrorCode.OUTPUT_VALIDATION_FAILED,
|
|
"the monthly XLSX builder runtime is unavailable",
|
|
)
|
|
work_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
os.chmod(work_dir, 0o700)
|
|
payload_path = work_dir / "workbook-payload.json"
|
|
output_path = work_dir / report.filename
|
|
preview_dir = work_dir / "previews"
|
|
summary_path = work_dir / "workbook-summary.json"
|
|
_private_json(payload_path, report.to_workbook_payload())
|
|
environment = os.environ.copy()
|
|
if self._artifact_tool_module is not None:
|
|
environment["MONTHLY_REPORT_ARTIFACT_TOOL_MODULE"] = str(
|
|
self._artifact_tool_module
|
|
)
|
|
try:
|
|
completed = subprocess.run(
|
|
[
|
|
self._node_binary,
|
|
str(self._builder_script),
|
|
str(payload_path),
|
|
str(output_path),
|
|
str(preview_dir),
|
|
str(summary_path),
|
|
],
|
|
cwd=self._builder_script.parent,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=self._timeout_seconds,
|
|
env=environment,
|
|
check=False,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
raise BuildError(
|
|
ErrorCode.OUTPUT_VALIDATION_FAILED,
|
|
"the monthly XLSX builder did not complete",
|
|
) from None
|
|
if completed.returncode != 0:
|
|
raise BuildError(
|
|
ErrorCode.OUTPUT_VALIDATION_FAILED,
|
|
"the monthly XLSX builder rejected the generated payload",
|
|
)
|
|
try:
|
|
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
|
expected_names = [channel.worksheet for channel in report.channels]
|
|
expected_rows = [len(channel.rows) for channel in report.channels]
|
|
if (
|
|
summary.get("status") != "success"
|
|
or summary.get("schema_version") != RESULT_SCHEMA_VERSION
|
|
or summary.get("filename") != report.filename
|
|
or summary.get("report_year") != report.report_year
|
|
or summary.get("report_month") != report.report_month
|
|
or summary.get("as_of_date") != report.as_of_date.isoformat()
|
|
or summary.get("sheet_names") != expected_names
|
|
or summary.get("row_counts") != expected_rows
|
|
or summary.get("formula_count") != report.row_count
|
|
or summary.get("preview_count") != len(report.channels)
|
|
or not isinstance(summary.get("semantic_sha256"), str)
|
|
or len(summary["semantic_sha256"]) != 64
|
|
or not output_path.is_file()
|
|
or output_path.stat().st_size <= 0
|
|
):
|
|
raise ValueError("builder summary mismatch")
|
|
except (OSError, ValueError, TypeError, json.JSONDecodeError, KeyError):
|
|
raise BuildError(
|
|
ErrorCode.OUTPUT_VALIDATION_FAILED,
|
|
"the monthly XLSX builder result could not be validated",
|
|
) from None
|
|
os.chmod(output_path, 0o600)
|
|
return BuiltWorkbook(
|
|
path=output_path,
|
|
sha256=sha256_file(output_path),
|
|
byte_size=output_path.stat().st_size,
|
|
summary=summary,
|
|
)
|
|
|
|
|
|
class AtomicReportPublisher:
|
|
def __init__(self, project_root: Path, output_root: Path) -> None:
|
|
self._project_root = project_root.resolve()
|
|
self._output_root = output_root.resolve()
|
|
try:
|
|
self._output_root.relative_to(self._project_root)
|
|
except ValueError:
|
|
raise ValueError("output_root must be inside project_root") from None
|
|
|
|
@staticmethod
|
|
def _fsync_parent(path: Path) -> None:
|
|
descriptor = os.open(path.parent, os.O_RDONLY)
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
@classmethod
|
|
def _atomic_copy(cls, source: Path, target: Path) -> None:
|
|
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
temporary = target.parent / f".{target.name}.tmp-{uuid.uuid4().hex}"
|
|
try:
|
|
shutil.copyfile(source, temporary)
|
|
os.chmod(temporary, 0o600)
|
|
with temporary.open("rb") as handle:
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary, target)
|
|
cls._fsync_parent(target)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
@classmethod
|
|
def _atomic_json(cls, target: Path, payload: Mapping[str, Any]) -> None:
|
|
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
temporary = target.parent / f".{target.name}.tmp-{uuid.uuid4().hex}"
|
|
try:
|
|
_private_json(temporary, payload)
|
|
os.replace(temporary, target)
|
|
cls._fsync_parent(target)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
@classmethod
|
|
def _install_once(cls, source: Path, target: Path, expected_sha256: str) -> bool:
|
|
if target.exists():
|
|
if sha256_file(target) != expected_sha256:
|
|
raise PublicationError(
|
|
ErrorCode.PUBLISH_FAILED,
|
|
"an immutable monthly archive path contains different content",
|
|
)
|
|
return False
|
|
cls._atomic_copy(source, target)
|
|
return True
|
|
|
|
def _storage_key(self, path: Path) -> str:
|
|
try:
|
|
return path.resolve().relative_to(self._project_root).as_posix()
|
|
except ValueError:
|
|
raise PublicationError(
|
|
ErrorCode.PUBLISH_FAILED,
|
|
"a monthly artifact escaped the controlled project root",
|
|
) from None
|
|
|
|
@staticmethod
|
|
def _success_result(
|
|
report: MonthlyReport,
|
|
reservation: ReservedReport,
|
|
artifact_storage_key: str,
|
|
artifact_sha256: str,
|
|
semantic_sha256: str,
|
|
) -> Dict[str, Any]:
|
|
return {
|
|
"schema_version": RESULT_SCHEMA_VERSION,
|
|
"status": "success",
|
|
"report_year": report.report_year,
|
|
"report_month": report.report_month,
|
|
"as_of_date": report.as_of_date.isoformat(),
|
|
"report_version_id": reservation.report_version_id,
|
|
"version_no": reservation.version_no,
|
|
"artifact": {
|
|
"filename": report.filename,
|
|
"storage_key": artifact_storage_key,
|
|
"sha256": artifact_sha256,
|
|
"semantic_sha256": semantic_sha256,
|
|
},
|
|
"row_count": report.row_count,
|
|
"channel_manifest": [
|
|
{
|
|
"worksheet": worksheet,
|
|
"worksheet_order": order,
|
|
"row_count": row_count,
|
|
}
|
|
for worksheet, order, row_count in report.channel_manifest
|
|
],
|
|
"daily_version_count": len(report.daily_versions),
|
|
"warnings": [],
|
|
"errors": [],
|
|
}
|
|
|
|
def publish(
|
|
self,
|
|
report: MonthlyReport,
|
|
reservation: ReservedReport,
|
|
built: BuiltWorkbook,
|
|
repository: ReportRepository,
|
|
work_dir: Path,
|
|
) -> PublicationOutcome:
|
|
month_root = self._output_root / f"{report.report_year:04d}" / f"{report.report_month:02d}"
|
|
version_root = month_root / "archive" / f"v{reservation.version_no:04d}"
|
|
archive_path = version_root / report.filename
|
|
result_path = version_root / "result.json"
|
|
latest_path = month_root / "latest.xlsx"
|
|
latest_result_path = month_root / "latest.result.json"
|
|
workbook_backup = work_dir / "previous-latest.xlsx"
|
|
result_backup = work_dir / "previous-latest.result.json"
|
|
latest_existed = latest_path.is_file()
|
|
latest_result_existed = latest_result_path.is_file()
|
|
archive_created = False
|
|
result_created = False
|
|
latest_replaced = False
|
|
latest_result_replaced = False
|
|
|
|
try:
|
|
month_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
if latest_existed:
|
|
shutil.copyfile(latest_path, workbook_backup)
|
|
os.chmod(workbook_backup, 0o600)
|
|
if latest_result_existed:
|
|
shutil.copyfile(latest_result_path, result_backup)
|
|
os.chmod(result_backup, 0o600)
|
|
|
|
archive_created = self._install_once(built.path, archive_path, built.sha256)
|
|
artifact_storage_key = self._storage_key(archive_path)
|
|
result_payload = self._success_result(
|
|
report,
|
|
reservation,
|
|
artifact_storage_key,
|
|
built.sha256,
|
|
str(built.summary.get("semantic_sha256", "")),
|
|
)
|
|
if result_path.exists():
|
|
expected = json.dumps(
|
|
result_payload,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
indent=2,
|
|
) + "\n"
|
|
if result_path.read_text(encoding="utf-8") != expected:
|
|
raise PublicationError(
|
|
ErrorCode.PUBLISH_FAILED,
|
|
"an immutable monthly result path contains different content",
|
|
)
|
|
else:
|
|
self._atomic_json(result_path, result_payload)
|
|
result_created = True
|
|
|
|
self._atomic_copy(built.path, latest_path)
|
|
latest_replaced = True
|
|
self._atomic_json(latest_result_path, result_payload)
|
|
latest_result_replaced = True
|
|
artifact = FileMetadata(
|
|
file_kind="monthly_xlsx",
|
|
original_filename=report.filename,
|
|
storage_key=artifact_storage_key,
|
|
sha256=built.sha256,
|
|
byte_size=built.byte_size,
|
|
mime_type=XLSX_MIME,
|
|
)
|
|
result_json = FileMetadata(
|
|
file_kind="result_json",
|
|
original_filename=result_path.name,
|
|
storage_key=self._storage_key(result_path),
|
|
sha256=sha256_file(result_path),
|
|
byte_size=result_path.stat().st_size,
|
|
mime_type="application/json",
|
|
)
|
|
repository.activate_report(
|
|
reservation,
|
|
artifact,
|
|
result_json,
|
|
str(built.summary.get("semantic_sha256", "")),
|
|
)
|
|
return PublicationOutcome(
|
|
latest_path=latest_path,
|
|
latest_result_path=latest_result_path,
|
|
archive_path=archive_path,
|
|
result_path=result_path,
|
|
artifact=artifact,
|
|
result_json=result_json,
|
|
)
|
|
except Exception as error:
|
|
if latest_result_replaced:
|
|
if latest_result_existed and result_backup.is_file():
|
|
self._atomic_copy(result_backup, latest_result_path)
|
|
else:
|
|
latest_result_path.unlink(missing_ok=True)
|
|
if latest_replaced:
|
|
if latest_existed and workbook_backup.is_file():
|
|
self._atomic_copy(workbook_backup, latest_path)
|
|
else:
|
|
latest_path.unlink(missing_ok=True)
|
|
if result_created:
|
|
result_path.unlink(missing_ok=True)
|
|
if archive_created:
|
|
archive_path.unlink(missing_ok=True)
|
|
code = getattr(error, "code", ErrorCode.PUBLISH_FAILED)
|
|
safe_message = getattr(error, "safe_message", "monthly report publication failed")
|
|
try:
|
|
repository.mark_failed(reservation, code, safe_message)
|
|
except Exception:
|
|
pass
|
|
if isinstance(error, PublicationError):
|
|
raise
|
|
if isinstance(error, RepositoryError):
|
|
raise PublicationError(error.code, error.safe_message) from None
|
|
raise PublicationError(
|
|
ErrorCode.PUBLISH_FAILED,
|
|
"monthly report publication failed",
|
|
) from None
|
|
|
|
|
|
def private_staging_directory(parent: Path):
|
|
parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
return tempfile.TemporaryDirectory(prefix="monthly-report-", dir=parent)
|