feat: move report artifacts to OSS storage

This commit is contained in:
Wyndham ARR
2026-08-04 12:37:26 +08:00
parent 727643f1f2
commit a2b86cdf10
45 changed files with 1288 additions and 1417 deletions

View File

@@ -17,6 +17,7 @@ from openpyxl import Workbook, load_workbook
from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
from openpyxl.worksheet.worksheet import Worksheet
from arr_storage.store import ManagedObjectStore
from company_reports.contracts import CompanyReport, ErrorCode, RESULT_SCHEMA_VERSION
from company_reports.repository import (
FileMetadata,
@@ -100,16 +101,11 @@ def _private_json(path: Path, payload: Mapping[str, Any]) -> None:
raise
class ArtifactToolBuilder:
def __init__(
self,
builder_script: Path,
node_binary: Optional[str] = None,
artifact_tool_module: Optional[Path] = None,
timeout_seconds: int = 120,
) -> None:
_ = (node_binary, artifact_tool_module, timeout_seconds)
self._builder_script = builder_script.resolve()
class OpenpyxlWorkbookBuilder:
def __init__(self, *_legacy_args: Any, **_legacy_options: Any) -> None:
# Retain import/call compatibility for older wrappers; no external
# workbook runtime is consulted.
_ = (_legacy_args, _legacy_options)
@staticmethod
def _safe_text(value: Any) -> str:
@@ -324,11 +320,19 @@ class ArtifactToolBuilder:
summary=summary,
)
class AtomicReportPublisher:
def __init__(self, project_root: Path, output_root: Path) -> None:
def __init__(
self,
project_root: Path,
output_root: Path,
*,
object_store: Optional[ManagedObjectStore] = None,
bucket_alias: str = "arr-private",
) -> None:
self._project_root = project_root.resolve()
self._output_root = output_root.resolve()
self._object_store = object_store
self._bucket_alias = bucket_alias
try:
self._output_root.relative_to(self._project_root)
except ValueError:
@@ -441,13 +445,41 @@ class AtomicReportPublisher:
if archive_sha256 != existing_sha256:
raise ValueError("archive and result JSON hashes differ")
artifact_storage_key = self._storage_key(archive_path)
storage_provider = str(artifact_payload.get("storage_provider") or "local")
bucket_alias = str(
artifact_payload.get("bucket_alias") or "arr-project-root"
)
if storage_provider in {"oss", "s3"}:
if self._object_store is None:
raise ValueError("OSS artifact reader is unavailable")
artifact_storage_key = str(artifact_payload.get("storage_key") or "")
stored = self._object_store.inspect_committed(
artifact_storage_key,
report.filename,
)
if (
stored.role != "company_report"
or stored.sha256 != existing_sha256
or stored.byte_size != archive_path.stat().st_size
or stored.mime_type != XLSX_MIME
):
raise ValueError("OSS archive and local cache differ")
artifact_byte_size = stored.byte_size
artifact_mime_type = stored.mime_type
elif storage_provider in {"local", "local_fixture"}:
artifact_storage_key = self._storage_key(archive_path)
artifact_byte_size = archive_path.stat().st_size
artifact_mime_type = XLSX_MIME
else:
raise ValueError("artifact storage provider is invalid")
expected_payload = self._success_result(
report,
reservation,
artifact_storage_key,
existing_sha256,
semantic_sha256,
storage_provider,
bucket_alias,
)
if result_payload != expected_payload:
raise ValueError("result JSON identity differs")
@@ -457,16 +489,38 @@ class AtomicReportPublisher:
original_filename=report.filename,
storage_key=artifact_storage_key,
sha256=existing_sha256,
byte_size=archive_path.stat().st_size,
mime_type=XLSX_MIME,
byte_size=artifact_byte_size,
mime_type=artifact_mime_type,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
result_sha256 = sha256_file(result_path)
result_byte_size = result_path.stat().st_size
result_storage_key = self._storage_key(result_path)
result_mime_type = "application/json"
if storage_provider in {"oss", "s3"}:
uploaded_result = self._object_store.upload_committed(
job_id=f"company-report-{reservation.report_version_id}",
attempt_no=1,
role="result_json",
source=result_path,
original_filename=result_path.name,
expected_sha256=result_sha256,
expected_byte_size=result_byte_size,
)
result_storage_key = uploaded_result.object_key
result_sha256 = uploaded_result.sha256
result_byte_size = uploaded_result.byte_size
result_mime_type = uploaded_result.mime_type
result_json = FileMetadata(
file_kind="result_json",
original_filename=result_path.name,
storage_key=self._storage_key(result_path),
sha256=sha256_file(result_path),
byte_size=result_path.stat().st_size,
mime_type="application/json",
storage_key=result_storage_key,
sha256=result_sha256,
byte_size=result_byte_size,
mime_type=result_mime_type,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
artifact.validate()
result_json.validate()
@@ -491,6 +545,8 @@ class AtomicReportPublisher:
artifact_storage_key: str,
artifact_sha256: str,
semantic_sha256: str,
storage_provider: str,
bucket_alias: str,
) -> Dict[str, Any]:
return {
"schema_version": RESULT_SCHEMA_VERSION,
@@ -506,6 +562,8 @@ class AtomicReportPublisher:
"storage_key": artifact_storage_key,
"sha256": artifact_sha256,
"semantic_sha256": semantic_sha256,
"storage_provider": storage_provider,
"bucket_alias": bucket_alias,
},
"row_count": report.row_count,
"period_row_counts": {
@@ -581,13 +639,30 @@ class AtomicReportPublisher:
archive_created = self._install_once(
built.path, archive_path, built.sha256
)
storage_provider = "local"
bucket_alias = "arr-project-root"
artifact_storage_key = self._storage_key(archive_path)
if self._object_store is not None:
uploaded_artifact = self._object_store.upload_committed(
job_id=f"company-report-{reservation.report_version_id}",
attempt_no=1,
role="company_report",
source=archive_path,
original_filename=report.filename,
expected_sha256=built.sha256,
expected_byte_size=built.byte_size,
)
storage_provider = "oss"
bucket_alias = self._bucket_alias
artifact_storage_key = uploaded_artifact.object_key
result_payload = self._success_result(
report,
reservation,
artifact_storage_key,
built.sha256,
str(built.summary.get("semantic_sha256", "")),
storage_provider,
bucket_alias,
)
if result_path.exists():
expected = json.dumps(
@@ -614,6 +689,8 @@ class AtomicReportPublisher:
sha256=built.sha256,
byte_size=built.byte_size,
mime_type=XLSX_MIME,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
result_json = FileMetadata(
file_kind="result_json",
@@ -622,7 +699,29 @@ class AtomicReportPublisher:
sha256=sha256_file(result_path),
byte_size=result_path.stat().st_size,
mime_type="application/json",
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
if self._object_store is not None:
uploaded_result = self._object_store.upload_committed(
job_id=f"company-report-{reservation.report_version_id}",
attempt_no=1,
role="result_json",
source=result_path,
original_filename=result_path.name,
expected_sha256=result_json.sha256,
expected_byte_size=result_json.byte_size,
)
result_json = FileMetadata(
file_kind="result_json",
original_filename=result_path.name,
storage_key=uploaded_result.object_key,
sha256=uploaded_result.sha256,
byte_size=uploaded_result.byte_size,
mime_type=uploaded_result.mime_type,
storage_provider=storage_provider,
bucket_alias=bucket_alias,
)
repository.activate_report(reservation, artifact, result_json)
return PublicationOutcome(
current_path=current_path,