feat: move report artifacts to OSS storage
This commit is contained in:
@@ -17,6 +17,7 @@ from openpyxl import Workbook, load_workbook
|
||||
from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
|
||||
from openpyxl.worksheet.worksheet import Worksheet
|
||||
|
||||
from arr_storage.store import ManagedObjectStore
|
||||
from company_reports.contracts import CompanyReport, ErrorCode, RESULT_SCHEMA_VERSION
|
||||
from company_reports.repository import (
|
||||
FileMetadata,
|
||||
@@ -100,16 +101,11 @@ def _private_json(path: Path, payload: Mapping[str, Any]) -> None:
|
||||
raise
|
||||
|
||||
|
||||
class ArtifactToolBuilder:
|
||||
def __init__(
|
||||
self,
|
||||
builder_script: Path,
|
||||
node_binary: Optional[str] = None,
|
||||
artifact_tool_module: Optional[Path] = None,
|
||||
timeout_seconds: int = 120,
|
||||
) -> None:
|
||||
_ = (node_binary, artifact_tool_module, timeout_seconds)
|
||||
self._builder_script = builder_script.resolve()
|
||||
class OpenpyxlWorkbookBuilder:
|
||||
def __init__(self, *_legacy_args: Any, **_legacy_options: Any) -> None:
|
||||
# Retain import/call compatibility for older wrappers; no external
|
||||
# workbook runtime is consulted.
|
||||
_ = (_legacy_args, _legacy_options)
|
||||
|
||||
@staticmethod
|
||||
def _safe_text(value: Any) -> str:
|
||||
@@ -324,11 +320,19 @@ class ArtifactToolBuilder:
|
||||
summary=summary,
|
||||
)
|
||||
|
||||
|
||||
class AtomicReportPublisher:
|
||||
def __init__(self, project_root: Path, output_root: Path) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
project_root: Path,
|
||||
output_root: Path,
|
||||
*,
|
||||
object_store: Optional[ManagedObjectStore] = None,
|
||||
bucket_alias: str = "arr-private",
|
||||
) -> None:
|
||||
self._project_root = project_root.resolve()
|
||||
self._output_root = output_root.resolve()
|
||||
self._object_store = object_store
|
||||
self._bucket_alias = bucket_alias
|
||||
try:
|
||||
self._output_root.relative_to(self._project_root)
|
||||
except ValueError:
|
||||
@@ -441,13 +445,41 @@ class AtomicReportPublisher:
|
||||
if archive_sha256 != existing_sha256:
|
||||
raise ValueError("archive and result JSON hashes differ")
|
||||
|
||||
artifact_storage_key = self._storage_key(archive_path)
|
||||
storage_provider = str(artifact_payload.get("storage_provider") or "local")
|
||||
bucket_alias = str(
|
||||
artifact_payload.get("bucket_alias") or "arr-project-root"
|
||||
)
|
||||
if storage_provider in {"oss", "s3"}:
|
||||
if self._object_store is None:
|
||||
raise ValueError("OSS artifact reader is unavailable")
|
||||
artifact_storage_key = str(artifact_payload.get("storage_key") or "")
|
||||
stored = self._object_store.inspect_committed(
|
||||
artifact_storage_key,
|
||||
report.filename,
|
||||
)
|
||||
if (
|
||||
stored.role != "company_report"
|
||||
or stored.sha256 != existing_sha256
|
||||
or stored.byte_size != archive_path.stat().st_size
|
||||
or stored.mime_type != XLSX_MIME
|
||||
):
|
||||
raise ValueError("OSS archive and local cache differ")
|
||||
artifact_byte_size = stored.byte_size
|
||||
artifact_mime_type = stored.mime_type
|
||||
elif storage_provider in {"local", "local_fixture"}:
|
||||
artifact_storage_key = self._storage_key(archive_path)
|
||||
artifact_byte_size = archive_path.stat().st_size
|
||||
artifact_mime_type = XLSX_MIME
|
||||
else:
|
||||
raise ValueError("artifact storage provider is invalid")
|
||||
expected_payload = self._success_result(
|
||||
report,
|
||||
reservation,
|
||||
artifact_storage_key,
|
||||
existing_sha256,
|
||||
semantic_sha256,
|
||||
storage_provider,
|
||||
bucket_alias,
|
||||
)
|
||||
if result_payload != expected_payload:
|
||||
raise ValueError("result JSON identity differs")
|
||||
@@ -457,16 +489,38 @@ class AtomicReportPublisher:
|
||||
original_filename=report.filename,
|
||||
storage_key=artifact_storage_key,
|
||||
sha256=existing_sha256,
|
||||
byte_size=archive_path.stat().st_size,
|
||||
mime_type=XLSX_MIME,
|
||||
byte_size=artifact_byte_size,
|
||||
mime_type=artifact_mime_type,
|
||||
storage_provider=storage_provider,
|
||||
bucket_alias=bucket_alias,
|
||||
)
|
||||
result_sha256 = sha256_file(result_path)
|
||||
result_byte_size = result_path.stat().st_size
|
||||
result_storage_key = self._storage_key(result_path)
|
||||
result_mime_type = "application/json"
|
||||
if storage_provider in {"oss", "s3"}:
|
||||
uploaded_result = self._object_store.upload_committed(
|
||||
job_id=f"company-report-{reservation.report_version_id}",
|
||||
attempt_no=1,
|
||||
role="result_json",
|
||||
source=result_path,
|
||||
original_filename=result_path.name,
|
||||
expected_sha256=result_sha256,
|
||||
expected_byte_size=result_byte_size,
|
||||
)
|
||||
result_storage_key = uploaded_result.object_key
|
||||
result_sha256 = uploaded_result.sha256
|
||||
result_byte_size = uploaded_result.byte_size
|
||||
result_mime_type = uploaded_result.mime_type
|
||||
result_json = FileMetadata(
|
||||
file_kind="result_json",
|
||||
original_filename=result_path.name,
|
||||
storage_key=self._storage_key(result_path),
|
||||
sha256=sha256_file(result_path),
|
||||
byte_size=result_path.stat().st_size,
|
||||
mime_type="application/json",
|
||||
storage_key=result_storage_key,
|
||||
sha256=result_sha256,
|
||||
byte_size=result_byte_size,
|
||||
mime_type=result_mime_type,
|
||||
storage_provider=storage_provider,
|
||||
bucket_alias=bucket_alias,
|
||||
)
|
||||
artifact.validate()
|
||||
result_json.validate()
|
||||
@@ -491,6 +545,8 @@ class AtomicReportPublisher:
|
||||
artifact_storage_key: str,
|
||||
artifact_sha256: str,
|
||||
semantic_sha256: str,
|
||||
storage_provider: str,
|
||||
bucket_alias: str,
|
||||
) -> Dict[str, Any]:
|
||||
return {
|
||||
"schema_version": RESULT_SCHEMA_VERSION,
|
||||
@@ -506,6 +562,8 @@ class AtomicReportPublisher:
|
||||
"storage_key": artifact_storage_key,
|
||||
"sha256": artifact_sha256,
|
||||
"semantic_sha256": semantic_sha256,
|
||||
"storage_provider": storage_provider,
|
||||
"bucket_alias": bucket_alias,
|
||||
},
|
||||
"row_count": report.row_count,
|
||||
"period_row_counts": {
|
||||
@@ -581,13 +639,30 @@ class AtomicReportPublisher:
|
||||
archive_created = self._install_once(
|
||||
built.path, archive_path, built.sha256
|
||||
)
|
||||
storage_provider = "local"
|
||||
bucket_alias = "arr-project-root"
|
||||
artifact_storage_key = self._storage_key(archive_path)
|
||||
if self._object_store is not None:
|
||||
uploaded_artifact = self._object_store.upload_committed(
|
||||
job_id=f"company-report-{reservation.report_version_id}",
|
||||
attempt_no=1,
|
||||
role="company_report",
|
||||
source=archive_path,
|
||||
original_filename=report.filename,
|
||||
expected_sha256=built.sha256,
|
||||
expected_byte_size=built.byte_size,
|
||||
)
|
||||
storage_provider = "oss"
|
||||
bucket_alias = self._bucket_alias
|
||||
artifact_storage_key = uploaded_artifact.object_key
|
||||
result_payload = self._success_result(
|
||||
report,
|
||||
reservation,
|
||||
artifact_storage_key,
|
||||
built.sha256,
|
||||
str(built.summary.get("semantic_sha256", "")),
|
||||
storage_provider,
|
||||
bucket_alias,
|
||||
)
|
||||
if result_path.exists():
|
||||
expected = json.dumps(
|
||||
@@ -614,6 +689,8 @@ class AtomicReportPublisher:
|
||||
sha256=built.sha256,
|
||||
byte_size=built.byte_size,
|
||||
mime_type=XLSX_MIME,
|
||||
storage_provider=storage_provider,
|
||||
bucket_alias=bucket_alias,
|
||||
)
|
||||
result_json = FileMetadata(
|
||||
file_kind="result_json",
|
||||
@@ -622,7 +699,29 @@ class AtomicReportPublisher:
|
||||
sha256=sha256_file(result_path),
|
||||
byte_size=result_path.stat().st_size,
|
||||
mime_type="application/json",
|
||||
storage_provider=storage_provider,
|
||||
bucket_alias=bucket_alias,
|
||||
)
|
||||
if self._object_store is not None:
|
||||
uploaded_result = self._object_store.upload_committed(
|
||||
job_id=f"company-report-{reservation.report_version_id}",
|
||||
attempt_no=1,
|
||||
role="result_json",
|
||||
source=result_path,
|
||||
original_filename=result_path.name,
|
||||
expected_sha256=result_json.sha256,
|
||||
expected_byte_size=result_json.byte_size,
|
||||
)
|
||||
result_json = FileMetadata(
|
||||
file_kind="result_json",
|
||||
original_filename=result_path.name,
|
||||
storage_key=uploaded_result.object_key,
|
||||
sha256=uploaded_result.sha256,
|
||||
byte_size=uploaded_result.byte_size,
|
||||
mime_type=uploaded_result.mime_type,
|
||||
storage_provider=storage_provider,
|
||||
bucket_alias=bucket_alias,
|
||||
)
|
||||
repository.activate_report(reservation, artifact, result_json)
|
||||
return PublicationOutcome(
|
||||
current_path=current_path,
|
||||
|
||||
Reference in New Issue
Block a user