feat: sync latest ARR implementation
This commit is contained in:
Wyndham ARR
236
arr_web/programmatic.py
Normal file
236
arr_web/programmatic.py
Normal file
@@ -0,0 +1,236 @@
|
||||
"""Programmatic XML upload, processing, validation, and ingestion boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
from arr_ingestion.contracts import (
|
||||
ARTIFACT_ROLES,
|
||||
DELIVERY_SCHEMA_VERSION,
|
||||
RESULT_SCHEMA_VERSION,
|
||||
DeliveryEnvelope,
|
||||
IngestionError,
|
||||
)
|
||||
from arr_ingestion.repository import IngestionRepository, JobRegistration
|
||||
from arr_ingestion.service import IngestionService
|
||||
from arr_processing.local import LocalDailyProcessor
|
||||
from arr_storage.store import ManagedObjectStore, StoredObject
|
||||
from arr_web.contracts import PortalError, validate_upload_filename, validate_xml_payload
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProgrammaticUploadCoordinator:
|
||||
"""Own the complete upload-to-terminal transaction without Agent or MCP."""
|
||||
|
||||
object_store: ManagedObjectStore
|
||||
ingestion_repository: IngestionRepository
|
||||
ingestion_service: IngestionService
|
||||
processor: LocalDailyProcessor
|
||||
processor_version: str
|
||||
rule_set_sha256: str
|
||||
|
||||
def submit(self, original_filename: str, payload: bytes) -> Dict[str, Any]:
|
||||
uploaded_filename = validate_upload_filename(original_filename)
|
||||
validate_xml_payload(payload)
|
||||
job_id = "arrjob-" + uuid.uuid4().hex
|
||||
attempt_no = 1
|
||||
registered = False
|
||||
terminal = False
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="arr-programmatic-") as temporary:
|
||||
root = Path(temporary)
|
||||
upload_path = root / "upload" / "source.xml"
|
||||
self._write_private(upload_path, payload)
|
||||
source = self.object_store.upload_committed(
|
||||
job_id=job_id,
|
||||
attempt_no=attempt_no,
|
||||
role="source_xml",
|
||||
source=upload_path,
|
||||
original_filename="source.xml",
|
||||
)
|
||||
self.ingestion_repository.register_job(
|
||||
JobRegistration(
|
||||
job_id=job_id,
|
||||
source=source.to_artifact_ref(),
|
||||
processor_version=self.processor_version,
|
||||
rule_set_sha256=self.rule_set_sha256,
|
||||
attempt_no=attempt_no,
|
||||
idempotency_key=self._idempotency_key(
|
||||
job_id, attempt_no, source
|
||||
),
|
||||
uploaded_filename=uploaded_filename,
|
||||
)
|
||||
)
|
||||
registered = True
|
||||
self.ingestion_repository.mark_running(job_id, attempt_no)
|
||||
|
||||
materialized_source = root / "processor-input" / "source.xml"
|
||||
self.object_store.materialize(
|
||||
source.object_key,
|
||||
materialized_source,
|
||||
source.byte_size,
|
||||
)
|
||||
processed = self.processor.run(
|
||||
materialized_source,
|
||||
root / "processor-output",
|
||||
)
|
||||
stored_outputs: Dict[str, StoredObject] = {}
|
||||
for role, path in processed.artifacts.items():
|
||||
stored_outputs[role] = self.object_store.upload_committed(
|
||||
job_id=job_id,
|
||||
attempt_no=attempt_no,
|
||||
role=role,
|
||||
source=path,
|
||||
original_filename=path.name,
|
||||
)
|
||||
envelope = self._delivery(
|
||||
job_id=job_id,
|
||||
attempt_no=attempt_no,
|
||||
source=source,
|
||||
outputs=stored_outputs,
|
||||
status=processed.status,
|
||||
business_date=(
|
||||
processed.business_date.isoformat()
|
||||
if processed.business_date is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
raw_envelope = (
|
||||
json.dumps(
|
||||
envelope.to_dict(),
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
+ "\n"
|
||||
).encode("utf-8")
|
||||
outcome = self.ingestion_service.ingest(raw_envelope)
|
||||
terminal = True
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"status": (
|
||||
"failed"
|
||||
if outcome.status == "recorded_failure"
|
||||
else "succeeded"
|
||||
),
|
||||
"ingestion_status": outcome.status,
|
||||
"attempt_no": attempt_no,
|
||||
"business_date": (
|
||||
outcome.business_date.isoformat()
|
||||
if outcome.business_date is not None
|
||||
else None
|
||||
),
|
||||
"daily_version_id": outcome.daily_version_id,
|
||||
"version_no": outcome.version_no,
|
||||
"source_sha256": source.sha256,
|
||||
"source_byte_size": source.byte_size,
|
||||
}
|
||||
except PortalError:
|
||||
raise
|
||||
except IngestionError as error:
|
||||
if registered and not terminal:
|
||||
self._best_effort_failure(job_id, attempt_no, error.code)
|
||||
retryable = (
|
||||
error.retryable
|
||||
or error.code.startswith("DATABASE_")
|
||||
or error.code in {"OBJECT_STORE_UNAVAILABLE", "PROCESSOR_UNAVAILABLE"}
|
||||
)
|
||||
raise PortalError(
|
||||
error.code,
|
||||
error.safe_message,
|
||||
503 if retryable else 422,
|
||||
) from None
|
||||
except Exception:
|
||||
if registered and not terminal:
|
||||
self._best_effort_failure(
|
||||
job_id, attempt_no, "PROGRAMMATIC_PROCESSING_FAILED"
|
||||
)
|
||||
raise PortalError(
|
||||
"PROGRAMMATIC_PROCESSING_FAILED",
|
||||
"文件已安全接收，但程序化处理未能完成",
|
||||
503,
|
||||
) from None
|
||||
|
||||
def _best_effort_failure(
|
||||
self,
|
||||
job_id: str,
|
||||
attempt_no: int,
|
||||
failure_code: str,
|
||||
) -> None:
|
||||
try:
|
||||
self.ingestion_repository.record_failure(
|
||||
job_id, attempt_no, failure_code
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _delivery(
|
||||
self,
|
||||
*,
|
||||
job_id: str,
|
||||
attempt_no: int,
|
||||
source: StoredObject,
|
||||
outputs: Dict[str, StoredObject],
|
||||
status: str,
|
||||
business_date: object,
|
||||
) -> DeliveryEnvelope:
|
||||
artifacts: Dict[str, object] = {
|
||||
role: None for role in ARTIFACT_ROLES
|
||||
}
|
||||
artifacts["source_xml"] = source.to_artifact_ref().to_dict()
|
||||
for role, stored in outputs.items():
|
||||
artifacts[role] = stored.to_artifact_ref().to_dict()
|
||||
return DeliveryEnvelope.from_dict(
|
||||
{
|
||||
"delivery_schema_version": DELIVERY_SCHEMA_VERSION,
|
||||
"delivery_id": f"local-{job_id}-a{attempt_no}",
|
||||
"job_id": job_id,
|
||||
"attempt_no": attempt_no,
|
||||
"status": status,
|
||||
"processor_version": self.processor_version,
|
||||
"rule_set_sha256": self.rule_set_sha256,
|
||||
"result_schema_version": RESULT_SCHEMA_VERSION,
|
||||
"business_date": business_date,
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
)
|
||||
|
||||
def _idempotency_key(
|
||||
self,
|
||||
job_id: str,
|
||||
attempt_no: int,
|
||||
source: StoredObject,
|
||||
) -> str:
|
||||
value = "\x00".join(
|
||||
(
|
||||
"arr-programmatic-v1",
|
||||
job_id,
|
||||
str(attempt_no),
|
||||
source.sha256,
|
||||
self.processor_version,
|
||||
self.rule_set_sha256,
|
||||
)
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _write_private(path: Path, payload: bytes) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as target:
|
||||
descriptor = -1
|
||||
target.write(payload)
|
||||
target.flush()
|
||||
os.fsync(target.fileno())
|
||||
finally:
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
Reference in New Issue
Block a user