1439 lines
48 KiB
Python
1439 lines
48 KiB
Python
"""PostgreSQL implementation of validate-before-write ARR daily ingestion."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
from dataclasses import dataclass
|
|
from datetime import date
|
|
from decimal import Decimal, InvalidOperation
|
|
from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Tuple
|
|
|
|
from arr_ingestion.contracts import ArtifactRef, IngestionError, SHA256_RE
|
|
from arr_ingestion.repository import (
|
|
IngestionOutcome,
|
|
IngestionRepository,
|
|
JobRegistration,
|
|
)
|
|
from arr_ingestion.validation import VerifiedDelivery
|
|
|
|
|
|
TARGET_DATABASE = "booking_test"
|
|
DATABASE_ENV = "ARR_DATABASE_URL"
|
|
ARTIFACT_STORAGE_PROVIDER = "oss"
|
|
ARTIFACT_BUCKET_ALIAS = "arr-private"
|
|
TRANSIENT_SQLSTATES = frozenset({"23505", "40001", "40P01", "55P03"})
|
|
MAX_TRANSACTION_ATTEMPTS = 4
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DatabaseConfig:
|
|
dsn: str
|
|
|
|
@classmethod
|
|
def from_environment(cls) -> "DatabaseConfig":
|
|
dsn = os.environ.get(DATABASE_ENV, "").strip()
|
|
if not dsn:
|
|
raise IngestionError(
|
|
"DATABASE_CONFIG_MISSING",
|
|
f"{DATABASE_ENV} is required",
|
|
)
|
|
return cls(dsn=dsn)
|
|
|
|
|
|
def _default_connect(dsn: str) -> Any:
|
|
try:
|
|
import psycopg # type: ignore[import-not-found]
|
|
except ImportError:
|
|
raise IngestionError(
|
|
"DATABASE_DRIVER_UNAVAILABLE",
|
|
"PostgreSQL driver is unavailable; install requirements-arr-ingestion.txt",
|
|
) from None
|
|
try:
|
|
return psycopg.connect(dsn, autocommit=False)
|
|
except Exception:
|
|
raise IngestionError(
|
|
"DATABASE_UNAVAILABLE",
|
|
"database connection failed",
|
|
) from None
|
|
|
|
|
|
def _decimal_or_none(value: Any) -> Optional[Decimal]:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
number = Decimal(str(value))
|
|
except (InvalidOperation, ValueError):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"numeric result value is invalid",
|
|
) from None
|
|
if not number.is_finite():
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"numeric result value is invalid",
|
|
)
|
|
return number
|
|
|
|
|
|
def _date_or_none(value: Any) -> Optional[date]:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, date):
|
|
return value
|
|
if isinstance(value, str):
|
|
try:
|
|
return date.fromisoformat(value)
|
|
except ValueError:
|
|
pass
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"date result value is invalid",
|
|
)
|
|
|
|
|
|
def _safe_failure_code(payload: Mapping[str, Any]) -> str:
|
|
errors = payload.get("errors")
|
|
if isinstance(errors, list) and errors and isinstance(errors[0], Mapping):
|
|
value = errors[0].get("code")
|
|
if isinstance(value, str) and re.fullmatch(
|
|
r"[A-Z][A-Z0-9_]{0,63}",
|
|
value,
|
|
):
|
|
return value
|
|
return "PROCESSING_FAILED"
|
|
|
|
|
|
def _is_transient_database_error(error: Exception) -> bool:
|
|
sqlstate = getattr(error, "sqlstate", None)
|
|
if not isinstance(sqlstate, str):
|
|
diagnostic = getattr(error, "diag", None)
|
|
sqlstate = getattr(diagnostic, "sqlstate", None)
|
|
return isinstance(sqlstate, str) and sqlstate in TRANSIENT_SQLSTATES
|
|
|
|
|
|
def _outcome_counts(payload: Mapping[str, Any]) -> Tuple[int, int, int, int, int]:
|
|
values = payload.get("outcome_counts")
|
|
if not isinstance(values, Mapping):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"structured outcome counts are invalid",
|
|
)
|
|
try:
|
|
counts = tuple(
|
|
int(values[name])
|
|
for name in (
|
|
"retained",
|
|
"excluded_rate_code",
|
|
"duplicate",
|
|
"validation_failed",
|
|
"price_unmatched",
|
|
)
|
|
)
|
|
except (KeyError, TypeError, ValueError):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"structured outcome counts are invalid",
|
|
) from None
|
|
if any(value < 0 for value in counts):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"structured outcome counts are invalid",
|
|
)
|
|
return counts # type: ignore[return-value]
|
|
|
|
|
|
class PostgresIngestionRepository(IngestionRepository):
|
|
def __init__(
|
|
self,
|
|
config: DatabaseConfig,
|
|
connect: Optional[Callable[[str], Any]] = None,
|
|
) -> None:
|
|
self._config = config
|
|
self._connect = connect or _default_connect
|
|
|
|
def _open(self) -> Any:
|
|
try:
|
|
return self._connect(self._config.dsn)
|
|
except IngestionError:
|
|
raise
|
|
except Exception:
|
|
raise IngestionError(
|
|
"DATABASE_UNAVAILABLE",
|
|
"database connection failed",
|
|
) from None
|
|
|
|
def _run_transaction(
|
|
self,
|
|
operation: Callable[[Any], Any],
|
|
failure_message: str,
|
|
) -> Any:
|
|
for attempt_index in range(MAX_TRANSACTION_ATTEMPTS):
|
|
connection = self._open()
|
|
retry = False
|
|
try:
|
|
with connection.transaction():
|
|
with connection.cursor() as cursor:
|
|
self._begin(cursor)
|
|
return operation(cursor)
|
|
except IngestionError:
|
|
raise
|
|
except Exception as error:
|
|
retry = (
|
|
attempt_index + 1 < MAX_TRANSACTION_ATTEMPTS
|
|
and _is_transient_database_error(error)
|
|
)
|
|
if not retry:
|
|
raise IngestionError(
|
|
"DATABASE_WRITE_FAILED",
|
|
failure_message,
|
|
) from None
|
|
finally:
|
|
connection.close()
|
|
if retry:
|
|
time.sleep(0.02 * (2**attempt_index))
|
|
raise IngestionError(
|
|
"DATABASE_WRITE_FAILED",
|
|
failure_message,
|
|
)
|
|
|
|
@staticmethod
|
|
def _begin(cursor: Any) -> None:
|
|
cursor.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE")
|
|
cursor.execute("SET LOCAL lock_timeout = '10s'")
|
|
cursor.execute("SET LOCAL statement_timeout = '30s'")
|
|
cursor.execute("SELECT current_database()")
|
|
row = cursor.fetchone()
|
|
if not row or row[0] != TARGET_DATABASE:
|
|
raise IngestionError(
|
|
"DATABASE_TARGET_INVALID",
|
|
"database target is not booking_test",
|
|
)
|
|
|
|
@staticmethod
|
|
def _ensure_artifact(cursor: Any, reference: ArtifactRef) -> int:
|
|
cursor.execute(
|
|
"""
|
|
SELECT
|
|
id,
|
|
artifact_kind,
|
|
original_filename,
|
|
sha256,
|
|
byte_size,
|
|
mime_type
|
|
FROM ingestion.artifacts
|
|
WHERE storage_provider = %s
|
|
AND bucket_alias = %s
|
|
AND object_key = %s
|
|
AND object_version_id IS NULL
|
|
FOR SHARE
|
|
""",
|
|
(
|
|
ARTIFACT_STORAGE_PROVIDER,
|
|
ARTIFACT_BUCKET_ALIAS,
|
|
reference.object_key,
|
|
),
|
|
)
|
|
row = cursor.fetchone()
|
|
if row:
|
|
if (
|
|
str(row[1]) != reference.file_kind
|
|
or str(row[2]) != reference.original_filename
|
|
or str(row[3]) != reference.sha256
|
|
or int(row[4]) != reference.byte_size
|
|
or (row[5] or "") != reference.mime_type
|
|
):
|
|
raise IngestionError(
|
|
"ARTIFACT_CONFLICT",
|
|
"stored artifact identity conflicts",
|
|
)
|
|
return int(row[0])
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO ingestion.artifacts (
|
|
artifact_kind,
|
|
storage_provider,
|
|
bucket_alias,
|
|
object_key,
|
|
original_filename,
|
|
sha256,
|
|
byte_size,
|
|
mime_type
|
|
)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
reference.file_kind,
|
|
ARTIFACT_STORAGE_PROVIDER,
|
|
ARTIFACT_BUCKET_ALIAS,
|
|
reference.object_key,
|
|
reference.original_filename,
|
|
reference.sha256,
|
|
reference.byte_size,
|
|
reference.mime_type,
|
|
),
|
|
)
|
|
return int(cursor.fetchone()[0])
|
|
|
|
def register_job(self, registration: JobRegistration) -> None:
|
|
if (
|
|
registration.source.role != "source_xml"
|
|
or registration.attempt_no < 1
|
|
or not SHA256_RE.fullmatch(registration.idempotency_key)
|
|
):
|
|
raise IngestionError(
|
|
"JOB_INVALID",
|
|
"processing job registration is invalid",
|
|
)
|
|
self._run_transaction(
|
|
lambda cursor: self._register_job(cursor, registration),
|
|
"processing job could not be stored",
|
|
)
|
|
|
|
def mark_running(self, job_id: str, attempt_no: int) -> None:
|
|
self._run_transaction(
|
|
lambda cursor: self._mark_running(cursor, job_id, attempt_no),
|
|
"processing job could not be started",
|
|
)
|
|
|
|
def _mark_running(self, cursor: Any, job_id: str, attempt_no: int) -> None:
|
|
run_id, run_status, attempt_id, attempt_status = self._lock_attempt(
|
|
cursor, job_id, attempt_no
|
|
)
|
|
if run_status in {"accepted", "rejected", "failed", "cancelled"}:
|
|
raise IngestionError("JOB_TERMINAL", "processing job is already terminal")
|
|
if attempt_status in {"succeeded", "failed", "cancelled"}:
|
|
raise IngestionError("JOB_TERMINAL", "processing attempt is already terminal")
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_attempts
|
|
SET attempt_status = 'running',
|
|
started_at = COALESCE(started_at, now())
|
|
WHERE id = %s
|
|
""",
|
|
(attempt_id,),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_runs
|
|
SET run_status = 'running',
|
|
failure_code = NULL,
|
|
failure_message = NULL,
|
|
updated_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(run_id,),
|
|
)
|
|
|
|
def record_failure(self, job_id: str, attempt_no: int, failure_code: str) -> None:
|
|
if not isinstance(failure_code, str) or re.fullmatch(
|
|
r"[A-Z][A-Z0-9_]{0,63}", failure_code
|
|
) is None:
|
|
failure_code = "PROCESSING_FAILED"
|
|
self._run_transaction(
|
|
lambda cursor: self._record_runtime_failure(
|
|
cursor, job_id, attempt_no, failure_code
|
|
),
|
|
"processing failure could not be stored",
|
|
)
|
|
|
|
def _record_runtime_failure(
|
|
self,
|
|
cursor: Any,
|
|
job_id: str,
|
|
attempt_no: int,
|
|
failure_code: str,
|
|
) -> None:
|
|
run_id, run_status, attempt_id, _attempt_status = self._lock_attempt(
|
|
cursor, job_id, attempt_no
|
|
)
|
|
if run_status == "failed":
|
|
return
|
|
if run_status in {"accepted", "rejected", "cancelled"}:
|
|
raise IngestionError("JOB_TERMINAL", "processing job is already terminal")
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_attempts
|
|
SET attempt_status = 'failed',
|
|
failure_code = %s,
|
|
failure_message = 'programmatic processing failed',
|
|
finished_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(failure_code, attempt_id),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_runs
|
|
SET run_status = 'failed',
|
|
failure_code = %s,
|
|
failure_message = 'programmatic processing failed',
|
|
updated_at = now(),
|
|
finished_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(failure_code, run_id),
|
|
)
|
|
self._insert_outbox(
|
|
cursor,
|
|
f"processing-run:{run_id}:failed",
|
|
"processing_run",
|
|
run_id,
|
|
"arr.processing_failed",
|
|
{
|
|
"job_id": job_id,
|
|
"business_date": None,
|
|
"daily_version_id": None,
|
|
"failure_code": failure_code,
|
|
},
|
|
)
|
|
|
|
@staticmethod
|
|
def _lock_attempt(
|
|
cursor: Any,
|
|
job_id: str,
|
|
attempt_no: int,
|
|
) -> Tuple[int, str, int, str]:
|
|
cursor.execute(
|
|
"""
|
|
SELECT run.id, run.run_status, attempt.id, attempt.attempt_status
|
|
FROM ingestion.processing_runs AS run
|
|
JOIN ingestion.processing_attempts AS attempt
|
|
ON attempt.processing_run_id = run.id
|
|
AND attempt.attempt_no = %s
|
|
WHERE run.run_key = %s
|
|
FOR UPDATE OF run, attempt
|
|
""",
|
|
(attempt_no, job_id),
|
|
)
|
|
row = cursor.fetchone()
|
|
if not row:
|
|
raise IngestionError("JOB_NOT_FOUND", "processing job was not registered")
|
|
return int(row[0]), str(row[1]), int(row[2]), str(row[3])
|
|
|
|
def _register_job(self, cursor: Any, registration: JobRegistration) -> None:
|
|
source_artifact_id = self._ensure_artifact(cursor, registration.source)
|
|
cursor.execute(
|
|
"""
|
|
SELECT
|
|
id,
|
|
source_artifact_id,
|
|
requested_processor_version,
|
|
requested_rule_set_sha256,
|
|
uploaded_filename,
|
|
run_status
|
|
FROM ingestion.processing_runs
|
|
WHERE run_key = %s
|
|
FOR UPDATE
|
|
""",
|
|
(registration.job_id,),
|
|
)
|
|
existing = cursor.fetchone()
|
|
if existing is None:
|
|
if registration.attempt_no != 1:
|
|
raise IngestionError(
|
|
"JOB_CONFLICT",
|
|
"first processing attempt must be attempt 1",
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO ingestion.processing_runs (
|
|
run_key,
|
|
pipeline_type,
|
|
source_artifact_id,
|
|
run_status,
|
|
requested_processor_version,
|
|
requested_rule_set_sha256,
|
|
uploaded_filename
|
|
)
|
|
VALUES (%s, 'opera_daily', %s, 'queued', %s, %s, %s)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
registration.job_id,
|
|
source_artifact_id,
|
|
registration.processor_version,
|
|
registration.rule_set_sha256,
|
|
registration.uploaded_filename,
|
|
),
|
|
)
|
|
processing_run_id = int(cursor.fetchone()[0])
|
|
else:
|
|
if (
|
|
int(existing[1]) != source_artifact_id
|
|
or existing[2] != registration.processor_version
|
|
or str(existing[3]) != registration.rule_set_sha256
|
|
or existing[4] != registration.uploaded_filename
|
|
):
|
|
raise IngestionError(
|
|
"JOB_CONFLICT",
|
|
"processing job identity conflicts",
|
|
)
|
|
processing_run_id = int(existing[0])
|
|
cursor.execute(
|
|
"""
|
|
SELECT idempotency_key
|
|
FROM ingestion.processing_attempts
|
|
WHERE processing_run_id = %s
|
|
AND attempt_no = %s
|
|
""",
|
|
(processing_run_id, registration.attempt_no),
|
|
)
|
|
attempt = cursor.fetchone()
|
|
if attempt is not None:
|
|
if str(attempt[0]) != registration.idempotency_key:
|
|
raise IngestionError(
|
|
"JOB_CONFLICT",
|
|
"processing attempt identity conflicts",
|
|
)
|
|
return
|
|
if str(existing[5]) in {
|
|
"accepted",
|
|
"rejected",
|
|
"failed",
|
|
"cancelled",
|
|
}:
|
|
raise IngestionError(
|
|
"JOB_TERMINAL",
|
|
"processing job is already terminal",
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
SELECT COALESCE(max(attempt_no), 0) + 1
|
|
FROM ingestion.processing_attempts
|
|
WHERE processing_run_id = %s
|
|
""",
|
|
(processing_run_id,),
|
|
)
|
|
if int(cursor.fetchone()[0]) != registration.attempt_no:
|
|
raise IngestionError(
|
|
"JOB_CONFLICT",
|
|
"processing attempt sequence conflicts",
|
|
)
|
|
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO ingestion.processing_attempts (
|
|
processing_run_id,
|
|
attempt_no,
|
|
attempt_status,
|
|
idempotency_key
|
|
)
|
|
VALUES (%s, %s, 'queued', %s)
|
|
""",
|
|
(
|
|
processing_run_id,
|
|
registration.attempt_no,
|
|
registration.idempotency_key,
|
|
),
|
|
)
|
|
|
|
def commit_delivery(self, delivery: VerifiedDelivery) -> IngestionOutcome:
|
|
return self._run_transaction(
|
|
lambda cursor: self._commit(cursor, delivery),
|
|
"validated delivery was not committed",
|
|
)
|
|
|
|
def _commit(
|
|
self,
|
|
cursor: Any,
|
|
delivery: VerifiedDelivery,
|
|
) -> IngestionOutcome:
|
|
envelope = delivery.envelope
|
|
cursor.execute(
|
|
"""
|
|
SELECT
|
|
run.id,
|
|
run.source_artifact_id,
|
|
run.requested_processor_version,
|
|
run.requested_rule_set_sha256,
|
|
run.run_status,
|
|
attempt.id,
|
|
attempt.attempt_status
|
|
FROM ingestion.processing_runs AS run
|
|
JOIN ingestion.processing_attempts AS attempt
|
|
ON attempt.processing_run_id = run.id
|
|
AND attempt.attempt_no = %s
|
|
WHERE run.run_key = %s
|
|
FOR UPDATE OF run, attempt
|
|
""",
|
|
(envelope.attempt_no, envelope.job_id),
|
|
)
|
|
run_row = cursor.fetchone()
|
|
if not run_row:
|
|
raise IngestionError(
|
|
"JOB_NOT_FOUND",
|
|
"processing job was not registered",
|
|
)
|
|
processing_run_id = int(run_row[0])
|
|
attempt_id = int(run_row[5])
|
|
|
|
cursor.execute(
|
|
"""
|
|
SELECT
|
|
delivery.id,
|
|
delivery.envelope_sha256,
|
|
delivery.delivery_status,
|
|
delivery.daily_version_id,
|
|
delivery.business_date,
|
|
version.version_no,
|
|
version.processing_run_id
|
|
FROM ingestion.processing_deliveries AS delivery
|
|
LEFT JOIN finance.daily_versions AS version
|
|
ON version.id = delivery.daily_version_id
|
|
WHERE delivery.delivery_key = %s
|
|
FOR UPDATE OF delivery
|
|
""",
|
|
(envelope.delivery_id,),
|
|
)
|
|
delivery_row = cursor.fetchone()
|
|
if delivery_row:
|
|
if str(delivery_row[1]) != delivery.envelope_sha256:
|
|
raise IngestionError(
|
|
"CALLBACK_CONFLICT",
|
|
"delivery identifier conflicts",
|
|
)
|
|
delivery_status = str(delivery_row[2])
|
|
if delivery_status == "recorded_failure":
|
|
return IngestionOutcome(
|
|
"recorded_failure",
|
|
envelope.job_id,
|
|
delivery_row[4],
|
|
(
|
|
int(delivery_row[3])
|
|
if delivery_row[3] is not None
|
|
else None
|
|
),
|
|
None,
|
|
)
|
|
if delivery_status == "committed":
|
|
disposition = (
|
|
"committed"
|
|
if (
|
|
delivery_row[6] is not None
|
|
and int(delivery_row[6]) == processing_run_id
|
|
)
|
|
else "already_committed"
|
|
)
|
|
return IngestionOutcome(
|
|
disposition,
|
|
envelope.job_id,
|
|
delivery_row[4],
|
|
int(delivery_row[3]),
|
|
int(delivery_row[5]),
|
|
)
|
|
if delivery_status == "rejected":
|
|
raise IngestionError(
|
|
"CALLBACK_REJECTED",
|
|
"delivery was previously rejected",
|
|
)
|
|
delivery_id = int(delivery_row[0])
|
|
else:
|
|
delivery_id = None
|
|
|
|
if (
|
|
delivery_id is None
|
|
and str(run_row[4])
|
|
in {"accepted", "rejected", "failed", "cancelled"}
|
|
):
|
|
raise IngestionError(
|
|
"JOB_TERMINAL",
|
|
"processing job is already terminal",
|
|
)
|
|
if (
|
|
run_row[2] != envelope.processor_version
|
|
or str(run_row[3]) != envelope.rule_set_sha256
|
|
):
|
|
raise IngestionError(
|
|
"JOB_DELIVERY_MISMATCH",
|
|
"delivery does not match its job",
|
|
)
|
|
|
|
source_ref = envelope.artifacts["source_xml"]
|
|
result_ref = envelope.artifacts["result_json"]
|
|
structured_ref = envelope.artifacts["structured_result_json"]
|
|
if source_ref is None or result_ref is None or structured_ref is None:
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"required artifact is missing",
|
|
)
|
|
source_artifact_id = self._ensure_artifact(cursor, source_ref)
|
|
if source_artifact_id != int(run_row[1]):
|
|
raise IngestionError(
|
|
"JOB_DELIVERY_MISMATCH",
|
|
"delivery does not match its job",
|
|
)
|
|
daily_ref = envelope.artifacts["daily_report"]
|
|
exception_ref = envelope.artifacts["exception_report"]
|
|
daily_artifact_id = (
|
|
self._ensure_artifact(cursor, daily_ref)
|
|
if daily_ref is not None
|
|
else None
|
|
)
|
|
result_artifact_id = self._ensure_artifact(cursor, result_ref)
|
|
structured_artifact_id = self._ensure_artifact(
|
|
cursor,
|
|
structured_ref,
|
|
)
|
|
exception_artifact_id = (
|
|
self._ensure_artifact(cursor, exception_ref)
|
|
if exception_ref is not None
|
|
else None
|
|
)
|
|
envelope_json = json.dumps(
|
|
envelope.to_dict(),
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
)
|
|
|
|
if delivery_id is None:
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO ingestion.processing_deliveries (
|
|
delivery_key,
|
|
processing_run_id,
|
|
attempt_id,
|
|
envelope_sha256,
|
|
envelope_json,
|
|
delivery_status,
|
|
result_status,
|
|
processor_version,
|
|
rule_set_sha256,
|
|
result_schema_version,
|
|
business_date,
|
|
source_artifact_id,
|
|
daily_report_artifact_id,
|
|
result_json_artifact_id,
|
|
structured_result_artifact_id,
|
|
exception_report_artifact_id,
|
|
validated_at
|
|
)
|
|
VALUES (
|
|
%s, %s, %s, %s, %s::jsonb, 'validating', %s,
|
|
%s, %s, %s, %s, %s, %s, %s, %s, %s, now()
|
|
)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
envelope.delivery_id,
|
|
processing_run_id,
|
|
attempt_id,
|
|
delivery.envelope_sha256,
|
|
envelope_json,
|
|
envelope.status,
|
|
envelope.processor_version,
|
|
envelope.rule_set_sha256,
|
|
envelope.result_schema_version,
|
|
envelope.business_date,
|
|
source_artifact_id,
|
|
daily_artifact_id,
|
|
result_artifact_id,
|
|
structured_artifact_id,
|
|
exception_artifact_id,
|
|
),
|
|
)
|
|
delivery_id = int(cursor.fetchone()[0])
|
|
else:
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_deliveries
|
|
SET delivery_status = 'validating',
|
|
result_status = %s,
|
|
processor_version = %s,
|
|
rule_set_sha256 = %s,
|
|
result_schema_version = %s,
|
|
business_date = %s,
|
|
source_artifact_id = %s,
|
|
daily_report_artifact_id = %s,
|
|
result_json_artifact_id = %s,
|
|
structured_result_artifact_id = %s,
|
|
exception_report_artifact_id = %s,
|
|
validated_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(
|
|
envelope.status,
|
|
envelope.processor_version,
|
|
envelope.rule_set_sha256,
|
|
envelope.result_schema_version,
|
|
envelope.business_date,
|
|
source_artifact_id,
|
|
daily_artifact_id,
|
|
result_artifact_id,
|
|
structured_artifact_id,
|
|
exception_artifact_id,
|
|
delivery_id,
|
|
),
|
|
)
|
|
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_runs
|
|
SET run_status = 'validating',
|
|
result_artifact_id = %s,
|
|
delivered_processor_version = %s,
|
|
delivered_rule_set_sha256 = %s,
|
|
result_schema_version = %s,
|
|
delivery_sha256 = %s,
|
|
delivery_json = %s::jsonb,
|
|
business_date = %s,
|
|
failure_code = NULL,
|
|
failure_message = NULL,
|
|
updated_at = now(),
|
|
validated_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(
|
|
structured_artifact_id,
|
|
envelope.processor_version,
|
|
envelope.rule_set_sha256,
|
|
envelope.result_schema_version,
|
|
delivery.envelope_sha256,
|
|
envelope_json,
|
|
envelope.business_date,
|
|
processing_run_id,
|
|
),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_attempts
|
|
SET attempt_status = 'delivered'
|
|
WHERE id = %s
|
|
""",
|
|
(attempt_id,),
|
|
)
|
|
|
|
if envelope.status == "failed":
|
|
return self._record_failure(
|
|
cursor,
|
|
delivery,
|
|
processing_run_id,
|
|
attempt_id,
|
|
delivery_id,
|
|
source_artifact_id,
|
|
result_artifact_id,
|
|
structured_artifact_id,
|
|
exception_artifact_id,
|
|
)
|
|
if envelope.business_date is None or daily_artifact_id is None:
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"successful delivery is incomplete",
|
|
)
|
|
return self._commit_success(
|
|
cursor,
|
|
delivery,
|
|
processing_run_id,
|
|
attempt_id,
|
|
delivery_id,
|
|
source_artifact_id,
|
|
daily_artifact_id,
|
|
result_artifact_id,
|
|
structured_artifact_id,
|
|
)
|
|
|
|
def _record_failure(
|
|
self,
|
|
cursor: Any,
|
|
delivery: VerifiedDelivery,
|
|
processing_run_id: int,
|
|
attempt_id: int,
|
|
delivery_id: int,
|
|
source_artifact_id: int,
|
|
result_artifact_id: int,
|
|
structured_artifact_id: int,
|
|
exception_artifact_id: Optional[int],
|
|
) -> IngestionOutcome:
|
|
envelope = delivery.envelope
|
|
payload = delivery.structured_payload
|
|
code = _safe_failure_code(payload)
|
|
counts = _outcome_counts(payload)
|
|
records = payload.get("records")
|
|
if not isinstance(records, list):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"structured records are invalid",
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO finance.daily_versions (
|
|
business_date,
|
|
version_no,
|
|
processing_run_id,
|
|
source_artifact_id,
|
|
result_json_artifact_id,
|
|
structured_result_artifact_id,
|
|
exception_report_artifact_id,
|
|
version_status,
|
|
processor_version,
|
|
rule_set_sha256,
|
|
result_schema_version,
|
|
result_sha256,
|
|
source_rows,
|
|
retained_rows,
|
|
excluded_rate_code_rows,
|
|
duplicate_rows,
|
|
validation_failed_rows,
|
|
price_unmatched_rows,
|
|
failure_code,
|
|
failure_message,
|
|
validated_at
|
|
)
|
|
VALUES (
|
|
NULL, NULL, %s, %s, %s, %s, %s, 'rejected',
|
|
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
|
%s, 'deterministic processing failed', now()
|
|
)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
processing_run_id,
|
|
source_artifact_id,
|
|
result_artifact_id,
|
|
structured_artifact_id,
|
|
exception_artifact_id,
|
|
envelope.processor_version,
|
|
envelope.rule_set_sha256,
|
|
envelope.result_schema_version,
|
|
envelope.artifacts["structured_result_json"].sha256,
|
|
int(payload["source_rows"]),
|
|
*counts,
|
|
code,
|
|
),
|
|
)
|
|
daily_version_id = int(cursor.fetchone()[0])
|
|
booking_links = self._load_booking_links(cursor, records)
|
|
self._insert_records(
|
|
cursor,
|
|
daily_version_id,
|
|
records,
|
|
booking_links,
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_deliveries
|
|
SET delivery_status = 'recorded_failure',
|
|
daily_version_id = %s,
|
|
failure_code = %s,
|
|
failure_message = 'deterministic processing failed',
|
|
committed_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(daily_version_id, code, delivery_id),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_attempts
|
|
SET attempt_status = 'failed',
|
|
failure_code = %s,
|
|
failure_message = 'deterministic processing failed',
|
|
finished_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(code, attempt_id),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_runs
|
|
SET run_status = 'failed',
|
|
failure_code = %s,
|
|
failure_message = 'deterministic processing failed',
|
|
updated_at = now(),
|
|
finished_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(code, processing_run_id),
|
|
)
|
|
self._insert_outbox(
|
|
cursor,
|
|
f"processing-run:{processing_run_id}:failed",
|
|
"processing_run",
|
|
processing_run_id,
|
|
"arr.processing_failed",
|
|
{
|
|
"job_id": envelope.job_id,
|
|
"business_date": (
|
|
envelope.business_date.isoformat()
|
|
if envelope.business_date
|
|
else None
|
|
),
|
|
"daily_version_id": daily_version_id,
|
|
"failure_code": code,
|
|
},
|
|
)
|
|
return IngestionOutcome(
|
|
"recorded_failure",
|
|
envelope.job_id,
|
|
envelope.business_date,
|
|
daily_version_id,
|
|
None,
|
|
)
|
|
|
|
def _commit_success(
|
|
self,
|
|
cursor: Any,
|
|
delivery: VerifiedDelivery,
|
|
processing_run_id: int,
|
|
attempt_id: int,
|
|
delivery_id: int,
|
|
source_artifact_id: int,
|
|
daily_artifact_id: int,
|
|
result_artifact_id: int,
|
|
structured_artifact_id: int,
|
|
) -> IngestionOutcome:
|
|
envelope = delivery.envelope
|
|
business_date = envelope.business_date
|
|
assert business_date is not None
|
|
cursor.execute(
|
|
"SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))",
|
|
(f"finance-daily:{business_date.isoformat()}",),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
SELECT id, version_no, version_status
|
|
FROM finance.daily_versions
|
|
WHERE source_artifact_id = %s
|
|
AND business_date = %s
|
|
AND processor_version = %s
|
|
AND rule_set_sha256 = %s
|
|
FOR UPDATE
|
|
""",
|
|
(
|
|
source_artifact_id,
|
|
business_date,
|
|
envelope.processor_version,
|
|
envelope.rule_set_sha256,
|
|
),
|
|
)
|
|
existing = cursor.fetchone()
|
|
if existing:
|
|
if str(existing[2]) not in {"active", "superseded"}:
|
|
raise IngestionError(
|
|
"DATABASE_STATE_INVALID",
|
|
"existing daily version is invalid",
|
|
)
|
|
daily_version_id = int(existing[0])
|
|
version_no = int(existing[1])
|
|
disposition = "already_committed"
|
|
else:
|
|
payload = delivery.structured_payload
|
|
counts = _outcome_counts(payload)
|
|
cursor.execute(
|
|
"""
|
|
SELECT COALESCE(MAX(version_no), 0) + 1
|
|
FROM finance.daily_versions
|
|
WHERE business_date = %s
|
|
""",
|
|
(business_date,),
|
|
)
|
|
version_no = int(cursor.fetchone()[0])
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO finance.daily_versions (
|
|
business_date,
|
|
version_no,
|
|
processing_run_id,
|
|
source_artifact_id,
|
|
daily_report_artifact_id,
|
|
result_json_artifact_id,
|
|
structured_result_artifact_id,
|
|
version_status,
|
|
processor_version,
|
|
rule_set_sha256,
|
|
result_schema_version,
|
|
result_sha256,
|
|
source_rows,
|
|
retained_rows,
|
|
excluded_rate_code_rows,
|
|
duplicate_rows,
|
|
validation_failed_rows,
|
|
price_unmatched_rows,
|
|
validated_at
|
|
)
|
|
VALUES (
|
|
%s, %s, %s, %s, %s, %s, %s, 'validated',
|
|
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, now()
|
|
)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
business_date,
|
|
version_no,
|
|
processing_run_id,
|
|
source_artifact_id,
|
|
daily_artifact_id,
|
|
result_artifact_id,
|
|
structured_artifact_id,
|
|
envelope.processor_version,
|
|
envelope.rule_set_sha256,
|
|
envelope.result_schema_version,
|
|
envelope.artifacts["structured_result_json"].sha256,
|
|
int(payload["source_rows"]),
|
|
*counts,
|
|
),
|
|
)
|
|
daily_version_id = int(cursor.fetchone()[0])
|
|
records = payload.get("records")
|
|
if not isinstance(records, list):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"structured records are invalid",
|
|
)
|
|
booking_links = self._load_booking_links(cursor, records)
|
|
self._insert_records(
|
|
cursor,
|
|
daily_version_id,
|
|
records,
|
|
booking_links,
|
|
)
|
|
channels = payload.get("channels")
|
|
if not isinstance(channels, list):
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"structured channels are invalid",
|
|
)
|
|
self._insert_channel_metrics(
|
|
cursor,
|
|
daily_version_id,
|
|
channels,
|
|
)
|
|
self._activate_daily_version(
|
|
cursor,
|
|
business_date,
|
|
daily_version_id,
|
|
)
|
|
disposition = "committed"
|
|
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_deliveries
|
|
SET delivery_status = 'committed',
|
|
daily_version_id = %s,
|
|
committed_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(daily_version_id, delivery_id),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_attempts
|
|
SET attempt_status = 'succeeded',
|
|
failure_code = NULL,
|
|
failure_message = NULL,
|
|
finished_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(attempt_id,),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
UPDATE ingestion.processing_runs
|
|
SET run_status = 'accepted',
|
|
business_date = %s,
|
|
failure_code = NULL,
|
|
failure_message = NULL,
|
|
updated_at = now(),
|
|
finished_at = now()
|
|
WHERE id = %s
|
|
""",
|
|
(business_date, processing_run_id),
|
|
)
|
|
self._insert_outbox(
|
|
cursor,
|
|
f"processing-run:{processing_run_id}:accepted",
|
|
"processing_run",
|
|
processing_run_id,
|
|
"arr.daily_version_committed",
|
|
{
|
|
"job_id": envelope.job_id,
|
|
"business_date": business_date.isoformat(),
|
|
"daily_version_id": daily_version_id,
|
|
"version_no": version_no,
|
|
"disposition": disposition,
|
|
},
|
|
)
|
|
return IngestionOutcome(
|
|
disposition,
|
|
envelope.job_id,
|
|
business_date,
|
|
daily_version_id,
|
|
version_no,
|
|
)
|
|
|
|
@staticmethod
|
|
def _insert_channel_metrics(
|
|
cursor: Any,
|
|
daily_version_id: int,
|
|
channels: Sequence[Mapping[str, Any]],
|
|
) -> None:
|
|
for channel_order, channel in enumerate(channels, 1):
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO finance.daily_channel_metrics (
|
|
daily_version_id,
|
|
channel_key,
|
|
channel_order,
|
|
row_count
|
|
)
|
|
VALUES (%s, %s, %s, %s)
|
|
""",
|
|
(
|
|
daily_version_id,
|
|
str(channel["worksheet"]),
|
|
channel_order,
|
|
int(channel["rows"]),
|
|
),
|
|
)
|
|
|
|
@staticmethod
|
|
def _activate_daily_version(
|
|
cursor: Any,
|
|
business_date: date,
|
|
daily_version_id: int,
|
|
) -> None:
|
|
cursor.execute(
|
|
"""
|
|
SELECT daily_version_id
|
|
FROM finance.current_daily_versions
|
|
WHERE business_date = %s
|
|
FOR UPDATE
|
|
""",
|
|
(business_date,),
|
|
)
|
|
current = cursor.fetchone()
|
|
if current and int(current[0]) != daily_version_id:
|
|
cursor.execute(
|
|
"""
|
|
UPDATE finance.daily_versions
|
|
SET version_status = 'superseded',
|
|
superseded_at = now()
|
|
WHERE id = %s
|
|
AND version_status = 'active'
|
|
""",
|
|
(int(current[0]),),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
UPDATE finance.daily_versions
|
|
SET version_status = 'active',
|
|
activated_at = now()
|
|
WHERE id = %s
|
|
AND version_status = 'validated'
|
|
""",
|
|
(daily_version_id,),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO finance.current_daily_versions (
|
|
business_date,
|
|
daily_version_id,
|
|
activated_at
|
|
)
|
|
VALUES (%s, %s, now())
|
|
ON CONFLICT (business_date) DO UPDATE
|
|
SET daily_version_id = EXCLUDED.daily_version_id,
|
|
activated_at = EXCLUDED.activated_at
|
|
""",
|
|
(business_date, daily_version_id),
|
|
)
|
|
|
|
@staticmethod
|
|
def _load_booking_links(
|
|
cursor: Any,
|
|
records: Sequence[Mapping[str, Any]],
|
|
) -> Mapping[str, Tuple[str, int]]:
|
|
group_codes = sorted(
|
|
{
|
|
str(record["group_code_key"])
|
|
for record in records
|
|
if record.get("group_code_key")
|
|
}
|
|
)
|
|
if not group_codes:
|
|
return {}
|
|
cursor.execute(
|
|
"""
|
|
SELECT
|
|
source.group_code_key,
|
|
count(DISTINCT source.id)
|
|
FROM booking.source_rows AS source
|
|
JOIN booking.current_source_batch AS active_source
|
|
ON active_source.source_batch_id = source.source_batch_id
|
|
AND active_source.singleton
|
|
JOIN booking.source_batches AS batch
|
|
ON batch.id = source.source_batch_id
|
|
AND batch.batch_status = 'accepted'
|
|
JOIN booking.current_row_parses AS current_parse
|
|
ON current_parse.source_row_id = source.id
|
|
JOIN booking.parse_versions AS parsed
|
|
ON parsed.id = current_parse.parse_version_id
|
|
AND parsed.parse_status = 'accepted'
|
|
WHERE source.group_code_key = ANY(%s)
|
|
GROUP BY source.group_code_key
|
|
""",
|
|
(group_codes,),
|
|
)
|
|
matches: Dict[str, Tuple[str, int]] = {}
|
|
for group_code, count in cursor.fetchall():
|
|
match_count = int(count)
|
|
matches[str(group_code)] = ("matched", match_count)
|
|
for group_code in group_codes:
|
|
matches.setdefault(group_code, ("unmatched", 0))
|
|
return matches
|
|
|
|
@staticmethod
|
|
def _insert_records(
|
|
cursor: Any,
|
|
daily_version_id: int,
|
|
records: Sequence[Mapping[str, Any]],
|
|
booking_links: Mapping[str, Tuple[str, int]],
|
|
) -> None:
|
|
inserted_ids: Dict[int, int] = {}
|
|
for record in records:
|
|
source_sequence = int(record["source_sequence"])
|
|
duplicate_sequence = record.get(
|
|
"duplicate_of_source_sequence",
|
|
)
|
|
duplicate_id = (
|
|
inserted_ids.get(int(duplicate_sequence))
|
|
if duplicate_sequence is not None
|
|
else None
|
|
)
|
|
if duplicate_sequence is not None and duplicate_id is None:
|
|
raise IngestionError(
|
|
"RESULT_CONTRACT_INVALID",
|
|
"duplicate lineage is invalid",
|
|
)
|
|
group_code = record.get("group_code_key")
|
|
if group_code is None:
|
|
source_status, source_match_count = (
|
|
"missing_group_code",
|
|
0,
|
|
)
|
|
else:
|
|
source_status, source_match_count = booking_links.get(
|
|
str(group_code),
|
|
("unmatched", 0),
|
|
)
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO finance.daily_records (
|
|
daily_version_id,
|
|
source_sequence,
|
|
source_location,
|
|
source_worksheet,
|
|
source_row_no,
|
|
outcome,
|
|
decision_codes,
|
|
duplicate_of_record_id,
|
|
block_code,
|
|
adults,
|
|
children,
|
|
company_name,
|
|
company_key,
|
|
confirmation_no,
|
|
disp_room_no,
|
|
effective_rate_amount,
|
|
full_name,
|
|
res_comment,
|
|
trace_text,
|
|
no_of_rooms,
|
|
products,
|
|
rate_code,
|
|
room_category_label,
|
|
arrival,
|
|
departure,
|
|
nights,
|
|
real_price,
|
|
total_price,
|
|
kb_amount,
|
|
channel_key,
|
|
pricing_method,
|
|
booking_source_match_status,
|
|
booking_source_match_count
|
|
)
|
|
VALUES (
|
|
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
|
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
|
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
|
%s, %s, %s
|
|
)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
daily_version_id,
|
|
source_sequence,
|
|
record.get("source_location"),
|
|
record.get("source_worksheet"),
|
|
record.get("source_row_no"),
|
|
record.get("outcome"),
|
|
list(record.get("decision_codes") or []),
|
|
duplicate_id,
|
|
record.get("block_code"),
|
|
record.get("adults"),
|
|
record.get("children"),
|
|
record.get("company_name"),
|
|
record.get("company_key"),
|
|
record.get("confirmation_no"),
|
|
record.get("disp_room_no"),
|
|
_decimal_or_none(record.get("effective_rate_amount")),
|
|
record.get("full_name"),
|
|
record.get("res_comment"),
|
|
record.get("trace_text"),
|
|
record.get("no_of_rooms"),
|
|
record.get("products"),
|
|
record.get("rate_code"),
|
|
record.get("room_category_label"),
|
|
_date_or_none(record.get("arrival")),
|
|
_date_or_none(record.get("departure")),
|
|
record.get("nights"),
|
|
_decimal_or_none(record.get("real_price")),
|
|
_decimal_or_none(record.get("total_price")),
|
|
_decimal_or_none(record.get("kb_amount")),
|
|
record.get("channel_key"),
|
|
record.get("pricing_method"),
|
|
source_status,
|
|
source_match_count,
|
|
),
|
|
)
|
|
inserted_ids[source_sequence] = int(cursor.fetchone()[0])
|
|
|
|
@staticmethod
|
|
def _insert_outbox(
|
|
cursor: Any,
|
|
event_key: str,
|
|
aggregate_type: str,
|
|
aggregate_id: int,
|
|
event_type: str,
|
|
payload: Mapping[str, Any],
|
|
) -> None:
|
|
cursor.execute(
|
|
"""
|
|
INSERT INTO ingestion.outbox_events (
|
|
event_key,
|
|
aggregate_type,
|
|
aggregate_id,
|
|
event_type,
|
|
payload
|
|
)
|
|
VALUES (%s, %s, %s, %s, %s::jsonb)
|
|
ON CONFLICT (event_key) DO NOTHING
|
|
""",
|
|
(
|
|
event_key,
|
|
aggregate_type,
|
|
aggregate_id,
|
|
event_type,
|
|
json.dumps(
|
|
payload,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
),
|
|
),
|
|
)
|