feat: sync latest ARR implementation
This commit is contained in:
Wyndham ARR
@@ -6,11 +6,12 @@ import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from arr_ingestion.contracts import IngestionError
|
||||
from arr_ingestion.contracts import ArtifactRef, IngestionError
|
||||
from arr_ingestion.postgres import (
|
||||
DatabaseConfig,
|
||||
PostgresIngestionRepository,
|
||||
)
|
||||
from arr_ingestion.repository import JobRegistration
|
||||
from tests.test_arr_opera_daily_ingest import run_processor, success_xml
|
||||
|
||||
|
||||
@@ -51,6 +52,26 @@ class BookingCursor:
|
||||
]
|
||||
|
||||
|
||||
class ArtifactCursor:
|
||||
def __init__(self, existing=None, inserted_id: int = 101) -> None:
|
||||
self.existing = existing
|
||||
self.inserted_id = inserted_id
|
||||
self.last_query = ""
|
||||
self.executed: list[tuple[str, tuple[object, ...]]] = []
|
||||
|
||||
def execute(self, query: str, params: tuple[object, ...]) -> None:
|
||||
InsertCursor.assert_placeholder_count(query, params)
|
||||
self.last_query = " ".join(query.split())
|
||||
self.executed.append((self.last_query, params))
|
||||
|
||||
def fetchone(self):
|
||||
if self.last_query.startswith("SELECT"):
|
||||
return self.existing
|
||||
if self.last_query.startswith("INSERT INTO ingestion.artifacts"):
|
||||
return (self.inserted_id,)
|
||||
return None
|
||||
|
||||
|
||||
class SqlStateError(RuntimeError):
|
||||
def __init__(self, sqlstate: str):
|
||||
super().__init__("synthetic database error")
|
||||
@@ -77,7 +98,64 @@ class RetryConnection:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class LifecycleCursor:
|
||||
def __init__(self, run_status: str = "queued", attempt_status: str = "queued") -> None:
|
||||
self.run_status = run_status
|
||||
self.attempt_status = attempt_status
|
||||
self.last_query = ""
|
||||
self.executed: list[tuple[str, tuple[object, ...]]] = []
|
||||
|
||||
def execute(self, query: str, params: tuple[object, ...]) -> None:
|
||||
InsertCursor.assert_placeholder_count(query, params)
|
||||
self.last_query = " ".join(query.split())
|
||||
self.executed.append((self.last_query, params))
|
||||
|
||||
def fetchone(self):
|
||||
if self.last_query.startswith("SELECT run.id, run.run_status"):
|
||||
return (12, self.run_status, 13, self.attempt_status)
|
||||
return None
|
||||
|
||||
|
||||
class RegistrationCursor:
|
||||
def __init__(self) -> None:
|
||||
self.last_query = ""
|
||||
self.executed: list[tuple[str, tuple[object, ...]]] = []
|
||||
|
||||
def execute(self, query: str, params: tuple[object, ...]) -> None:
|
||||
InsertCursor.assert_placeholder_count(query, params)
|
||||
self.last_query = " ".join(query.split())
|
||||
self.executed.append((self.last_query, params))
|
||||
|
||||
def fetchone(self):
|
||||
if self.last_query.startswith(
|
||||
"SELECT id, artifact_kind, original_filename"
|
||||
):
|
||||
return None
|
||||
if self.last_query.startswith("INSERT INTO ingestion.artifacts"):
|
||||
return (101,)
|
||||
if self.last_query.startswith(
|
||||
"SELECT id, source_artifact_id, requested_processor_version"
|
||||
):
|
||||
return None
|
||||
if self.last_query.startswith("INSERT INTO ingestion.processing_runs"):
|
||||
return (202,)
|
||||
return None
|
||||
|
||||
|
||||
class PostgresIngestionTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def artifact(object_key: str, sha256: str = "a" * 64) -> ArtifactRef:
|
||||
return ArtifactRef.from_dict(
|
||||
"source_xml",
|
||||
{
|
||||
"object_key": object_key,
|
||||
"original_filename": "source.xml",
|
||||
"sha256": sha256,
|
||||
"byte_size": 321,
|
||||
"mime_type": "application/xml",
|
||||
},
|
||||
)
|
||||
|
||||
def test_database_config_is_environment_only(self):
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
with self.assertRaisesRegex(IngestionError, "ARR_DATABASE_URL"):
|
||||
@@ -135,6 +213,96 @@ class PostgresIngestionTests(unittest.TestCase):
|
||||
self.assertEqual(links["GROUP-MISSING"], ("unmatched", 0))
|
||||
self.assertIsNotNone(cursor.executed)
|
||||
|
||||
def test_artifact_identity_is_storage_object_not_content_hash(self):
|
||||
key = (
|
||||
"arr/jobs/arrjob-repeat/attempts/0001/committed/"
|
||||
"source_xml/source.xml"
|
||||
)
|
||||
cursor = ArtifactCursor()
|
||||
|
||||
artifact_id = PostgresIngestionRepository._ensure_artifact(
|
||||
cursor,
|
||||
self.artifact(key),
|
||||
)
|
||||
|
||||
self.assertEqual(artifact_id, 101)
|
||||
select_query, select_params = cursor.executed[0]
|
||||
self.assertIn("object_key = %s", select_query)
|
||||
self.assertIn("object_version_id IS NULL", select_query)
|
||||
self.assertNotIn("WHERE artifact_kind = %s", select_query)
|
||||
self.assertEqual(select_params, ("oss", "arr-private", key))
|
||||
self.assertTrue(
|
||||
any(query.startswith("INSERT INTO ingestion.artifacts") for query, _ in cursor.executed)
|
||||
)
|
||||
|
||||
def test_existing_storage_identity_is_reused_only_when_all_metadata_matches(self):
|
||||
key = (
|
||||
"arr/jobs/arrjob-existing/attempts/0001/committed/"
|
||||
"source_xml/source.xml"
|
||||
)
|
||||
cursor = ArtifactCursor(
|
||||
(77, "opera_xml", "source.xml", "a" * 64, 321, "application/xml")
|
||||
)
|
||||
|
||||
artifact_id = PostgresIngestionRepository._ensure_artifact(
|
||||
cursor,
|
||||
self.artifact(key),
|
||||
)
|
||||
|
||||
self.assertEqual(artifact_id, 77)
|
||||
self.assertEqual(len(cursor.executed), 1)
|
||||
|
||||
def test_existing_storage_identity_rejects_conflicting_content(self):
|
||||
key = (
|
||||
"arr/jobs/arrjob-conflict/attempts/0001/committed/"
|
||||
"source_xml/source.xml"
|
||||
)
|
||||
cursor = ArtifactCursor(
|
||||
(77, "opera_xml", "source.xml", "b" * 64, 321, "application/xml")
|
||||
)
|
||||
|
||||
with self.assertRaises(IngestionError) as raised:
|
||||
PostgresIngestionRepository._ensure_artifact(
|
||||
cursor,
|
||||
self.artifact(key),
|
||||
)
|
||||
|
||||
self.assertEqual(raised.exception.code, "ARTIFACT_CONFLICT")
|
||||
|
||||
def test_job_registration_persists_uploaded_filename_separately(self):
|
||||
cursor = RegistrationCursor()
|
||||
registration = JobRegistration(
|
||||
job_id="arrjob-upload-name-001",
|
||||
source=self.artifact(
|
||||
"arr/jobs/arrjob-upload-name-001/attempts/0001/"
|
||||
"committed/source_xml/source.xml"
|
||||
),
|
||||
processor_version="3.0.0",
|
||||
rule_set_sha256="b" * 64,
|
||||
attempt_no=1,
|
||||
idempotency_key="c" * 64,
|
||||
uploaded_filename="0723-Opera.XML",
|
||||
)
|
||||
|
||||
repository = PostgresIngestionRepository(
|
||||
DatabaseConfig("postgresql://synthetic")
|
||||
)
|
||||
repository._register_job(cursor, registration)
|
||||
|
||||
run_insert = next(
|
||||
(query, params)
|
||||
for query, params in cursor.executed
|
||||
if query.startswith("INSERT INTO ingestion.processing_runs")
|
||||
)
|
||||
self.assertIn("uploaded_filename", run_insert[0])
|
||||
self.assertEqual(run_insert[1][-1], "0723-Opera.XML")
|
||||
artifact_insert = next(
|
||||
(query, params)
|
||||
for query, params in cursor.executed
|
||||
if query.startswith("INSERT INTO ingestion.artifacts")
|
||||
)
|
||||
self.assertEqual(artifact_insert[1][4], "source.xml")
|
||||
|
||||
def test_transaction_reopens_after_serialization_failure(self):
|
||||
connections = [RetryConnection(), RetryConnection()]
|
||||
connect_count = 0
|
||||
@@ -180,6 +348,40 @@ class PostgresIngestionTests(unittest.TestCase):
|
||||
self.assertEqual(raised.exception.code, "DATABASE_WRITE_FAILED")
|
||||
self.assertTrue(connection.closed)
|
||||
|
||||
def test_programmatic_start_marks_attempt_and_run_running(self):
|
||||
cursor = LifecycleCursor()
|
||||
repository = PostgresIngestionRepository(DatabaseConfig("postgresql://synthetic"))
|
||||
|
||||
repository._mark_running(
|
||||
cursor,
|
||||
"arrjob-programmatic-001",
|
||||
1,
|
||||
)
|
||||
|
||||
statements = "\n".join(query for query, _params in cursor.executed)
|
||||
self.assertIn("attempt_status = 'running'", statements)
|
||||
self.assertIn("run_status = 'running'", statements)
|
||||
self.assertIn("started_at = COALESCE(started_at, now())", statements)
|
||||
|
||||
def test_programmatic_infrastructure_failure_is_terminal_and_outboxed(self):
|
||||
cursor = LifecycleCursor(run_status="running", attempt_status="running")
|
||||
repository = PostgresIngestionRepository(DatabaseConfig("postgresql://synthetic"))
|
||||
|
||||
repository._record_runtime_failure(
|
||||
cursor,
|
||||
"arrjob-programmatic-002",
|
||||
1,
|
||||
"PROCESSOR_TIMEOUT",
|
||||
)
|
||||
|
||||
statements = "\n".join(query for query, _params in cursor.executed)
|
||||
self.assertIn("attempt_status = 'failed'", statements)
|
||||
self.assertIn("run_status = 'failed'", statements)
|
||||
self.assertIn("INSERT INTO ingestion.outbox_events", statements)
|
||||
rendered_params = repr([params for _query, params in cursor.executed])
|
||||
self.assertIn("PROCESSOR_TIMEOUT", rendered_params)
|
||||
self.assertNotIn("password", rendered_params.lower())
|
||||
|
||||
|
||||
class Migration008ContractTests(unittest.TestCase):
|
||||
def test_rebuild_targets_v3_and_all_source_record_storage(self):
|
||||
|
||||
Reference in New Issue
Block a user