338 lines
12 KiB
Python
338 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import copy
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from datetime import date, datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from arr_ingestion.contracts import ArtifactRef, IngestionError
|
|
from arr_ingestion.direct_contracts import (
|
|
DIRECT_CONTRACT_VERSION,
|
|
MAX_DIRECT_PAYLOAD_BYTES,
|
|
DirectSubmissionReceipt,
|
|
DirectSubmissionRequest,
|
|
ReceivedDirectSubmission,
|
|
SubmissionGrant,
|
|
VerifiedDirectSubmission,
|
|
)
|
|
from arr_ingestion.direct_service import DirectSubmissionService
|
|
from arr_ingestion.direct_validation import DirectResultValidator
|
|
from tests.test_arr_ingestion_validation import MemoryStore, policy, processor
|
|
from tests.test_arr_opera_daily_ingest import success_xml
|
|
|
|
|
|
GRANT = "G" * 43
|
|
JOB_ID = "arrjob-direct-001"
|
|
SOURCE_KEY = (
|
|
"arr/jobs/arrjob-direct-001/attempts/0001/committed/"
|
|
"source_xml/source.xml"
|
|
)
|
|
|
|
|
|
def run_direct_processor(
|
|
root: Path,
|
|
) -> tuple[bytes, dict[str, object]]:
|
|
source = root / "source.xml"
|
|
output = root / "output"
|
|
result = output / "result.json"
|
|
structured = output / "structured-result.json"
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
source.write_text(success_xml(), encoding="utf-8")
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
exit_code = processor.process(
|
|
argparse.Namespace(
|
|
xml=str(source.resolve()),
|
|
output_dir=str(output.resolve()),
|
|
result_json=str(result.resolve()),
|
|
structured_result_json=str(structured.resolve()),
|
|
)
|
|
)
|
|
if exit_code != 0:
|
|
raise AssertionError(f"synthetic processor failed: {exit_code}")
|
|
return source.read_bytes(), json.loads(structured.read_text(encoding="utf-8"))
|
|
|
|
|
|
def request_dict(payload: dict[str, object]) -> dict[str, object]:
|
|
return {
|
|
"submission_grant": GRANT,
|
|
"job_id": JOB_ID,
|
|
"attempt_no": 1,
|
|
"payload": payload,
|
|
}
|
|
|
|
|
|
def source_ref(source: bytes) -> ArtifactRef:
|
|
return ArtifactRef.from_dict(
|
|
"source_xml",
|
|
{
|
|
"object_key": SOURCE_KEY,
|
|
"original_filename": "source.xml",
|
|
"sha256": hashlib.sha256(source).hexdigest(),
|
|
"byte_size": len(source),
|
|
"mime_type": "application/xml",
|
|
},
|
|
)
|
|
|
|
|
|
def received(
|
|
payload: dict[str, object],
|
|
source: bytes,
|
|
*,
|
|
receipt: DirectSubmissionReceipt | None = None,
|
|
) -> ReceivedDirectSubmission:
|
|
return ReceivedDirectSubmission(
|
|
submission_id=17,
|
|
status="committed" if receipt is not None else "validating",
|
|
request=DirectSubmissionRequest.from_dict(request_dict(payload)),
|
|
source=source_ref(source),
|
|
receipt=receipt,
|
|
)
|
|
|
|
|
|
def receipt() -> DirectSubmissionReceipt:
|
|
return DirectSubmissionReceipt(
|
|
status="committed",
|
|
job_id=JOB_ID,
|
|
attempt_no=1,
|
|
business_date=date(2026, 7, 27),
|
|
daily_version_id=91,
|
|
version_no=3,
|
|
record_count=5,
|
|
)
|
|
|
|
|
|
class FakeRepository:
|
|
def __init__(self, state: ReceivedDirectSubmission):
|
|
self.state = state
|
|
self.received_requests: list[DirectSubmissionRequest] = []
|
|
self.rejections: list[tuple[int, str]] = []
|
|
self.commits: list[VerifiedDirectSubmission] = []
|
|
self.expiry_limits: list[int] = []
|
|
self.ready_checks = 0
|
|
|
|
def assert_ready(self) -> None:
|
|
self.ready_checks += 1
|
|
|
|
def issue_grant(
|
|
self,
|
|
job_id: str,
|
|
attempt_no: int,
|
|
*,
|
|
ttl_seconds: int,
|
|
) -> SubmissionGrant:
|
|
self.issued = (job_id, attempt_no, ttl_seconds)
|
|
return SubmissionGrant(
|
|
GRANT,
|
|
job_id,
|
|
attempt_no,
|
|
datetime(2026, 7, 29, tzinfo=timezone.utc),
|
|
)
|
|
|
|
def receive(self, request: DirectSubmissionRequest) -> ReceivedDirectSubmission:
|
|
self.received_requests.append(request)
|
|
return self.state
|
|
|
|
def reject(
|
|
self,
|
|
submission: ReceivedDirectSubmission,
|
|
failure_code: str,
|
|
) -> None:
|
|
self.rejections.append((submission.submission_id, failure_code))
|
|
|
|
def commit(
|
|
self,
|
|
verified: VerifiedDirectSubmission,
|
|
) -> DirectSubmissionReceipt:
|
|
self.commits.append(verified)
|
|
return receipt()
|
|
|
|
def expire_stale(self, *, limit: int) -> int:
|
|
self.expiry_limits.append(limit)
|
|
return 2
|
|
|
|
|
|
class FakeValidator:
|
|
def __init__(self, error: IngestionError | None = None):
|
|
self.error = error
|
|
self.calls: list[ReceivedDirectSubmission] = []
|
|
|
|
def validate(
|
|
self,
|
|
submission: ReceivedDirectSubmission,
|
|
) -> VerifiedDirectSubmission:
|
|
self.calls.append(submission)
|
|
if self.error is not None:
|
|
raise self.error
|
|
return VerifiedDirectSubmission(submission, "a" * 64)
|
|
|
|
|
|
class DirectContractTests(unittest.TestCase):
|
|
def test_request_is_canonical_bounded_and_keeps_grant_out_of_repr(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
_source, payload = run_direct_processor(Path(temp_dir))
|
|
request = DirectSubmissionRequest.from_dict(request_dict(payload))
|
|
self.assertEqual(request.business_date, date(2026, 7, 27))
|
|
self.assertEqual(request.record_count, 5)
|
|
self.assertEqual(
|
|
request.payload_sha256,
|
|
hashlib.sha256(request.payload_bytes).hexdigest(),
|
|
)
|
|
self.assertLess(len(request.payload_bytes), MAX_DIRECT_PAYLOAD_BYTES)
|
|
self.assertNotIn(GRANT, repr(request))
|
|
self.assertTrue(request.submission_key.startswith("direct:"))
|
|
|
|
def test_request_rejects_extra_fields_nonfinite_json_and_oversize(self) -> None:
|
|
minimal = {
|
|
"status": "success",
|
|
"activation_eligible": True,
|
|
"business_date": "2026-07-27",
|
|
"processor_version": "3.0.0",
|
|
"rule_set_sha256": "a" * 64,
|
|
"result_schema_version": "3.0",
|
|
"records": [],
|
|
}
|
|
extra = request_dict(minimal)
|
|
extra["payload_sha256"] = "a" * 64
|
|
with self.assertRaises(IngestionError) as raised:
|
|
DirectSubmissionRequest.from_dict(extra)
|
|
self.assertEqual(raised.exception.code, "DIRECT_SUBMISSION_INVALID")
|
|
|
|
nonfinite = copy.deepcopy(minimal)
|
|
nonfinite["number"] = float("nan")
|
|
with self.assertRaises(IngestionError) as raised:
|
|
DirectSubmissionRequest.from_dict(request_dict(nonfinite))
|
|
self.assertEqual(raised.exception.code, "DIRECT_SUBMISSION_INVALID")
|
|
|
|
oversize = copy.deepcopy(minimal)
|
|
oversize["padding"] = "x" * MAX_DIRECT_PAYLOAD_BYTES
|
|
with self.assertRaises(IngestionError) as raised:
|
|
DirectSubmissionRequest.from_dict(request_dict(oversize))
|
|
self.assertEqual(raised.exception.code, "DIRECT_PAYLOAD_TOO_LARGE")
|
|
|
|
def test_receipt_round_trip_is_minimal_and_strict(self) -> None:
|
|
value = receipt()
|
|
payload = value.to_dict()
|
|
self.assertEqual(payload["contract_version"], DIRECT_CONTRACT_VERSION)
|
|
self.assertEqual(DirectSubmissionReceipt.from_dict(payload), value)
|
|
payload["guest_name"] = "must-not-be-accepted"
|
|
with self.assertRaises(IngestionError):
|
|
DirectSubmissionReceipt.from_dict(payload)
|
|
|
|
|
|
class DirectReplayValidationTests(unittest.TestCase):
|
|
def test_source_is_replayed_and_daily_zip_hash_variation_is_normalized(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
source, payload = run_direct_processor(Path(temp_dir) / "agent")
|
|
state = received(payload, source)
|
|
validator = DirectResultValidator(
|
|
MemoryStore({SOURCE_KEY: source}),
|
|
policy(),
|
|
)
|
|
verified = validator.validate(state)
|
|
self.assertEqual(verified.submission.submission_id, 17)
|
|
self.assertEqual(len(verified.replay_payload_sha256), 64)
|
|
|
|
def test_business_fact_tamper_is_rejected_by_replay(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
source, payload = run_direct_processor(Path(temp_dir) / "agent")
|
|
tampered = copy.deepcopy(payload)
|
|
retained = next(
|
|
row
|
|
for row in tampered["records"]
|
|
if row["outcome"] == "retained"
|
|
)
|
|
retained["total_price"] += 1
|
|
validator = DirectResultValidator(
|
|
MemoryStore({SOURCE_KEY: source}),
|
|
policy(),
|
|
)
|
|
with self.assertRaises(IngestionError) as raised:
|
|
validator.validate(received(tampered, source))
|
|
self.assertEqual(raised.exception.code, "DIRECT_RESULT_MISMATCH")
|
|
|
|
|
|
class DirectSubmissionServiceTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.source, self.payload = run_direct_processor(Path(self.temp.name))
|
|
|
|
def tearDown(self) -> None:
|
|
self.temp.cleanup()
|
|
|
|
def test_commit_and_terminal_replay(self) -> None:
|
|
state = received(self.payload, self.source)
|
|
repository = FakeRepository(state)
|
|
validator = FakeValidator()
|
|
service = DirectSubmissionService(validator, repository)
|
|
result = service.submit(request_dict(self.payload))
|
|
self.assertEqual(result.status, "committed")
|
|
self.assertEqual(len(validator.calls), 1)
|
|
self.assertEqual(len(repository.commits), 1)
|
|
|
|
terminal = received(self.payload, self.source, receipt=receipt())
|
|
terminal_repository = FakeRepository(terminal)
|
|
terminal_validator = FakeValidator()
|
|
result = DirectSubmissionService(
|
|
terminal_validator,
|
|
terminal_repository,
|
|
).submit(request_dict(self.payload))
|
|
self.assertEqual(result, receipt())
|
|
self.assertEqual(terminal_validator.calls, [])
|
|
|
|
def test_terminal_validation_error_rejects_and_retryable_error_stays_open(self) -> None:
|
|
state = received(self.payload, self.source)
|
|
repository = FakeRepository(state)
|
|
service = DirectSubmissionService(
|
|
FakeValidator(
|
|
IngestionError(
|
|
"DIRECT_RESULT_MISMATCH",
|
|
"direct result does not match replay",
|
|
)
|
|
),
|
|
repository,
|
|
)
|
|
with self.assertRaises(IngestionError):
|
|
service.submit(request_dict(self.payload))
|
|
self.assertEqual(
|
|
repository.rejections,
|
|
[(17, "DIRECT_RESULT_MISMATCH")],
|
|
)
|
|
|
|
retry_repository = FakeRepository(state)
|
|
retry_service = DirectSubmissionService(
|
|
FakeValidator(
|
|
IngestionError(
|
|
"DIRECT_REPLAY_UNAVAILABLE",
|
|
"replay unavailable",
|
|
retryable=True,
|
|
)
|
|
),
|
|
retry_repository,
|
|
)
|
|
with self.assertRaises(IngestionError):
|
|
retry_service.submit(request_dict(self.payload))
|
|
self.assertEqual(retry_repository.rejections, [])
|
|
|
|
def test_grant_and_expiry_are_repository_owned(self) -> None:
|
|
repository = FakeRepository(received(self.payload, self.source))
|
|
service = DirectSubmissionService(FakeValidator(), repository)
|
|
service.assert_ready()
|
|
self.assertEqual(repository.ready_checks, 1)
|
|
grant = service.issue_grant(JOB_ID, 1, ttl_seconds=600)
|
|
self.assertEqual(grant.job_id, JOB_ID)
|
|
self.assertEqual(repository.issued, (JOB_ID, 1, 600))
|
|
self.assertEqual(service.expire_stale(limit=20), 2)
|
|
self.assertEqual(repository.expiry_limits, [20])
|
|
with self.assertRaises(IngestionError):
|
|
service.expire_stale(limit=0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|