444 lines
21 KiB
Python
444 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
from datetime import datetime, timedelta, timezone
|
|
from email.utils import format_datetime
|
|
import hashlib
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import stat
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import urllib.error
|
|
import urllib.parse
|
|
|
|
from integrations.ohip import collect_arr_source as collector
|
|
|
|
|
|
DAY, HOTEL = "2026-09-15", "TEST_HOTEL"
|
|
SECRET = "private-test-key-never-print"
|
|
|
|
|
|
def row(identity="id1"):
|
|
return {"reservationIdList": [{"type": "Reservation", "id": identity},
|
|
{"type": "Confirmation", "id": "confirmation-" + identity}],
|
|
"hotelId": HOTEL, "roomStay": {"arrivalDate": DAY}, "reservationStatus": "InHouse",
|
|
"lastModifyDateTime": "2026-09-15 10:00:00", "comments": [{"comment": {
|
|
"type": "GEN", "internal": True, "text": {"value": "PRIVATE_NOTE"}}}]}
|
|
|
|
|
|
def envelope(operation, data):
|
|
return {"operation_id": operation, "hotel_id": HOTEL, "oracle_request_id": "test-oracle-id", "data": data}
|
|
|
|
|
|
class FakeService:
|
|
def __init__(self, count=3):
|
|
self.rows = [row(f"id{i}") for i in range(count)]
|
|
self.calls = []
|
|
self.responses = []
|
|
self.edit = lambda document, index: document
|
|
|
|
def __call__(self, method, path, raw):
|
|
self.calls.append((method, path, raw))
|
|
if method == "POST":
|
|
body = json.loads(raw)
|
|
offset, limit = body["offset"], body["limit"]
|
|
page = {"reservationInfo": self.rows[offset:offset + limit], "offset": offset + limit,
|
|
"limit": limit, "totalResults": len(self.rows),
|
|
"totalPages": (len(self.rows) + limit - 1) // limit,
|
|
"hasMore": offset + limit < len(self.rows)}
|
|
doc = envelope(collector.SEARCH, {"reservations": page})
|
|
else:
|
|
identity = urllib.parse.urlsplit(path).path.rsplit("/", 1)[-1]
|
|
details = [item for item in self.rows if collector.reservation_id(item) == identity]
|
|
doc = envelope(collector.DETAIL, {"reservations": {"reservation": details}})
|
|
doc = self.edit(copy.deepcopy(doc), len(self.calls))
|
|
raw = collector.json_bytes(doc)
|
|
self.responses.append(raw)
|
|
return 200, {"Content-Type": "application/json"}, raw
|
|
|
|
|
|
class CollectionTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.output = self.root / "capture"
|
|
self.service = FakeService()
|
|
|
|
def run_capture(self, service=None, options=None):
|
|
archive = collector.Archive(self.output)
|
|
reader = collector.Reader(archive, HOTEL, service or self.service, key=SECRET, sleep=lambda _: None)
|
|
result = collector.collect(options or collector.Options(DAY, HOTEL, page_size=2), archive, reader)
|
|
saved = json.loads((self.output / "result.json").read_bytes())
|
|
self.assertNotIn(SECRET, (self.output / "result.json").read_text())
|
|
return result, saved
|
|
|
|
def assert_failed(self, code):
|
|
result, saved = self.run_capture()
|
|
self.assertFalse(result["candidate_capture_complete"])
|
|
self.assertFalse(result["finance_ready"])
|
|
self.assertEqual(result["manifest_sha256"], hashlib.sha256((self.output / "result.json").read_bytes()).hexdigest())
|
|
self.assertEqual(result["error"], code)
|
|
self.assertEqual(saved["error"], code)
|
|
|
|
def test_complete_pagination_detail_and_recheck_preserve_exact_bytes(self):
|
|
result, saved = self.run_capture()
|
|
self.assertEqual(result["status"], "complete_candidate_capture")
|
|
self.assertEqual((result["search_records"], result["verified_details"], result["http_attempts"]), (3, 3, 7))
|
|
self.assertTrue(result["search_recheck_equal"])
|
|
self.assertFalse(result["atomic_snapshot"])
|
|
self.assertFalse(result["report_equivalence_verified"])
|
|
self.assertFalse(result["finance_ready"])
|
|
self.assertEqual(stat.S_IMODE(self.output.stat().st_mode), 0o700)
|
|
for entry in saved["files"]:
|
|
target = self.output / entry["name"]
|
|
raw = target.read_bytes()
|
|
self.assertEqual(len(raw), entry["bytes"])
|
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), entry["sha256"])
|
|
self.assertEqual(stat.S_IMODE(target.stat().st_mode), 0o600)
|
|
self.assertNotIn(SECRET.encode(), raw)
|
|
self.assertEqual([p.read_bytes() for p in sorted(self.output.glob("*.response.bin"))], self.service.responses)
|
|
searches = [json.loads(raw) for method, _, raw in self.service.calls if method == "POST"]
|
|
self.assertEqual([body["offset"] for body in searches], [0, 2, 0, 2])
|
|
self.assertTrue(all(set(body) == {"arrivalStartDate", "arrivalEndDate", "limit", "offset", "orderBy", "sortOrder"}
|
|
for body in searches))
|
|
for method, path, raw in self.service.calls:
|
|
if method == "GET":
|
|
self.assertEqual(urllib.parse.parse_qs(urllib.parse.urlsplit(path).query), {"fetchInstructions": list(collector.FETCH)})
|
|
self.assertNotIn("confirmation", path)
|
|
self.assertIsNone(raw)
|
|
|
|
def test_existing_output_never_overwritten(self):
|
|
self.run_capture()
|
|
original = (self.output / "result.json").read_bytes()
|
|
with self.assertRaises(FileExistsError):
|
|
collector.Archive(self.output)
|
|
self.assertEqual((self.output / "result.json").read_bytes(), original)
|
|
|
|
def test_duplicate_across_pages_fails(self):
|
|
self.service.rows[2] = copy.deepcopy(self.service.rows[0])
|
|
self.assert_failed("duplicate_reservation")
|
|
|
|
def test_empty_day_never_accepted(self):
|
|
self.service.rows = []
|
|
self.assert_failed("empty_source_requires_review")
|
|
|
|
def test_ambiguous_internal_id_not_replaced_with_confirmation(self):
|
|
self.service.rows[0]["reservationIdList"] = [{"type": "Confirmation", "id": "other"}]
|
|
self.assert_failed("ambiguous_reservation_identity")
|
|
|
|
def test_hotel_and_date_checks(self):
|
|
for key, value, expected in [("hotelId", "WRONG", "record_hotel_mismatch"),
|
|
("roomStay", {"arrivalDate": "2026-09-14"}, "record_arrival_mismatch")]:
|
|
with self.subTest(key=key):
|
|
self.output = self.root / key
|
|
self.service = FakeService()
|
|
self.service.rows[0][key] = value
|
|
self.assert_failed(expected)
|
|
|
|
def test_early_final_page_cannot_truncate_total(self):
|
|
def edit(doc, _):
|
|
doc["data"]["reservations"]["hasMore"] = False
|
|
return doc
|
|
self.service.edit = edit
|
|
self.assert_failed("incomplete_search_total")
|
|
|
|
def test_counts_and_next_offset_are_checked(self):
|
|
cases = [("offset", 0, "pagination_mismatch"), ("limit", 1, "pagination_mismatch"),
|
|
("count", 0, "pagination_mismatch"), ("hasMore", "false", "invalid_has_more"),
|
|
("totalResults", True, "invalid_total")]
|
|
for index, (name, value, expected) in enumerate(cases):
|
|
with self.subTest(name=name):
|
|
self.output = self.root / str(index)
|
|
self.service = FakeService()
|
|
def edit(doc, _, name=name, value=value):
|
|
doc["data"]["reservations"][name] = value
|
|
return doc
|
|
self.service.edit = edit
|
|
self.assert_failed(expected)
|
|
|
|
def test_changed_total_on_second_page_fails(self):
|
|
def edit(doc, index):
|
|
if index == 2:
|
|
doc["data"]["reservations"]["totalResults"] = 4
|
|
return doc
|
|
self.service.edit = edit
|
|
self.assert_failed("pagination_totals_changed")
|
|
|
|
def test_has_more_missing_accepted_only_with_consistent_other_evidence(self):
|
|
self.service = FakeService(count=1)
|
|
def edit(doc, _):
|
|
doc["data"]["reservations"].pop("hasMore", None)
|
|
return doc
|
|
self.service.edit = edit
|
|
result, _ = self.run_capture()
|
|
self.assertTrue(result["candidate_capture_complete"])
|
|
|
|
def test_short_nonfinal_page_and_page_bound(self):
|
|
def edit(doc, _):
|
|
doc["data"]["reservations"]["reservationInfo"].pop()
|
|
return doc
|
|
self.service.edit = edit
|
|
self.assert_failed("short_nonfinal_page")
|
|
self.output = self.root / "bounded"
|
|
self.service = FakeService()
|
|
result, _ = self.run_capture(options=collector.Options(DAY, HOTEL, page_size=2, max_pages=1))
|
|
self.assertEqual(result["error"], "page_limit_exceeded")
|
|
|
|
def test_record_bound(self):
|
|
result, _ = self.run_capture(options=collector.Options(DAY, HOTEL, page_size=2, max_records=2))
|
|
self.assertEqual(result["error"], "record_limit_exceeded")
|
|
|
|
def test_missing_duplicate_wrong_identity_or_changed_detail_fails(self):
|
|
cases = [([], "ambiguous_detail_count"), ([row(), row()], "ambiguous_detail_count"),
|
|
([row("wrong")], "detail_identity_mismatch"),
|
|
([{**row("id0"), "reservationStatus": "Cancelled"}], "search_detail_state_mismatch")]
|
|
for index, (details, expected) in enumerate(cases):
|
|
with self.subTest(index=index):
|
|
self.output = self.root / str(index)
|
|
self.service = FakeService()
|
|
def edit(doc, _, details=details):
|
|
if doc["operation_id"] == collector.DETAIL:
|
|
doc["data"]["reservations"]["reservation"] = details
|
|
return doc
|
|
self.service.edit = edit
|
|
self.assert_failed(expected)
|
|
|
|
def test_same_ids_but_changed_content_on_recheck_fails(self):
|
|
def edit(doc, index):
|
|
if index == 6:
|
|
doc["data"]["reservations"]["reservationInfo"][0]["comments"][0]["comment"]["text"]["value"] = "CHANGED"
|
|
return doc
|
|
self.service.edit = edit
|
|
self.assert_failed("source_changed_during_collection")
|
|
|
|
def test_explicit_note_and_trace_indicators_cannot_lose_rows(self):
|
|
for name in ("COMMENT", "TRACE"):
|
|
with self.subTest(name=name):
|
|
self.output = self.root / name.lower()
|
|
self.service = FakeService()
|
|
self.service.rows[0]["reservationIndicators"] = [{"indicatorName": name, "count": 2}]
|
|
self.assert_failed("note_or_trace_count_mismatch")
|
|
|
|
def test_note_counts_match_and_internal_note_is_preserved(self):
|
|
self.service.rows[0]["reservationIndicators"] = [{"indicatorName": "COMMENT", "count": 1},
|
|
{"indicatorName": "TRACE", "count": 0}]
|
|
result, _ = self.run_capture()
|
|
self.assertTrue(result["candidate_capture_complete"])
|
|
detail = json.loads((self.output / "request-000003.response.bin").read_bytes())
|
|
self.assertTrue(detail["data"]["reservations"]["reservation"][0]["comments"][0]["comment"]["internal"])
|
|
|
|
def test_warning_keeps_raw_and_fails(self):
|
|
def edit(doc, _):
|
|
doc["data"]["warnings"] = [{"code": "GEN10001", "message": "PRIVATE_DATA"}]
|
|
return doc
|
|
self.service.edit = edit
|
|
self.assert_failed("upstream_warning_or_error")
|
|
self.assertIn(b"PRIVATE_DATA", next(self.output.glob("*.response.bin")).read_bytes())
|
|
self.assertNotIn("PRIVATE_DATA", (self.output / "result.json").read_text())
|
|
|
|
def test_operation_and_hotel_envelope_mismatch(self):
|
|
for index, (key, expected) in enumerate([("operation_id", "operation_mismatch"), ("hotel_id", "hotel_mismatch")]):
|
|
with self.subTest(key=key):
|
|
self.output = self.root / str(index)
|
|
self.service = FakeService()
|
|
def edit(doc, _, key=key):
|
|
doc[key] = "WRONG"
|
|
return doc
|
|
self.service.edit = edit
|
|
self.assert_failed(expected)
|
|
|
|
def test_malformed_and_duplicate_json_fail_without_payload_in_diagnostic(self):
|
|
for index, raw in enumerate([b"PRIVATE_DATA", b'{"a":1,"a":2}', b'{"a":NaN}']):
|
|
self.output = self.root / str(index)
|
|
result, _ = self.run_capture(service=lambda *args: (200, {}, raw))
|
|
self.assertFalse(result["candidate_capture_complete"])
|
|
self.assertNotIn("PRIVATE_DATA", json.dumps(result))
|
|
|
|
def test_reflected_key_is_not_archived(self):
|
|
result, _ = self.run_capture(service=lambda *args: (403, {}, SECRET.encode()))
|
|
self.assertEqual(result["error"], "secret_in_response")
|
|
self.assertFalse(list(self.output.glob("*.response.bin")))
|
|
self.assertNotIn(SECRET, "".join(p.read_text() for p in self.output.iterdir()))
|
|
|
|
def test_bounded_response_fails(self):
|
|
with patch.object(collector, "MAX_RESPONSE_BYTES", 20):
|
|
result, _ = self.run_capture(service=lambda *args: (200, {}, b"x" * 21))
|
|
self.assertEqual(result["error"], "response_too_large")
|
|
|
|
def test_invalid_options_fail_before_request(self):
|
|
for day in ("not-a-date", "20260915", "2026-02-30"):
|
|
with self.assertRaises(collector.CollectionError):
|
|
collector.Options(day, HOTEL).validate()
|
|
for size in (True, 0, 101):
|
|
with self.assertRaises(collector.CollectionError):
|
|
collector.Options(DAY, HOTEL, page_size=size).validate()
|
|
self.assertEqual(self.service.calls, [])
|
|
|
|
|
|
class CredentialTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
os.chmod(self.root, 0o700)
|
|
self.file = self.root / "credential.json"
|
|
self.doc = {"version": "ohip.application-credential/v1", "status": "ready", "value": SECRET,
|
|
"service_url": collector.SERVICE, "application_id": collector.APPLICATION}
|
|
self.save()
|
|
|
|
def save(self):
|
|
self.file.write_bytes(collector.json_bytes(self.doc))
|
|
os.chmod(self.file, 0o600)
|
|
|
|
def test_valid_private_credential(self):
|
|
self.assertEqual(collector.load_key(self.file), SECRET)
|
|
self.assertNotIn(SECRET, repr(collector.HTTPTransport(SECRET)))
|
|
|
|
def test_wrong_identity_service_or_pending_refused(self):
|
|
for field in ("application_id", "service_url", "status", "version"):
|
|
with self.subTest(field=field):
|
|
original = self.doc[field]
|
|
self.doc[field] = "wrong"
|
|
self.save()
|
|
with self.assertRaisesRegex(collector.CollectionError, "credential_identity_mismatch"):
|
|
collector.load_key(self.file)
|
|
self.doc[field] = original
|
|
|
|
def test_unsafe_file_parent_permissions_and_symlink_refused(self):
|
|
os.chmod(self.file, 0o644)
|
|
with self.assertRaisesRegex(collector.CollectionError, "credential_file_permissions"):
|
|
collector.load_key(self.file)
|
|
os.chmod(self.file, 0o600)
|
|
os.chmod(self.root, 0o755)
|
|
with self.assertRaisesRegex(collector.CollectionError, "credential_directory_permissions"):
|
|
collector.load_key(self.file)
|
|
os.chmod(self.root, 0o700)
|
|
link = self.root / "link.json"
|
|
link.symlink_to(self.file)
|
|
with self.assertRaises(collector.CollectionError):
|
|
collector.load_key(link)
|
|
|
|
|
|
class HTTPStubTests(unittest.TestCase):
|
|
"""Exercise real urllib request construction, auth, failures and redirect policy."""
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.service = FakeService(count=1)
|
|
self.replies = []
|
|
self.headers_received = []
|
|
owner = self
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
def do_POST(self):
|
|
self.respond()
|
|
|
|
def do_GET(self):
|
|
self.respond()
|
|
|
|
def respond(self):
|
|
owner.headers_received.append(dict(self.headers))
|
|
length = int(self.headers.get("Content-Length", 0))
|
|
raw = self.rfile.read(length) if length else None
|
|
status, headers, body = owner.replies.pop(0) if owner.replies else owner.service(self.command, self.path, raw)
|
|
self.send_response(status)
|
|
for key, value in headers.items():
|
|
self.send_header(key, value)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
|
self.thread.start()
|
|
self.addCleanup(self.stop)
|
|
self.delays = []
|
|
|
|
def stop(self):
|
|
self.server.shutdown()
|
|
self.server.server_close()
|
|
self.thread.join()
|
|
|
|
def run_capture(self):
|
|
archive = collector.Archive(self.root / "capture")
|
|
with patch.object(collector, "SERVICE", f"http://127.0.0.1:{self.server.server_port}"):
|
|
reader = collector.Reader(archive, HOTEL, collector.HTTPTransport(SECRET), key=SECRET, sleep=self.delays.append)
|
|
return collector.collect(collector.Options(DAY, HOTEL), archive, reader)
|
|
|
|
def test_real_http_auth_body_and_repeat_query_keys(self):
|
|
result = self.run_capture()
|
|
self.assertTrue(result["candidate_capture_complete"])
|
|
self.assertEqual(len(self.headers_received), 3)
|
|
self.assertTrue(all(headers["X-Api-Key"] == SECRET for headers in self.headers_received))
|
|
self.assertEqual(json.loads(self.service.calls[0][2])["arrivalStartDate"], DAY)
|
|
self.assertEqual(urllib.parse.parse_qs(urllib.parse.urlsplit(self.service.calls[1][1]).query)["fetchInstructions"], list(collector.FETCH))
|
|
|
|
def test_429_and_503_retry_and_archive_every_attempt(self):
|
|
self.replies = [(429, {"Retry-After": "2"}, b"limited"), (503, {}, b"unavailable")]
|
|
result = self.run_capture()
|
|
self.assertTrue(result["candidate_capture_complete"])
|
|
self.assertEqual(result["http_attempts"], 5)
|
|
self.assertEqual(self.delays, [2, 2])
|
|
self.assertEqual(len(list((self.root / "capture").glob("*.response.bin"))), 5)
|
|
|
|
def test_permission_denial_not_retried(self):
|
|
self.replies = [(403, {}, b"PRIVATE_DENIAL")]
|
|
result = self.run_capture()
|
|
self.assertEqual(result["error"], "http_failure")
|
|
self.assertEqual(result["http_attempts"], 1)
|
|
self.assertEqual(self.delays, [])
|
|
self.assertNotIn("PRIVATE_DENIAL", json.dumps(result))
|
|
|
|
def test_401_not_retried(self):
|
|
self.replies = [(401, {}, b"unauthorized")]
|
|
result = self.run_capture()
|
|
self.assertEqual(result["http_attempts"], 1)
|
|
self.assertEqual(result["error"], "http_failure")
|
|
|
|
def test_transient_failures_stop_after_three(self):
|
|
self.replies = [(503, {}, b"unavailable")] * 3
|
|
result = self.run_capture()
|
|
self.assertEqual(result["error"], "http_failure")
|
|
self.assertEqual(result["http_attempts"], 3)
|
|
|
|
def test_redirect_not_followed_even_same_origin(self):
|
|
self.replies = [(302, {"Location": "/capture-secret"}, b"")]
|
|
result = self.run_capture()
|
|
self.assertEqual(result["error"], "redirect_refused")
|
|
self.assertEqual(len(self.headers_received), 1)
|
|
|
|
def test_long_retry_after_stops_without_retrying_early(self):
|
|
self.replies = [(429, {"Retry-After": "600"}, b"limited")]
|
|
result = self.run_capture()
|
|
self.assertEqual(result["error"], "retry_after_exceeds_budget")
|
|
self.assertEqual(len(self.headers_received), 1)
|
|
self.assertEqual(self.delays, [])
|
|
|
|
def test_http_date_retry_after(self):
|
|
future = format_datetime(datetime.now(timezone.utc) + timedelta(seconds=5), usegmt=True)
|
|
self.assertTrue(0 <= collector.retry_delay({"retry-after": future}, 1) <= 5)
|
|
|
|
def test_transport_timeout_bounded_without_secret_in_diagnostics(self):
|
|
archive = collector.Archive(self.root / "capture")
|
|
def fail(*args):
|
|
raise urllib.error.URLError(SECRET)
|
|
reader = collector.Reader(archive, HOTEL, fail, key=SECRET, sleep=self.delays.append)
|
|
result = collector.collect(collector.Options(DAY, HOTEL), archive, reader)
|
|
self.assertEqual(result["error"], "transport_retry_exhausted")
|
|
self.assertEqual(result["http_attempts"], 3)
|
|
self.assertNotIn(SECRET, "".join(path.read_text() for path in archive.path.iterdir()))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|