299 lines
17 KiB
Python
299 lines
17 KiB
Python
"""Integrated source candidates preserve all records and never approve ARR."""
|
|
import contextlib
|
|
import copy
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from integrations.ohip import audit_arr_day, audit_arr_named_day
|
|
from integrations.ohip import collect_arr_day as day, collect_arr_named_day as named
|
|
from integrations.ohip import collect_arr_source as capture, profile_reader, rate_info
|
|
from integrations.ohip import prepare_arr_source as prep, source_facts, source_fields, arr_xml
|
|
from integrations.ohip.source_facts_contract import FIELDS, canonical
|
|
from tests.test_ohip_named_day import NamedService, DAY, HOTEL, SECRET
|
|
|
|
|
|
class SourcePreparationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.root = Path(temp.name)
|
|
self.directory = self.root / 'capture'
|
|
self.service = NamedService()
|
|
for i, row in enumerate(self.service.base.rows):
|
|
stay = row['roomStay']
|
|
stay.update(departureDate='2026-09-16', roomId='0010',
|
|
currentRoomInfo={'roomId': '0010'}, guestCounts={'adults': 2, 'children': 0})
|
|
stay['roomRates'][0].update(start=DAY, end=DAY, roomId='0010', roomType='TYPE',
|
|
ratePlanCode=' NOT_WHITELISTED ', guestCounts={'adults': 2, 'children': 0}, numberOfUnits=1)
|
|
row['comments'] = [{'comment': {'type': 'GEN', 'notificationLocation': 'RESERVATION',
|
|
'internal': True, 'text': {'value': ' \r\nPRIVATE_NOTE ไทย < & '}}}]
|
|
|
|
def collect(self, version='v3', transport=None):
|
|
sink = capture.Archive(self.directory)
|
|
transport = transport or self.service
|
|
readers = [capture.Reader(sink, HOTEL, transport, key=SECRET, sleep=lambda _: None),
|
|
rate_info.RateInfoReader(sink, HOTEL, transport, key=SECRET, sleep=lambda _: None)]
|
|
if version == 'v3':
|
|
options = named.Options(DAY, DAY, DAY, HOTEL, max_profiles=3, page_size=2)
|
|
result = named.collect(options, sink, *readers,
|
|
profile_reader.ProfileSummaryReader(sink, HOTEL, transport, max_profiles=3,
|
|
key=SECRET, sleep=lambda _: None))
|
|
else:
|
|
result = day.collect(day.Options(DAY, DAY, DAY, HOTEL, page_size=2), sink, *readers)
|
|
self.pin = result['manifest_sha256']
|
|
return result
|
|
|
|
def prepare(self, version='v3', transport=None):
|
|
self.assertTrue(self.collect(version, transport)['candidate_capture_complete'])
|
|
protocol = audit_arr_named_day if version == 'v3' else audit_arr_day
|
|
self.archive = protocol.VerifiedArchive(self.directory, self.pin)
|
|
self.evidence = source_facts.build_source_facts(self.archive)
|
|
with (patch.object(capture, 'HTTPTransport', side_effect=AssertionError('offline')),
|
|
patch.object(capture, 'load_key', side_effect=AssertionError('no credentials')),
|
|
patch.object(arr_xml, 'serialize', side_effect=AssertionError('no XML'))):
|
|
self.raw = prep.prepare(self.archive, self.evidence)
|
|
self.document = json.loads(self.raw)
|
|
return self.document['records']
|
|
|
|
def test_known_fields_exact_values_all_rows_and_unresolved_display_fields(self):
|
|
rows = self.prepare()
|
|
self.assertEqual([r['reservation_id'] for r in rows], ['id0', 'id1', 'id2'])
|
|
for i, row in enumerate(rows):
|
|
values = row['fields']
|
|
self.assertEqual(set(values), set(FIELDS))
|
|
expected = {'ARRIVAL': DAY, 'DEPARTURE': '2026-09-16', 'ADULTS': '2', 'CHILDREN': '0',
|
|
'NO_OF_ROOMS': '1', 'DISP_ROOM_NO': '0010', 'RATE_CODE': ' NOT_WHITELISTED ',
|
|
'CONFIRMATION_NO': f'confirmation-id{i}', 'EFFECTIVE_RATE_AMOUNT': str(i),
|
|
'FULL_NAME': f' Surname{i}, Given · 测试 ', 'TRACE_TEXT': [],
|
|
'RES_COMMENT': [' \r\nPRIVATE_NOTE ไทย < & ']}
|
|
self.assertEqual({k: values[k]['value'] for k in expected}, expected)
|
|
self.assertEqual(row['effective_rate_currency'], 'USD')
|
|
for name, code in prep.UNRESOLVED.items():
|
|
self.assertEqual(values[name], {'state': 'gap', 'reason': code})
|
|
for key in ('finance_ready', 'source_mapping_verified', 'report_equivalence_verified', 'complete_arr_output'):
|
|
self.assertIs(self.document[key], False)
|
|
report = prep.summary(self.raw)
|
|
self.assertEqual((report['records'], report['records_with_gaps']), (3, 3))
|
|
self.assertEqual(report['fields']['FULL_NAME']['candidates'], 3)
|
|
self.assertNotIn('trace_scope_order', report['report_gaps'])
|
|
self.assertEqual(report['fields']['BLOCK_CODE']['gaps'], 3)
|
|
self.assertTrue(all(r['fields']['BLOCK_CODE']['reason'] ==
|
|
'source_field_missing_or_invalid_reservation_block' for r in rows))
|
|
|
|
def test_typed_block_code_reaches_versioned_candidates_without_new_reads(self):
|
|
from tests.test_ohip_block_fields import block
|
|
for i, row in enumerate(self.service.base.rows[:2]):
|
|
value = block(' RAW_CODE ' if i == 0 else '')
|
|
row['roomStay']['reservationBlock'] = copy.deepcopy(value)
|
|
row['roomStay']['roomRates'][0]['reservationBlock'] = copy.deepcopy(value)
|
|
rows = self.prepare('v2')
|
|
self.assertEqual(self.document['version'], 'arr-source-preparation/v4')
|
|
self.assertEqual([r['fields']['BLOCK_CODE']['value'] for r in rows[:2]], [' RAW_CODE ', ''])
|
|
self.assertEqual(rows[2]['fields']['BLOCK_CODE']['state'], 'gap')
|
|
self.assertEqual([r['reservation_id'] for r in rows], ['id0', 'id1', 'id2'])
|
|
self.assertEqual(prep.summary(self.raw)['fields']['BLOCK_CODE']['candidates'], 2)
|
|
self.assertFalse(self.document['source_mapping_verified'])
|
|
self.assertFalse(self.document['complete_arr_output'])
|
|
|
|
def test_room_type_candidate_does_not_use_charged_type_or_descriptions(self):
|
|
first = self.service.base.rows[0]['roomStay']
|
|
first.update(roomType=' ACTUAL ', roomTypeCharged='CHARGED')
|
|
first['currentRoomInfo'].update(roomType=' ACTUAL ', roomTypeDescription='Description')
|
|
first['roomRates'][0].update(roomType=' ACTUAL ', roomTypeCharged='CHARGED')
|
|
second = self.service.base.rows[1]['roomStay']
|
|
second.update(roomType='SEARCH')
|
|
second['currentRoomInfo'].update(roomType='CURRENT')
|
|
rows = self.prepare('v2')
|
|
self.assertEqual(rows[0]['fields']['ROOM_CATEGORY_LABEL'],
|
|
{'state': 'candidate', 'kind': 'text', 'value': ' ACTUAL '})
|
|
self.assertEqual(rows[1]['fields']['ROOM_CATEGORY_LABEL'],
|
|
{'state': 'gap', 'reason': 'source_field_room_type_disagreement'})
|
|
self.assertEqual(rows[2]['fields']['ROOM_CATEGORY_LABEL']['reason'],
|
|
'source_field_missing_or_invalid_room_type')
|
|
self.assertEqual(len(rows), 3)
|
|
self.assertFalse(self.document['report_equivalence_verified'])
|
|
|
|
def test_v2_has_explicit_name_gap_and_current_trace_scope(self):
|
|
self.service.base.rows[0]['traces'] = [{'traceText': 'HISTORICAL_TRACE'}]
|
|
rows = self.prepare('v2')
|
|
self.assertEqual(rows[0]['fields']['FULL_NAME'],
|
|
{'state': 'gap', 'reason': 'profile_source_not_acquired'})
|
|
self.assertEqual(rows[0]['fields']['TRACE_TEXT']['value'], [])
|
|
self.assertNotIn('HISTORICAL_TRACE', self.raw.decode())
|
|
self.assertEqual(self.service.profile_calls, [])
|
|
|
|
def test_missing_and_conflicting_names_never_fall_back_or_drop_records(self):
|
|
self.service.profiles['p0']['fullName'] = ' '
|
|
self.service.profiles['p1']['givenName'] = 'Different'
|
|
self.service.base.rows[2]['reservationGuests'] = []
|
|
rows = self.prepare()
|
|
self.assertEqual([r['fields']['FULL_NAME']['reason'] for r in rows],
|
|
['profile_full_name_unavailable', 'profile_name_components_mismatch',
|
|
'profile_ambiguous_primary_guest'])
|
|
self.assertTrue(all(r['fields']['CONFIRMATION_NO']['state'] == 'candidate' for r in rows))
|
|
|
|
def test_all_diagnostics_accumulate_without_suppressing_unrelated_fields(self):
|
|
first = self.service.base.rows[0]
|
|
first['roomStay']['roomRates'][0].pop('ratePlanCode')
|
|
first['roomStay']['guestCounts']['children'] = True
|
|
first['roomStay'].pop('currentRoomInfo')
|
|
first['comments'][0]['comment'].pop('notificationLocation')
|
|
def rate_edit(data, body):
|
|
if body['id'] == 'id0':
|
|
data['detail']['rateSuppressed'] = True
|
|
return data
|
|
self.service.rate_edit = rate_edit
|
|
values = self.prepare()[0]['fields']
|
|
for name in ('RATE_CODE', 'ADULTS', 'CHILDREN', 'DISP_ROOM_NO', 'RES_COMMENT', 'EFFECTIVE_RATE_AMOUNT'):
|
|
self.assertEqual(values[name]['state'], 'gap', name)
|
|
self.assertNotIn('value', values[name])
|
|
self.assertEqual(values['EFFECTIVE_RATE_AMOUNT']['reason'], 'rate_suppressed')
|
|
self.assertIsNone(self.document['records'][0]['effective_rate_currency'])
|
|
self.assertEqual(values['FULL_NAME']['state'], 'candidate')
|
|
self.assertEqual(len(self.document['records']), 3)
|
|
|
|
def test_room_blanks_do_not_mask_missing_and_duplicates_are_not_removed(self):
|
|
for row in self.service.base.rows[:2]:
|
|
stay = row['roomStay']
|
|
stay['roomId'] = stay['currentRoomInfo']['roomId'] = stay['roomRates'][0]['roomId'] = ''
|
|
self.service.base.rows[2]['roomStay']['roomRates'][0].pop('roomId')
|
|
rows = self.prepare()
|
|
self.assertEqual([row['fields']['DISP_ROOM_NO']['value'] for row in rows[:2]], ['', ''])
|
|
self.assertEqual(rows[2]['fields']['DISP_ROOM_NO']['state'], 'gap')
|
|
self.assertEqual([row['capture_sequence'] for row in rows], [1, 2, 3])
|
|
|
|
def test_non_single_day_segments_remain_explicitly_unsupported(self):
|
|
self.service.base.rows[0]['roomStay']['roomRates'][0]['end'] = '2026-09-16'
|
|
values = self.prepare()[0]['fields']
|
|
self.assertEqual(values['RATE_CODE']['reason'], 'source_field_unsupported_rate_interval')
|
|
self.assertEqual(values['DISP_ROOM_NO']['reason'], 'source_field_unsupported_rate_interval')
|
|
self.assertEqual(values['ARRIVAL']['value'], DAY)
|
|
self.assertEqual(values['EFFECTIVE_RATE_AMOUNT']['value'], '0')
|
|
|
|
def test_decimal_raw_rate_never_rounds_through_float(self):
|
|
amount = b'12345678901234567890.0012300'
|
|
def transport(method, path, raw):
|
|
status, headers, body = self.service(method, path, raw)
|
|
if path.endswith('rate-info/searches'):
|
|
for digit in b'012':
|
|
body = body.replace(b'"totalRateAmount": ' + bytes([digit]) + b',',
|
|
b'"totalRateAmount": ' + amount + b',')
|
|
return status, headers, body
|
|
rows = self.prepare(transport=transport)
|
|
self.assertEqual([row['fields']['EFFECTIVE_RATE_AMOUNT']['value'] for row in rows],
|
|
[amount.decode()] * 3)
|
|
|
|
def test_tiny_decimal_exponent_stays_compact_in_candidate_text(self):
|
|
def transport(method, path, raw):
|
|
status, headers, body = self.service(method, path, raw)
|
|
if path.endswith('rate-info/searches'):
|
|
for digit in b'012':
|
|
body = body.replace(b'"totalRateAmount": ' + bytes([digit]) + b',',
|
|
b'"totalRateAmount": 1E-1000000,')
|
|
return status, headers, body
|
|
rows = self.prepare(transport=transport)
|
|
self.assertEqual([row['fields']['EFFECTIVE_RATE_AMOUNT']['value'] for row in rows], ['1E-1000000'] * 3)
|
|
self.assertLess(len(self.raw), 10000)
|
|
|
|
def test_successful_rate_and_profile_retry_references_survive_preparation(self):
|
|
attempts = {}
|
|
def transport(method, path, raw):
|
|
if path.endswith(('rate-info/searches', 'profiles/searches')):
|
|
body = json.loads(raw)
|
|
identity = body.get('id') or body['profileIds'][0]
|
|
key = (path, identity)
|
|
attempts[key] = attempts.get(key, 0) + 1
|
|
if identity in ('id0', 'p0') and attempts[key] == 1:
|
|
return 503, {}, b'{"error": "retry only, not a field value"}'
|
|
return self.service(method, path, raw)
|
|
rows = self.prepare(transport=transport)
|
|
references = json.loads(self.evidence)['records'][0]['sources']
|
|
self.assertEqual(references['rate']['request']['file'], 'rate-000002.json')
|
|
self.assertEqual(references['profile']['request']['file'], 'profile-000002.json')
|
|
self.assertEqual(references['rate']['request']['attempt'], 2)
|
|
self.assertEqual(references['profile']['request']['attempt'], 2)
|
|
self.assertEqual(rows[0]['fields']['EFFECTIVE_RATE_AMOUNT']['value'], '0')
|
|
self.assertEqual(rows[0]['fields']['FULL_NAME']['value'], ' Surname0, Given · 测试 ')
|
|
|
|
def test_tampered_evidence_rejected_before_any_selection(self):
|
|
self.prepare()
|
|
original = json.loads(self.evidence)
|
|
for mutate in (
|
|
lambda d: d['records'].reverse(),
|
|
lambda d: d.update(finance_ready=True),
|
|
lambda d: d['records'][0]['sources']['detail'].update(file='private-other'),
|
|
lambda d: d['records'][0]['fields']['FULL_NAME']['variants'][0].update(path='wrong'),
|
|
):
|
|
doc = copy.deepcopy(original); mutate(doc)
|
|
with patch.object(prep, '_select', side_effect=AssertionError('no selection before verification')):
|
|
with self.assertRaises(capture.CollectionError):
|
|
prep.prepare(self.archive, canonical(doc))
|
|
|
|
def test_cli_private_deterministic_output_no_business_claim_and_no_overwrite(self):
|
|
self.prepare()
|
|
before = {p.name: p.read_bytes() for p in self.directory.iterdir()}
|
|
outputs = []
|
|
for name in ('first', 'second'):
|
|
out = self.root / name
|
|
text = io.StringIO()
|
|
with contextlib.redirect_stdout(text):
|
|
code = prep.main(['--capture-dir', str(self.directory), '--capture-sha256', self.pin,
|
|
'--capture-version', 'v3', '--output-dir', str(out)])
|
|
self.assertEqual(code, 0)
|
|
self.assertEqual(out.stat().st_mode & 0o777, 0o700)
|
|
for file in out.iterdir():
|
|
self.assertEqual(file.stat().st_mode & 0o777, 0o600)
|
|
for private in (SECRET, 'Surname', 'PRIVATE_NOTE', '0010', 'confirmation-id', 'NOT_WHITELISTED'):
|
|
self.assertNotIn(private, text.getvalue())
|
|
report = json.loads(text.getvalue())
|
|
self.assertFalse(report['complete_arr_output'])
|
|
self.assertEqual(report['fields']['COMPANY_NAME']['gaps'], 3)
|
|
self.assertEqual((out / 'source-facts.json').read_bytes(), self.evidence)
|
|
outputs.append((out / 'source-candidates.json').read_bytes())
|
|
self.assertEqual(outputs[0], outputs[1])
|
|
with self.assertRaises(FileExistsError):
|
|
prep.export(self.directory, self.pin, self.root / 'first', capture_version='v3')
|
|
self.assertEqual(before, {p.name: p.read_bytes() for p in self.directory.iterdir()})
|
|
|
|
def test_failed_capture_cannot_prepare_partial_rows(self):
|
|
def transport(method, path, raw):
|
|
if path.endswith('profiles/searches'):
|
|
return 503, {}, b'{}'
|
|
return self.service(method, path, raw)
|
|
self.assertFalse(self.collect(transport=transport)['candidate_capture_complete'])
|
|
with self.assertRaisesRegex(capture.CollectionError, 'incomplete_capture'):
|
|
prep.export(self.directory, self.pin, self.root / 'output', capture_version='v3')
|
|
self.assertFalse((self.root / 'output').exists())
|
|
|
|
def test_wrong_version_pin_and_output_location_refused(self):
|
|
self.prepare()
|
|
for pin, version in ((self.pin, 'v2'), ('0' * 64, 'v3'), (self.pin, 'v9')):
|
|
with self.subTest(version=version), self.assertRaises(capture.CollectionError):
|
|
prep.export(self.directory, pin, self.root / 'bad', capture_version=version)
|
|
self.assertFalse((self.root / 'bad').exists())
|
|
with self.assertRaisesRegex(capture.CollectionError, 'output_must_be_outside_repository'):
|
|
prep.export(self.directory, self.pin, Path(__file__).parent / 'never-created', capture_version='v3')
|
|
|
|
def test_budget_failure_and_unexpected_exception_are_fatal_not_field_gaps(self):
|
|
self.prepare()
|
|
with patch.object(prep, 'MAX_FACTS_BYTES', 1), self.assertRaisesRegex(capture.CollectionError, 'byte_budget'):
|
|
prep.export(self.directory, self.pin, self.root / 'budget', capture_version='v3')
|
|
self.assertFalse((self.root / 'budget').exists())
|
|
text = io.StringIO()
|
|
with (patch.object(source_fields, 'confirmation_no', side_effect=RuntimeError('PRIVATE_CONTENT')),
|
|
contextlib.redirect_stdout(text)):
|
|
code = prep.main(['--capture-dir', str(self.directory), '--capture-sha256', self.pin,
|
|
'--capture-version', 'v3', '--output-dir', str(self.root / 'broken')])
|
|
self.assertEqual(code, 1)
|
|
self.assertNotIn('PRIVATE_CONTENT', text.getvalue())
|
|
self.assertEqual(json.loads(text.getvalue())['error'], 'source_preparation_failed')
|
|
self.assertFalse((self.root / 'broken').exists())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|