feat: prepare ARR for controlled public deployment
This commit is contained in:
Wyndham ARR
15
booking_ingestion/__init__.py
Normal file
15
booking_ingestion/__init__.py
Normal file
@@ -0,0 +1,15 @@
|
||||
"""Booking-source ingestion for ARR PostgreSQL."""
|
||||
|
||||
from .md_fixture import (
|
||||
FixtureDocument,
|
||||
FixtureRow,
|
||||
build_fixture_parse_result,
|
||||
parse_fixture_markdown,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"FixtureDocument",
|
||||
"FixtureRow",
|
||||
"build_fixture_parse_result",
|
||||
"parse_fixture_markdown",
|
||||
]
|
||||
4
booking_ingestion/__main__.py
Normal file
4
booking_ingestion/__main__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
from .cli import main
|
||||
|
||||
|
||||
raise SystemExit(main())
|
||||
71
booking_ingestion/cli.py
Normal file
71
booking_ingestion/cli.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""CLI for importing the approved booking Markdown fixture."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .postgres import DatabaseConfig, import_fixture
|
||||
|
||||
|
||||
ENV_KEYS = {
|
||||
"ARR_DB_HOST",
|
||||
"ARR_DB_PORT",
|
||||
"ARR_DB_USER",
|
||||
"ARR_DB_PASSWORD",
|
||||
"ARR_DB_NAME",
|
||||
}
|
||||
|
||||
|
||||
def load_config(path: Path) -> DatabaseConfig:
|
||||
values: dict[str, str] = {}
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
key, separator, value = line.partition("=")
|
||||
if not separator or key.strip() in values:
|
||||
raise RuntimeError("invalid database config structure")
|
||||
values[key.strip()] = value.strip().strip('"').strip("'")
|
||||
if set(values) != ENV_KEYS or not all(values.values()):
|
||||
raise RuntimeError("database config keys are incomplete")
|
||||
return DatabaseConfig(
|
||||
host=values["ARR_DB_HOST"],
|
||||
port=int(values["ARR_DB_PORT"]),
|
||||
user=values["ARR_DB_USER"],
|
||||
password=values["ARR_DB_PASSWORD"],
|
||||
database=values["ARR_DB_NAME"],
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--fixture", required=True, type=Path)
|
||||
parser.add_argument("--contract", required=True, type=Path)
|
||||
parser.add_argument("--db-env", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
result = import_fixture(
|
||||
load_config(args.db_env),
|
||||
args.fixture,
|
||||
args.contract,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": result.status,
|
||||
"source_batch_id": result.source_batch_id,
|
||||
"source_rows": result.source_rows,
|
||||
"current_parses": result.current_parses,
|
||||
"room_items": result.room_items,
|
||||
"distinct_group_codes": result.distinct_group_codes,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
198
booking_ingestion/md_fixture.py
Normal file
198
booking_ingestion/md_fixture.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""Deterministic parser for the approved booking-room Markdown fixture."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROCESSOR_NAME = "booking-md-fixture-importer"
|
||||
PROCESSOR_VERSION = "1.0.0"
|
||||
RESULT_SCHEMA_VERSION = "1.0"
|
||||
|
||||
_WORKSHEET_RE = re.compile(r"^## 工作表：(.*\S)\s*$")
|
||||
_ROW_RE = re.compile(
|
||||
r"^\|\s*第\s*(\d+)\s*行\s*"
|
||||
r"\|\s*([^|]+?)\s*"
|
||||
r"\|\s*([^|]+?)\s*"
|
||||
r"\|\s*(\d+)\s*\|\s*$"
|
||||
)
|
||||
_SOURCE_FILE_RE = re.compile(r"^来源文件：`([^`]+)`\s*$")
|
||||
_MAPPING_FILE_RE = re.compile(r"^房型映射文件：`([^`]+)`\s*$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FixtureRow:
|
||||
worksheet: str
|
||||
row_no: int
|
||||
group_code_raw: str
|
||||
type_of_room_raw: str
|
||||
no_of_rooms: int
|
||||
|
||||
@property
|
||||
def group_code_key(self) -> str:
|
||||
return self.group_code_raw.strip().upper()
|
||||
|
||||
def canonical_source(self) -> dict[str, object]:
|
||||
return {
|
||||
"worksheet": self.worksheet,
|
||||
"row_no": self.row_no,
|
||||
"group_code_raw": self.group_code_raw,
|
||||
"group_code_key": self.group_code_key,
|
||||
"type_of_room_raw": self.type_of_room_raw,
|
||||
"no_of_rooms": self.no_of_rooms,
|
||||
}
|
||||
|
||||
def sha256(self) -> str:
|
||||
return hashlib.sha256(
|
||||
canonical_json_bytes(self.canonical_source())
|
||||
).hexdigest()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FixtureDocument:
|
||||
source_file: str
|
||||
mapping_file: str
|
||||
rows: tuple[FixtureRow, ...]
|
||||
|
||||
@property
|
||||
def worksheet_count(self) -> int:
|
||||
return len({row.worksheet for row in self.rows})
|
||||
|
||||
@property
|
||||
def distinct_group_code_count(self) -> int:
|
||||
return len({row.group_code_key for row in self.rows})
|
||||
|
||||
|
||||
def canonical_json_bytes(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def parse_fixture_markdown(text: str) -> FixtureDocument:
|
||||
source_file = ""
|
||||
mapping_file = ""
|
||||
worksheet: str | None = None
|
||||
rows: list[FixtureRow] = []
|
||||
coordinates: set[tuple[str, int]] = set()
|
||||
|
||||
for physical_line_no, raw_line in enumerate(text.splitlines(), start=1):
|
||||
line = raw_line.strip()
|
||||
source_match = _SOURCE_FILE_RE.fullmatch(line)
|
||||
if source_match:
|
||||
source_file = source_match.group(1).strip()
|
||||
continue
|
||||
mapping_match = _MAPPING_FILE_RE.fullmatch(line)
|
||||
if mapping_match:
|
||||
mapping_file = mapping_match.group(1).strip()
|
||||
continue
|
||||
worksheet_match = _WORKSHEET_RE.fullmatch(line)
|
||||
if worksheet_match:
|
||||
worksheet = worksheet_match.group(1).strip()
|
||||
if not worksheet:
|
||||
raise ValueError(
|
||||
f"empty worksheet at Markdown line {physical_line_no}"
|
||||
)
|
||||
continue
|
||||
row_match = _ROW_RE.fullmatch(line)
|
||||
if not row_match:
|
||||
continue
|
||||
if worksheet is None:
|
||||
raise ValueError(
|
||||
f"data row before worksheet at Markdown line {physical_line_no}"
|
||||
)
|
||||
row = FixtureRow(
|
||||
worksheet=worksheet,
|
||||
row_no=int(row_match.group(1)),
|
||||
group_code_raw=row_match.group(2).strip(),
|
||||
type_of_room_raw=row_match.group(3).strip(),
|
||||
no_of_rooms=int(row_match.group(4)),
|
||||
)
|
||||
if not row.group_code_key:
|
||||
raise ValueError(
|
||||
f"empty Group Code at Markdown line {physical_line_no}"
|
||||
)
|
||||
if not row.type_of_room_raw:
|
||||
raise ValueError(
|
||||
f"empty type of room at Markdown line {physical_line_no}"
|
||||
)
|
||||
if row.no_of_rooms <= 0:
|
||||
raise ValueError(
|
||||
f"invalid room quantity at Markdown line {physical_line_no}"
|
||||
)
|
||||
coordinate = (row.worksheet, row.row_no)
|
||||
if coordinate in coordinates:
|
||||
raise ValueError(
|
||||
"duplicate worksheet/row coordinate "
|
||||
f"{row.worksheet!r}/{row.row_no}"
|
||||
)
|
||||
coordinates.add(coordinate)
|
||||
rows.append(row)
|
||||
|
||||
if not source_file:
|
||||
raise ValueError("fixture source file metadata is missing")
|
||||
if not mapping_file:
|
||||
raise ValueError("fixture mapping file metadata is missing")
|
||||
if not rows:
|
||||
raise ValueError("fixture contains no data rows")
|
||||
return FixtureDocument(
|
||||
source_file=source_file,
|
||||
mapping_file=mapping_file,
|
||||
rows=tuple(rows),
|
||||
)
|
||||
|
||||
|
||||
def parse_fixture_file(path: Path) -> FixtureDocument:
|
||||
return parse_fixture_markdown(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def rule_set_sha256(contract_path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
digest.update(contract_path.read_bytes())
|
||||
digest.update(b"\0")
|
||||
digest.update(Path(__file__).read_bytes())
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def build_fixture_parse_result(
|
||||
row: FixtureRow,
|
||||
*,
|
||||
rule_sha256: str,
|
||||
) -> dict[str, object]:
|
||||
source_sha256 = row.sha256()
|
||||
return {
|
||||
"result_schema_version": RESULT_SCHEMA_VERSION,
|
||||
"status": "accepted",
|
||||
"source_row": {
|
||||
"worksheet": row.worksheet,
|
||||
"row_no": row.row_no,
|
||||
"sha256": source_sha256,
|
||||
},
|
||||
"processor_version": PROCESSOR_VERSION,
|
||||
"rule_set_sha256": rule_sha256,
|
||||
"group_code_raw": row.group_code_raw,
|
||||
"group_code_key": row.group_code_key,
|
||||
"type_of_room_raw": row.type_of_room_raw,
|
||||
"no_of_rooms": row.no_of_rooms,
|
||||
"room_items": [
|
||||
{
|
||||
"item_no": 1,
|
||||
"room_type_raw": row.type_of_room_raw,
|
||||
"room_type_code": row.type_of_room_raw,
|
||||
"quantity": row.no_of_rooms,
|
||||
"unit_price": None,
|
||||
"currency_code": None,
|
||||
"price_token_raw": None,
|
||||
"source_fragment": row.type_of_room_raw,
|
||||
}
|
||||
],
|
||||
"warnings": [],
|
||||
"errors": [],
|
||||
}
|
||||
435
booking_ingestion/postgres.py
Normal file
435
booking_ingestion/postgres.py
Normal file
@@ -0,0 +1,435 @@
|
||||
"""PostgreSQL adapter for the approved booking Markdown fixture."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .md_fixture import (
|
||||
PROCESSOR_NAME,
|
||||
PROCESSOR_VERSION,
|
||||
RESULT_SCHEMA_VERSION,
|
||||
FixtureDocument,
|
||||
build_fixture_parse_result,
|
||||
canonical_json_bytes,
|
||||
parse_fixture_file,
|
||||
rule_set_sha256,
|
||||
)
|
||||
|
||||
|
||||
TARGET_DATABASE = "booking_test"
|
||||
EXPECTED_FIXTURE_ROWS = 867
|
||||
EXPECTED_FIXTURE_WORKSHEETS = 6
|
||||
EXPECTED_FIXTURE_GROUP_CODES = 348
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatabaseConfig:
|
||||
host: str
|
||||
port: int
|
||||
user: str
|
||||
password: str
|
||||
database: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportResult:
|
||||
status: str
|
||||
source_batch_id: int
|
||||
source_rows: int
|
||||
current_parses: int
|
||||
room_items: int
|
||||
distinct_group_codes: int
|
||||
|
||||
|
||||
def _artifact_id(
|
||||
cursor,
|
||||
*,
|
||||
fixture_path: Path,
|
||||
content_sha256: str,
|
||||
byte_size: int,
|
||||
) -> int:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT id, byte_size
|
||||
FROM ingestion.artifacts
|
||||
WHERE artifact_kind = 'booking_source_md'
|
||||
AND sha256 = %s
|
||||
""",
|
||||
(content_sha256,),
|
||||
)
|
||||
existing = cursor.fetchone()
|
||||
if existing is not None:
|
||||
if int(existing[1]) != byte_size:
|
||||
raise RuntimeError("existing booking artifact metadata conflicts")
|
||||
return int(existing[0])
|
||||
|
||||
try:
|
||||
object_key = fixture_path.resolve().relative_to(PROJECT_ROOT).as_posix()
|
||||
except ValueError:
|
||||
raise RuntimeError("booking fixture must be inside the project root") from None
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO ingestion.artifacts (
|
||||
artifact_kind, storage_provider, bucket_alias, object_key,
|
||||
original_filename, sha256, byte_size, mime_type
|
||||
)
|
||||
VALUES (
|
||||
'booking_source_md', 'local_fixture', 'workspace', %s,
|
||||
%s, %s, %s, 'text/markdown'
|
||||
)
|
||||
RETURNING id
|
||||
""",
|
||||
(object_key, fixture_path.name, content_sha256, byte_size),
|
||||
)
|
||||
return int(cursor.fetchone()[0])
|
||||
|
||||
|
||||
def _existing_result(
|
||||
cursor,
|
||||
*,
|
||||
artifact_id: int,
|
||||
expected: FixtureDocument,
|
||||
) -> ImportResult | None:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT id, batch_status, source_rows, accepted_rows, failed_rows
|
||||
FROM booking.source_batches
|
||||
WHERE source_artifact_id = %s
|
||||
""",
|
||||
(artifact_id,),
|
||||
)
|
||||
batch = cursor.fetchone()
|
||||
if batch is None:
|
||||
return None
|
||||
batch_id = int(batch[0])
|
||||
if (
|
||||
batch[1] != "accepted"
|
||||
or int(batch[2]) != len(expected.rows)
|
||||
or int(batch[3]) != len(expected.rows)
|
||||
or int(batch[4]) != 0
|
||||
):
|
||||
raise RuntimeError("existing booking source batch is incomplete")
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
count(*) AS source_rows,
|
||||
count(current_parse.source_row_id) AS current_parses,
|
||||
count(item.id) AS room_items,
|
||||
count(DISTINCT source.group_code_key) AS distinct_group_codes
|
||||
FROM booking.source_rows AS source
|
||||
LEFT JOIN booking.current_row_parses AS current_parse
|
||||
ON current_parse.source_row_id = source.id
|
||||
LEFT JOIN booking.room_items AS item
|
||||
ON item.parse_version_id = current_parse.parse_version_id
|
||||
WHERE source.source_batch_id = %s
|
||||
""",
|
||||
(batch_id,),
|
||||
)
|
||||
counts = tuple(int(value) for value in cursor.fetchone())
|
||||
expected_counts = (
|
||||
len(expected.rows),
|
||||
len(expected.rows),
|
||||
len(expected.rows),
|
||||
expected.distinct_group_code_count,
|
||||
)
|
||||
if counts != expected_counts:
|
||||
raise RuntimeError("existing booking fixture rows are inconsistent")
|
||||
return ImportResult(
|
||||
status="already_applied_and_verified",
|
||||
source_batch_id=batch_id,
|
||||
source_rows=counts[0],
|
||||
current_parses=counts[1],
|
||||
room_items=counts[2],
|
||||
distinct_group_codes=counts[3],
|
||||
)
|
||||
|
||||
|
||||
def import_fixture(
|
||||
config: DatabaseConfig,
|
||||
fixture_path: Path,
|
||||
contract_path: Path,
|
||||
) -> ImportResult:
|
||||
if config.database != TARGET_DATABASE:
|
||||
raise RuntimeError("database target is invalid")
|
||||
fixture_path = fixture_path.resolve()
|
||||
contract_path = contract_path.resolve()
|
||||
document = parse_fixture_file(fixture_path)
|
||||
if (
|
||||
len(document.rows) != EXPECTED_FIXTURE_ROWS
|
||||
or document.worksheet_count != EXPECTED_FIXTURE_WORKSHEETS
|
||||
or document.distinct_group_code_count
|
||||
!= EXPECTED_FIXTURE_GROUP_CODES
|
||||
):
|
||||
raise RuntimeError("approved booking fixture baseline changed")
|
||||
|
||||
fixture_bytes = fixture_path.read_bytes()
|
||||
fixture_sha256 = hashlib.sha256(fixture_bytes).hexdigest()
|
||||
rule_sha256 = rule_set_sha256(contract_path)
|
||||
run_key = f"booking-fixture:{fixture_sha256[:24]}"
|
||||
delivery_payload = {
|
||||
"source_kind": "expected_fixture",
|
||||
"source_rows": len(document.rows),
|
||||
"worksheet_count": document.worksheet_count,
|
||||
"distinct_group_codes": document.distinct_group_code_count,
|
||||
"source_file": document.source_file,
|
||||
"mapping_file": document.mapping_file,
|
||||
"contract": f"booking-row-parse-result/{RESULT_SCHEMA_VERSION}",
|
||||
}
|
||||
delivery_json = canonical_json_bytes(delivery_payload).decode("utf-8")
|
||||
delivery_sha256 = hashlib.sha256(
|
||||
delivery_json.encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
try:
|
||||
import psycopg
|
||||
except ImportError:
|
||||
raise RuntimeError("psycopg is required for PostgreSQL import") from None
|
||||
|
||||
connection = psycopg.connect(
|
||||
host=config.host,
|
||||
port=config.port,
|
||||
user=config.user,
|
||||
password=config.password,
|
||||
dbname=config.database,
|
||||
connect_timeout=5,
|
||||
options=(
|
||||
"-c statement_timeout=120000 "
|
||||
"-c lock_timeout=5000 "
|
||||
"-c idle_in_transaction_session_timeout=180000 "
|
||||
"-c application_name=arr_booking_fixture_import"
|
||||
),
|
||||
)
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE")
|
||||
cursor.execute(
|
||||
"SELECT current_database(), current_setting('transaction_read_only')"
|
||||
)
|
||||
if cursor.fetchone() != (TARGET_DATABASE, "off"):
|
||||
raise RuntimeError("database target/write guard failed")
|
||||
cursor.execute(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))",
|
||||
("arr_booking_fixture_import",),
|
||||
)
|
||||
cursor.execute("SELECT to_regclass('booking.source_rows')")
|
||||
if cursor.fetchone()[0] is None:
|
||||
raise RuntimeError("ARR MVP v1 booking schema is not installed")
|
||||
|
||||
artifact_id = _artifact_id(
|
||||
cursor,
|
||||
fixture_path=fixture_path,
|
||||
content_sha256=fixture_sha256,
|
||||
byte_size=len(fixture_bytes),
|
||||
)
|
||||
existing = _existing_result(
|
||||
cursor,
|
||||
artifact_id=artifact_id,
|
||||
expected=document,
|
||||
)
|
||||
if existing is not None:
|
||||
connection.rollback()
|
||||
return existing
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO ingestion.processing_runs (
|
||||
run_key, pipeline_type, source_artifact_id, run_status,
|
||||
requested_processor_version, requested_rule_set_sha256,
|
||||
delivered_processor_version, delivered_rule_set_sha256,
|
||||
result_schema_version, delivery_sha256, delivery_json
|
||||
)
|
||||
VALUES (
|
||||
%s, 'booking_source_import', %s, 'validating',
|
||||
%s, %s, %s, %s, %s, %s, %s::jsonb
|
||||
)
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
run_key,
|
||||
artifact_id,
|
||||
PROCESSOR_VERSION,
|
||||
rule_sha256,
|
||||
PROCESSOR_VERSION,
|
||||
rule_sha256,
|
||||
RESULT_SCHEMA_VERSION,
|
||||
delivery_sha256,
|
||||
delivery_json,
|
||||
),
|
||||
)
|
||||
processing_run_id = int(cursor.fetchone()[0])
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO booking.source_batches (
|
||||
source_artifact_id, source_kind, source_format_version,
|
||||
batch_status, source_rows, accepted_rows, failed_rows
|
||||
)
|
||||
VALUES (
|
||||
%s, 'expected_fixture', 'md/1.0',
|
||||
'processing', %s, 0, 0
|
||||
)
|
||||
RETURNING id
|
||||
""",
|
||||
(artifact_id, len(document.rows)),
|
||||
)
|
||||
source_batch_id = int(cursor.fetchone()[0])
|
||||
|
||||
for row in document.rows:
|
||||
row_sha256 = row.sha256()
|
||||
parse_result = build_fixture_parse_result(
|
||||
row,
|
||||
rule_sha256=rule_sha256,
|
||||
)
|
||||
result_bytes = canonical_json_bytes(parse_result)
|
||||
result_sha256 = hashlib.sha256(result_bytes).hexdigest()
|
||||
cursor.execute(
|
||||
"""
|
||||
WITH source AS (
|
||||
INSERT INTO booking.source_rows (
|
||||
source_batch_id, source_worksheet, source_row_no,
|
||||
group_code_raw, type_of_room_raw, no_of_rooms,
|
||||
source_row_sha256
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING id
|
||||
),
|
||||
parsed AS (
|
||||
INSERT INTO booking.parse_versions (
|
||||
source_row_id, version_no, parse_status,
|
||||
result_schema_version, processor_name,
|
||||
processor_version, rule_set_sha256,
|
||||
input_row_sha256, result_sha256,
|
||||
result_json, validated_at
|
||||
)
|
||||
SELECT
|
||||
source.id, 1, 'accepted', %s, %s, %s, %s,
|
||||
%s, %s, %s::jsonb, now()
|
||||
FROM source
|
||||
RETURNING id, source_row_id
|
||||
),
|
||||
item AS (
|
||||
INSERT INTO booking.room_items (
|
||||
parse_version_id, item_no, room_type_raw,
|
||||
room_type_code, quantity, unit_price,
|
||||
currency_code, price_token_raw, source_fragment
|
||||
)
|
||||
SELECT
|
||||
parsed.id, 1, %s, %s, %s,
|
||||
NULL, NULL, NULL, %s
|
||||
FROM parsed
|
||||
RETURNING id
|
||||
)
|
||||
SELECT source_row_id, id
|
||||
FROM parsed
|
||||
""",
|
||||
(
|
||||
source_batch_id,
|
||||
row.worksheet,
|
||||
row.row_no,
|
||||
row.group_code_raw,
|
||||
row.type_of_room_raw,
|
||||
row.no_of_rooms,
|
||||
row_sha256,
|
||||
RESULT_SCHEMA_VERSION,
|
||||
PROCESSOR_NAME,
|
||||
PROCESSOR_VERSION,
|
||||
rule_sha256,
|
||||
row_sha256,
|
||||
result_sha256,
|
||||
result_bytes.decode("utf-8"),
|
||||
row.type_of_room_raw,
|
||||
row.type_of_room_raw,
|
||||
row.no_of_rooms,
|
||||
row.type_of_room_raw,
|
||||
),
|
||||
)
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE booking.source_batches
|
||||
SET batch_status = 'accepted',
|
||||
accepted_rows = source_rows,
|
||||
failed_rows = 0,
|
||||
validated_at = now(),
|
||||
finished_at = now()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(source_batch_id,),
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO booking.current_row_parses (
|
||||
source_row_id, parse_version_id
|
||||
)
|
||||
SELECT source.id, parsed.id
|
||||
FROM booking.source_rows AS source
|
||||
JOIN booking.parse_versions AS parsed
|
||||
ON parsed.source_row_id = source.id
|
||||
AND parsed.version_no = 1
|
||||
AND parsed.parse_status = 'accepted'
|
||||
WHERE source.source_batch_id = %s
|
||||
""",
|
||||
(source_batch_id,),
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE ingestion.processing_runs
|
||||
SET run_status = 'accepted',
|
||||
updated_at = now(),
|
||||
validated_at = now(),
|
||||
finished_at = now()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(processing_run_id,),
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO ingestion.outbox_events (
|
||||
event_key, aggregate_type, aggregate_id,
|
||||
event_type, payload
|
||||
)
|
||||
VALUES (
|
||||
%s, 'booking_source_batch', %s,
|
||||
'booking.source_batch.accepted', %s::jsonb
|
||||
)
|
||||
""",
|
||||
(
|
||||
f"booking-source-batch:{source_batch_id}:accepted",
|
||||
source_batch_id,
|
||||
json.dumps(
|
||||
{
|
||||
"source_batch_id": source_batch_id,
|
||||
"source_rows": len(document.rows),
|
||||
"distinct_group_codes":
|
||||
document.distinct_group_code_count,
|
||||
},
|
||||
separators=(",", ":"),
|
||||
),
|
||||
),
|
||||
)
|
||||
result = _existing_result(
|
||||
cursor,
|
||||
artifact_id=artifact_id,
|
||||
expected=document,
|
||||
)
|
||||
if result is None:
|
||||
raise RuntimeError("booking fixture post-write validation failed")
|
||||
connection.commit()
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
return ImportResult(
|
||||
status="imported_and_verified",
|
||||
source_batch_id=result.source_batch_id,
|
||||
source_rows=result.source_rows,
|
||||
current_parses=result.current_parses,
|
||||
room_items=result.room_items,
|
||||
distinct_group_codes=result.distinct_group_codes,
|
||||
)
|
||||
Reference in New Issue
Block a user