feat: add daily manual price review workflow

This commit is contained in:
Wyndham ARR
2026-08-06 22:40:18 +08:00
parent aae3d8e1db
commit ca3e8e18fa
77 changed files with 7750 additions and 602 deletions

View File

@@ -4,6 +4,7 @@
from __future__ import annotations
import argparse
import copy
import json
import sys
import traceback
@@ -251,6 +252,40 @@ def expected_from_xml(
return business_date, len(reservations), removed_by_rate, removed_duplicates, records
def replay_classified_from_xml(
xml_path: Path,
price_path: Path,
manual_prices: Optional[Dict[Tuple[str, str, core.Decimal], core.Decimal]] = None,
) -> Tuple[
date,
int,
int,
int,
List[Dict[str, Any]],
List[Dict[str, Any]],
List[core.ErrorItem],
]:
"""Freshly derive all source records, including non-output outcomes, from XML."""
business_date, reservations = core.read_xml(xml_path)
all_records, records, removed_by_rate, removed_duplicates, classification_errors = (
core.classify_source_records(reservations, business_date)
)
if classification_errors:
raise core.ProcessingFailure(classification_errors)
price_map = core.load_price_map(price_path)
pricing_errors = core.apply_prices_classified(records, price_map, manual_prices)
return (
business_date,
len(reservations),
removed_by_rate,
removed_duplicates,
all_records,
records,
pricing_errors,
)
def validate_result_contract(
payload: Dict[str, Any],
business_date: date,
@@ -258,10 +293,16 @@ def validate_result_contract(
removed_by_rate: int,
removed_duplicates: int,
expected_records: Sequence[Dict[str, Any]],
daily_path: Path,
daily_path: Optional[Path],
structured_result_path: Path,
expected_channels: Sequence[Dict[str, Any]],
errors: List[core.ErrorItem],
*,
status: str = "success",
candidate_rows: int = 0,
review_required_rows: int = 0,
review_issue_count: int = 0,
expected_errors: Sequence[core.ErrorItem] = (),
) -> None:
required = {
"version",
@@ -279,10 +320,10 @@ def validate_result_contract(
)
)
return
if payload.get("version") != core.RESULT_VERSION or payload.get("status") != "success":
if payload.get("version") != core.RESULT_VERSION or payload.get("status") != status:
errors.append(
validation_error(
"OUTPUT_RESULT_CONTRACT_MISMATCH", "成功校验时result版本或status不正确"
"OUTPUT_RESULT_CONTRACT_MISMATCH", "独立校验时result版本或status不正确"
)
)
if payload.get("business_date") != business_date.isoformat():
@@ -297,6 +338,9 @@ def validate_result_contract(
"removed_by_rate_code": removed_by_rate,
"removed_as_duplicates": removed_duplicates,
"output_rows": len(expected_records),
"candidate_rows": candidate_rows,
"review_required_rows": review_required_rows,
"review_issue_count": review_issue_count,
}
if not isinstance(metrics, dict):
errors.append(validation_error("OUTPUT_RESULT_CONTRACT_MISMATCH", "metrics必须是对象"))
@@ -324,10 +368,10 @@ def validate_result_contract(
if not isinstance(outputs, dict):
errors.append(validation_error("OUTPUT_RESULT_CONTRACT_MISMATCH", "outputs必须是对象"))
else:
if outputs.get("daily_report") != daily_path.name:
errors.append(
validation_error("OUTPUT_RESULT_FILENAME_MISMATCH", "result日报文件名不一致")
)
if daily_path is not None and outputs.get("daily_report") != daily_path.name:
errors.append(validation_error("OUTPUT_RESULT_FILENAME_MISMATCH", "result日报文件名不一致"))
if daily_path is None and outputs.get("daily_report") is not None:
errors.append(validation_error("OUTPUT_RESULT_CONTRACT_MISMATCH", "待复核结果不得包含日报文件名"))
if outputs.get("structured_result") != structured_result_path.name:
errors.append(
validation_error(
@@ -341,7 +385,7 @@ def validate_result_contract(
"OUTPUT_RESULT_CONTRACT_MISMATCH", "成功结果不得包含异常清单文件名"
)
)
for field in ("daily_report", "structured_result"):
for field in ("structured_result",):
value = outputs.get(field)
if not isinstance(value, str) or Path(value).name != value:
errors.append(
@@ -349,9 +393,17 @@ def validate_result_contract(
"OUTPUT_RESULT_PATH_FORBIDDEN", f"outputs.{field} 必须是相对文件名"
)
)
if payload.get("errors") != []:
if daily_path is not None:
value = outputs.get("daily_report")
if not isinstance(value, str) or Path(value).name != value:
errors.append(
validation_error(
"OUTPUT_RESULT_PATH_FORBIDDEN", "outputs.daily_report 必须是相对文件名"
)
)
if payload.get("errors") != [item.to_dict() for item in expected_errors]:
errors.append(
validation_error("OUTPUT_RESULT_CONTRACT_MISMATCH", "成功结果的errors必须为空数组")
validation_error("OUTPUT_RESULT_CONTRACT_MISMATCH", "result的errors与独立重放不一致")
)
@@ -386,6 +438,9 @@ def validate_structured_result_contract(
expected_records: Sequence[Dict[str, Any]],
result_payload: Dict[str, Any],
errors: List[core.ErrorItem],
*,
manual_manifest: Optional[core.ManualOverrideManifest] = None,
manual_override_path: Optional[Path] = None,
) -> None:
required = {
"result_schema_version",
@@ -400,6 +455,13 @@ def validate_structured_result_contract(
"removed_as_duplicates",
"output_rows",
"outcome_counts",
"candidate_rows",
"review_required_rows",
"review_issue_count",
"review_issues",
"review_case_id",
"manual_override_sha256",
"manually_priced_rows",
"channels",
"artifacts",
"records",
@@ -436,6 +498,15 @@ def validate_structured_result_contract(
"removed_by_rate_code": removed_by_rate,
"removed_as_duplicates": removed_duplicates,
"output_rows": len(expected_records),
"candidate_rows": 0,
"review_required_rows": 0,
"review_issue_count": 0,
"review_issues": [],
"review_case_id": manual_manifest.review_case_id if manual_manifest else None,
"manual_override_sha256": manual_manifest.sha256 if manual_manifest else None,
"manually_priced_rows": sum(
1 for record in expected_records if record.get("_PRICING_METHOD") == "manual_review"
),
"channels": result_payload.get("metrics", {}).get("channels"),
"errors": [],
}
@@ -450,6 +521,7 @@ def validate_structured_result_contract(
expected_outcome_counts = {
"duplicate": removed_duplicates,
"excluded_rate_code": removed_by_rate,
"candidate": 0,
"price_unmatched": 0,
"retained": len(expected_records),
"validation_failed": 0,
@@ -488,6 +560,29 @@ def validate_structured_result_contract(
"成功structured-result.json不得引用异常清单",
)
)
if manual_manifest is not None and manual_override_path is not None:
validate_artifact(
artifacts.get("manual_override_json"),
manual_override_path,
"manual_override_json",
"application/json",
"manual_override_json",
errors,
)
if payload.get("manual_override_sha256") != core.sha256_file(manual_override_path):
errors.append(
validation_error(
"OUTPUT_STRUCTURED_ARTIFACT_MISMATCH",
"人工价格清单哈希与清单文件不一致",
)
)
elif artifacts.get("manual_override_json") is not None:
errors.append(
validation_error(
"OUTPUT_STRUCTURED_ARTIFACT_MISMATCH",
"非人工定价成功结果不得引用人工价格清单",
)
)
else:
errors.append(
validation_error(
@@ -508,7 +603,11 @@ def validate_structured_result_contract(
)
return
price_map = core.load_price_map(Path(core.PRICE_REFERENCE).resolve())
pricing_errors = core.apply_prices_classified(retained, price_map)
pricing_errors = core.apply_prices_classified(
retained,
price_map,
manual_manifest.prices if manual_manifest is not None else None,
)
if pricing_errors:
errors.append(
validation_error(
@@ -554,6 +653,345 @@ def validate_structured_result_contract(
)
def validate_review_structured_result_contract(
payload: Dict[str, Any],
xml_path: Path,
result_json: Path,
business_date: date,
source_rows: int,
removed_by_rate: int,
removed_duplicates: int,
all_records: Sequence[Dict[str, Any]],
records: Sequence[Dict[str, Any]],
pricing_errors: Sequence[core.ErrorItem],
price_map: Dict[Tuple[str, str, core.Decimal], core.Decimal],
errors: List[core.ErrorItem],
) -> None:
"""Validate the initial no-XLSX review result against a second XML replay."""
required = {
"result_schema_version",
"status",
"activation_eligible",
"ingestion_mode",
"business_date",
"processor_version",
"rule_set_sha256",
"source_rows",
"removed_by_rate_code",
"removed_as_duplicates",
"output_rows",
"outcome_counts",
"candidate_rows",
"review_required_rows",
"review_issue_count",
"review_issues",
"review_case_id",
"manual_override_sha256",
"manually_priced_rows",
"channels",
"artifacts",
"records",
"errors",
}
if set(payload) != required:
errors.append(
validation_error(
"OUTPUT_STRUCTURED_CONTRACT_MISMATCH",
"review structured-result.json 顶层字段不符合固定Schema",
)
)
return
try:
core.validate_structured_completeness(payload)
except core.ProcessingFailure as exc:
errors.extend(
validation_error("OUTPUT_STRUCTURED_CONTRACT_MISMATCH", item.message, item.source_location)
for item in exc.errors
)
core.assign_channels(records)
expected_issues = core.review_issues(records, price_map)
core.finalize_review_outcomes(all_records)
candidate_rows = sum(1 for record in all_records if record.get("_OUTCOME") == "candidate")
review_rows = sum(1 for record in all_records if record.get("_OUTCOME") == "price_unmatched")
expected_scalars = {
"result_schema_version": core.STRUCTURED_RESULT_SCHEMA_VERSION,
"status": "review_required",
"activation_eligible": False,
"ingestion_mode": "opera_xml",
"business_date": business_date.isoformat(),
"processor_version": core.PROCESSOR_VERSION,
"rule_set_sha256": core.rule_set_sha256(),
"source_rows": source_rows,
"removed_by_rate_code": removed_by_rate,
"removed_as_duplicates": removed_duplicates,
"output_rows": 0,
"candidate_rows": candidate_rows,
"review_required_rows": review_rows,
"review_issue_count": len(expected_issues),
"review_issues": expected_issues,
"review_case_id": None,
"manual_override_sha256": None,
"manually_priced_rows": 0,
"channels": [],
"errors": [item.to_dict() for item in pricing_errors],
}
for field, expected in expected_scalars.items():
if payload.get(field) != expected:
errors.append(
validation_error(
"OUTPUT_STRUCTURED_VALUE_MISMATCH",
f"review structured-result.json 的 {field} 应为 {expected!r}",
)
)
expected_outcome_counts = {
"candidate": candidate_rows,
"duplicate": removed_duplicates,
"excluded_rate_code": removed_by_rate,
"price_unmatched": review_rows,
"retained": 0,
"validation_failed": 0,
}
if payload.get("outcome_counts") != expected_outcome_counts:
errors.append(
validation_error(
"OUTPUT_STRUCTURED_OUTCOME_MISMATCH",
f"review structured outcome计数应为 {expected_outcome_counts}",
)
)
artifacts = payload.get("artifacts")
if not isinstance(artifacts, dict):
errors.append(validation_error("OUTPUT_STRUCTURED_CONTRACT_MISMATCH", "review artifacts必须是对象"))
else:
validate_artifact(artifacts.get("source_xml"), xml_path, "opera_xml", "application/xml", "source_xml", errors)
validate_artifact(
artifacts.get("result_json"), result_json, "result_json", "application/json", "result_json", errors
)
for name in ("daily_report", "exception_report", "manual_override_json"):
if artifacts.get(name) is not None:
errors.append(
validation_error(
"OUTPUT_STRUCTURED_ARTIFACT_MISMATCH",
f"review structured-result.json不得引用{name}",
)
)
actual_records = payload.get("records")
if not isinstance(actual_records, list) or len(actual_records) != len(all_records):
errors.append(
validation_error(
"OUTPUT_STRUCTURED_RECORD_COUNT_MISMATCH",
f"review structured records应保留全部 {len(all_records)} 条XML源记录",
)
)
return
actual_by_sequence = {
item.get("source_sequence"): item for item in actual_records if isinstance(item, dict)
}
for expected_record in all_records:
sequence = expected_record["_SOURCE_INDEX"]
actual = actual_by_sequence.get(sequence)
if actual != core.structured_record(expected_record):
errors.append(
validation_error(
"OUTPUT_STRUCTURED_RECORD_MISMATCH",
f"review source_sequence={sequence}的结构化字段与XML确定性推导不一致",
f"reservation[{sequence}]",
)
)
def validate_legacy_direct_success_contracts(
result_payload: Dict[str, Any],
structured_payload: Dict[str, Any],
xml_path: Path,
daily_path: Path,
result_json: Path,
structured_result_json: Path,
business_date: date,
source_rows: int,
removed_by_rate: int,
removed_duplicates: int,
expected_records: Sequence[Dict[str, Any]],
expected_channels: Sequence[Dict[str, Any]],
errors: List[core.ErrorItem],
) -> None:
"""Validate the retired v3 direct-MCP success contract through v4 replay.
The active replay logic remains the source of truth for XML, workbook and
row calculations. This adapter first rejects any field outside the frozen
v3 shape, then adds only zero-valued v4 review fields in memory so the
shared independent checks can replay the same deterministic business facts.
"""
result_fields = {
"version",
"status",
"business_date",
"message",
"metrics",
"outputs",
"errors",
}
structured_fields = {
"result_schema_version",
"status",
"activation_eligible",
"ingestion_mode",
"business_date",
"processor_version",
"rule_set_sha256",
"source_rows",
"removed_by_rate_code",
"removed_as_duplicates",
"output_rows",
"outcome_counts",
"channels",
"artifacts",
"records",
"errors",
}
result_metrics = {
"source_rows",
"removed_by_rate_code",
"removed_as_duplicates",
"output_rows",
"channels",
}
outcome_fields = {
"duplicate",
"excluded_rate_code",
"price_unmatched",
"retained",
"validation_failed",
}
artifact_fields = {
"source_xml",
"daily_report",
"result_json",
"exception_report",
}
if set(result_payload) != result_fields or set(structured_payload) != structured_fields:
errors.append(
validation_error(
"OUTPUT_LEGACY_DIRECT_CONTRACT_MISMATCH",
"旧direct_mcp结果字段集不符合冻结v3契约",
)
)
return
if (
result_payload.get("version") != core.LEGACY_DIRECT_RESULT_VERSION
or result_payload.get("status") != "success"
or not isinstance(result_payload.get("message"), str)
or result_payload.get("business_date") != business_date.isoformat()
or result_payload.get("errors") != []
):
errors.append(
validation_error(
"OUTPUT_LEGACY_DIRECT_CONTRACT_MISMATCH",
"旧direct_mcp result.json身份或成功状态无效",
)
)
return
metrics = result_payload.get("metrics")
outputs = result_payload.get("outputs")
if (
not isinstance(metrics, dict)
or set(metrics) != result_metrics
or not isinstance(outputs, dict)
or set(outputs) != {"daily_report", "structured_result", "exception_report"}
):
errors.append(
validation_error(
"OUTPUT_LEGACY_DIRECT_CONTRACT_MISMATCH",
"旧direct_mcp result指标或输出字段不符合冻结v3契约",
)
)
return
artifacts = structured_payload.get("artifacts")
outcomes = structured_payload.get("outcome_counts")
if (
not isinstance(artifacts, dict)
or set(artifacts) != artifact_fields
or not isinstance(outcomes, dict)
or set(outcomes) != outcome_fields
or structured_payload.get("result_schema_version") != core.LEGACY_DIRECT_RESULT_VERSION
or structured_payload.get("status") != "success"
or structured_payload.get("activation_eligible") is not True
or structured_payload.get("ingestion_mode") != "opera_xml"
or structured_payload.get("business_date") != business_date.isoformat()
or structured_payload.get("processor_version") != core.LEGACY_DIRECT_PROCESSOR_VERSION
or structured_payload.get("rule_set_sha256") != core.LEGACY_DIRECT_RULE_SET_SHA256
or structured_payload.get("errors") != []
):
errors.append(
validation_error(
"OUTPUT_LEGACY_DIRECT_CONTRACT_MISMATCH",
"旧direct_mcp structured-result身份或字段无效",
)
)
return
replay_result = copy.deepcopy(result_payload)
replay_result["version"] = core.RESULT_VERSION
replay_metrics = dict(metrics)
replay_metrics.update(
{
"candidate_rows": 0,
"review_required_rows": 0,
"review_issue_count": 0,
}
)
replay_result["metrics"] = replay_metrics
replay_structured = copy.deepcopy(structured_payload)
replay_structured.update(
{
"result_schema_version": core.STRUCTURED_RESULT_SCHEMA_VERSION,
"processor_version": core.PROCESSOR_VERSION,
"rule_set_sha256": core.rule_set_sha256(),
"candidate_rows": 0,
"review_required_rows": 0,
"review_issue_count": 0,
"review_issues": [],
"review_case_id": None,
"manual_override_sha256": None,
"manually_priced_rows": 0,
}
)
replay_outcomes = dict(outcomes)
replay_outcomes["candidate"] = 0
replay_structured["outcome_counts"] = replay_outcomes
replay_artifacts = dict(artifacts)
replay_artifacts["manual_override_json"] = None
replay_structured["artifacts"] = replay_artifacts
validate_result_contract(
replay_result,
business_date,
source_rows,
removed_by_rate,
removed_duplicates,
expected_records,
daily_path,
structured_result_json,
expected_channels,
errors,
)
validate_structured_result_contract(
replay_structured,
xml_path,
daily_path,
result_json,
business_date,
source_rows,
removed_by_rate,
removed_duplicates,
expected_records,
replay_result,
errors,
)
def validate_daily(
daily_path: Path,
business_date: date,
@@ -653,17 +1091,43 @@ def validate_daily(
def validate(args: argparse.Namespace) -> List[core.ErrorItem]:
xml_path = Path(args.xml)
daily_path = Path(args.daily)
result_json = Path(args.result_json)
structured_result_json = Path(args.structured_result_json)
price_path = Path(args.price_reference)
review_only = bool(getattr(args, "review_only", False))
legacy_v3_output = getattr(args, "legacy_v3_output", False)
daily_arg = getattr(args, "daily", None)
daily_path = Path(daily_arg) if daily_arg else None
manual_override_arg = getattr(args, "manual_override_json", None)
review_job_id = getattr(args, "review_job_id", None)
review_case_id = getattr(args, "review_case_id", None)
manual_override_sha256 = getattr(args, "manual_override_sha256", None)
manual_values = (manual_override_arg, review_job_id, review_case_id, manual_override_sha256)
if not isinstance(legacy_v3_output, bool):
return [
validation_error(
"OUTPUT_VALIDATOR_INPUT_INVALID", "旧direct_mcp兼容开关必须是布尔值"
)
]
if legacy_v3_output and (review_only or any(value is not None for value in manual_values)):
return [
validation_error(
"OUTPUT_VALIDATOR_INPUT_INVALID",
"旧direct_mcp兼容校验不支持待复核或人工价格清单",
)
]
required_paths = [
(xml_path, ".xml", "XML"),
(daily_path, ".xlsx", "日报"),
(result_json, ".json", "result.json"),
(structured_result_json, ".json", "structured-result.json"),
(price_path, ".xlsx", "价格对照"),
]
if not review_only:
if daily_path is None:
return [validation_error("OUTPUT_VALIDATOR_INPUT_INVALID", "成功重放必须提供日报路径")]
required_paths.append((daily_path, ".xlsx", "日报"))
elif daily_path is not None:
return [validation_error("OUTPUT_VALIDATOR_INPUT_INVALID", "待复核重放不得提供日报路径")]
for path, suffix, label in required_paths:
if not path.is_absolute() or not path.is_file() or path.suffix.lower() != suffix:
return [
@@ -673,6 +1137,16 @@ def validate(args: argparse.Namespace) -> List[core.ErrorItem]:
str(path),
)
]
if any(value is not None for value in manual_values) and not all(
isinstance(value, str) and value for value in manual_values
):
return [
validation_error(
"OUTPUT_VALIDATOR_INPUT_INVALID", "人工价格重放必须同时提供清单、任务、case和哈希"
)
]
if review_only and any(value is not None for value in manual_values):
return [validation_error("OUTPUT_VALIDATOR_INPUT_INVALID", "待复核校验不得提供人工价格清单")]
try:
payload = json.loads(result_json.read_text(encoding="utf-8"))
except Exception as exc:
@@ -700,11 +1174,111 @@ def validate(args: argparse.Namespace) -> List[core.ErrorItem]:
)
]
business_date, source_rows, removed_rate, removed_duplicates, records = expected_from_xml(
xml_path, price_path
)
expected_channels = core.channel_metrics(core.assign_channels(records))
manual_manifest: Optional[core.ManualOverrideManifest] = None
manual_override_path = Path(manual_override_arg) if manual_override_arg else None
try:
business_date, reservations = core.read_xml(xml_path)
all_records, records, removed_rate, removed_duplicates, classification_errors = (
core.classify_source_records(reservations, business_date)
)
if classification_errors:
raise core.ProcessingFailure(classification_errors)
price_map = core.load_price_map(price_path)
if manual_override_path is not None:
manual_manifest = core.load_manual_override_manifest(
manual_override_path,
job_id=str(review_job_id),
review_case_id=str(review_case_id),
expected_sha256=str(manual_override_sha256),
xml_path=xml_path,
business_date=business_date,
)
if core.missing_price_keys(records, price_map) != set(manual_manifest.prices):
raise core.ProcessingFailure(
[
validation_error(
"OUTPUT_MANUAL_OVERRIDE_KEYSET_MISMATCH",
"人工价格清单问题键集合与XML独立重放不一致",
)
]
)
pricing_errors = core.apply_prices_classified(
records,
price_map,
manual_manifest.prices if manual_manifest is not None else None,
)
except core.ProcessingFailure as exc:
return list(exc.errors)
source_rows = len(reservations)
errors: List[core.ErrorItem] = []
if review_only:
if not pricing_errors or not all(item.code == "PRICE_UNMATCHED" for item in pricing_errors):
return [
validation_error(
"OUTPUT_REVIEW_SOURCE_REPLAY_FAILED",
"待复核结果只能来自纯PRICE_UNMATCHED的XML重放",
)
]
candidate_rows = len(records) - len(pricing_errors)
validate_result_contract(
payload,
business_date,
source_rows,
removed_rate,
removed_duplicates,
[],
None,
structured_result_json,
[],
errors,
status="review_required",
candidate_rows=candidate_rows,
review_required_rows=len(pricing_errors),
review_issue_count=len(core.review_issues(records, price_map)),
expected_errors=pricing_errors,
)
validate_review_structured_result_contract(
structured_payload,
xml_path,
result_json,
business_date,
source_rows,
removed_rate,
removed_duplicates,
all_records,
records,
pricing_errors,
price_map,
errors,
)
return errors
if pricing_errors:
return [
validation_error(
"OUTPUT_STRUCTURED_SOURCE_REPLAY_FAILED",
"成功批次的XML独立重放不应出现定价错误",
)
]
expected_channels = core.channel_metrics(core.assign_channels(records))
assert daily_path is not None
if legacy_v3_output:
validate_legacy_direct_success_contracts(
payload,
structured_payload,
xml_path,
daily_path,
result_json,
structured_result_json,
business_date,
source_rows,
removed_rate,
removed_duplicates,
records,
expected_channels,
errors,
)
validate_daily(daily_path, business_date, records, errors)
return errors
validate_result_contract(
payload,
business_date,
@@ -729,6 +1303,8 @@ def validate(args: argparse.Namespace) -> List[core.ErrorItem]:
records,
payload,
errors,
manual_manifest=manual_manifest,
manual_override_path=manual_override_path,
)
validate_daily(daily_path, business_date, records, errors)
return errors
@@ -737,10 +1313,16 @@ def validate(args: argparse.Namespace) -> List[core.ErrorItem]:
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--xml", required=True)
parser.add_argument("--daily", required=True)
parser.add_argument("--daily")
parser.add_argument("--result-json", required=True)
parser.add_argument("--structured-result-json", required=True)
parser.add_argument("--price-reference", required=True)
parser.add_argument("--review-only", action="store_true")
parser.add_argument("--manual-override-json")
parser.add_argument("--review-job-id")
parser.add_argument("--review-case-id")
parser.add_argument("--manual-override-sha256")
parser.add_argument("--legacy-v3-output", action="store_true", help=argparse.SUPPRESS)
return parser