fix: recognize corroborated empty Oracle associations

arr_web/arr_data_review.py

@@ -28,6 +28,8 @@ from integrations.ohip.collect_arr_source import json_bytes, require, strict_jso
 from integrations.ohip.collect_arr_source import CollectionError
 
 VERSION = "arr-source-field-review/v1"
+REANALYSIS_POLICY = "oracle-optional-association/v1"
+REANALYSIS_VERSION = "arr-source-reanalysis/v1"
 LIMIT = 100 * 1024 * 1024
 OPTIONAL = frozenset({"BLOCK_CODE", "RES_COMMENT", "PRODUCTS", "ROOM_CATEGORY_LABEL"})
 LABELS = {
@@ -128,6 +130,12 @@ class DataFieldReviews:
         state = strict_json(state_raw)
         require(state["identity_sha256"] == fingerprint(meta) and type(state["revision"]) is int
                 and state["status"] in {"editing", "finalized"}, "data_review_state_context_changed")
+        if "source_reanalysis" in state:
+            reference = state["source_reanalysis"]
+            require(type(reference) is dict and reference.get("parent_original_sha256") == meta["original_sha256"]
+                    and reference.get("source_manifest_sha256") == meta["source_manifest_sha256"],
+                    "data_review_reanalysis_context_changed")
+            self._reanalyzed_source(directory, original, state)
         return meta, original, state
 
     def _publish(self, directory, state):
@@ -148,8 +156,74 @@ class DataFieldReviews:
             value = [value]
         return {"state": "available", "value": value}
 
-    def _derive(self, original, state):
-        data = copy.deepcopy(original)
+    @staticmethod
+    def _reanalysis_projection(original, reanalyzed, policy_id):
+        require(policy_id == REANALYSIS_POLICY and type(reanalyzed) is dict,
+                "data_review_reanalysis_policy_invalid")
+        expected = copy.deepcopy(original)
+        require(type(reanalyzed.get("records")) is list
+                and len(reanalyzed["records"]) == len(original["records"]),
+                "data_review_reanalysis_records_changed")
+        changes = []
+        allowed = {"BLOCK_CODE": ("missing_reservation_block", ""),
+                   "PRODUCTS": ("missing_reservation_packages", [])}
+        for index, (old, new) in enumerate(zip(original["records"], reanalyzed["records"])):
+            require(type(new) is dict and type(new.get("fields")) is dict,
+                    "data_review_reanalysis_record_invalid")
+            for field, (reason, empty_value) in allowed.items():
+                before, after = old["fields"][field], new["fields"].get(field)
+                if fingerprint(before) == fingerprint(after):
+                    continue
+                empty = {"state": "empty", "value": empty_value}
+                require(before.get("state") == "missing" and before.get("reason") == reason
+                        and fingerprint(after) == fingerprint(empty),
+                        "data_review_reanalysis_observation_invalid")
+                expected["records"][index]["fields"][field] = copy.deepcopy(empty)
+                changes.append({"item_id": f"{old['source_sequence']}:{field}",
+                    "source_sequence": old["source_sequence"], "field": field,
+                    "original_observation": copy.deepcopy(before), "reanalyzed_observation": copy.deepcopy(empty)})
+        require(bool(changes), "data_review_reanalysis_no_changes")
+        unresolved = any(observation.get("state") != "available"
+                         and not (field in OPTIONAL and observation.get("state") == "empty")
+                         for row in expected["records"] for field, observation in row["fields"].items())
+        expected["input_complete"] = not unresolved
+        expected["status"] = "collected_with_gaps" if unresolved else "collected"
+        expected["optional_omission_policy"] = REANALYSIS_POLICY
+        require(fingerprint(expected) == fingerprint(reanalyzed), "data_review_reanalysis_source_changed")
+        return changes
+
+    def _reanalyzed_source(self, directory, original, state):
+        reference = state.get("source_reanalysis")
+        if reference is None:
+            return original, None
+        require(type(reference) is dict and reference.get("policy_id") == REANALYSIS_POLICY,
+                "data_review_reanalysis_reference_invalid")
+        digest, receipt_digest = reference.get("sha256", ""), reference.get("receipt_sha256", "")
+        manifest_digest = reference.get("manifest_sha256", "")
+        require(type(digest) is str and bool(re.fullmatch(r"[0-9a-f]{64}", digest))
+                and type(receipt_digest) is str and bool(re.fullmatch(r"[0-9a-f]{64}", receipt_digest))
+                and type(manifest_digest) is str and bool(re.fullmatch(r"[0-9a-f]{64}", manifest_digest))
+                and reference.get("file") == f"reanalyzed-source-{digest}.json"
+                and reference.get("receipt_file") == f"source-reanalysis-{receipt_digest}.json",
+                "data_review_reanalysis_reference_invalid")
+        payload = protected_read(directory / reference["file"], LIMIT)
+        receipt_raw = protected_read(directory / reference["receipt_file"], LIMIT)
+        require(_hash(payload) == digest and _hash(receipt_raw) == receipt_digest,
+                "data_review_reanalysis_changed")
+        reanalyzed, receipt = strict_json(payload), strict_json(receipt_raw)
+        changes = self._reanalysis_projection(original, reanalyzed, REANALYSIS_POLICY)
+        expected_receipt = {"version": REANALYSIS_VERSION, "policy_id": REANALYSIS_POLICY,
+            "request_id": directory.name, "report_date": original["report_date"], "context": self.context,
+            "parent_original_sha256": reference["parent_original_sha256"],
+            "source_manifest_sha256": reference["source_manifest_sha256"],
+            "reanalyzed_sha256": digest, "reanalysis_manifest_sha256": reference["manifest_sha256"],
+            "changes": changes}
+        require(fingerprint(receipt) == fingerprint(expected_receipt), "data_review_reanalysis_receipt_changed")
+        return reanalyzed, receipt
+
+    def _derive(self, directory, original, state):
+        source, _receipt = self._reanalyzed_source(directory, original, state)
+        data = copy.deepcopy(source)
         for item_id, decision in state["decisions"].items():
             sequence, field = item_id.split(":")
             row = data["records"][int(sequence) - 1]
@@ -197,7 +271,7 @@ class DataFieldReviews:
         return value if observation.get("state") == "available" and isinstance(value, (str, int)) else ""
 
     def _public(self, directory, meta, original, state):
-        data = self._derive(original, state)
+        data = self._derive(directory, original, state)
         issues = self._issues(directory, data)
         keys = set(issues) | set(state["decisions"])
         items = []
@@ -209,6 +283,7 @@ class DataFieldReviews:
                 "field_label": LABELS[field], "confirmation_no": self._display(row, "CONFIRMATION_NO"),
                 "room_no": self._display(row, "DISP_ROOM_NO"), "company_name": self._display(row, "COMPANY_NAME"),
                 "rate_code": self._display(row, "RATE_CODE"), "reason_code": issues.get(item_id, "confirmed"),
+                "source_state": row["fields"][field].get("state"),
                 "can_be_empty": field in OPTIONAL, "value": decision["value"] if decision else None,
                 "confirmed": decision is not None and item_id not in issues})
         return {"request_id": meta["request_id"], "report_date": meta["report_date"],
@@ -245,13 +320,66 @@ class DataFieldReviews:
             require(fingerprint(actual) == fingerprint(meta), "data_review_prepare_conflict")
             review = self._public(directory, actual, original, state)
             # A complete ordinary source needs no human step or derived source.
-            return review if review["total_count"] else None
+            # A reanalysis remains an explicit review until finalized, even if
+            # its validated source interpretation resolves every listed gap.
+            return review if review["total_count"] or "source_reanalysis" in state else None
 
     def get(self, request_id):
         directory = self._directory(request_id)
         with self._lock(directory):
             return self._public(directory, *self._read(directory))
 
+    def apply_source_reanalysis(self, request_id, payload, manifest_sha256, policy_id, *, expected_revision):
+        """Attach a validated offline reinterpretation, without changing acquisition or staff decisions.
+
+        This internal maintenance operation has no HTTP route. Its caller must
+        establish the policy's omission evidence from the saved complete capture.
+        The projection here prevents any other source or business-value edits.
+        """
+        require(policy_id == REANALYSIS_POLICY, "data_review_reanalysis_policy_invalid")
+        require(type(payload) is bytes and len(payload) <= LIMIT,
+                "data_review_reanalysis_payload_invalid")
+        require(type(manifest_sha256) is str and bool(re.fullmatch(r"[0-9a-f]{64}", manifest_sha256)),
+                "data_review_reanalysis_manifest_invalid")
+        reanalyzed = strict_json(payload)
+        directory = self._directory(request_id)
+        with self._lock(directory):
+            meta, original, state = self._read(directory)
+            if state["status"] != "editing" or (directory / "finalize-intent.json").exists():
+                raise _error("FROZEN", "已确认生成，来源解释不能再修改", 409)
+            if type(expected_revision) is not int:
+                self._revision(state, expected_revision)
+            digest = _hash(payload)
+            prior = state.get("source_reanalysis")
+            if prior is not None:
+                if prior["sha256"] == digest and prior["manifest_sha256"] == manifest_sha256:
+                    return self._public(directory, meta, original, state)
+                raise _error("CONFLICT", "该来源已完成重新解释，请核对现有记录", 409)
+            self._revision(state, expected_revision)
+            changes = self._reanalysis_projection(original, reanalyzed, policy_id)
+            receipt = {"version": REANALYSIS_VERSION, "policy_id": policy_id,
+                "request_id": request_id, "report_date": meta["report_date"], "context": self.context,
+                "parent_original_sha256": meta["original_sha256"],
+                "source_manifest_sha256": meta["source_manifest_sha256"], "reanalyzed_sha256": digest,
+                "reanalysis_manifest_sha256": manifest_sha256, "changes": changes}
+            receipt_raw = json_bytes(receipt)
+            receipt_digest = _hash(receipt_raw)
+            reference = {"policy_id": policy_id, "file": f"reanalyzed-source-{digest}.json", "sha256": digest,
+                "receipt_file": f"source-reanalysis-{receipt_digest}.json", "receipt_sha256": receipt_digest,
+                "parent_original_sha256": meta["original_sha256"],
+                "source_manifest_sha256": meta["source_manifest_sha256"], "manifest_sha256": manifest_sha256}
+            _write_once(directory / reference["file"], payload)
+            _write_once(directory / reference["receipt_file"], receipt_raw)
+            state["source_reanalysis"] = reference
+            state["revision"] += 1
+            state["events"].append({"revision": state["revision"], "action": "source_reanalysis",
+                "policy_id": policy_id, "parent_original_sha256": meta["original_sha256"],
+                "reanalyzed_sha256": digest, "reanalysis_manifest_sha256": manifest_sha256,
+                "receipt_sha256": receipt_digest, "at": datetime.now(timezone.utc).isoformat()})
+            result = self._public(directory, meta, original, state)
+            self._publish(directory, state)
+            return result
+
     @staticmethod
     def _revision(state, revision):
         if type(revision) is not int or revision != state["revision"]:
@@ -312,7 +440,7 @@ class DataFieldReviews:
                 raise _error("FROZEN", "已确认生成，字段不能再修改", 409)
             self._revision(state, revision)
             actor = self._actor(actor)
-            data = self._derive(original, state)
+            data = self._derive(directory, original, state)
             if item_id not in self._issues(directory, data) and item_id not in state["decisions"]:
                 raise _error(message="只能完善当前任务列出的异常字段")
             sequence, field = item_id.split(":")
@@ -328,7 +456,7 @@ class DataFieldReviews:
             self._publish(directory, state)
             return result
 
-    def _frozen(self, meta, original, state):
+    def _frozen(self, directory, meta, original, state):
         changes = []
         for item_id, decision in sorted(state["decisions"].items()):
             sequence, field = item_id.split(":")
@@ -339,7 +467,10 @@ class DataFieldReviews:
                  "context": self.context, "original_sha256": meta["original_sha256"],
                  "source_manifest_sha256": meta["source_manifest_sha256"], "revision": state["revision"],
                  "changes": changes, "events": state["events"]}
-        data = self._derive(original, state)
+        _source, receipt = self._reanalyzed_source(directory, original, state)
+        if receipt is not None:
+            audit["source_reanalysis"] = {**receipt, "receipt_sha256": state["source_reanalysis"]["receipt_sha256"]}
+        data = self._derive(directory, original, state)
         data["manual_data_review"] = {"manifest": audit, "manifest_sha256": fingerprint(audit)}
         binding = fingerprint({"source_manifest_sha256": meta["source_manifest_sha256"],
                                "review_manifest_sha256": fingerprint(audit), "data_sha256": _hash(json_bytes(data))})
@@ -354,7 +485,7 @@ class DataFieldReviews:
                 return self._public(directory, meta, original, state)
             self._revision(state, revision)
             actor = self._actor(actor)
-            if self._issues(directory, self._derive(original, state)):
+            if self._issues(directory, self._derive(directory, original, state)):
                 raise _error("INCOMPLETE", "请先完善并确认所有异常字段", 409)
             intent = directory / "finalize-intent.json"
             prior_sha256 = fingerprint(state)
@@ -368,14 +499,14 @@ class DataFieldReviews:
                 state["events"].append({"revision": state["revision"], "action": "finalize", "actor": actor,
                                         "at": datetime.now(timezone.utc).isoformat()})
                 atomic_json(intent, {"prior_sha256": prior_sha256, "state": state}, replace=False)
-            payload, binding = self._frozen(meta, original, state)
+            payload, binding = self._frozen(directory, meta, original, state)
             _write_once(directory / "reviewed-source.json", payload)
             state["derived_sha256"], state["binding_manifest_sha256"] = _hash(payload), binding
             self._publish(directory, state)
             return self._public(directory, meta, original, state)
 
     def _verify_frozen(self, directory, meta, original, state):
-        payload, binding = self._frozen(meta, original, state)
+        payload, binding = self._frozen(directory, meta, original, state)
         require(_hash(payload) == state["derived_sha256"] and binding == state["binding_manifest_sha256"]
                 and protected_read(directory / "reviewed-source.json", LIMIT) == payload,
                 "data_review_frozen_source_changed")

arr_web/static/app.js

@@ -633,6 +633,23 @@
     return Array.isArray(item.value) ? item.value.join("\n") : String(item.value ?? "");
   }
 
+  function arrDataReviewReasonKey(item) {
+    if (item.confirmed) return "confirmed_value";
+    const sourceState = String(item.source_state || "").toLowerCase();
+    const reason = String(item.reason_code || "").toUpperCase();
+    // Older review snapshots do not expose source_state. Recognize only the
+    // collector's explicit failure codes; a missing optional field is not a
+    // failed request and does not make the field required.
+    const failedReasons = ["HTTP_FAILURE", "HTTP_PERMISSION_DENIED", "TRANSPORT_RETRY_EXHAUSTED",
+      "REQUEST_LIMIT_EXCEEDED", "RESPONSE_TOO_LARGE", "OPERATION_MISMATCH", "HOTEL_MISMATCH",
+      "INVALID_DATA_ENVELOPE", "MISSING_ORACLE_REQUEST_ID", "SECRET_IN_RESPONSE", "INVALID_JSON",
+      "UPSTREAM_WARNING_OR_ERROR"];
+    if (sourceState === "failed" || failedReasons.includes(reason)) return "fetch_failed";
+    if (sourceState === "ambiguous" || /(?:^|_)(?:CONFLICT|AMBIGUOUS|MISMATCH|DISAGREEMENT|CHANGED)(?:_|$)/.test(reason)) return "conflict_value";
+    if (/(?:^|_)(?:INVALID|MALFORMED)(?:_|$)/.test(reason) || ["XML_NEGATIVE_NIGHTS", "DATA_NEGATIVE_NIGHTS"].includes(reason)) return "invalid_value";
+    return item.can_be_empty ? "source_unconfirmed" : "required_value_missing";
+  }
+
   function arrDataReviewCanFinalize(review = state.arrDataReview) {
     return Boolean(arrDownloadNeedsDataReview() && state.arrDownloadTask.request_id === state.arrDataReviewRequestId
       && !state.arrDownloadBusy && review && review.status === "editing" && review.can_finalize && Number(review.pending_count) === 0
@@ -675,8 +692,8 @@
       const labelKey = `data_review.field_${field.toLowerCase()}`;
       const translatedLabel = I18N.t(labelKey);
       const label = translatedLabel === labelKey ? (item.field_label || field) : translatedLabel;
-      const reason = String(item.reason_code || "").toUpperCase();
-      const reasonKey = item.confirmed ? "confirmed_value" : /CONFLICT|AMBIGUOUS/.test(reason) ? "conflict_value" : /INVALID|MALFORMED/.test(reason) ? "invalid_value" : "missing_value";
+      const reasonKey = arrDataReviewReasonKey(item);
+      const ruleKey = item.can_be_empty ? "optional_field_rule" : "required_field_rule";
       const inputId = `arr-data-review-value-${index}`;
       const value = Object.prototype.hasOwnProperty.call(state.arrDataReviewDrafts, item.item_id)
         ? state.arrDataReviewDrafts[item.item_id] : arrDataReviewInputValue(item);
@@ -687,7 +704,7 @@
         : `<input id="${inputId}" data-arr-data-review-value type="text" inputmode="${inputMode}" autocomplete="off" value="${escapeHtml(value)}" placeholder="${escapeHtml(placeholder)}" ${editable ? "" : "disabled"} />`;
       return `<tr data-arr-data-review-item-id="${escapeHtml(item.item_id)}" class="${item.confirmed ? "is-complete" : ""}">
         <td data-label="${escapeHtml(I18N.t("data_review.reservation"))}"><strong>${escapeHtml(item.confirmation_no || "—")}</strong><small>${escapeHtml(I18N.t("data_review.source_row", { sequence: item.source_sequence }))} · ${escapeHtml(I18N.t("data_review.room", { room: item.room_no || "—" }))}</small><small>${escapeHtml(item.company_name || "—")} · ${escapeHtml(item.rate_code || "—")}</small></td>
-        <td data-label="${escapeHtml(I18N.t("data_review.field"))}"><label for="${inputId}">${escapeHtml(label)}</label><small>${escapeHtml(I18N.t(`data_review.${reasonKey}`))}</small></td>
+        <td data-label="${escapeHtml(I18N.t("data_review.field"))}"><label for="${inputId}">${escapeHtml(label)}</label><small>${escapeHtml(I18N.t(`data_review.${ruleKey}`))}</small><small>${escapeHtml(I18N.t(`data_review.${reasonKey}`))}</small></td>
         <td data-label="${escapeHtml(I18N.t("data_review.value"))}">${input}</td>
         <td class="arr-data-review-row-actions" data-label="${escapeHtml(I18N.t("common.operation"))}"><span class="arr-data-review-item-status">${escapeHtml(I18N.t(item.confirmed ? "data_review.confirmed" : "data_review.pending"))}</span><button type="button" data-arr-data-review-save ${editable ? "" : "disabled"}>${escapeHtml(I18N.t("data_review.save"))}</button>${item.can_be_empty ? `<button type="button" data-arr-data-review-empty ${editable ? "" : "disabled"}>${escapeHtml(I18N.t("data_review.confirm_empty"))}</button>` : ""}</td>
       </tr>`;

arr_web/static/i18n.js

@@ -146,16 +146,20 @@
     "arr_download.retry_unavailable": ["该任务不能继续重试，请查看任务日志", "This task cannot be retried. Check the task log.", "ไม่สามารถลองงานนี้อีกครั้ง โปรดดูบันทึกงาน"],
     "arr_download.other_active": ["该日期已有进行中的下载任务", "A download for this date is already in progress", "มีงานดาวน์โหลดสำหรับวันที่นี้กำลังดำเนินการอยู่"],
     "data_review.title": ["人工完善报表字段", "Complete report fields", "ตรวจสอบและกรอกข้อมูลรายงาน"],
-    "data_review.copy": ["请核实并逐项保存缺失或有冲突的字段。允许为空的字段也须明确确认，全部完成后才能生成日报。", "Verify and save each missing or conflicting field. Explicitly confirm any allowed empty values, then generate the daily report when all fields are complete.", "ตรวจสอบและบันทึกข้อมูลที่ขาดหายหรือขัดแย้งทีละรายการ ต้องยืนยันค่าที่เว้นว่างได้อย่างชัดเจน ก่อนสร้างรายงานเมื่อครบทุกช่อง"],
+    "data_review.copy": ["请按各项提示核实来源与原处理规则。可空字段不必补成有值；核实无此信息后，可明确确认留空。全部确认后可生成日报。", "Check each field's source and original processing requirement. Optional fields need no value when none exists; explicitly confirm this after verification. Generate the daily report after all items are confirmed.", "ตรวจสอบแหล่งข้อมูลและข้อกำหนดเดิมของแต่ละช่อง ช่องที่เว้นว่างได้ไม่จำเป็นต้องกรอกค่า หากตรวจสอบแล้วว่าไม่มีข้อมูล ให้ยืนยันการเว้นว่าง สร้างรายงานรายวันได้เมื่อยืนยันครบทุกช่อง"],
     "data_review.progress": ["{date} · 已确认 {completed} / {total} 项", "{date} · {completed} of {total} confirmed", "{date} · ยืนยันแล้ว {completed} / {total} รายการ"],
     "data_review.reservation": ["预订信息", "Reservation", "ข้อมูลการจอง"],
     "data_review.field": ["待完善字段", "Field to review", "ข้อมูลที่ต้องตรวจสอบ"],
     "data_review.value": ["核实后的值", "Verified value", "ค่าที่ตรวจสอบแล้ว"],
     "data_review.source_row": ["原始记录 {sequence}", "Source row {sequence}", "รายการต้นฉบับ {sequence}"],
     "data_review.room": ["房号 {room}", "Room {room}", "ห้อง {room}"],
-    "data_review.missing_value": ["来源字段缺失，请核实", "Source value missing. Verify it.", "ไม่มีข้อมูลต้นฉบับ โปรดตรวจสอบ"],
-    "data_review.conflict_value": ["来源值有冲突，请核实", "Source values conflict. Verify them.", "ข้อมูลต้นฉบับขัดแย้งกัน โปรดตรวจสอบ"],
-    "data_review.invalid_value": ["来源值格式无效，请核实", "Source value is invalid. Verify it.", "รูปแบบข้อมูลต้นฉบับไม่ถูกต้อง โปรดตรวจสอบ"],
+    "data_review.required_field_rule": ["原处理规则必填，不可为空。", "Required by the original processing rules; cannot be blank.", "กฎการประมวลผลเดิมกำหนดให้กรอก ห้ามเว้นว่าง"],
+    "data_review.optional_field_rule": ["原处理规则允许为空。", "The original processing rules allow this field to be blank.", "กฎการประมวลผลเดิมอนุญาตให้เว้นช่องนี้ว่างได้"],
+    "data_review.required_value_missing": ["本次未取得该值，请填写核实后的值。", "This value was not obtained. Enter the verified value.", "ครั้งนี้ไม่ได้รับค่านี้ โปรดกรอกค่าที่ตรวจสอบแล้ว"],
+    "data_review.source_unconfirmed": ["来源尚未确认。请核实有无此信息；没有则点击“确认无此信息”。", "The source is unconfirmed. Check whether this information exists; if it does not, choose “Confirm no value”.", "ยังยืนยันข้อมูลจากแหล่งต้นทางไม่ได้ โปรดตรวจสอบว่ามีข้อมูลนี้หรือไม่ หากไม่มี ให้เลือก “ยืนยันว่าไม่มีข้อมูล”"],
+    "data_review.fetch_failed": ["接口读取未完成，请先核实来源，不能直接认定没有此信息。", "The interface read did not complete. Verify the source before concluding that no information exists.", "การอ่านข้อมูลผ่านอินเทอร์เฟซยังไม่เสร็จ โปรดตรวจสอบแหล่งข้อมูลก่อนสรุปว่าไม่มีข้อมูลนี้"],
+    "data_review.conflict_value": ["来源值有冲突，请核实后填写正确的值。", "Source values conflict. Verify and enter the correct value.", "ข้อมูลต้นฉบับขัดแย้งกัน โปรดตรวจสอบและกรอกค่าที่ถูกต้อง"],
+    "data_review.invalid_value": ["来源值格式无效，请核实后填写有效的值。", "The source value is invalid. Verify and enter a valid value.", "รูปแบบข้อมูลต้นฉบับไม่ถูกต้อง โปรดตรวจสอบและกรอกค่าที่ใช้ได้"],
     "data_review.confirmed_value": ["已使用人工核实值", "Manually verified value saved", "บันทึกค่าที่ตรวจสอบด้วยตนเองแล้ว"],
     "data_review.confirmed": ["已确认", "Confirmed", "ยืนยันแล้ว"],
     "data_review.pending": ["待确认", "Awaiting confirmation", "รอยืนยัน"],