fix: stream large graph JSON imports

app/project_lifecycle.py

@@ -7,6 +7,7 @@ they happen to use the same project id.
 from __future__ import annotations
 
 import json
+import math
 import re
 from collections import defaultdict
 from collections.abc import Mapping
@@ -99,6 +100,11 @@ def _is_spatial_map_node(node: Mapping[str, Any]) -> bool:
     properties = node.get("properties")
     if not isinstance(properties, Mapping):
         return False
+    # A semantic graph can contain coordinates without verified map-region
+    # metadata. Keep those nodes in the graph without adding them to the POI
+    # layer. Old bundles without this optional flag retain their behavior.
+    if properties.get("map_poi") is False:
+        return False
     has_coordinates = (
         properties.get("lng") is not None
         and properties.get("lat") is not None
@@ -265,8 +271,8 @@ def _validate_properties(
             errors.append(f"{path}.{field_name} 应为 {expected} 类型")
 
 
-def normalize_provision_payload(body: Any) -> dict[str, Any]:
-    """Normalize and fully validate a project provision request.
+def normalize_provision_header(body: Any) -> dict[str, Any]:
+    """Validate project metadata and Schema before reading graph records.
 
     The function is intentionally pure so validation can be completed before
     PostgreSQL or FalkorDB is touched.
@@ -389,168 +395,6 @@ def normalize_provision_payload(body: Any) -> dict[str, Any]:
         )
         normalized_relations[relation_name] = meta
 
-    raw_graph_data = body.get("graph_data")
-    if raw_graph_data is None:
-        raw_graph_data = {"nodes": [], "relations": []}
-    if not isinstance(raw_graph_data, Mapping):
-        errors.append("graph_data 必须是 JSON 对象")
-        raw_graph_data = {}
-    raw_nodes = raw_graph_data.get("nodes", [])
-    raw_relations = raw_graph_data.get("relations", [])
-    if not isinstance(raw_nodes, list):
-        errors.append("graph_data.nodes 必须是数组")
-        raw_nodes = []
-    if not isinstance(raw_relations, list):
-        errors.append("graph_data.relations 必须是数组")
-        raw_relations = []
-    if not raw_nodes:
-        errors.append("graph_data.nodes 至少需要一个节点，FalkorDB 不支持持久化空图")
-
-    nodes: list[dict[str, Any]] = []
-    node_types: dict[str, str] = {}
-    for index, raw_node in enumerate(raw_nodes):
-        path = f"graph_data.nodes[{index}]"
-        if not isinstance(raw_node, Mapping):
-            errors.append(f"{path} 必须是 JSON 对象")
-            continue
-        node = dict(raw_node)
-        node_id = str(node.get("id") or "").strip()
-        entity_type = str(node.get("type") or node.get("entity_type") or "").strip()
-        properties = node.get("properties", {})
-        raw_labels = node.get("labels")
-        normalized_labels: list[str] | None = None
-        if not node_id:
-            errors.append(f"{path}.id 不能为空")
-        elif node_id in node_types:
-            errors.append(f"节点 ID {node_id!r} 重复")
-        if entity_type not in normalized_entities:
-            errors.append(f"{path}.type {entity_type!r} 未在 Schema 中定义")
-        if raw_labels is not None:
-            if not isinstance(raw_labels, list) or not raw_labels:
-                errors.append(f"{path}.labels 必须是非空数组")
-            else:
-                normalized_labels = []
-                for label_index, raw_label in enumerate(raw_labels):
-                    label = str(raw_label or "").strip()
-                    label_path = f"{path}.labels[{label_index}]"
-                    if not label or not SAFE_SCHEMA_IDENTIFIER.fullmatch(label):
-                        errors.append(f"{label_path} 不是安全的 Schema/Cypher 标识符")
-                        continue
-                    if label not in normalized_entities:
-                        errors.append(f"{label_path} {label!r} 未在 Schema 中定义")
-                        continue
-                    if label not in normalized_labels:
-                        normalized_labels.append(label)
-                if entity_type and entity_type not in normalized_labels:
-                    errors.append(f"{path}.labels 必须包含主类型 {entity_type!r}")
-        if not isinstance(properties, Mapping):
-            errors.append(f"{path}.properties 必须是 JSON 对象")
-            properties = {}
-        normalized_properties = dict(properties)
-        if entity_type in normalized_entities:
-            _validate_properties(
-                normalized_properties,
-                normalized_entities[entity_type].get("fields", {}),
-                path=path,
-                errors=errors,
-            )
-        if node_id and node_id not in node_types:
-            node_types[node_id] = entity_type
-        normalized_node = {
-            "id": node_id,
-            "type": entity_type,
-            "properties": normalized_properties,
-        }
-        if normalized_labels is not None:
-            normalized_node["labels"] = normalized_labels
-        nodes.append(normalized_node)
-
-    if map_enabled:
-        map_nodes = [node for node in nodes if _is_spatial_map_node(node)]
-        if not map_nodes:
-            errors.append("启用知识地图时，图谱数据至少需要一个带经纬度和业态的空间节点")
-        elif SAFE_ADCODE.fullmatch(map_region_adcode) and map_region_level in SPATIAL_REGION_LEVELS:
-            unverified_region = 0
-            mismatched_adcodes: set[str] = set()
-            for node in map_nodes:
-                properties = node["properties"]
-                adcode = str(properties.get("adcode") or "").strip()
-                if adcode and not _adcode_belongs_to_region(
-                    adcode,
-                    map_region_adcode,
-                    map_region_level,
-                ):
-                    mismatched_adcodes.add(adcode)
-                if adcode:
-                    continue
-                region_field = {
-                    "province": properties.get("province"),
-                    "city": properties.get("city"),
-                    "district": properties.get("district"),
-                }[map_region_level]
-                if not _region_names_match(region_field, map_region_name):
-                    unverified_region += 1
-            if unverified_region:
-                errors.append(
-                    f"知识地图空间节点有 {unverified_region} 条既无可用 adcode，"
-                    "也没有与所选区域匹配的省市区县字段"
-                )
-            if mismatched_adcodes:
-                examples = "、".join(sorted(mismatched_adcodes)[:5])
-                errors.append(
-                    f"知识地图数据与所选区域 {map_region_name} 不一致，发现行政区划代码：{examples}"
-                )
-
-    relations: list[dict[str, Any]] = []
-    for index, raw_relation in enumerate(raw_relations):
-        path = f"graph_data.relations[{index}]"
-        if not isinstance(raw_relation, Mapping):
-            errors.append(f"{path} 必须是 JSON 对象")
-            continue
-        relation = dict(raw_relation)
-        relation_type = str(relation.get("type") or relation.get("relation_type") or "").strip()
-        source = str(relation.get("source") or "").strip()
-        target = str(relation.get("target") or "").strip()
-        properties = relation.get("properties", {})
-        definition = normalized_relations.get(relation_type)
-        if not definition:
-            errors.append(f"{path}.type {relation_type!r} 未在 Schema 中定义")
-        if source not in node_types:
-            errors.append(f"{path}.source {source!r} 不存在")
-        if target not in node_types:
-            errors.append(f"{path}.target {target!r} 不存在")
-        if (
-            definition
-            and source in node_types
-            and node_types[source] not in _endpoint_types(definition.get("from"))
-        ):
-            errors.append(f"{path}.source 的节点类型必须是 {definition.get('from')}")
-        if (
-            definition
-            and target in node_types
-            and node_types[target] not in _endpoint_types(definition.get("to"))
-        ):
-            errors.append(f"{path}.target 的节点类型必须是 {definition.get('to')}")
-        if not isinstance(properties, Mapping):
-            errors.append(f"{path}.properties 必须是 JSON 对象")
-            properties = {}
-        normalized_properties = dict(properties)
-        if definition:
-            _validate_properties(
-                normalized_properties,
-                definition.get("properties", {}),
-                path=path,
-                errors=errors,
-            )
-        relations.append(
-            {
-                "type": relation_type,
-                "source": source,
-                "target": target,
-                "properties": normalized_properties,
-            }
-        )
-
     if errors:
         raise ProjectValidationError(errors)
 
@@ -572,14 +416,205 @@ def normalize_provision_payload(body: Any) -> dict[str, Any]:
             "region_level": map_region_level or "province",
         },
         "schema": schema,
-        "graph_data": {"nodes": nodes, "relations": relations},
+        "graph_data": {"nodes": [], "relations": []},
         "counts": {
             "entity_types": len(normalized_entities),
             "relation_types": len(normalized_relations),
-            "nodes": len(nodes),
-            "relations": len(relations),
+            "nodes": 0,
+            "relations": 0,
         },
     }
+def normalize_graph_node(raw_node: Any, index: int, normalized_entities: Mapping,
+                         node_types: Mapping, errors: list[str]) -> dict | None:
+    path = f"graph_data.nodes[{index}]"
+    if not isinstance(raw_node, Mapping):
+        errors.append(f"{path} 必须是 JSON 对象")
+        return None
+    node = dict(raw_node)
+    node_id = str(node.get("id") or "").strip()
+    entity_type = str(node.get("type") or node.get("entity_type") or "").strip()
+    properties = node.get("properties", {})
+    raw_labels = node.get("labels")
+    normalized_labels: list[str] | None = None
+    if not node_id:
+        errors.append(f"{path}.id 不能为空")
+    elif node_id in node_types:
+        errors.append(f"节点 ID {node_id!r} 重复")
+    if entity_type not in normalized_entities:
+        errors.append(f"{path}.type {entity_type!r} 未在 Schema 中定义")
+    if raw_labels is not None:
+        if not isinstance(raw_labels, list) or not raw_labels:
+            errors.append(f"{path}.labels 必须是非空数组")
+        else:
+            normalized_labels = []
+            for label_index, raw_label in enumerate(raw_labels):
+                label = str(raw_label or "").strip()
+                label_path = f"{path}.labels[{label_index}]"
+                if not label or not SAFE_SCHEMA_IDENTIFIER.fullmatch(label):
+                    errors.append(f"{label_path} 不是安全的 Schema/Cypher 标识符")
+                    continue
+                if label not in normalized_entities:
+                    errors.append(f"{label_path} {label!r} 未在 Schema 中定义")
+                    continue
+                if label not in normalized_labels:
+                    normalized_labels.append(label)
+            if entity_type and entity_type not in normalized_labels:
+                errors.append(f"{path}.labels 必须包含主类型 {entity_type!r}")
+    if not isinstance(properties, Mapping):
+        errors.append(f"{path}.properties 必须是 JSON 对象")
+        properties = {}
+    normalized_properties = dict(properties)
+    if (
+        normalized_properties.get("map_poi") is not None
+        and not isinstance(normalized_properties["map_poi"], bool)
+    ):
+        errors.append(f"{path}.properties.map_poi 必须是布尔值")
+    if entity_type in normalized_entities:
+        _validate_properties(
+            normalized_properties,
+            normalized_entities[entity_type].get("fields", {}),
+            path=path,
+            errors=errors,
+        )
+    normalized_node = {
+        "id": node_id,
+        "type": entity_type,
+        "properties": normalized_properties,
+    }
+    if normalized_labels is not None:
+        normalized_node["labels"] = normalized_labels
+    return normalized_node
+
+
+def normalize_graph_relation(raw_relation: Any, index: int, normalized_relations: Mapping,
+                             node_types: Mapping, errors: list[str]) -> dict | None:
+    path = f"graph_data.relations[{index}]"
+    if not isinstance(raw_relation, Mapping):
+        errors.append(f"{path} 必须是 JSON 对象")
+        return None
+    relation = dict(raw_relation)
+    relation_type = str(relation.get("type") or relation.get("relation_type") or "").strip()
+    source = str(relation.get("source") or "").strip()
+    target = str(relation.get("target") or "").strip()
+    properties = relation.get("properties", {})
+    definition = normalized_relations.get(relation_type)
+    if not definition:
+        errors.append(f"{path}.type {relation_type!r} 未在 Schema 中定义")
+    if source not in node_types:
+        errors.append(f"{path}.source {source!r} 不存在")
+    if target not in node_types:
+        errors.append(f"{path}.target {target!r} 不存在")
+    if (
+        definition
+        and source in node_types
+        and node_types[source] not in _endpoint_types(definition.get("from"))
+    ):
+        errors.append(f"{path}.source 的节点类型必须是 {definition.get('from')}")
+    if (
+        definition
+        and target in node_types
+        and node_types[target] not in _endpoint_types(definition.get("to"))
+    ):
+        errors.append(f"{path}.target 的节点类型必须是 {definition.get('to')}")
+    if not isinstance(properties, Mapping):
+        errors.append(f"{path}.properties 必须是 JSON 对象")
+        properties = {}
+    normalized_properties = dict(properties)
+    if definition:
+        _validate_properties(
+            normalized_properties,
+            definition.get("properties", {}),
+            path=path,
+            errors=errors,
+        )
+    return {
+        "type": relation_type,
+        "source": source,
+        "target": target,
+        "properties": normalized_properties,
+    }
+
+
+class SpatialMapValidation:
+    """Incremental region validation shared by inline and streamed imports."""
+
+    def __init__(self, spatial_map: Mapping):
+        self.config = spatial_map
+        self.count = 0
+        self.unverified = 0
+        self.mismatched: set[str] = set()
+
+    def add(self, node: Mapping, errors: list[str]) -> None:
+        if not _is_spatial_map_node(node):
+            return
+        properties = node["properties"]
+        try:
+            lng = float(properties.get("lng") if properties.get("lng") is not None else properties.get("longitude"))
+            lat = float(properties.get("lat") if properties.get("lat") is not None else properties.get("latitude"))
+            valid = math.isfinite(lng) and math.isfinite(lat) and -180 <= lng <= 180 and -90 <= lat <= 90
+        except (TypeError, ValueError):
+            valid = False
+        if not valid:
+            if self.config["enabled"]:
+                errors.append(f"节点 {node['id']!r} 的地图经纬度无效")
+            return
+        self.count += 1
+        if not self.config["enabled"]:
+            return
+        code = str(properties.get("adcode") or "").strip()
+        level = self.config["region_level"]
+        if code:
+            if not _adcode_belongs_to_region(code, self.config["region_adcode"], level):
+                if len(self.mismatched) < 5:
+                    self.mismatched.add(code)
+        elif not _region_names_match(properties.get({"province": "province", "city": "city", "district": "district"}[level]), self.config["region_name"]):
+            self.unverified += 1
+
+    def finish(self, errors: list[str]) -> None:
+        if not self.config["enabled"]:
+            return
+        if not self.count:
+            errors.append("启用知识地图时，图谱数据至少需要一个带经纬度和业态的空间节点")
+        if self.unverified:
+            errors.append(f"知识地图空间节点有 {self.unverified} 条既无可用 adcode，也没有与所选区域匹配的省市区县字段")
+        if self.mismatched:
+            errors.append(f"知识地图数据与所选区域 {self.config['region_name']} 不一致，发现行政区划代码：{'、'.join(sorted(self.mismatched))}")
+
+
+def normalize_provision_payload(body: Any) -> dict[str, Any]:
+    """Validate small inline requests using the same rules as file imports."""
+    payload = normalize_provision_header(body)
+    errors: list[str] = []
+    data = body.get("graph_data") or {}
+    if not isinstance(data, Mapping):
+        raise ProjectValidationError(["graph_data 必须是 JSON 对象"])
+    raw_nodes, raw_relations = data.get("nodes", []), data.get("relations", [])
+    if not isinstance(raw_nodes, list):
+        errors.append("graph_data.nodes 必须是数组")
+        raw_nodes = []
+    if not isinstance(raw_relations, list):
+        errors.append("graph_data.relations 必须是数组")
+        raw_relations = []
+    if not raw_nodes:
+        errors.append("graph_data.nodes 至少需要一个节点，FalkorDB 不支持持久化空图")
+    nodes, relations, node_types = [], [], {}
+    spatial = SpatialMapValidation(payload["spatial_map"])
+    for index, raw in enumerate(raw_nodes):
+        node = normalize_graph_node(raw, index, payload["schema"]["entity_types"], node_types, errors)
+        if node is not None:
+            nodes.append(node)
+            node_types.setdefault(node["id"], node["type"])
+            spatial.add(node, errors)
+    spatial.finish(errors)
+    for index, raw in enumerate(raw_relations):
+        relation = normalize_graph_relation(raw, index, payload["schema"]["relation_types"], node_types, errors)
+        if relation is not None:
+            relations.append(relation)
+    if errors:
+        raise ProjectValidationError(errors)
+    payload["graph_data"] = {"nodes": nodes, "relations": relations}
+    payload["counts"].update(nodes=len(nodes), relations=len(relations))
+    return payload
 
 
 def _falkor_client() -> FalkorDB: