fix: stream large graph JSON imports

app/api/__init__.py

@@ -20,8 +20,10 @@ api_router.include_router(auth_router, tags=["auth"])
 
 # Projects & Graph Releases
 from app.api.projects import router as projects_router  # noqa: E402
+from app.api.graph_file_import import router as graph_file_import_router  # noqa: E402
 
 api_router.include_router(projects_router, tags=["projects"])
+api_router.include_router(graph_file_import_router, tags=["graph-imports"])
 
 # Plaza
 api_router.include_router(plaza_router, tags=["plaza"])

app/api/graph_file_import.py

@@ -0,0 +1,184 @@
+"""Request-scoped file import: spool, validate, batch-write, report progress.
+
+No daemon, persisted job queue or startup changes. The normal project
+transaction/compensation boundary publishes only a completely imported graph.
+"""
+from __future__ import annotations
+
+import asyncio
+from functools import partial
+import json
+import logging
+from pathlib import Path
+import shutil
+import tempfile
+
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import StreamingResponse
+from starlette.datastructures import FormData, UploadFile
+from starlette.formparsers import MultiPartException, MultiPartParser
+
+from app.auth import AdminUser
+from app.config import settings
+from app.graph_import_files import import_staged_graph, validate_graph_file
+from app.project_lifecycle import ProjectValidationError, normalize_provision_header
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+SCHEMA_BYTES = 2 * 1024 * 1024
+MIN_FREE_BYTES = 512 * 1024 * 1024
+# Bound concurrent upload/index allocations within each existing API process.
+_import_lock = asyncio.Lock()
+
+
+def event(kind: str, data: dict) -> str:
+    return json.dumps({"event": kind, "data": data}, ensure_ascii=False, default=str) + "\n"
+
+
+async def read_import_form(request: Request) -> FormData:
+    limit = settings.graph_import_max_bytes + SCHEMA_BYTES + 65536
+    length = request.headers.get("content-length", "")
+    if length.isdigit():
+        if int(length) > limit:
+            raise HTTPException(413, "图谱文件超过服务器文件大小限制")
+        free = shutil.disk_usage(tempfile.gettempdir()).free
+        needed = int(length) * 3 + MIN_FREE_BYTES
+        if free < needed:
+            raise HTTPException(507, f"服务器临时磁盘空间不足（可用 {free // 1048576} MiB，预计至少需要 {needed // 1048576} MiB），未接收文件")
+    if not request.headers.get("content-type", "").lower().startswith("multipart/form-data"):
+        raise HTTPException(415, "请使用文件上传，而不是把整个图谱放入 JSON 请求体")
+    status = 400
+
+    async def bounded_stream():
+        nonlocal status
+        received = 0
+        async for block in request.stream():
+            received += len(block)
+            if received > limit:
+                status = 413
+                raise MultiPartException("图谱文件超过服务器文件大小限制")
+            if shutil.disk_usage(tempfile.gettempdir()).free < len(block) + MIN_FREE_BYTES:
+                status = 507
+                raise MultiPartException("服务器临时磁盘空间不足，上传已停止")
+            yield block
+
+    parser = MultiPartParser(request.headers, bounded_stream(), max_files=1, max_fields=1,
+                             max_part_size=SCHEMA_BYTES)
+    try:
+        return await parser.parse()
+    except BaseException as exc:
+        # Starlette closes partial files on MultiPartException, but a network
+        # disconnect/cancellation can also interrupt parsing. Close those too.
+        for stream in parser._files_to_close_on_error:
+            stream.close()
+        if isinstance(exc, MultiPartException):
+            raise HTTPException(status, exc.message) from exc
+        raise
+
+
+@router.get("/projects/import-options")
+async def import_options(_user: AdminUser):
+    return {"max_file_bytes": settings.graph_import_max_bytes, "editor_max_bytes": SCHEMA_BYTES}
+
+
+@router.post("/projects/provision-file")
+async def provision_file(request: Request, user: AdminUser):
+    # Admin authentication finishes before reading the potentially large body.
+    if _import_lock.locked():
+        raise HTTPException(429, "已有文件正在上传或导入，请等待完成后再试")
+    await _import_lock.acquire()
+    form = None
+    handed_to_response = False
+    try:
+        form = await read_import_form(request)
+        source, metadata = form.get("file"), form.get("metadata")
+        if not isinstance(source, UploadFile) or not str(source.filename or "").lower().endswith(".json"):
+            raise HTTPException(422, "请选择 .json 图谱数据文件")
+        if not source.size or source.size > settings.graph_import_max_bytes:
+            raise HTTPException(413, "图谱文件为空或超过服务器文件大小限制")
+        if not isinstance(metadata, str):
+            raise HTTPException(422, "缺少项目与 Schema 信息")
+        try:
+            body = json.loads(metadata)
+            if not isinstance(body, dict):
+                raise ValueError("项目信息必须是对象")
+            payload = await asyncio.to_thread(normalize_provision_header, body)
+        except (ValueError, ProjectValidationError) as exc:
+            raise HTTPException(422, {"message": "项目或 Schema 校验失败",
+                                      "errors": getattr(exc, "errors", [str(exc)])[:100]}) from exc
+
+        async def response_stream():
+            from app.api.projects import _run_async_cleanup_to_completion, provision_validated_project
+
+            queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=1)
+            loop = asyncio.get_running_loop()
+            latest = {"phase": "服务器校验 JSON", "nodes": 0, "relations": 0}
+
+            def progress(phase: str, nodes: int, relations: int):
+                free = shutil.disk_usage(tempfile.gettempdir()).free
+                if free < MIN_FREE_BYTES:
+                    logger.warning("graph import disk reserve reached: project=%s phase=%s free_bytes=%s",
+                                   payload["project_id"], phase, free)
+                    raise RuntimeError(f"服务器临时磁盘空间不足（{phase}阶段可用 {free // 1048576} MiB，须保留 512 MiB），未发布图谱")
+
+                def publish():
+                    if queue.full():
+                        queue.get_nowait()
+                    queue.put_nowait({"phase": phase, "nodes": nodes, "relations": relations})
+                loop.call_soon_threadsafe(publish)
+
+            async def execute():
+                # A private, request-owned index validates references before
+                # any graph write. No uploaded path can select server files.
+                with tempfile.TemporaryDirectory(prefix="kg-graph-import-") as directory:
+                    from app.api.projects import _run_blocking_to_completion
+                    prepared = await _run_blocking_to_completion(
+                        validate_graph_file, source.file, Path(directory) / "index.sqlite", payload, progress,
+                    )
+                    return await provision_validated_project(
+                        prepared, user, graph_importer=partial(import_staged_graph, progress=progress),
+                    )
+
+            operation = asyncio.create_task(execute())
+            try:
+                yield event("progress", latest)
+                while not operation.done():
+                    try:
+                        latest = await asyncio.wait_for(queue.get(), 1)
+                    except TimeoutError:
+                        pass
+                    yield event("progress", latest)
+                result = await operation
+                logger.info("graph file import completed: project=%s counts=%s", payload["project_id"], result.get("counts"))
+                yield event("result", result)
+            except Exception as exc:
+                # Do not log uploaded records, credentials or the JSON body.
+                logger.warning("graph file import failed: project=%s phase=%s error_type=%s",
+                               payload["project_id"], latest["phase"], type(exc).__name__)
+                if isinstance(exc, HTTPException):
+                    detail = exc.detail if isinstance(exc.detail, dict) else {"message": str(exc.detail)}
+                else:
+                    detail = {"message": "图谱导入失败", "errors": getattr(exc, "errors", [str(exc)])[:100]}
+                yield event("error", detail)
+            finally:
+                # Do not delete a spool while a DB worker still reads it. An
+                # interrupted browser must check the project list before retry.
+                async def cleanup():
+                    await asyncio.gather(operation, return_exceptions=True)
+                    try:
+                        await form.close()
+                    finally:
+                        _import_lock.release()
+                await _run_async_cleanup_to_completion(cleanup())
+
+        response = StreamingResponse(response_stream(), media_type="application/x-ndjson",
+                                     headers={"Cache-Control": "no-store", "X-Accel-Buffering": "no"})
+        handed_to_response = True
+        return response
+    finally:
+        if not handed_to_response:
+            try:
+                if form is not None:
+                    await form.close()
+            finally:
+                _import_lock.release()

app/api/projects.py

@@ -4,7 +4,7 @@ import json
 from pathlib import Path
 from typing import Any
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Request
 from psycopg.types.json import Jsonb
 
 from app.auth import CurrentUser
@@ -895,16 +895,46 @@ async def list_projects(_user: CurrentUser):
             return await cur.fetchall()
 
 
-@router.post("/projects/provision")
-async def provision_project(body: dict, _user: CurrentUser):
+@router.post("/projects/provision", openapi_extra={
+    "requestBody": {"required": True, "content": {"application/json": {"schema": {"type": "object"}}}},
+})
+async def provision_project(body: Request, _user: CurrentUser):
     """Atomically provision project metadata, ontology, release, and a new graph."""
 
     _require_project_admin(_user)
+    if isinstance(body, Request):
+        # Check before JSON decoding. Old clients must not exhaust memory by
+        # submitting a whole large graph to the inline compatibility endpoint.
+        limit = 8 * 1024 * 1024
+        length = body.headers.get("content-length", "")
+        detail = "内联 JSON 超过 8 MiB，请在创建项目页面选择文件上传"
+        if length.isdigit() and int(length) > limit:
+            raise HTTPException(413, detail)
+        raw = bytearray()
+        async for block in body.stream():
+            if len(raw) + len(block) > limit:
+                raise HTTPException(413, detail)
+            raw.extend(block)
+        try:
+            body = await asyncio.to_thread(json.loads, raw)
+        except (ValueError, UnicodeDecodeError) as exc:
+            raise HTTPException(422, "项目 JSON 格式无效") from exc
     try:
-        payload = normalize_provision_payload(body)
+        payload = await asyncio.to_thread(normalize_provision_payload, body)
     except ProjectValidationError as exc:
         raise HTTPException(422, {"message": "项目数据校验失败", "errors": exc.errors}) from exc
 
+    return await provision_validated_project(payload, _user)
+
+
+async def provision_validated_project(payload: dict, _user: dict, *,
+                                      graph_importer: Any = None):
+    """Shared commit/compensation boundary for inline and validated file imports.
+
+    This is not a public route: only trusted server-side validation may supply
+    a normalized payload or a staged-file importer.
+    """
+    _require_project_admin(_user)
     try:
         existing_graphs = await asyncio.to_thread(list_falkor_graphs)
     except Exception as exc:
@@ -1146,7 +1176,7 @@ async def provision_project(body: dict, _user: CurrentUser):
 
                 try:
                     graph_counts = await _run_blocking_to_completion(
-                        import_falkor_graph,
+                        graph_importer or import_falkor_graph,
                         payload["graph_name"],
                         payload["graph_data"],
                     )