fix: stream large graph JSON imports

This commit is contained in:
xuelong committed 2026-09-01 23:26:16 -07:00
1 parent 712ca1418c
commit 4b19638ff8
16 files changed
+1595 -193

No files matched your search

+35 -5
View File
@@ -4,7 +4,7 @@ import json
from pathlib import Path
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Request
from psycopg.types.json import Jsonb
from app.auth import CurrentUser
@@ -895,16 +895,46 @@ async def list_projects(_user: CurrentUser):
return await cur.fetchall()
@router.post("/projects/provision")
async def provision_project(body: dict, _user: CurrentUser):
@router.post("/projects/provision", openapi_extra={
"requestBody": {"required": True, "content": {"application/json": {"schema": {"type": "object"}}}},
})
async def provision_project(body: Request, _user: CurrentUser):
"""Atomically provision project metadata, ontology, release, and a new graph."""
_require_project_admin(_user)
if isinstance(body, Request):
# Check before JSON decoding. Old clients must not exhaust memory by
# submitting a whole large graph to the inline compatibility endpoint.
limit = 8 * 1024 * 1024
length = body.headers.get("content-length", "")
detail = "内联 JSON 超过 8 MiB,请在创建项目页面选择文件上传"
if length.isdigit() and int(length) > limit:
raise HTTPException(413, detail)
raw = bytearray()
async for block in body.stream():
if len(raw) + len(block) > limit:
raise HTTPException(413, detail)
raw.extend(block)
try:
body = await asyncio.to_thread(json.loads, raw)
except (ValueError, UnicodeDecodeError) as exc:
raise HTTPException(422, "项目 JSON 格式无效") from exc
try:
payload = normalize_provision_payload(body)
payload = await asyncio.to_thread(normalize_provision_payload, body)
except ProjectValidationError as exc:
raise HTTPException(422, {"message": "项目数据校验失败", "errors": exc.errors}) from exc
return await provision_validated_project(payload, _user)
async def provision_validated_project(payload: dict, _user: dict, *,
graph_importer: Any = None):
"""Shared commit/compensation boundary for inline and validated file imports.
This is not a public route: only trusted server-side validation may supply
a normalized payload or a staged-file importer.
"""
_require_project_admin(_user)
try:
existing_graphs = await asyncio.to_thread(list_falkor_graphs)
except Exception as exc:
@@ -1146,7 +1176,7 @@ async def provision_project(body: dict, _user: CurrentUser):
try:
graph_counts = await _run_blocking_to_completion(
import_falkor_graph,
graph_importer or import_falkor_graph,
payload["graph_name"],
payload["graph_data"],
)