fix: stream large graph JSON imports
This commit is contained in:
1 parent
712ca1418c
commit
4b19638ff8
16 files changed
+1595
-193
No files matched your search
+35
-5
@@ -4,7 +4,7 @@ import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from app.auth import CurrentUser
|
||||
@@ -895,16 +895,46 @@ async def list_projects(_user: CurrentUser):
|
||||
return await cur.fetchall()
|
||||
|
||||
|
||||
@router.post("/projects/provision")
|
||||
async def provision_project(body: dict, _user: CurrentUser):
|
||||
@router.post("/projects/provision", openapi_extra={
|
||||
"requestBody": {"required": True, "content": {"application/json": {"schema": {"type": "object"}}}},
|
||||
})
|
||||
async def provision_project(body: Request, _user: CurrentUser):
|
||||
"""Atomically provision project metadata, ontology, release, and a new graph."""
|
||||
|
||||
_require_project_admin(_user)
|
||||
if isinstance(body, Request):
|
||||
# Check before JSON decoding. Old clients must not exhaust memory by
|
||||
# submitting a whole large graph to the inline compatibility endpoint.
|
||||
limit = 8 * 1024 * 1024
|
||||
length = body.headers.get("content-length", "")
|
||||
detail = "内联 JSON 超过 8 MiB,请在创建项目页面选择文件上传"
|
||||
if length.isdigit() and int(length) > limit:
|
||||
raise HTTPException(413, detail)
|
||||
raw = bytearray()
|
||||
async for block in body.stream():
|
||||
if len(raw) + len(block) > limit:
|
||||
raise HTTPException(413, detail)
|
||||
raw.extend(block)
|
||||
try:
|
||||
body = await asyncio.to_thread(json.loads, raw)
|
||||
except (ValueError, UnicodeDecodeError) as exc:
|
||||
raise HTTPException(422, "项目 JSON 格式无效") from exc
|
||||
try:
|
||||
payload = normalize_provision_payload(body)
|
||||
payload = await asyncio.to_thread(normalize_provision_payload, body)
|
||||
except ProjectValidationError as exc:
|
||||
raise HTTPException(422, {"message": "项目数据校验失败", "errors": exc.errors}) from exc
|
||||
|
||||
return await provision_validated_project(payload, _user)
|
||||
|
||||
|
||||
async def provision_validated_project(payload: dict, _user: dict, *,
|
||||
graph_importer: Any = None):
|
||||
"""Shared commit/compensation boundary for inline and validated file imports.
|
||||
|
||||
This is not a public route: only trusted server-side validation may supply
|
||||
a normalized payload or a staged-file importer.
|
||||
"""
|
||||
_require_project_admin(_user)
|
||||
try:
|
||||
existing_graphs = await asyncio.to_thread(list_falkor_graphs)
|
||||
except Exception as exc:
|
||||
@@ -1146,7 +1176,7 @@ async def provision_project(body: dict, _user: CurrentUser):
|
||||
|
||||
try:
|
||||
graph_counts = await _run_blocking_to_completion(
|
||||
import_falkor_graph,
|
||||
graph_importer or import_falkor_graph,
|
||||
payload["graph_name"],
|
||||
payload["graph_data"],
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user