From 4b19638ff836371c4fed467567b434302b9af47e Mon Sep 17 00:00:00 2001 From: 3452078359-xuexue <3452078359@qq.com> Date: Tue, 1 Sep 2026 23:26:16 -0700 Subject: [PATCH] fix: stream large graph JSON imports --- admin-web/src/ProjectLanding.tsx | 139 +++++++-- admin-web/src/api.ts | 41 +++ admin-web/src/graphFileUpload.ts | 49 ++++ admin-web/tests/graphFileUpload.test.ts | 52 ++++ app/api/__init__.py | 2 + app/api/graph_file_import.py | 184 ++++++++++++ app/api/projects.py | 40 ++- app/config.py | 3 + app/graph_import_files.py | 311 ++++++++++++++++++++ app/project_lifecycle.py | 369 +++++++++++++----------- deploy/nginx/travel-kg.conf.example | 15 + docs/GRAPH_FILE_IMPORT.md | 93 ++++++ requirements.txt | 1 + scripts/verify_graph_file_import.py | 167 +++++++++++ tests/test_graph_file_import.py | 315 ++++++++++++++++++++ tests/test_project_lifecycle.py | 7 + 16 files changed, 1595 insertions(+), 193 deletions(-) create mode 100644 admin-web/src/graphFileUpload.ts create mode 100644 admin-web/tests/graphFileUpload.test.ts create mode 100644 app/api/graph_file_import.py create mode 100644 app/graph_import_files.py create mode 100644 docs/GRAPH_FILE_IMPORT.md create mode 100644 scripts/verify_graph_file_import.py create mode 100644 tests/test_graph_file_import.py diff --git a/admin-web/src/ProjectLanding.tsx b/admin-web/src/ProjectLanding.tsx index f24bb97..7f5711c 100644 --- a/admin-web/src/ProjectLanding.tsx +++ b/admin-web/src/ProjectLanding.tsx @@ -10,6 +10,7 @@ import { Input, List, Modal, + Progress, Select, Space, Spin, @@ -42,6 +43,8 @@ import { listGraphReleases, listProjects, provisionGraphProject, + provisionGraphProjectFile, + graphFileImportOptions, setProjectContext, updateGraphProject, type ProjectDeletionImpact, @@ -57,6 +60,7 @@ import { mergeProjectCatalogItem, notifyProjectCatalogChanged, } from "./projectCatalog"; +import { JSON_EDITOR_MAX_BYTES, importHttpErrorDetail, jsonEditorFits, useGraphFileTransfer, type ImportProgress } from "./graphFileUpload"; const { Paragraph, Text, Title } = Typography; const { TextArea } = Input; @@ -157,7 +161,7 @@ function projectName(project: ProjectSummary): string { } function apiErrorText(error: unknown, fallback: string): string { - const detail = (error as { response?: { data?: { detail?: unknown } } })?.response?.data?.detail; + const detail = importHttpErrorDetail(error); if (typeof detail === "string" && detail.trim()) return detail; if (Array.isArray(detail)) { const messages = detail @@ -192,6 +196,7 @@ function apiErrorText(error: unknown, fallback: string): string { } function parseJsonObject(text: string, label: string): ValidationResult> { + if (!jsonEditorFits(text)) return { errors: [`${label}超过编辑器 2 MiB 上限,请选择 JSON 文件上传`] }; if (!text.trim()) return { errors: [`请上传或输入${label} JSON`] }; try { const parsed: unknown = JSON.parse(text); @@ -651,6 +656,10 @@ export default function ProjectLanding() { const [schemaErrors, setSchemaErrors] = useState([]); const [graphText, setGraphText] = useState(""); const [graphFileName, setGraphFileName] = useState(""); + const [graphFile, setGraphFile] = useState(null); + const [loadingFile, setLoadingFile] = useState(false); + const [uploadPercent, setUploadPercent] = useState(0); + const [importProgress, setImportProgress] = useState(null); const [graphObject, setGraphObject] = useState(null); const [graphErrors, setGraphErrors] = useState([]); const [provisioning, setProvisioning] = useState(false); @@ -669,6 +678,7 @@ export default function ProjectLanding() { const mapEnabled = Form.useWatch("map_enabled", basicForm); const [renameForm] = Form.useForm<{ display_name: string }>(); const impactRequest = useRef(0); + const fileReadRequest = useRef(0); const navigate = useNavigate(); const refresh = async (): Promise => { @@ -727,6 +737,11 @@ export default function ProjectLanding() { setSchemaErrors([]); setGraphText(""); setGraphFileName(""); + setGraphFile(null); + setLoadingFile(false); + fileReadRequest.current += 1; + setUploadPercent(0); + setImportProgress(null); setGraphObject(null); setGraphErrors([]); setProvisionError(""); @@ -780,28 +795,63 @@ export default function ProjectLanding() { }; const loadJsonFile = async (file: File, kind: "schema" | "graph") => { + const requestId = ++fileReadRequest.current; + setLoadingFile(true); + if (kind === "schema") { + setSchemaText(""); + setSchemaObject(null); + setSchemaErrors([]); + setSchemaFileName(file.name); + } else { + setGraphFile(null); + setGraphText(""); + setGraphObject(null); + setGraphErrors([]); + setGraphFileName(file.name); + } try { + if (!file.name.toLowerCase().endsWith(".json")) throw new Error("请选择 .json 文件"); + if (!file.size) throw new Error("文件为空"); + if (useGraphFileTransfer(file)) { + if (kind === "schema") throw new Error("Schema 超过 2 MiB 上限,请确认未误选图谱 data 文件"); + const { data: options } = await graphFileImportOptions(); + if (requestId !== fileReadRequest.current) return; + if (file.size > options.max_file_bytes) { + throw new Error(`超过服务器 ${Math.floor(options.max_file_bytes / 1024 / 1024)} MiB 文件上限`); + } + setGraphFile(file); + setGraphFileName(file.name); + setGraphText(""); + setGraphObject(null); + setGraphErrors([]); + message.success("已选择文件,将在确认创建后上传并由服务器校验"); + return; + } const text = await file.text(); + if (requestId !== fileReadRequest.current) return; const parsed: unknown = JSON.parse(text); if (!isRecord(parsed)) throw new Error("JSON 顶层必须是对象"); - const formatted = JSON.stringify(parsed, null, 2); if (kind === "schema") { - setSchemaText(formatted); + setSchemaText(text); setSchemaFileName(file.name); setSchemaObject(null); setSchemaErrors([]); } else { - setGraphText(formatted); + setGraphFile(null); + setGraphText(text); setGraphFileName(file.name); setGraphObject(null); setGraphErrors([]); } message.success(`${file.name} 已读取,可继续编辑`); } catch (error) { - const errorText = error instanceof Error ? error.message : "无法读取文件"; - if (kind === "schema") setSchemaErrors([`本体 Schema JSON 格式错误:${errorText}`]); - else setGraphErrors([`图谱数据 JSON 格式错误:${errorText}`]); - message.error(`${file.name} 不是有效的 JSON 文件`); + if (requestId !== fileReadRequest.current) return; + const errorText = apiErrorText(error, error instanceof Error ? error.message : "无法读取文件,请确认后端已更新"); + if (kind === "schema") setSchemaErrors([errorText]); + else setGraphErrors([errorText]); + message.error(errorText); + } finally { + if (requestId === fileReadRequest.current) setLoadingFile(false); } }; @@ -814,6 +864,10 @@ export default function ProjectLanding() { return; } const formatted = JSON.stringify(parsed.value, null, 2); + if (!jsonEditorFits(formatted)) { + message.warning("格式化后的文本超过编辑器上限,已保留原文"); + return; + } if (kind === "schema") { setSchemaText(formatted); setSchemaErrors([]); @@ -836,6 +890,10 @@ export default function ProjectLanding() { setWizardStep(1); return false; } + if (graphFile) { + setGraphErrors([]); + return true; // Full graph validation happens server-side before any write. + } const result = validateGraphText(graphText, schemaObject); setGraphErrors(result.errors); setGraphObject(result.value || null); @@ -853,8 +911,8 @@ export default function ProjectLanding() { setWizardStep(1); return; } - const graphResult = validateGraphText(graphText, schemaResult.value); - if (!graphResult.value) { + const graphResult = graphFile ? { value: undefined, errors: [] } : validateGraphText(graphText, schemaResult.value); + if (!graphFile && !graphResult.value) { setGraphErrors(graphResult.errors); setWizardStep(2); return; @@ -862,8 +920,10 @@ export default function ProjectLanding() { setProvisioning(true); setProvisionError(""); + setUploadPercent(0); + setImportProgress(graphFile ? { phase: "上传文件" } : null); try { - const { data } = await provisionGraphProject({ + const metadata = { project_id: basicValues.project_id, display_name: basicValues.display_name, spatial_map: spatialMapConfigForRegion( @@ -871,8 +931,10 @@ export default function ProjectLanding() { basicValues.map_region_adcode, ), schema: schemaResult.value, - graph_data: graphResult.value, - }); + }; + const data = graphFile + ? await provisionGraphProjectFile(metadata, graphFile, setImportProgress, setUploadPercent) + : (await provisionGraphProject({ ...metadata, graph_data: graphResult.value! })).data; const createdProject = data.project; const projectId = String(createdProject?.project_id || basicValues.project_id).trim(); const tenantId = String( @@ -896,7 +958,11 @@ export default function ProjectLanding() { message.warning("项目已创建,但接口未返回可用图谱版本;请从项目列表重新进入"); } } catch (error) { - const detail = apiErrorText(error, "项目创建失败,请检查输入后重试"); + const status = (error as { response?: { status?: number } })?.response?.status; + const fallback = status === 413 + ? "服务器或反向代理拒绝了文件大小,请检查上传限制" + : "连接中断或创建失败,请先刷新项目列表确认结果,再决定是否重试"; + const detail = apiErrorText(error, fallback); setProvisionError(detail); message.error(detail); } finally { @@ -1188,8 +1254,13 @@ export default function ProjectLanding() {