fix: stream large graph JSON imports

admin-web/src/ProjectLanding.tsx

@@ -10,6 +10,7 @@ import {
   Input,
   List,
   Modal,
+  Progress,
   Select,
   Space,
   Spin,
@@ -42,6 +43,8 @@ import {
   listGraphReleases,
   listProjects,
   provisionGraphProject,
+  provisionGraphProjectFile,
+  graphFileImportOptions,
   setProjectContext,
   updateGraphProject,
   type ProjectDeletionImpact,
@@ -57,6 +60,7 @@ import {
   mergeProjectCatalogItem,
   notifyProjectCatalogChanged,
 } from "./projectCatalog";
+import { JSON_EDITOR_MAX_BYTES, importHttpErrorDetail, jsonEditorFits, useGraphFileTransfer, type ImportProgress } from "./graphFileUpload";
 
 const { Paragraph, Text, Title } = Typography;
 const { TextArea } = Input;
@@ -157,7 +161,7 @@ function projectName(project: ProjectSummary): string {
 }
 
 function apiErrorText(error: unknown, fallback: string): string {
-  const detail = (error as { response?: { data?: { detail?: unknown } } })?.response?.data?.detail;
+  const detail = importHttpErrorDetail(error);
   if (typeof detail === "string" && detail.trim()) return detail;
   if (Array.isArray(detail)) {
     const messages = detail
@@ -192,6 +196,7 @@ function apiErrorText(error: unknown, fallback: string): string {
 }
 
 function parseJsonObject(text: string, label: string): ValidationResult<Record<string, unknown>> {
+  if (!jsonEditorFits(text)) return { errors: [`${label}超过编辑器 2 MiB 上限，请选择 JSON 文件上传`] };
   if (!text.trim()) return { errors: [`请上传或输入${label} JSON`] };
   try {
     const parsed: unknown = JSON.parse(text);
@@ -651,6 +656,10 @@ export default function ProjectLanding() {
   const [schemaErrors, setSchemaErrors] = useState<string[]>([]);
   const [graphText, setGraphText] = useState("");
   const [graphFileName, setGraphFileName] = useState("");
+  const [graphFile, setGraphFile] = useState<File | null>(null);
+  const [loadingFile, setLoadingFile] = useState(false);
+  const [uploadPercent, setUploadPercent] = useState(0);
+  const [importProgress, setImportProgress] = useState<ImportProgress | null>(null);
   const [graphObject, setGraphObject] = useState<GraphImportData | null>(null);
   const [graphErrors, setGraphErrors] = useState<string[]>([]);
   const [provisioning, setProvisioning] = useState(false);
@@ -669,6 +678,7 @@ export default function ProjectLanding() {
   const mapEnabled = Form.useWatch("map_enabled", basicForm);
   const [renameForm] = Form.useForm<{ display_name: string }>();
   const impactRequest = useRef(0);
+  const fileReadRequest = useRef(0);
   const navigate = useNavigate();
 
   const refresh = async (): Promise<ProjectSummary[] | null> => {
@@ -727,6 +737,11 @@ export default function ProjectLanding() {
     setSchemaErrors([]);
     setGraphText("");
     setGraphFileName("");
+    setGraphFile(null);
+    setLoadingFile(false);
+    fileReadRequest.current += 1;
+    setUploadPercent(0);
+    setImportProgress(null);
     setGraphObject(null);
     setGraphErrors([]);
     setProvisionError("");
@@ -780,28 +795,63 @@ export default function ProjectLanding() {
   };
 
   const loadJsonFile = async (file: File, kind: "schema" | "graph") => {
+    const requestId = ++fileReadRequest.current;
+    setLoadingFile(true);
+    if (kind === "schema") {
+      setSchemaText("");
+      setSchemaObject(null);
+      setSchemaErrors([]);
+      setSchemaFileName(file.name);
+    } else {
+      setGraphFile(null);
+      setGraphText("");
+      setGraphObject(null);
+      setGraphErrors([]);
+      setGraphFileName(file.name);
+    }
     try {
+      if (!file.name.toLowerCase().endsWith(".json")) throw new Error("请选择 .json 文件");
+      if (!file.size) throw new Error("文件为空");
+      if (useGraphFileTransfer(file)) {
+        if (kind === "schema") throw new Error("Schema 超过 2 MiB 上限，请确认未误选图谱 data 文件");
+        const { data: options } = await graphFileImportOptions();
+        if (requestId !== fileReadRequest.current) return;
+        if (file.size > options.max_file_bytes) {
+          throw new Error(`超过服务器 ${Math.floor(options.max_file_bytes / 1024 / 1024)} MiB 文件上限`);
+        }
+        setGraphFile(file);
+        setGraphFileName(file.name);
+        setGraphText("");
+        setGraphObject(null);
+        setGraphErrors([]);
+        message.success("已选择文件，将在确认创建后上传并由服务器校验");
+        return;
+      }
       const text = await file.text();
+      if (requestId !== fileReadRequest.current) return;
       const parsed: unknown = JSON.parse(text);
       if (!isRecord(parsed)) throw new Error("JSON 顶层必须是对象");
-      const formatted = JSON.stringify(parsed, null, 2);
       if (kind === "schema") {
-        setSchemaText(formatted);
+        setSchemaText(text);
         setSchemaFileName(file.name);
         setSchemaObject(null);
         setSchemaErrors([]);
       } else {
-        setGraphText(formatted);
+        setGraphFile(null);
+        setGraphText(text);
         setGraphFileName(file.name);
         setGraphObject(null);
         setGraphErrors([]);
       }
       message.success(`${file.name} 已读取，可继续编辑`);
     } catch (error) {
-      const errorText = error instanceof Error ? error.message : "无法读取文件";
-      if (kind === "schema") setSchemaErrors([`本体 Schema JSON 格式错误：${errorText}`]);
-      else setGraphErrors([`图谱数据 JSON 格式错误：${errorText}`]);
-      message.error(`${file.name} 不是有效的 JSON 文件`);
+      if (requestId !== fileReadRequest.current) return;
+      const errorText = apiErrorText(error, error instanceof Error ? error.message : "无法读取文件，请确认后端已更新");
+      if (kind === "schema") setSchemaErrors([errorText]);
+      else setGraphErrors([errorText]);
+      message.error(errorText);
+    } finally {
+      if (requestId === fileReadRequest.current) setLoadingFile(false);
     }
   };
 
@@ -814,6 +864,10 @@ export default function ProjectLanding() {
       return;
     }
     const formatted = JSON.stringify(parsed.value, null, 2);
+    if (!jsonEditorFits(formatted)) {
+      message.warning("格式化后的文本超过编辑器上限，已保留原文");
+      return;
+    }
     if (kind === "schema") {
       setSchemaText(formatted);
       setSchemaErrors([]);
@@ -836,6 +890,10 @@ export default function ProjectLanding() {
       setWizardStep(1);
       return false;
     }
+    if (graphFile) {
+      setGraphErrors([]);
+      return true; // Full graph validation happens server-side before any write.
+    }
     const result = validateGraphText(graphText, schemaObject);
     setGraphErrors(result.errors);
     setGraphObject(result.value || null);
@@ -853,8 +911,8 @@ export default function ProjectLanding() {
       setWizardStep(1);
       return;
     }
-    const graphResult = validateGraphText(graphText, schemaResult.value);
-    if (!graphResult.value) {
+    const graphResult = graphFile ? { value: undefined, errors: [] } : validateGraphText(graphText, schemaResult.value);
+    if (!graphFile && !graphResult.value) {
       setGraphErrors(graphResult.errors);
       setWizardStep(2);
       return;
@@ -862,8 +920,10 @@ export default function ProjectLanding() {
 
     setProvisioning(true);
     setProvisionError("");
+    setUploadPercent(0);
+    setImportProgress(graphFile ? { phase: "上传文件" } : null);
     try {
-      const { data } = await provisionGraphProject({
+      const metadata = {
         project_id: basicValues.project_id,
         display_name: basicValues.display_name,
         spatial_map: spatialMapConfigForRegion(
@@ -871,8 +931,10 @@ export default function ProjectLanding() {
           basicValues.map_region_adcode,
         ),
         schema: schemaResult.value,
-        graph_data: graphResult.value,
-      });
+      };
+      const data = graphFile
+        ? await provisionGraphProjectFile(metadata, graphFile, setImportProgress, setUploadPercent)
+        : (await provisionGraphProject({ ...metadata, graph_data: graphResult.value! })).data;
       const createdProject = data.project;
       const projectId = String(createdProject?.project_id || basicValues.project_id).trim();
       const tenantId = String(
@@ -896,7 +958,11 @@ export default function ProjectLanding() {
         message.warning("项目已创建，但接口未返回可用图谱版本；请从项目列表重新进入");
       }
     } catch (error) {
-      const detail = apiErrorText(error, "项目创建失败，请检查输入后重试");
+      const status = (error as { response?: { status?: number } })?.response?.status;
+      const fallback = status === 413
+        ? "服务器或反向代理拒绝了文件大小，请检查上传限制"
+        : "连接中断或创建失败，请先刷新项目列表确认结果，再决定是否重试";
+      const detail = apiErrorText(error, fallback);
       setProvisionError(detail);
       message.error(detail);
     } finally {
@@ -1188,8 +1254,13 @@ export default function ProjectLanding() {
               </div>
               <TextArea
                 aria-labelledby="project-schema-json-label"
+                maxLength={JSON_EDITOR_MAX_BYTES}
                 value={schemaText}
                 onChange={(event) => {
+                  if (!jsonEditorFits(event.target.value)) {
+                    setSchemaErrors(["Schema 超过编辑器 2 MiB 上限"]);
+                    return;
+                  }
                   setSchemaText(event.target.value);
                   setSchemaObject(null);
                   setSchemaErrors([]);
@@ -1225,17 +1296,30 @@ export default function ProjectLanding() {
             >
               <p className="ant-upload-drag-icon"><InboxOutlined /></p>
               <p className="ant-upload-text">上传节点与关系 JSON</p>
-              <p className="ant-upload-hint">{graphFileName || "点击或拖拽文件；读取后仍可编辑"}</p>
+              <p className="ant-upload-hint">{graphFileName || "点击或拖拽文件；大文件直接上传，不放入编辑框"}</p>
             </Upload.Dragger>
             <div className="project-json-editor-shell">
               <div className="project-json-editor-toolbar">
                 <span id="project-graph-json-label">Graph Data JSON</span>
-                <Button size="small" onClick={() => formatJson("graph")}>格式化</Button>
+                <Button size="small" disabled={Boolean(graphFile) || loadingFile} onClick={() => formatJson("graph")}>格式化</Button>
               </div>
-              <TextArea
+              {graphFile ? (
+                <Space direction="vertical" size={16} style={{ padding: 24 }} role="status">
+                  <InboxOutlined style={{ fontSize: 36, color: "#1677ff" }} />
+                  <Text strong>{graphFile.name}</Text>
+                  <Text>{(graphFile.size / 1024 / 1024).toFixed(1)} MiB</Text>
+                  <Text type="secondary">已选择原文件。确认创建后上传，由服务器校验节点、关系和地图数据。</Text>
+                  <Button onClick={() => { setGraphFile(null); setGraphFileName(""); }}>移除文件</Button>
+                </Space>
+              ) : <TextArea
                 aria-labelledby="project-graph-json-label"
+                maxLength={JSON_EDITOR_MAX_BYTES}
                 value={graphText}
                 onChange={(event) => {
+                  if (!jsonEditorFits(event.target.value)) {
+                    setGraphErrors(["文本超过 2 MiB，请选择 JSON 文件上传"]);
+                    return;
+                  }
                   setGraphText(event.target.value);
                   setGraphObject(null);
                   setGraphErrors([]);
@@ -1243,7 +1327,7 @@ export default function ProjectLanding() {
                 className="project-json-editor"
                 placeholder={'{\n  "nodes": [\n    { "id": "node-1", "type": "Place", "properties": { "name": "示例地点" } }\n  ],\n  "relations": []\n}'}
                 spellCheck={false}
-              />
+              />}
             </div>
           </div>
           {validationAlert("图谱数据校验未通过", graphErrors)}
@@ -1262,6 +1346,9 @@ export default function ProjectLanding() {
         <Descriptions bordered size="small" column={2} className="project-confirm-descriptions">
           <Descriptions.Item label="项目名称">{basicValues?.display_name || "-"}</Descriptions.Item>
           <Descriptions.Item label="项目英文标识">{basicValues?.project_id || "-"}</Descriptions.Item>
+          {graphFile && <Descriptions.Item label="图谱数据文件" span={2}>
+            {graphFile.name} · {(graphFile.size / 1024 / 1024).toFixed(1)} MiB
+          </Descriptions.Item>}
           <Descriptions.Item label="知识地图" span={2}>
             {basicValues?.map_enabled ? "显示" : "不显示"}
           </Descriptions.Item>
@@ -1275,8 +1362,18 @@ export default function ProjectLanding() {
           type="warning"
           showIcon
           message="项目创建和数据导入将由系统统一完成"
-          description="任一步失败都会返回明确错误；窗口会保留当前输入，便于修正后重试。"
+          description={graphFile
+            ? "先上传并校验原文件，通过后分批导入。处理期间请保持窗口打开。"
+            : "任一步失败都会返回明确错误；窗口会保留当前输入，便于修正后重试。"}
         />
+        {importProgress && <div role="status" aria-live="polite" style={{ width: "100%" }}>
+          <Text>{importProgress.phase}</Text>
+          <Progress percent={uploadPercent} status={provisionError ? "exception" : provisioning ? "active" : "normal"}
+            format={(percent) => `上传 ${percent}%`} />
+          {importProgress.nodes !== undefined && <Text type="secondary">
+            当前阶段：{importProgress.nodes.toLocaleString()} 个节点，{(importProgress.relations || 0).toLocaleString()} 条关系
+          </Text>}
+        </div>}
         {provisionError && (
           <Alert className="project-provision-error" type="error" showIcon message="创建失败" description={provisionError} />
         )}
@@ -1393,7 +1490,7 @@ export default function ProjectLanding() {
             <Button onClick={closeCreateWizard} disabled={checkingBasics || provisioning}>取消</Button>
             {wizardStep > 0 && (
               <Button
-                disabled={provisioning}
+                disabled={provisioning || loadingFile}
                 onClick={() => {
                   setProvisionError("");
                   setWizardStep((current) => Math.max(0, current - 1));
@@ -1404,7 +1501,7 @@ export default function ProjectLanding() {
             )}
             <Button
               type="primary"
-              loading={checkingBasics || provisioning}
+              loading={checkingBasics || provisioning || loadingFile}
               onClick={() => void nextWizardStep()}
             >
               {wizardStep === 3 ? "确认创建" : "下一步"}

admin-web/src/api.ts

@@ -1,4 +1,5 @@
 import axios from "axios";
+import { importEventReader, type ImportProgress } from "./graphFileUpload";
 
 const api = axios.create({ baseURL: "/v1/admin" });
 
@@ -226,6 +227,46 @@ export const createGraphRelease = (projectId: string, data: unknown) =>
   api.post(`/projects/${encodeURIComponent(projectId)}/graph-releases`, data);
 export const provisionGraphProject = (data: GraphProjectProvisionRequest) =>
   api.post<GraphProjectProvisionResponse>("/projects/provision", data);
+export const graphFileImportOptions = () =>
+  api.get<{ max_file_bytes: number; editor_max_bytes: number }>("/projects/import-options");
+
+export async function provisionGraphProjectFile(
+  metadata: Omit<GraphProjectProvisionRequest, "graph_data">,
+  file: File,
+  onProgress: (progress: ImportProgress) => void,
+  onUpload: (percent: number) => void,
+): Promise<GraphProjectProvisionResponse> {
+  const form = new FormData();
+  // Serialize only the small project/Schema header. The data File is opaque.
+  form.append("metadata", JSON.stringify(metadata));
+  form.append("file", file, file.name);
+  let result: GraphProjectProvisionResponse | undefined;
+  let failure: unknown;
+  let parseFailure = false;
+  const read = importEventReader((item) => {
+    if (item.event === "progress") onProgress(item.data as ImportProgress);
+    if (item.event === "result") result = item.data as GraphProjectProvisionResponse;
+    if (item.event === "error") failure = item.data;
+  });
+  const response = await api.post<string>("/projects/provision-file", form, {
+    responseType: "text",
+    timeout: 0,
+    onUploadProgress: (event) => onUpload(event.total ? Math.floor(event.loaded * 100 / event.total) : 0),
+    onDownloadProgress: (event) => {
+      const text = event.event?.target?.responseText;
+      if (typeof text === "string") {
+        try { read(text); } catch { parseFailure = true; }
+      }
+    },
+  });
+  try { read(response.data); } catch { parseFailure = true; }
+  if (failure || parseFailure || !result) {
+    throw { response: { data: { detail: failure || {
+      message: "导入响应不完整，请刷新项目列表确认结果，勿重复提交",
+    } } } };
+  }
+  return result;
+}
 export const updateGraphProject = (projectId: string, data: { display_name: string }) =>
   api.patch<ProjectSummary>(`/projects/${encodeURIComponent(projectId)}`, data);
 export const getGraphProjectDeletionImpact = (projectId: string) =>

admin-web/src/graphFileUpload.ts

@@ -0,0 +1,49 @@
+/** Large graph JSON stays a File/Blob: never parse/stringify it on the UI thread. */
+export const JSON_EDITOR_MAX_BYTES = 2 * 1024 * 1024;
+
+export function jsonEditorFits(text: string): boolean {
+  // The fast character check avoids another huge allocation on pasted input.
+  return text.length <= JSON_EDITOR_MAX_BYTES
+    && new TextEncoder().encode(text).byteLength <= JSON_EDITOR_MAX_BYTES;
+}
+
+export function useGraphFileTransfer(file: Pick<Blob, "size">): boolean {
+  return file.size > JSON_EDITOR_MAX_BYTES;
+}
+
+/** Streaming uploads use responseType=text, including small HTTP errors. */
+export function importHttpErrorDetail(error: unknown): unknown {
+  let data = (error as { response?: { data?: unknown } })?.response?.data;
+  if (typeof data === "string") {
+    if (data.length > 65536) return undefined;
+    try { data = JSON.parse(data); } catch { return undefined; }
+  }
+  return data !== null && typeof data === "object" && "detail" in data
+    ? (data as { detail: unknown }).detail : undefined;
+}
+
+export type ImportProgress = {
+  phase: string;
+  nodes?: number;
+  relations?: number;
+  counts?: Record<string, number>;
+};
+
+export type ImportEvent = {
+  event: "progress" | "result" | "error";
+  data: Record<string, unknown>;
+};
+
+/** Progress responses are small NDJSON, not the uploaded graph body. */
+export function importEventReader(onEvent: (event: ImportEvent) => void) {
+  let offset = 0;
+  return (responseText: string) => {
+    let end = responseText.indexOf("\n", offset);
+    while (end !== -1) {
+      const line = responseText.slice(offset, end).trim();
+      offset = end + 1;
+      if (line) onEvent(JSON.parse(line) as ImportEvent);
+      end = responseText.indexOf("\n", offset);
+    }
+  };
+}

admin-web/tests/graphFileUpload.test.ts

@@ -0,0 +1,52 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+import { JSON_EDITOR_MAX_BYTES, importEventReader, importHttpErrorDetail, jsonEditorFits, useGraphFileTransfer } from "../src/graphFileUpload.ts";
+
+test("280 MB and larger files select file transfer without reading content", () => {
+  for (const size of [287527160, 1024 ** 3, 10 * 1024 ** 3]) {
+    const file = { size, text() { throw new Error("large file must not be read"); } };
+    assert.equal(useGraphFileTransfer(file), true);
+  }
+  assert.equal(useGraphFileTransfer({ size: JSON_EDITOR_MAX_BYTES }), false);
+});
+
+test("editor limits account for UTF-8 bytes, not only characters", () => {
+  assert.equal(jsonEditorFits("a".repeat(JSON_EDITOR_MAX_BYTES)), true);
+  assert.equal(jsonEditorFits("a".repeat(JSON_EDITOR_MAX_BYTES + 1)), false);
+  assert.equal(jsonEditorFits("中".repeat(800000)), false);
+});
+
+test("HTTP errors are readable for both JSON and streaming text responses", () => {
+  const detail = { message: "磁盘空间不足", errors: ["未接收文件"] };
+  assert.deepEqual(importHttpErrorDetail({ response: { data: { detail } } }), detail);
+  assert.deepEqual(importHttpErrorDetail({ response: { data: JSON.stringify({ detail }) } }), detail);
+  assert.equal(importHttpErrorDetail({ response: { data: '<html>proxy error</html>' } }), undefined);
+  assert.equal(importHttpErrorDetail({ response: { data: 'x'.repeat(65537) } }), undefined);
+  assert.equal(importHttpErrorDetail(null), undefined);
+});
+
+test("partial progress lines parse once and result is not confused with progress", () => {
+  const events: string[] = [];
+  const read = importEventReader((item) => events.push(item.event));
+  const progress = JSON.stringify({ event: "progress", data: { phase: "校验", nodes: 1 } }) + "\n";
+  const result = JSON.stringify({ event: "result", data: { project: { project_id: "test" } } }) + "\n";
+  read(progress.slice(0, 12));
+  assert.deepEqual(events, []);
+  read(progress);
+  read(progress + result.slice(0, 6));
+  read(progress + result);
+  read(progress + result);
+  assert.deepEqual(events, ["progress", "result"]);
+});
+
+test("wizard bypasses full text parsing for large files and disables formatting", () => {
+  const source = readFileSync(new URL("../src/ProjectLanding.tsx", import.meta.url), "utf8");
+  const start = source.indexOf("const loadJsonFile =");
+  const load = source.slice(start, source.indexOf("const formatJson", start));
+  assert.ok(load.indexOf("if (useGraphFileTransfer(file))") < load.indexOf("await file.text()"));
+  assert.ok(load.indexOf("setGraphFile(file)") < load.indexOf("await file.text()"));
+  assert.doesNotMatch(load, /JSON\.stringify/);
+  assert.match(source, /disabled=\{Boolean\(graphFile\) \|\| loadingFile\}/);
+  assert.match(source, /provisionGraphProjectFile\(metadata, graphFile/);
+});