数据中心<\/Text>/);
+
+ const treeStart = dataCenterSource.indexOf('');
+ const projectNode = dataCenterSource.indexOf("data-center-project-node", treeStart);
+ const createDatabase = dataCenterSource.indexOf("data-center-create-database", projectNode);
+
+ assert.ok(treeStart >= 0);
+ assert.ok(projectNode > treeStart);
+ assert.ok(createDatabase > projectNode);
+ assert.match(
+ dataCenterSource.slice(createDatabase, createDatabase + 260),
+ /onClick=\{openDatabaseCreate\}[\s\S]*新建数据库/,
+ );
+});
diff --git a/app/agents/auditor.py b/app/agents/auditor.py
deleted file mode 100644
index a78569d..0000000
--- a/app/agents/auditor.py
+++ /dev/null
@@ -1,486 +0,0 @@
-"""STEP 02 — Quality Audit Agent (GraphRAG).
-
-For each tourist question we actually retrieve evidence from the FalkorDB
-knowledge graph, then let the LLM judge coverage/quality against that real
-evidence. Domain is scoped to **Guizhou tourism visitor service**.
-
-Suggested actions: hit / gap / low_quality / conflict.
-"""
-from __future__ import annotations
-
-import asyncio
-import json
-from dataclasses import dataclass
-from typing import Literal
-
-from app.config import settings
-from app.db import get_question_trace, update_question_trace, get_agent_settings
-from app.llm_client import LlmClient
-
-SYSTEM_PROMPT = """你是「贵州旅游知识图谱」的质量稽查员。该图谱服务于来贵州旅游的游客,
-覆盖:地点(Place)、区域(Area,贵州行政区)、体验标签(ExperienceTag)、线路模板(RouteTemplate)。
-
-给你一个游客问题,以及【从图谱真实检索到的实体证据】。请只依据这些真实证据判断
-图谱能否答好这个问题——检索不到相关实体即为缺失,不要凭常识脑补。
-
-判定动作:
-- hit: 命中且关键字段齐全、证据充分
-- gap: 图谱缺失相关实体(需要补藏新数据)
-- low_quality: 命中但关键字段空缺/证据不足(需要完善纠错)
-- conflict: 命中但数据互相矛盾(需要冲突处理)
-
-只输出 JSON。"""
-
-USER_TEMPLATE = """游客问题:{question}
-
-从贵州旅游图谱检索到的实体(name/type/key_fields/evidence_count/empty_fields):
-{retrieved}
-
-输出 schema(只输出 JSON):
-{{
- "coverage_score": 0.0~1.0,
- "confidence": 0.0~1.0,
- "evidence_count": int,
- "matched_entity_ids": ["..."],
- "missing_fields": ["field_name", ...],
- "scenario_tags": ["雨天","夜间","亲子","美食","夜市","路线"...],
- "suggested_action": "hit|gap|low_quality|conflict",
- "explanation": "一句话中文解释,说明依据哪些实体/缺什么"
-}}"""
-
-# curated per-type fields shown to the LLM, and which count as "important"
-_KEY_FIELDS = {
- "Place": ["place_type", "area_id", "best_time_start", "best_time_end",
- "price_band", "rain_suitability", "must_try_items", "night_vibe_score"],
- "Area": ["area_type", "night_identity", "first_timer_friendliness",
- "walkability_score", "rain_backup_strength"],
- "ExperienceTag": ["tag_id"],
- "RouteTemplate": ["theme", "target_user_type", "ideal_total_minutes",
- "rhythm_type", "is_rain_compatible"],
-}
-_IMPORTANT = {
- "Place": ["place_type", "area_id", "best_time_start", "must_try_items"],
- "Area": ["area_type", "night_identity"],
- "ExperienceTag": [],
- "RouteTemplate": ["theme", "target_user_type"],
-}
-_SCENARIOS = ["雨天", "夜间", "夜晚", "亲子", "美食", "小吃", "夜市", "路线",
- "地铁", "公交", "第一次", "本地", "打卡", "周边", "室内", "带娃"]
-
-# controlled vocabulary the LLM must map the question onto (proper GraphRAG)
-_TAG_VOCAB = [
- "authentic_local", "worth_first_visit", "rain_ok", "less_queue",
- "night_walk_friendly", "good_after_dinner", "late_night_friendly",
- "breakfast_friendly", "daytime_food_friendly", "morning_walk_friendly",
- "low_walk", "day_walk_friendly", "daytime_drink_friendly",
- "quiet_refreshment", "transit", "metro",
-]
-_PLACE_TYPES = ["eat", "walk", "drink", "transit_stop"]
-
-_INTENT_SYSTEM = """你是贵州旅游知识图谱的检索意图解析器。把游客问题映射到图谱的受控词表,
-只输出 JSON,不要解释。
-
-place_type 仅可选:eat(吃/美食/小吃/餐馆) / walk(逛/散步/夜市/步道/景点/打卡) /
-drink(喝/酒吧/咖啡/茶) / transit_stop(地铁/公交/交通站)
-
-tags 仅可从此列表选(英文键):
-authentic_local(本地正宗) worth_first_visit(第一次必去) rain_ok(雨天可去)
-less_queue(不排队) night_walk_friendly(适合夜间散步) good_after_dinner(饭后)
-late_night_friendly(深夜) breakfast_friendly(早餐) daytime_food_friendly(白天吃)
-morning_walk_friendly(晨间散步) low_walk(少走路) day_walk_friendly(白天散步)
-daytime_drink_friendly(白天喝) quiet_refreshment(安静小憩) transit(交通枢纽) metro(地铁)
-
-areas:问题里出现的贵州区域/片区中文名(如 甲秀楼、青云市集、南明区、贵阳…),没有就空。
-keywords:其它有用中文检索词。
-
-输出:{"areas":[],"place_types":[],"tags":[],"keywords":[]}"""
-
-
-@dataclass(frozen=True)
-class AuditResult:
- trace_id: int
- coverage_score: float
- confidence: float
- evidence_count: int
- matched_entity_ids: list[str]
- missing_fields: list[str]
- scenario_tags: list[str]
- suggested_action: Literal["hit", "gap", "low_quality", "conflict"]
- explanation: str | None = None
-
-
-async def _build_llm() -> LlmClient | None:
- """Construct the LLM client from Agent settings (global + auditor override).
-
- This is the "consumer wiring": the key/model/base_url an admin saves on the
- Agent 设置 page is what actually drives the audit.
- """
- try:
- cfg = await get_agent_settings()
- except Exception:
- cfg = {}
- g = cfg.get("global", {}) if cfg else {}
- a = (cfg.get("agents", {}) or {}).get("auditor", {}) if cfg else {}
-
- api_key = a.get("api_key") or g.get("api_key") or settings.llm_api_key
- if not api_key:
- return None
- base = (a.get("base_url") or g.get("base_url")
- or settings.llm_api_base or "https://api.deepseek.com/v1")
- model = a.get("model") or g.get("model") or settings.llm_model or "deepseek-chat"
- timeout = int(g.get("timeout") or settings.llm_timeout_seconds or 30)
- return LlmClient(api_base=base, api_key=api_key, model=model, timeout=timeout)
-
-
-def _load_catalog() -> list[dict]:
- """Pull all named nodes from FalkorDB once per audit run."""
- from app.api.graph import _get_graph
-
- g = _get_graph()
- res = g.query(
- "MATCH (n) WHERE n.name IS NOT NULL "
- "RETURN labels(n)[0] AS t, n AS node LIMIT 5000"
- )
- catalog: list[dict] = []
- for row in res.result_set:
- node = row[1]
- props = getattr(node, "properties", {}) or {}
- nm = str(props.get("name") or "").strip()
- if nm:
- catalog.append({"type": row[0] or "Node", "name": nm, "props": props})
- return catalog
-
-
-def _evidence_counts(names: list[str]) -> dict:
- if not names:
- return {}
- from app.api.graph import _get_graph
-
- g = _get_graph()
- try:
- res = g.query(
- "MATCH (n)-[r]-() WHERE n.name IN $names "
- "RETURN n.name AS nm, count(r) AS c",
- {"names": names},
- )
- return {row[0]: int(row[1]) for row in res.result_set}
- except Exception:
- return {}
-
-
-def _retrieve(question: str, catalog: list[dict]) -> tuple[list[dict], list[str]]:
- """Substring-match the question against graph entity names; gather evidence."""
- # non-Area entities first (Area names are noisy at 1600+ rows)
- hits: list[dict] = []
- for c in catalog:
- nm = c["name"]
- if len(nm) >= 2 and nm in question:
- hits.append(c)
- hits.sort(key=lambda c: 0 if c["type"] != "Area" else 1)
- hits = hits[:20]
-
- names = [h["name"] for h in hits]
- deg = _evidence_counts(names)
-
- retrieved = []
- for h in hits:
- t, props = h["type"], h["props"]
- kf = {k: props.get(k) for k in _KEY_FIELDS.get(t, []) if props.get(k) not in (None, "")}
- empty = [k for k in _IMPORTANT.get(t, []) if props.get(k) in (None, "", [], "[]")]
- retrieved.append({
- "name": h["name"],
- "type": t,
- "key_fields": kf,
- "evidence_count": deg.get(h["name"], 0),
- "empty_fields": empty,
- })
-
- scen = [s for s in _SCENARIOS if s in question]
- return retrieved, scen
-
-
-def _extract_intent(question: str, llm: LlmClient) -> dict | None:
- """Step 1 of GraphRAG: LLM maps the question to the controlled vocabulary."""
- try:
- out = llm.chat_json(system=_INTENT_SYSTEM, user=f"问题:{question}")
- return {
- "areas": [str(x) for x in (out.get("areas") or [])],
- "place_types": [x for x in (out.get("place_types") or []) if x in _PLACE_TYPES],
- "tags": [x for x in (out.get("tags") or []) if x in _TAG_VOCAB],
- "keywords": [str(x) for x in (out.get("keywords") or [])],
- }
- except Exception:
- return None
-
-
-def _graph_search(intent: dict) -> list[dict]:
- """Step 2 of GraphRAG: query FalkorDB by the structured intent."""
- from app.api.graph import _get_graph
-
- g = _get_graph()
- areas = intent.get("areas") or []
- ptypes = intent.get("place_types") or []
- tags = intent.get("tags") or []
- kws = [k for k in (intent.get("keywords") or []) if len(k) >= 2]
-
- # one enriched pass over all places (107 rows — cheap)
- try:
- res = g.query(
- "MATCH (p:Place) "
- "OPTIONAL MATCH (p)-[:LOCATED_IN]->(a:Area) "
- "OPTIONAL MATCH (p)-[:HAS_TAG]->(t:ExperienceTag) "
- "RETURN p AS p, a.name AS area, collect(DISTINCT t.name) AS tags"
- )
- except Exception:
- return []
-
- scored: list[tuple[int, dict]] = []
- for row in res.result_set:
- node = row[0]
- props = getattr(node, "properties", {}) or {}
- area = row[1]
- ptags = [x for x in (row[2] or []) if x]
- nm = str(props.get("name") or "")
- score = 0
- if area and any(a and a in area for a in areas):
- score += 2
- if props.get("place_type") in ptypes:
- score += 2
- tag_hit = len(set(ptags) & set(tags))
- score += tag_hit
- if any(k in nm for k in kws):
- score += 1
- if score <= 0:
- continue
- scored.append((score, {
- "name": nm, "type": "Place",
- "key_fields": {k: props.get(k) for k in _KEY_FIELDS["Place"]
- if props.get(k) not in (None, "")},
- "tags": ptags,
- "area": area,
- "empty_fields": [k for k in _IMPORTANT["Place"]
- if props.get(k) in (None, "", [], "[]")],
- }))
-
- scored.sort(key=lambda x: x[0], reverse=True)
- retrieved = [d for _, d in scored[:15]]
-
- # also surface matched Area nodes themselves
- if areas:
- try:
- ar = g.query(
- "MATCH (a:Area) WHERE a.name IN $names RETURN a",
- {"names": areas},
- )
- for row in ar.result_set:
- ap = getattr(row[0], "properties", {}) or {}
- retrieved.append({
- "name": str(ap.get("name") or ""), "type": "Area",
- "key_fields": {k: ap.get(k) for k in _KEY_FIELDS["Area"]
- if ap.get(k) not in (None, "")},
- "empty_fields": [k for k in _IMPORTANT["Area"]
- if ap.get(k) in (None, "", [], "[]")],
- })
- except Exception:
- pass
-
- names = [r["name"] for r in retrieved]
- deg = _evidence_counts(names)
- for r in retrieved:
- r["evidence_count"] = deg.get(r["name"], 0)
- return retrieved
-
-
-def _rule_based_audit(question: str, retrieved: list[dict], scen: list[str]) -> AuditResult:
- """Fallback when no LLM: score from real retrieval, not toy keywords."""
- n = len(retrieved)
- total_ev = sum(r["evidence_count"] for r in retrieved)
- any_empty = any(r["empty_fields"] for r in retrieved)
- if n == 0:
- action, score = "gap", 0.1
- elif any_empty or total_ev == 0:
- action, score = "low_quality", 0.45
- else:
- action, score = "hit", min(0.6 + 0.08 * n, 0.9)
- miss = sorted({f for r in retrieved for f in r["empty_fields"]}) or (["相关实体"] if n == 0 else [])
- return AuditResult(
- trace_id=0, coverage_score=score, confidence=0.4,
- evidence_count=total_ev,
- matched_entity_ids=[r["name"] for r in retrieved],
- missing_fields=miss, scenario_tags=scen,
- suggested_action=action,
- explanation=f"图谱检索评估(无 LLM):命中 {n} 个实体" + (",存在空字段" if any_empty else ""),
- )
-
-
-async def audit_single_trace(
- trace: dict, llm: LlmClient | None = None, catalog: list[dict] | None = None
-) -> AuditResult:
- question = trace["question_text"]
- trace_id = trace["id"]
-
- # ── GraphRAG retrieval ──
- if llm and llm.available():
- intent = await asyncio.to_thread(_extract_intent, question, llm)
- if intent is not None:
- retrieved = await asyncio.to_thread(_graph_search, intent)
- scen = (intent.get("tags") or []) + [s for s in _SCENARIOS if s in question]
- else:
- if catalog is None:
- try:
- catalog = _load_catalog()
- except Exception:
- catalog = []
- retrieved, scen = _retrieve(question, catalog)
- else:
- if catalog is None:
- try:
- catalog = _load_catalog()
- except Exception:
- catalog = []
- retrieved, scen = _retrieve(question, catalog)
-
- if llm and llm.available():
- try:
- result = await asyncio.to_thread(
- llm.chat_json,
- SYSTEM_PROMPT,
- USER_TEMPLATE.format(
- question=question,
- retrieved=json.dumps(retrieved, ensure_ascii=False, indent=2),
- ),
- )
- return AuditResult(
- trace_id=trace_id,
- coverage_score=float(result.get("coverage_score", 0.5)),
- confidence=float(result.get("confidence", 0.5)),
- evidence_count=int(result.get("evidence_count", len(retrieved))),
- matched_entity_ids=result.get("matched_entity_ids")
- or [r["name"] for r in retrieved],
- missing_fields=result.get("missing_fields", []),
- scenario_tags=result.get("scenario_tags") or scen,
- suggested_action=result.get("suggested_action", "gap"),
- explanation=result.get("explanation"),
- )
- except Exception:
- pass
-
- r = _rule_based_audit(question, retrieved, scen)
- return AuditResult(
- trace_id=trace_id, coverage_score=r.coverage_score, confidence=r.confidence,
- evidence_count=r.evidence_count, matched_entity_ids=r.matched_entity_ids,
- missing_fields=r.missing_fields, scenario_tags=r.scenario_tags,
- suggested_action=r.suggested_action, explanation=r.explanation,
- )
-
-
-# action → 工单元数据(gap=补藏 / low_quality=完善 / conflict=冲突核查)
-_ACTION_TASK = {
- "gap": {"prefix": "补藏", "desc": "AI稽查发现知识缺口(缺相关实体)",
- "priority": 4, "note": "该城区缺失数据,请落实采集"},
- "low_quality": {"prefix": "完善", "desc": "AI稽查:图谱已有数据但关键字段缺失/证据不足,需完善纠错",
- "priority": 3, "note": "该城区数据不完整,请完善"},
- "conflict": {"prefix": "冲突核查", "desc": "AI稽查:命中但数据相互矛盾,需冲突处理",
- "priority": 5, "note": "该城区数据存在矛盾,请核查"},
-}
-
-
-_BG_TASKS: set = set()
-
-
-async def _bg_audit(trace_ids: list[int], run_id: int) -> None:
- from app.db import finish_audit_run
- try:
- await run_audit_for_traces(trace_ids, run_id)
- await finish_audit_run(run_id, "done")
- except Exception as e: # noqa: BLE001
- try:
- await finish_audit_run(run_id, "error", str(e)[:300])
- except Exception:
- pass
-
-
-def schedule_audit(trace_ids: list[int], run_id: int) -> None:
- """Fire-and-forget background audit; progress tracked in audit_runs."""
- t = asyncio.create_task(_bg_audit(trace_ids, run_id))
- _BG_TASKS.add(t)
- t.add_done_callback(_BG_TASKS.discard)
-
-
-async def run_audit_for_traces(trace_ids: list[int], run_id: int | None = None) -> list[dict]:
- """Run audit on a batch of traces, persist results, route work-orders."""
- from app.db import (
- create_acquisition_task, create_notification, get_area_responsible,
- resolve_area_from_entities, set_task_routing, bump_audit_run,
- )
-
- llm = await _build_llm()
- try:
- catalog = await asyncio.to_thread(_load_catalog)
- except Exception:
- catalog = []
- results = []
-
- for tid in trace_ids:
- trace = await get_question_trace(tid)
- if not trace:
- continue
-
- result = await audit_single_trace(trace, llm, catalog)
-
- await update_question_trace(tid, {
- "coverage_score": result.coverage_score,
- "confidence": result.confidence,
- "evidence_count": result.evidence_count,
- "matched_entity_ids": json.dumps(result.matched_entity_ids),
- "missing_fields": json.dumps(result.missing_fields),
- "scenario_tags": json.dumps(result.scenario_tags),
- "suggested_action": result.suggested_action,
- "evaluated_at": "now()",
- })
-
- # Route gap / low_quality / conflict → a work-order; hit = no-op
- meta = _ACTION_TASK.get(result.suggested_action)
- if meta:
- task = await create_acquisition_task({
- "tenant_id": settings.default_tenant,
- "project_id": settings.default_project,
- "created_by": "auditor",
- "triggered_by_trace_id": tid,
- "title": f"{meta['prefix']}: {trace['question_text'][:60]}",
- "description": result.explanation or meta["desc"],
- "scenario_tags": json.dumps(result.scenario_tags),
- "target_entity_types": json.dumps([]),
- "target_fields": json.dumps(result.missing_fields),
- "suggested_collection_method": "manual",
- "priority": meta["priority"],
- })
- try:
- area_id = await resolve_area_from_entities(result.matched_entity_ids)
- responsible = await get_area_responsible(area_id) if area_id else None
- if task and responsible:
- await set_task_routing(task["id"], area_id, responsible["username"])
- await create_notification(
- user_id=responsible["id"],
- title=f"新{meta['prefix']}工单({area_id})",
- body=f"{trace['question_text'][:80]} — {meta['note']}。",
- ntype="task",
- related_task_id=task["id"],
- area_id=area_id,
- )
- except Exception:
- pass # routing/notification best-effort; never block audit
-
- if run_id is not None:
- try:
- await bump_audit_run(run_id, result.suggested_action)
- except Exception:
- pass
-
- results.append({
- "trace_id": tid,
- "coverage_score": result.coverage_score,
- "suggested_action": result.suggested_action,
- })
-
- return results
diff --git a/app/api/__init__.py b/app/api/__init__.py
index 8861e04..352f142 100644
--- a/app/api/__init__.py
+++ b/app/api/__init__.py
@@ -1,27 +1,14 @@
"""API router aggregation."""
from fastapi import APIRouter
-from app.api.source_profiles import router as source_profiles_router
-from app.api.batches import router as batches_router
-from app.api.question_traces import router as question_traces_router
-from app.api.simulation_panel import router as simulation_panel_router
-from app.api.audit import router as audit_router
from app.api.plaza import router as plaza_router
-from app.api.entities import router as entities_router
-from app.api.acquisition_tasks import router as acquisition_tasks_router
-from app.api.inventory import router as inventory_router
-from app.api.treatment_history import router as treatment_history_router
-from app.api.vocabulary import router as vocabulary_router
-from app.api.conflicts import router as conflicts_router
-from app.api.aligner import router as aligner_router
-from app.api.publish_jobs import router as publish_jobs_router
from app.api.graph import router as graph_router
-from app.api.evidence import router as evidence_router
-from app.api.manual_ingest import router as manual_ingest_router
from app.api.ocr import router as ocr_router
from app.api.travel_assistant import public_router as travel_assistant_public_router
from app.api.travel_assistant import router as travel_assistant_router
from app.api.data_platform import router as data_platform_router
+from app.api.interface_center import public_router as interface_center_public_router
+from app.api.interface_center import router as interface_center_router
api_router = APIRouter(prefix="/v1/admin")
openapi_router = APIRouter(prefix="/v1/openapi")
@@ -36,37 +23,15 @@ from app.api.projects import router as projects_router # noqa: E402
api_router.include_router(projects_router, tags=["projects"])
-# STEP 01 — Source profiles & batches
-api_router.include_router(source_profiles_router, tags=["source-profiles"])
-api_router.include_router(batches_router, tags=["batches"])
-
-# STEP 02 — Question traces, simulation, audit, plaza
-api_router.include_router(question_traces_router, tags=["question-traces"])
-api_router.include_router(simulation_panel_router, tags=["simulation-panel"])
-api_router.include_router(audit_router, tags=["audit"])
+# Plaza
api_router.include_router(plaza_router, tags=["plaza"])
-# STEP 03 — Entities & field decisions
-api_router.include_router(entities_router, tags=["entities"])
-
-# STEP 04 — Acquisition tasks, inventory, treatment history
-api_router.include_router(acquisition_tasks_router, tags=["acquisition-tasks"])
-api_router.include_router(inventory_router, tags=["inventory"])
-api_router.include_router(treatment_history_router, tags=["treatment-history"])
-
-# STEP 05 — Vocabulary, conflicts, aligner, publish rollback
-api_router.include_router(vocabulary_router, tags=["vocabulary"])
-api_router.include_router(conflicts_router, tags=["conflicts"])
-api_router.include_router(aligner_router, tags=["aligner"])
-api_router.include_router(publish_jobs_router, tags=["publish-jobs"])
-
# Graph browser
api_router.include_router(graph_router, tags=["graph"])
-api_router.include_router(evidence_router, tags=["evidence"])
-api_router.include_router(manual_ingest_router, tags=["manual-ingest"])
api_router.include_router(travel_assistant_router, tags=["travel-assistant"])
api_router.include_router(ocr_router, tags=["ocr"])
api_router.include_router(data_platform_router, tags=["data-platform"])
+api_router.include_router(interface_center_router, tags=["interface-center"])
# Agent call logs
from app.api.agent_call_logs import router as agent_call_logs_router # noqa: E402
@@ -78,21 +43,6 @@ from app.api.rbac import router as rbac_router # noqa: E402
api_router.include_router(rbac_router, tags=["rbac"])
-# City areas & responsibility (P3)
-from app.api.areas import router as areas_router # noqa: E402
-
-api_router.include_router(areas_router, tags=["areas"])
-
-# Notifications / inbox (P4)
-from app.api.notifications import router as notifications_router # noqa: E402
-
-api_router.include_router(notifications_router, tags=["notifications"])
-
-# Agent settings
-from app.api.agent_settings import router as agent_settings_router # noqa: E402
-
-api_router.include_router(agent_settings_router, tags=["agent-settings"])
-
# Super Agent (P1)
from app.api.super_agent import router as super_agent_router # noqa: E402
@@ -100,3 +50,4 @@ api_router.include_router(super_agent_router, tags=["super-agent"])
# Third-party integration APIs
openapi_router.include_router(travel_assistant_public_router, tags=["openapi-knowledge-qa"])
+openapi_router.include_router(interface_center_public_router, tags=["openapi-data"])
diff --git a/app/api/acquisition_tasks.py b/app/api/acquisition_tasks.py
deleted file mode 100644
index 74ec254..0000000
--- a/app/api/acquisition_tasks.py
+++ /dev/null
@@ -1,251 +0,0 @@
-"""STEP 04 — Acquisition Tasks (补藏队列) — work-order state machine + timeline."""
-from datetime import datetime, timedelta, timezone
-
-from fastapi import APIRouter, Depends, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.contracts import AcquisitionTaskCreate, TaskFromGap, TaskAssign, TaskComplete
-from app.db import (
- list_acquisition_tasks,
- create_acquisition_task,
- get_acquisition_task,
- update_acquisition_task,
- get_question_trace,
- log_task_event,
- list_task_events,
-)
-from app.project_context import ProjectContext, get_project_context
-
-router = APIRouter()
-
-# 工单状态流:待派单→已派单→处理中→待复核→已完成 / 已取消
-STATUS_LABELS = {
- "pending": "待派单",
- "assigned": "已派单",
- "in_progress": "处理中",
- "pending_review": "待复核",
- "done": "已完成",
- "cancelled": "已取消",
-}
-
-
-def _sla(task: dict) -> dict:
- st = task.get("status")
- if st in ("done", "cancelled"):
- return {"level": "closed", "label": "已关闭"}
- due = task.get("due_at")
- if not due:
- return {"level": "none", "label": "无时限"}
- now = datetime.now(timezone.utc)
- if now > due:
- return {"level": "overdue", "label": "已超时"}
- if now > due - timedelta(hours=24):
- return {"level": "soon", "label": "即将超时"}
- return {"level": "ok", "label": "正常"}
-
-
-async def _transition(task: dict, action: str, to_status: str,
- actor: str, note: str | None = None,
- extra: dict | None = None) -> dict:
- data = {"status": to_status}
- if extra:
- data.update(extra)
- updated = await update_acquisition_task(task["id"], data)
- await log_task_event(task["id"], action, task.get("status"), to_status, actor, note)
- return updated
-
-
-@router.get("/acquisition-tasks")
-async def _list(
- status: str | None = None,
- limit: int = 50,
- offset: int = 0,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- tasks = await list_acquisition_tasks(
- context.tenant_id, context.project_id, status, limit, offset
- )
- for t in tasks:
- t["sla"] = _sla(t)
- return tasks
-
-
-@router.post("/acquisition-tasks")
-async def _create(
- body: AcquisitionTaskCreate,
- user: CurrentUser,
- context: ProjectContext = Depends(get_project_context),
-):
- data = body.model_dump()
- data["tenant_id"] = context.tenant_id
- data["project_id"] = context.project_id
- data["created_by"] = user["username"]
- task = await create_acquisition_task(data)
- await log_task_event(task["id"], "created", None, task["status"], user["username"], "工单创建")
- return task
-
-
-@router.post("/acquisition-tasks/from-gap")
-async def _from_gap(
- body: TaskFromGap,
- user: CurrentUser,
- context: ProjectContext = Depends(get_project_context),
-):
- """Create acquisition task from an audit gap trace."""
- trace = await get_question_trace(body.trace_id)
- if not trace:
- raise HTTPException(404, "Trace not found")
-
- import json
- scenario_tags = trace.get("scenario_tags")
- if isinstance(scenario_tags, str):
- scenario_tags = json.loads(scenario_tags)
- missing_fields = trace.get("missing_fields")
- if isinstance(missing_fields, str):
- missing_fields = json.loads(missing_fields)
-
- data = {
- "tenant_id": context.tenant_id,
- "project_id": context.project_id,
- "created_by": user["username"],
- "triggered_by_trace_id": body.trace_id,
- "title": body.title or f"补藏: {trace['question_text'][:60]}",
- "scenario_tags": json.dumps(scenario_tags or []),
- "target_entity_types": json.dumps([]),
- "target_fields": json.dumps(missing_fields or []),
- "priority": body.priority,
- }
- task = await create_acquisition_task(data)
- await log_task_event(task["id"], "created", None, task["status"],
- user["username"], "由 AI 稽查缺口生成")
- return task
-
-
-@router.get("/acquisition-tasks/{task_id}")
-async def _get(task_id: int, _user: CurrentUser = None):
- task = await get_acquisition_task(task_id)
- if not task:
- raise HTTPException(404, "Task not found")
-
- events = await list_task_events(task_id)
- # synthetic "创建" entry only when there is no real one — covers
- # AI-auto-created tasks that predate event logging without duplicating
- if not any(e.get("action") == "created" for e in events):
- events = [{
- "id": 0,
- "task_id": task_id,
- "from_status": None,
- "to_status": "pending",
- "action": "created",
- "actor": task.get("created_by") or "系统",
- "note": "工单创建",
- "created_at": task.get("created_at"),
- }] + events
- timeline = events
-
- trace = None
- if task.get("triggered_by_trace_id"):
- tr = await get_question_trace(task["triggered_by_trace_id"])
- if tr:
- trace = {
- "id": tr.get("id"),
- "question_text": tr.get("question_text"),
- "suggested_action": tr.get("suggested_action"),
- "missing_fields": tr.get("missing_fields"),
- }
-
- return {**task, "events": timeline, "sla": _sla(task), "trace": trace,
- "status_label": STATUS_LABELS.get(task.get("status"), task.get("status"))}
-
-
-@router.post("/acquisition-tasks/{task_id}/assign")
-async def _assign(task_id: int, body: TaskAssign, user: CurrentUser):
- task = await get_acquisition_task(task_id)
- if not task:
- raise HTTPException(404, "Task not found")
- if task["status"] not in ("pending",):
- raise HTTPException(400, f"当前状态「{STATUS_LABELS.get(task['status'])}」不可派单")
- return await _transition(
- task, "assign", "assigned", user["username"],
- note=f"派单给 {body.assignee}",
- extra={"assignee": body.assignee, "assigned_at": datetime.now(timezone.utc)},
- )
-
-
-@router.post("/acquisition-tasks/{task_id}/start")
-async def _start(task_id: int, user: CurrentUser):
- task = await get_acquisition_task(task_id)
- if not task:
- raise HTTPException(404, "Task not found")
- if task["status"] not in ("assigned", "in_progress"):
- raise HTTPException(400, f"当前状态「{STATUS_LABELS.get(task['status'])}」不可开始")
- return await _transition(task, "start", "in_progress", user["username"], "开始处理")
-
-
-@router.post("/acquisition-tasks/{task_id}/complete")
-async def _submit_review(task_id: int, body: TaskComplete | None = None, user: CurrentUser = None):
- """采集员提交复核:处理中 → 待复核。"""
- task = await get_acquisition_task(task_id)
- if not task:
- raise HTTPException(404, "Task not found")
- if task["status"] not in ("in_progress", "assigned"):
- raise HTTPException(400, f"当前状态「{STATUS_LABELS.get(task['status'])}」不可提交复核")
- extra = {}
- note = "提交复核"
- if body and body.result_summary:
- extra["result_summary"] = body.result_summary
- note = f"提交复核:{body.result_summary[:80]}"
- return await _transition(task, "submit_review", "pending_review",
- user["username"] if user else "", note, extra)
-
-
-@router.post("/acquisition-tasks/{task_id}/review-approve")
-async def _approve(task_id: int, user: CurrentUser, body: dict | None = None):
- """运营复核通过:待复核 → 已完成。"""
- task = await get_acquisition_task(task_id)
- if not task:
- raise HTTPException(404, "Task not found")
- if task["status"] != "pending_review":
- raise HTTPException(400, "仅「待复核」工单可复核通过")
- note = (body or {}).get("note") or "复核通过,工单闭环"
- return await _transition(task, "approve", "done", user["username"], note,
- {"completed_at": datetime.now(timezone.utc)})
-
-
-@router.post("/acquisition-tasks/{task_id}/review-reject")
-async def _reject(task_id: int, user: CurrentUser, body: dict | None = None):
- """运营复核打回:待复核 → 处理中。"""
- task = await get_acquisition_task(task_id)
- if not task:
- raise HTTPException(404, "Task not found")
- if task["status"] != "pending_review":
- raise HTTPException(400, "仅「待复核」工单可打回")
- note = (body or {}).get("note") or "复核未通过,打回重做"
- return await _transition(task, "reject", "in_progress", user["username"], note)
-
-
-@router.post("/acquisition-tasks/{task_id}/urge")
-async def _urge(task_id: int, user: CurrentUser, body: dict | None = None):
- """催办:不改状态,仅记录一条催办事件。"""
- task = await get_acquisition_task(task_id)
- if not task:
- raise HTTPException(404, "Task not found")
- if task["status"] in ("done", "cancelled"):
- raise HTTPException(400, "已关闭工单无需催办")
- note = (body or {}).get("note") or "催办:请尽快处理"
- await log_task_event(task_id, "urge", task["status"], task["status"],
- user["username"], note)
- return {"ok": True, "task_id": task_id}
-
-
-@router.post("/acquisition-tasks/{task_id}/cancel")
-async def _cancel(task_id: int, user: CurrentUser, body: dict | None = None):
- task = await get_acquisition_task(task_id)
- if not task:
- raise HTTPException(404, "Task not found")
- if task["status"] in ("done", "cancelled"):
- raise HTTPException(400, "工单已关闭")
- note = (body or {}).get("note") or "工单取消"
- return await _transition(task, "cancel", "cancelled", user["username"], note)
diff --git a/app/api/agent_settings.py b/app/api/agent_settings.py
deleted file mode 100644
index bb52367..0000000
--- a/app/api/agent_settings.py
+++ /dev/null
@@ -1,17 +0,0 @@
-"""Agent settings — global LLM, thresholds, and per-sub-agent config."""
-from fastapi import APIRouter
-
-from app.auth import CurrentUser
-from app.db import get_agent_settings, save_agent_settings
-
-router = APIRouter()
-
-
-@router.get("/agent-settings")
-async def _get(_user: CurrentUser = None):
- return await get_agent_settings()
-
-
-@router.put("/agent-settings")
-async def _save(body: dict, _user: CurrentUser):
- return await save_agent_settings(body)
diff --git a/app/api/aligner.py b/app/api/aligner.py
deleted file mode 100644
index 55aa39c..0000000
--- a/app/api/aligner.py
+++ /dev/null
@@ -1,58 +0,0 @@
-"""STEP 05 — Entity Aligner (实体归一) endpoints."""
-from fastapi import APIRouter, Depends, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.contracts import AlignSuggestRequest, MergeEntities
-from app.db import get_candidate_entity
-from app.agents.aligner import align_candidates
-
-router = APIRouter()
-
-
-@router.post("/aligner/suggest")
-async def _suggest(body: AlignSuggestRequest, _user: CurrentUser = None):
- """Given candidate IDs, run the aligner agent to suggest merges."""
- candidates = []
- for cid in body.candidate_ids:
- c = await get_candidate_entity(cid)
- if c:
- candidates.append(c)
-
- if not candidates:
- raise HTTPException(404, "No candidates found")
-
- results = await align_candidates(candidates)
- return {"suggestions": results}
-
-
-@router.post("/aligner/auto-merge")
-async def _auto_merge(body: AlignSuggestRequest, user: CurrentUser):
- """Run aligner and auto-merge high-confidence matches."""
- from app.db import update_candidate_entity, create_review_action
-
- candidates = []
- for cid in body.candidate_ids:
- c = await get_candidate_entity(cid)
- if c:
- candidates.append(c)
-
- results = await align_candidates(candidates)
-
- merged = 0
- for r in results:
- if r.get("action") == "merge" and r.get("confidence", 0) >= 0.95:
- cid = r["candidate_id"]
- target = r.get("target_entity_id")
- if target:
- await update_candidate_entity(cid, {"status": "merged"})
- await create_review_action({
- "candidate_id": cid,
- "candidate_type": "entity",
- "action": "merged",
- "actor": user["username"],
- "note": f"Auto-merged into {target} by aligner (confidence={r['confidence']:.2f})",
- })
- merged += 1
-
- return {"total": len(results), "auto_merged": merged, "suggestions": results}
diff --git a/app/api/areas.py b/app/api/areas.py
deleted file mode 100644
index dab9bdc..0000000
--- a/app/api/areas.py
+++ /dev/null
@@ -1,113 +0,0 @@
-"""City areas & responsibility (P3) — list, tree, sync-from-graph, assign."""
-from fastapi import APIRouter, HTTPException
-
-from app.auth import CurrentUser
-from app.contracts import AreaUpdate
-from app.db import list_areas, update_area, upsert_area
-from app.api.graph import _get_graph
-
-router = APIRouter()
-
-
-def _area_level(area_id: str) -> str:
- """Derive admin level from the GB/T 2260 6-digit code."""
- if area_id.isdigit() and len(area_id) == 6:
- if area_id.endswith("0000"):
- return "province"
- if area_id.endswith("00"):
- return "city"
- return "county"
- return "poi" # semantic graph areas (e.g. jiaxiu_lou)
-
-
-def _area_parent(area_id: str, level: str) -> str | None:
- if level == "city":
- return area_id[:2] + "0000"
- if level == "county":
- return area_id[:4] + "00"
- return None
-
-
-@router.get("/areas")
-async def _list(_user: CurrentUser = None):
- return await list_areas()
-
-
-@router.get("/areas/tree")
-async def _tree(_user: CurrentUser = None):
- """省 > 市 > 区县 hierarchy, plus a group for 专题/自定义 areas."""
- rows = await list_areas()
- by_id = {r["area_id"]: r for r in rows}
-
- def node(r):
- return {
- "value": r["area_id"],
- "title": f'{r["name"]}({r["area_id"]})',
- "name": r["name"],
- "level": r["level"],
- "children": [],
- }
-
- nodes = {aid: node(r) for aid, r in by_id.items()}
- roots: list = []
- specials: list = []
- for aid, r in by_id.items():
- lvl = r["level"]
- if lvl == "province":
- roots.append(nodes[aid])
- elif lvl in ("city", "county"):
- parent = r.get("parent_id")
- if parent and parent in nodes:
- nodes[parent]["children"].append(nodes[aid])
- else:
- roots.append(nodes[aid])
- else:
- specials.append(nodes[aid])
-
- # collapse empty children
- def clean(n):
- n["children"] = [clean(c) for c in n["children"]]
- if not n["children"]:
- n.pop("children")
- return n
-
- tree = [clean(r) for r in roots]
- if specials:
- tree.append({
- "value": "__special__",
- "title": "专题 / 自定义区域",
- "selectable": False,
- "children": specials,
- })
- return tree
-
-
-@router.post("/areas/sync-from-graph")
-async def _sync(_user: CurrentUser):
- """Pull Area nodes from the knowledge graph, computing admin levels."""
- try:
- g = _get_graph()
- res = g.query("MATCH (a:Area) RETURN a")
- except Exception as e:
- raise HTTPException(400, f"读取图谱失败:{str(e)[:200]}")
-
- synced = 0
- for row in res.result_set:
- node = row[0]
- props = getattr(node, "properties", {}) or {}
- area_id = str(props.get("area_id") or "").strip()
- if not area_id:
- continue
- name = str(props.get("name") or area_id)
- lvl = _area_level(area_id)
- await upsert_area(area_id, name, lvl, _area_parent(area_id, lvl))
- synced += 1
- return {"synced": synced}
-
-
-@router.patch("/areas/{area_id}")
-async def _update(area_id: str, body: AreaUpdate, _user: CurrentUser):
- row = await update_area(area_id, body.model_dump(exclude_none=True))
- if not row:
- raise HTTPException(404, "区域不存在")
- return row
diff --git a/app/api/audit.py b/app/api/audit.py
deleted file mode 100644
index a9bb6df..0000000
--- a/app/api/audit.py
+++ /dev/null
@@ -1,100 +0,0 @@
-"""STEP 02 — AI Audit endpoints."""
-from fastapi import APIRouter, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.db import get_conn, create_audit_run, get_audit_run, get_latest_audit_run
-from app.agents.auditor import schedule_audit
-
-router = APIRouter()
-
-
-@router.post("/audit/run")
-async def run_audit(_user: CurrentUser = None):
- """Background-audit all un-evaluated question traces; returns run_id."""
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"SELECT id FROM {s}.question_traces "
- "WHERE tenant_id=%s AND project_id=%s AND coverage_score IS NULL "
- "ORDER BY created_at LIMIT 100",
- (settings.default_tenant, settings.default_project),
- )
- rows = await cur.fetchall()
-
- if not rows:
- return {"message": "没有待补审的未评估问题(题库稽查请用「运行稽查」)",
- "run_id": None, "total": 0}
-
- trace_ids = [r["id"] for r in rows]
- run_id = await create_audit_run("audit_run", len(trace_ids))
- schedule_audit(trace_ids, run_id)
- return {"run_id": run_id, "total": len(trace_ids)}
-
-
-@router.get("/audit-runs/latest")
-async def _latest_run(_user: CurrentUser = None):
- return await get_latest_audit_run() or {}
-
-
-@router.get("/audit-runs/{run_id}")
-async def _get_run(run_id: int, _user: CurrentUser = None):
- r = await get_audit_run(run_id)
- if not r:
- raise HTTPException(404, "audit run not found")
- return r
-
-
-@router.get("/audit/reports")
-async def list_reports(_user: CurrentUser = None):
- """Return historical audit snapshots (aggregated from question_traces)."""
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""SELECT
- DATE(evaluated_at) AS report_date,
- COUNT(*) AS total,
- COUNT(*) FILTER (WHERE suggested_action='hit') AS hits,
- COUNT(*) FILTER (WHERE suggested_action='gap') AS gaps,
- COUNT(*) FILTER (WHERE suggested_action='low_quality') AS low_quality,
- COUNT(*) FILTER (WHERE suggested_action='conflict') AS conflicts,
- ROUND(AVG(coverage_score)::numeric, 3) AS avg_coverage
- FROM {s}.question_traces
- WHERE evaluated_at IS NOT NULL
- GROUP BY DATE(evaluated_at)
- ORDER BY report_date DESC LIMIT 30""",
- )
- return await cur.fetchall()
-
-
-@router.get("/audit/reports/latest")
-async def latest_report(_user: CurrentUser = None):
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"SELECT * FROM {s}.question_traces "
- "WHERE evaluated_at IS NOT NULL "
- "ORDER BY evaluated_at DESC LIMIT 50"
- )
- rows = await cur.fetchall()
- if not rows:
- return {"message": "No audit data yet"}
- hits = sum(1 for r in rows if r.get("suggested_action") == "hit")
- gaps = sum(1 for r in rows if r.get("suggested_action") == "gap")
- avg_cov = sum(r.get("coverage_score") or 0 for r in rows) / len(rows) if rows else 0
- return {
- "total_evaluated": len(rows),
- "hits": hits,
- "gaps": gaps,
- "avg_coverage": round(avg_cov, 3),
- "coverage_rate": round(hits / len(rows), 3) if rows else 0,
- }
-
-
-@router.get("/audit/gaps")
-async def list_gaps(_user: CurrentUser = None):
- from app.db import get_audit_gaps
- return await get_audit_gaps(settings.default_tenant, settings.default_project)
diff --git a/app/api/auth.py b/app/api/auth.py
index 7dc31f2..4e8cd46 100644
--- a/app/api/auth.py
+++ b/app/api/auth.py
@@ -1,17 +1,43 @@
-"""Auth endpoints — login / me."""
-from fastapi import APIRouter, Depends, HTTPException
+"""Auth endpoints — login / me with brute-force protection."""
+from fastapi import APIRouter, HTTPException, Request
from app.auth import authenticate, create_access_token, CurrentUser, get_current_user
+from app.config import settings
from app.contracts import LoginRequest, TokenResponse
+from app.rate_limit import FixedWindowLimiter
router = APIRouter()
+_login_limiter = FixedWindowLimiter(
+ limit=settings.auth_login_max_attempts,
+ window_seconds=settings.auth_login_window_seconds,
+ block_seconds=settings.auth_login_lock_seconds,
+)
@router.post("/auth/login", response_model=TokenResponse)
-async def login(body: LoginRequest):
- user = await authenticate(body.username, body.password)
+async def login(body: LoginRequest, request: Request):
+ username = body.username.strip()
+ source_ip = request.client.host if request.client else "unknown"
+ keys = (f"ip:{source_ip}", f"account:{username.casefold()}")
+ retry_after = max([await _login_limiter.check(key) for key in keys], default=0)
+ if retry_after:
+ raise HTTPException(
+ 429,
+ detail="登录尝试过多,请稍后重试",
+ headers={"Retry-After": str(retry_after)},
+ )
+ user = await authenticate(username, body.password)
if not user:
+ retry_after = max([await _login_limiter.record(key) for key in keys], default=0)
+ if retry_after:
+ raise HTTPException(
+ 429,
+ detail="登录尝试过多,请稍后重试",
+ headers={"Retry-After": str(retry_after)},
+ )
raise HTTPException(401, detail="Invalid credentials")
+ for key in keys:
+ await _login_limiter.reset(key)
token = create_access_token({"sub": user["username"], "roles": user.get("roles", [])})
return TokenResponse(access_token=token)
diff --git a/app/api/batches.py b/app/api/batches.py
deleted file mode 100644
index 4301898..0000000
--- a/app/api/batches.py
+++ /dev/null
@@ -1,45 +0,0 @@
-"""STEP 01 — Import Batches (enhanced with source_code association)."""
-from fastapi import APIRouter, Depends, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.db import list_batches, get_batch, get_batch_raw_records, get_batch_quality_summary
-from app.project_context import ProjectContext, get_project_context
-
-router = APIRouter()
-
-
-@router.get("/batches")
-async def _list(
- source_code: str | None = None,
- tenant_id: str | None = None,
- project_id: str | None = None,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- return await list_batches(
- tenant_id or context.tenant_id,
- project_id or context.project_id,
- source_code,
- )
-
-
-@router.get("/batches/{batch_id}")
-async def _get(batch_id: int, _user: CurrentUser = None):
- batch = await get_batch(batch_id)
- if not batch:
- raise HTTPException(404, "Batch not found")
- return batch
-
-
-@router.get("/batches/{batch_id}/raw-records")
-async def _raw_records(batch_id: int, limit: int = 100, offset: int = 0, _user: CurrentUser = None):
- return await get_batch_raw_records(batch_id, limit, offset)
-
-
-@router.get("/batches/{batch_id}/quality-summary")
-async def _quality(batch_id: int, _user: CurrentUser = None):
- summary = await get_batch_quality_summary(batch_id)
- if not summary:
- raise HTTPException(404, "Batch not found")
- return summary
diff --git a/app/api/conflicts.py b/app/api/conflicts.py
deleted file mode 100644
index d93db97..0000000
--- a/app/api/conflicts.py
+++ /dev/null
@@ -1,71 +0,0 @@
-"""STEP 05 — Conflict Desk (冲突处理台)."""
-from fastapi import APIRouter, Depends, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.contracts import ConflictResolve
-from app.db import get_conn
-
-router = APIRouter()
-
-
-@router.get("/conflicts")
-async def _list(_user: CurrentUser = None):
- """List entities with conflicting field values from multiple sources."""
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""SELECT id, natural_key, entity_type, field_provenance_jsonb, confidence
- FROM {s}.candidate_entities
- WHERE tenant_id=%s AND project_id=%s
- AND status='pending_review'
- AND field_provenance_jsonb != '{{}}'::jsonb
- ORDER BY created_at DESC LIMIT 50""",
- (settings.default_tenant, settings.default_project),
- )
- rows = await cur.fetchall()
-
- import json
- conflicts = []
- for r in rows:
- prov = r.get("field_provenance_jsonb")
- if isinstance(prov, str):
- prov = json.loads(prov)
- # A field has conflict if multiple source_batch_ids exist for same field
- for field, info in (prov or {}).items():
- if isinstance(info, dict) and info.get("source_batch_id"):
- conflicts.append({
- "entity_id": r["id"],
- "natural_key": r["natural_key"],
- "entity_type": r["entity_type"],
- "field": field,
- "current_value": info.get("value"),
- "source": info.get("source_code"),
- "confidence": info.get("confidence"),
- })
-
- return conflicts
-
-
-@router.post("/conflicts/{conflict_id}/resolve")
-async def _resolve(conflict_id: int, body: ConflictResolve, user: CurrentUser):
- """Resolve a conflict — update the entity's field value."""
- from app.db import get_candidate_entity, update_candidate_entity
-
- entity = await get_candidate_entity(conflict_id)
- if not entity:
- raise HTTPException(404, "Entity not found")
-
- import json
- payload = entity.get("payload") or {}
- if isinstance(payload, str):
- payload = json.loads(payload)
-
- if body.chosen_value is not None:
- # Update the conflicted field in payload (field name from query param or body)
- payload["resolved_value"] = body.chosen_value
-
- await update_candidate_entity(conflict_id, {"payload": json.dumps(payload)})
-
- return {"resolved": conflict_id, "resolution": body.resolution}
diff --git a/app/api/data_platform.py b/app/api/data_platform.py
index 026c067..81ac784 100644
--- a/app/api/data_platform.py
+++ b/app/api/data_platform.py
@@ -3,44 +3,41 @@ from __future__ import annotations
from typing import Any
-from fastapi import APIRouter, File, HTTPException, Query, Response, UploadFile
+from fastapi import APIRouter, File, HTTPException, Query, Request, Response, UploadFile
-from app.auth import CurrentUser
-from app.data_platform.csv_service import (
- MAX_CSV_BYTES,
- export_csv_records,
- import_csv_records,
- preview_csv_import,
-)
-from app.data_platform.record_service import (
- create_record,
- delete_record,
- inspect_table,
- list_databases,
- list_records,
- list_tables,
- update_record,
-)
-from app.data_platform.schema import (
+from app.auth import AdminUser, CurrentUser, DataOperatorUser
+from app.data_platform.mysql_service import (
IDENTIFIER_PATTERN,
+ MAX_CSV_BYTES,
create_custom_table,
create_custom_table_from_sql,
+ create_record,
create_table_column,
delete_custom_table,
delete_project_database,
+ delete_record,
delete_table_column,
ensure_project_database,
+ execute_project_sql,
+ export_csv_records,
+ import_csv_records,
+ inspect_table,
+ list_admin_action_logs,
+ list_databases,
+ list_records,
+ list_tables,
+ preview_csv_import,
rename_custom_table,
rename_project_database,
+ update_record,
update_table_column,
)
-from app.data_platform.sql_console import execute_project_sql
router = APIRouter(prefix="/data-platform")
@router.get("/databases")
-async def databases(_user: CurrentUser):
+async def databases(_user: DataOperatorUser):
return await list_databases()
@@ -56,7 +53,10 @@ async def add_database(body: dict[str, Any], user: CurrentUser):
)
display_name = str(body.get("display_name") or database_id).strip()
tenant_id = str(body.get("tenant_id") or database_id).strip()
- return await ensure_project_database(database_id, tenant_id, display_name)
+ try:
+ return await ensure_project_database(database_id, tenant_id, display_name)
+ except ValueError as exc:
+ raise HTTPException(400, str(exc)) from exc
@router.post("/databases/initialize")
@@ -93,7 +93,7 @@ async def remove_database(project_id: str, body: dict[str, Any], user: CurrentUs
@router.get("/databases/{project_id}/tables")
-async def tables(project_id: str, _user: CurrentUser):
+async def tables(project_id: str, _user: DataOperatorUser):
return await list_tables(project_id)
@@ -101,18 +101,32 @@ async def tables(project_id: str, _user: CurrentUser):
async def execute_console_sql(
project_id: str,
body: dict[str, Any],
+ request: Request,
user: CurrentUser,
):
if "admin" not in user.get("roles", []):
raise HTTPException(403, "只有系统管理员可以使用 SQL 控制台")
try:
- return await execute_project_sql(project_id, str(body.get("sql") or ""))
+ return await execute_project_sql(
+ project_id,
+ str(body.get("sql") or ""),
+ actor=user["username"],
+ source_ip=request.client.host if request.client else None,
+ )
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
+@router.get("/security/audit-logs")
+async def security_audit_logs(
+ _user: AdminUser,
+ limit: int = Query(default=200, ge=1, le=1000),
+):
+ return await list_admin_action_logs(limit)
+
+
@router.get("/databases/{project_id}/tables/{table_code}/inspection")
-async def table_inspection(project_id: str, table_code: str, _user: CurrentUser):
+async def table_inspection(project_id: str, table_code: str, _user: DataOperatorUser):
return await inspect_table(project_id, table_code)
@@ -245,7 +259,7 @@ async def remove_table(
async def records(
project_id: str,
table_code: str,
- _user: CurrentUser,
+ _user: DataOperatorUser,
page: int = Query(default=1, ge=1),
page_size: int = Query(default=50, ge=1, le=5000),
search: str | None = None,
@@ -279,7 +293,7 @@ async def _read_csv_upload(file: UploadFile) -> tuple[bytes, str]:
async def preview_records_import(
project_id: str,
table_code: str,
- user: CurrentUser,
+ user: DataOperatorUser,
file: UploadFile = File(...),
):
content, file_name = await _read_csv_upload(file)
@@ -290,7 +304,7 @@ async def preview_records_import(
async def import_records(
project_id: str,
table_code: str,
- user: CurrentUser,
+ user: DataOperatorUser,
file: UploadFile = File(...),
):
content, file_name = await _read_csv_upload(file)
@@ -307,7 +321,7 @@ async def import_records(
async def export_records(
project_id: str,
table_code: str,
- _user: CurrentUser,
+ _user: DataOperatorUser,
search: str | None = None,
):
content, filename, total = await export_csv_records(
@@ -330,7 +344,7 @@ async def add_record(
project_id: str,
table_code: str,
body: dict[str, Any],
- user: CurrentUser,
+ user: DataOperatorUser,
):
return await create_record(project_id, table_code, body, user["username"])
@@ -341,7 +355,7 @@ async def edit_record(
table_code: str,
record_id: str,
body: dict[str, Any],
- user: CurrentUser,
+ user: DataOperatorUser,
):
return await update_record(project_id, table_code, record_id, body, user["username"])
@@ -351,6 +365,6 @@ async def remove_record(
project_id: str,
table_code: str,
record_id: str,
- user: CurrentUser,
+ user: DataOperatorUser,
):
return await delete_record(project_id, table_code, record_id, user["username"])
diff --git a/app/api/doc_restructure.py b/app/api/doc_restructure.py
deleted file mode 100644
index 2ecdd34..0000000
--- a/app/api/doc_restructure.py
+++ /dev/null
@@ -1,320 +0,0 @@
-"""文档 MD 的 ②结构化重排 + ③事实级校验(供 8765 上传转换后调用)。
-
-来自 doc-eval 压测验证过的逻辑,纯函数、无外部副作用:
- · restructure_md(md) 把"忠实转换"的宽表 MD 折叠成「每产品一节 + 干净明细表」的结构化 MD
- · validate(src, out) 事实级校验:数字保真(幻觉)、按产品价格不串位、内容不被删减、业务规则
-"""
-from __future__ import annotations
-
-import re
-from collections import Counter, defaultdict
-
-NUM = re.compile(r"\d+(?:\.\d+)?")
-
-
-def _clean(s: str) -> str:
- s = (s or "").replace("
", " ")
- s = re.sub(r"\s+", " ", s).strip()
- n = len(s)
- if n > 20: # 折叠"整串精确对折重复"(源 Excel 合并单元格常存双份)
- for mid in range(n // 2 - 2, n // 2 + 3):
- if 0 < mid < n:
- left, right = s[:mid].strip(), s[mid:].strip()
- if left and len(left) > 10 and left == right:
- return left
- return s
-
-
-def parse_blocks(md: str):
- lines = md.splitlines()
- blocks, i, n, buf = [], 0, len(lines), []
-
- def flush():
- if buf:
- for para in re.split(r"\n\s*\n", "\n".join(buf)):
- if para.strip():
- blocks.append(("text", para.strip()))
- buf.clear()
-
- def is_row(s):
- return s.strip().startswith("|") and s.count("|") >= 2
-
- def is_sep(s):
- return bool(re.match(r"^\s*\|[\s:|-]+\|\s*$", s)) and "-" in s
-
- while i < n:
- if is_row(lines[i]) and i + 1 < n and is_sep(lines[i + 1]):
- flush()
- header = [c.strip() for c in lines[i].strip().strip("|").split("|")]
- j, rows = i + 2, []
- while j < n and is_row(lines[j]) and not is_sep(lines[j]):
- row = [c.strip() for c in lines[j].strip().strip("|").split("|")]
- rows.append((row + [""] * len(header))[: len(header)])
- j += 1
- blocks.append(("table", header, rows))
- i = j
- else:
- buf.append(lines[i])
- i += 1
- flush()
- return blocks
-
-
-def classify_columns(header, rows):
- n, doc, keys, data = len(rows), [], [], []
- for c in range(len(header)):
- vals = [r[c] for r in rows]
- nonempty = [v for v in vals if v]
- if not nonempty:
- continue
- distinct = len(set(nonempty))
- changes = sum(1 for k in range(1, n) if vals[k] != vals[k - 1])
- if distinct <= 1:
- doc.append(c)
- elif n >= 4 and changes <= max(1, int(n * 0.45)):
- keys.append(c)
- else:
- data.append(c)
- return doc, keys, data
-
-
-def _restructure_table(header, rows):
- doc, keys, data = classify_columns(header, rows)
- docmeta = {}
- for c in doc:
- v = next((r[c] for r in rows if r[c]), "")
- if v:
- docmeta[_clean(header[c])] = _clean(v)
- if not keys or not data:
- keep = [c for c in range(len(header)) if any(r[c] for r in rows)]
- out = ["| " + " | ".join(_clean(header[c]) for c in keep) + " |",
- "| " + " | ".join("---" for _ in keep) + " |"]
- out += ["| " + " | ".join(r[c] for c in keep) + " |" for r in rows]
- return docmeta, ["\n".join(out)]
- title_col = max(keys, key=lambda c: len({r[c] for r in rows if r[c]}))
- attr_keys = [c for c in keys if c != title_col]
- sections, cur_key, grp = [], object(), []
-
- def emit(group):
- if not group:
- return
- seg = [f"## {_clean(group[0][title_col]) or '(未命名)'}"]
- for c in attr_keys:
- v = _clean(group[0][c])
- if v:
- seg.append(f"- {_clean(header[c])}: {v}")
- seg.append("")
- seg.append("| " + " | ".join(_clean(header[c]) for c in data) + " |")
- seg.append("| " + " | ".join("---" for _ in data) + " |")
- for r in group:
- seg.append("| " + " | ".join(_clean(r[c]) for c in data) + " |")
- sections.append("\n".join(seg))
-
- for r in rows:
- kt = tuple(r[c] for c in keys)
- if kt != cur_key:
- emit(grp)
- grp, cur_key = [], kt
- grp.append(r)
- emit(grp)
- return docmeta, sections
-
-
-def restructure_md(md: str, source: str = "") -> str:
- blocks = parse_blocks(md)
- docmeta, body, seen, title = {}, [], set(), ""
- for blk in blocks:
- if blk[0] == "text":
- para = blk[1]
- key = re.sub(r"\s+", "", para)
- if key in seen:
- continue
- seen.add(key)
- if not title and not para.startswith(("#", "|")):
- title = para.split("\n")[0]
- if "\n" not in para:
- continue
- if re.fullmatch(r"##\s*Sheet\d+\s*", para):
- continue
- body.append(("text", para))
- else:
- dm, secs = _restructure_table(blk[1], blk[2])
- docmeta.update(dm)
- body.append(("secs", secs))
- out = ["## 基本信息", ""]
- if source:
- out.append(f"- 来源文件: {source}")
- for k, v in docmeta.items():
- out.append(f"- {k}: {v}")
- out.append("")
- if title:
- out.append(f"# {title}\n")
- for kind, val in body:
- if kind == "text" and not val.startswith("#"):
- out.append(val + "\n")
- elif kind == "text":
- out.append(val + "\n")
- else:
- out.extend(s + "\n" for s in val)
- return "\n".join(out).strip() + "\n"
-
-
-# ---------- ②-LLM 语义结构化(类型自适应,用于行程单/复杂件;价格表走免费确定性)----------
-STRUCTURE_PROMPT = """你是旅游 ToB 文档结构化助手。把下面从源文件转出的原始 Markdown,重排成"结构化科学 MD",供后期 LLM 检索入库。
-
-【先判断文档类型,套用对应结构】
-· 报价单/价目表:顶部「## 基本信息」表(供应商/有效期/出发地/币种);每个产品一个 `## 产品:<名称>` 小节(拼团类型/起订人数/景区小交通费/退费政策等);价格用「房型 | 成人价 | 儿童价 | 单房差」表;通用规则、参考酒店各自独立小节。
-· 行程单:顶部「## 基本信息」表(产品名/天数/车型/出发地);按天分 `## D1 <当天路线>`、`## D2 …` 小节,每天完整保留 行程详情/餐食(早中晚)/住宿/交通;费用包含、费用不含、购物场所、预订须知、温馨提示 各自独立小节。
-· 资源表(酒店/车辆/餐厅):每个资源一个小节 + 属性。
-· 其它:用标题分层、表格如实保留。
-
-【铁律,违反即事故】
-1. 完整保留原文每一条信息、逐条照搬——严禁摘要、严禁概括、严禁删减合并、严禁省略。这是入库用的全量数据;输出篇幅应与原文相当。
-2. 所有数值/价格/日期/时间/电话严格来自原文,一字不改、不新增、不计算、不脑补;看不清就原样抄。
-3. 可纠正明显的"表头误标"(如"车型"列内容其实是"N人拼小团"→改标「拼团类型」),但只改标签、不改数值。
-4. 元数据用可见的「## 基本信息」表,不要 YAML frontmatter。
-只输出 MD,不要任何解释。"""
-
-
-def resolve_llm_key() -> str | None:
- """LLM key 解析顺序:环境变量 LLM_API_KEY / DEEPSEEK_API_KEY → ~/.ark_key 文件。"""
- import os
- k = os.environ.get("LLM_API_KEY") or os.environ.get("DEEPSEEK_API_KEY")
- if k:
- return k.strip()
- try:
- with open(os.path.expanduser("~/.ark_key")) as f:
- return f.read().strip() or None
- except Exception:
- return None
-
-
-def structure_llm(md: str, api_key: str) -> str:
- """调 LLM(默认火山豆包,可用 LLM_BASE_URL/LLM_MODEL 覆盖)做类型自适应结构化,带重试退避。"""
- import os
- import time
- import requests
- base_url = os.environ.get("LLM_BASE_URL", "https://ark.cn-beijing.volces.com/api/v3")
- model = os.environ.get("LLM_MODEL", "doubao-seed-2-0-lite-260428")
- payload = {"model": model, "temperature": 0,
- "messages": [{"role": "system", "content": STRUCTURE_PROMPT},
- {"role": "user", "content": md}]}
- last = None
- for attempt in range(4):
- try:
- r = requests.post(f"{base_url}/chat/completions",
- headers={"Authorization": f"Bearer {api_key}"},
- json=payload, timeout=180)
- r.raise_for_status()
- return r.json()["choices"][0]["message"]["content"]
- except requests.exceptions.RequestException as e:
- last = e
- if attempt < 3:
- time.sleep(2 * (attempt + 1))
- raise last
-
-
-# ---------- ③ 事实级校验 ----------
-def _wide_product_numbers(md: str) -> dict:
- out = defaultdict(set)
- for blk in parse_blocks(md):
- if blk[0] != "table":
- continue
- header, rows = blk[1], blk[2]
- _doc, keys, _data = classify_columns(header, rows)
- if not keys:
- continue
- tcol = max(keys, key=lambda c: len({r[c] for r in rows if r[c]}))
- for r in rows:
- for cell in r:
- out[_clean(r[tcol])].update(NUM.findall(cell))
- return out
-
-
-def _section_numbers(md: str) -> dict:
- out, cur = defaultdict(set), None
- for line in md.splitlines():
- m = re.match(r"^##\s+(?:产品[::])?\s*(.+)$", line)
- if m:
- cur = _clean(m.group(1))
- elif cur and line.count("|") >= 2:
- out[cur].update(NUM.findall(line))
- return out
-
-
-def _has_price_table(md: str) -> bool:
- for blk in parse_blocks(md):
- if blk[0] != "table":
- continue
- _doc, keys, data = classify_columns(blk[1], blk[2])
- if keys and len(data) >= 2:
- vals = [r[c] for r in blk[2] for c in data]
- nums = [float(v) for v in vals if re.fullmatch(r"\d+(?:\.\d+)?", v)]
- if nums and sum(x >= 100 for x in nums) >= len(nums) * 0.5:
- return True
- return False
-
-
-def _strip_fm(md: str) -> str:
- m = re.match(r"^---\n.*?\n---\n", md, re.S)
- return md[m.end():] if m else md
-
-
-def validate(src_md: str, out_md: str) -> dict:
- out_body = _strip_fm(out_md)
- # 去掉"来源文件"行(文件名里的日期是元数据、不是业务数据,不参与数字校验)
- out_body = "\n".join(l for l in out_body.splitlines() if not l.lstrip().startswith("- 来源文件"))
- src_nums, out_nums = Counter(NUM.findall(src_md)), Counter(NUM.findall(out_body))
- invented = {n: c for n, c in (out_nums - src_nums).items() if n not in src_nums}
-
- misassigned: dict = {}
- if _has_price_table(src_md):
- src_by = _wide_product_numbers(src_md)
- for prod, nums in _section_numbers(out_md).items():
- match = next((src_by[k] for k in src_by if k and (k in prod or prod in k)), None)
- if match is None:
- continue
- bad = {n for n in (nums - match) if float(n) >= 100 and not 1900 <= float(n) <= 2099}
- if bad:
- misassigned[prod] = sorted(bad)
- shared = {n for n, c in Counter(n for ns in misassigned.values() for n in ns).items()
- if c >= max(3, int(len(misassigned) * 0.5))}
- misassigned = {p: [n for n in ns if n not in shared]
- for p, ns in misassigned.items() if [n for n in ns if n not in shared]}
-
- def _prose(md):
- return sum(len(re.sub(r"\s", "", l)) for l in md.splitlines() if l.count("|") < 2)
-
- sp, op = _prose(src_md), _prose(out_body)
- content_loss = f"②正文{op}字 / ①{sp}字,疑似删减" if (sp > 300 and op < sp * 0.5) else None
-
- warn = []
- for line in out_md.splitlines():
- if line.count("|") >= 3:
- cells = [c.strip() for c in line.strip("|").split("|")]
- nums = [c for c in cells if re.fullmatch(r"\d+(?:\.\d+)?", c)]
- if len(nums) >= 2 and float(nums[0]) < float(nums[1]):
- warn.append(f"成人价<儿童价? {line.strip()[:50]}")
- return {
- "passed": not invented and not misassigned and not content_loss,
- "invented": invented,
- "misassigned": misassigned,
- "content_loss": content_loss,
- "business_warn": warn[:5],
- }
-
-
-def build_structured(md: str, source: str = "") -> tuple[str, dict, str]:
- """8765 用:自动路由结构化。价格表且确定性已过 → 免费确定性重排;
- 行程单/复杂件/确定性没过 → 若有 key 则上 LLM(类型自适应)。返回 (结构化MD, 校验, 模式)。"""
- deterministic = restructure_md(md, source=source)
- rep = validate(md, deterministic)
- key = resolve_llm_key()
- if key and not (_has_price_table(md) and rep["passed"]):
- try:
- llm_md = structure_llm(md, key)
- if llm_md and llm_md.strip():
- return llm_md, validate(md, llm_md), "llm"
- except Exception: # noqa: BLE001
- pass
- return deterministic, rep, "deterministic"
diff --git a/app/api/entities.py b/app/api/entities.py
deleted file mode 100644
index 2edc782..0000000
--- a/app/api/entities.py
+++ /dev/null
@@ -1,177 +0,0 @@
-"""STEP 03 — Entities with field-level provenance & field decisions.
-
-Covers:
-- GET /entities — list candidate entities
-- GET /entities/:id — single entity detail
-- GET /entities/:id/provenance — field-level provenance
-- PATCH /entities/:id/field-decisions — field-level review decisions
-- POST /entities/:id/field-decisions/:field/use-source/:sourceId — use a specific source for a field
-- POST /entities/:id/merge-with/:targetId — merge entities (STEP 05)
-"""
-from fastapi import APIRouter, Depends, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.contracts import FieldDecisionUpdate
-from app.db import (
- get_candidate_entity,
- list_candidate_entities,
- update_candidate_entity,
- create_review_action,
- list_review_actions,
-)
-from app.project_context import ProjectContext, get_project_context
-
-router = APIRouter()
-
-
-@router.get("/entities")
-async def _list(
- status: str | None = None,
- entity_type: str | None = None,
- limit: int = 50,
- offset: int = 0,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- return await list_candidate_entities(
- context.tenant_id,
- context.project_id,
- status,
- entity_type,
- limit,
- offset,
- )
-
-
-@router.get("/entities/{entity_id}")
-async def _get(entity_id: int, _user: CurrentUser = None):
- entity = await get_candidate_entity(entity_id)
- if not entity:
- raise HTTPException(404, "Entity not found")
- return entity
-
-
-@router.get("/entities/{entity_id}/provenance")
-async def _provenance(entity_id: int, _user: CurrentUser = None):
- """Return field-level provenance for this entity (core 4 fields)."""
- entity = await get_candidate_entity(entity_id)
- if not entity:
- raise HTTPException(404, "Entity not found")
- import json
- prov = entity.get("field_provenance_jsonb")
- if isinstance(prov, str):
- prov = json.loads(prov)
- return {"entity_id": entity_id, "natural_key": entity["natural_key"], "provenance": prov}
-
-
-@router.patch("/entities/{entity_id}/field-decisions")
-async def _field_decisions(entity_id: int, body: FieldDecisionUpdate, user: CurrentUser):
- """Record field-level review decisions."""
- entity = await get_candidate_entity(entity_id)
- if not entity:
- raise HTTPException(404, "Entity not found")
-
- # Determine overall action
- decisions = body.field_decisions
- if "_overall" in decisions:
- overall = decisions["_overall"]
- else:
- approved_count = sum(1 for v in decisions.values() if v in ("approved", "use_source"))
- overall = "approved" if approved_count >= len(decisions) * 0.5 else "rejected"
-
- # Record review action
- await create_review_action({
- "candidate_id": entity_id,
- "candidate_type": "entity",
- "action": overall,
- "actor": user["username"],
- "note": body.note,
- "field_decisions_jsonb": decisions,
- })
-
- # Update entity status if approved
- if overall == "approved":
- await update_candidate_entity(entity_id, {"status": "approved"})
-
- return {"entity_id": entity_id, "overall": overall, "field_decisions": decisions}
-
-
-@router.post("/entities/{entity_id}/field-decisions/{field}/use-source/{source_id}")
-async def _use_source(entity_id: int, field: str, source_id: int, user: CurrentUser):
- """Select a specific source's value for a field."""
- entity = await get_candidate_entity(entity_id)
- if not entity:
- raise HTTPException(404, "Entity not found")
-
- import json
- prov = entity.get("field_provenance_jsonb")
- if isinstance(prov, str):
- prov = json.loads(prov)
-
- if field not in prov:
- raise HTTPException(404, f"Field '{field}' not in provenance")
-
- prov[field]["chosen_reason"] = f"采用来源 #{source_id}"
- prov[field]["verified_by"] = user["username"]
-
- await update_candidate_entity(entity_id, {
- "field_provenance_jsonb": json.dumps(prov),
- })
-
- return {"entity_id": entity_id, "field": field, "source_id": source_id, "provenance": prov[field]}
-
-
-@router.get("/entities/{entity_id}/review-history")
-async def _review_history(entity_id: int, _user: CurrentUser = None):
- return await list_review_actions(entity_id)
-
-
-@router.post("/entities/{entity_id}/approve")
-async def _approve(entity_id: int, user: CurrentUser):
- entity = await get_candidate_entity(entity_id)
- if not entity:
- raise HTTPException(404, "Entity not found")
- await create_review_action({
- "candidate_id": entity_id,
- "candidate_type": "entity",
- "action": "approved",
- "actor": user["username"],
- })
- await update_candidate_entity(entity_id, {"status": "approved"})
- return {"ok": True}
-
-
-@router.post("/entities/{entity_id}/reject")
-async def _reject(entity_id: int, note: str | None = None, user: CurrentUser = None):
- entity = await get_candidate_entity(entity_id)
- if not entity:
- raise HTTPException(404, "Entity not found")
- await create_review_action({
- "candidate_id": entity_id,
- "candidate_type": "entity",
- "action": "rejected",
- "actor": user["username"] if user else "system",
- "note": note,
- })
- await update_candidate_entity(entity_id, {"status": "rejected"})
- return {"ok": True}
-
-
-@router.post("/entities/{entity_id}/merge-with/{target_id}")
-async def _merge(entity_id: int, target_id: int, note: str | None = None, user: CurrentUser = None):
- """Merge entity_id into target_id (STEP 05)."""
- source = await get_candidate_entity(entity_id)
- target = await get_candidate_entity(target_id)
- if not source or not target:
- raise HTTPException(404, "Entity not found")
-
- await create_review_action({
- "candidate_id": entity_id,
- "candidate_type": "entity",
- "action": "merged",
- "actor": user["username"] if user else "system",
- "note": note or f"Merged into entity #{target_id}",
- })
- await update_candidate_entity(entity_id, {"status": "merged"})
- return {"merged": entity_id, "into": target_id, "ok": True}
diff --git a/app/api/evidence.py b/app/api/evidence.py
deleted file mode 100644
index e6a528d..0000000
--- a/app/api/evidence.py
+++ /dev/null
@@ -1,297 +0,0 @@
-"""Evidence quality review API.
-
-Read-only endpoints that join graph POIs with their social raw evidence.
-"""
-from __future__ import annotations
-
-from typing import Any
-
-from falkordb import FalkorDB
-from fastapi import APIRouter, HTTPException, Query
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.db import get_conn
-
-router = APIRouter()
-
-
-def _get_graph():
- db = FalkorDB(host=settings.falkordb_host, port=settings.falkordb_port)
- return db.select_graph(settings.falkordb_graph)
-
-
-def _node_props(node: Any) -> dict[str, Any]:
- props = getattr(node, "properties", None) or {}
- return dict(props)
-
-
-def _normalize_platform(platform: str | None) -> str | None:
- p = (platform or "").strip()
- if not p or p == "all":
- return None
- if p == "xiaohongshu":
- return "xhs"
- return p
-
-
-def _tag_source_label(source: str | None) -> str:
- if source == "xiaohongshu":
- return "xhs"
- return source or "seed"
-
-
-def _row_int(row: dict[str, Any], key: str) -> int:
- return int(row.get(key) or 0)
-
-
-async def _evidence_summary(
- *,
- keyword: str | None,
- platform: str | None,
- limit: int,
- offset: int,
-) -> tuple[list[dict[str, Any]], int]:
- s = settings.db_schema
- kw = (keyword or "").strip()
- like = f"%{kw}%" if kw else None
- plat = _normalize_platform(platform)
- where = [
- "place_natural_key IS NOT NULL",
- "place_natural_key <> ''",
- ]
- params: list[Any] = []
- if plat:
- where.append("platform = %s")
- params.append(plat)
- if like:
- where.append(
- "(entity_name ILIKE %s OR keyword ILIKE %s OR "
- "title ILIKE %s OR content ILIKE %s)"
- )
- params.extend([like, like, like, like])
- where_sql = " AND ".join(where)
-
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""SELECT count(*) AS total
- FROM (
- SELECT place_natural_key
- FROM {s}.social_evidence
- WHERE {where_sql}
- GROUP BY place_natural_key
- ) x""",
- params,
- )
- total = _row_int(await cur.fetchone(), "total")
-
- await cur.execute(
- f"""WITH filtered AS (
- SELECT *
- FROM {s}.social_evidence
- WHERE {where_sql}
- ),
- grouped AS (
- SELECT place_natural_key,
- max(entity_name) FILTER (
- WHERE entity_name IS NOT NULL AND entity_name <> ''
- ) AS evidence_entity_name,
- count(*) AS evidence_count,
- max(captured_at) AS latest_captured_at
- FROM filtered
- GROUP BY place_natural_key
- ),
- platform_counts AS (
- SELECT place_natural_key,
- jsonb_object_agg(platform, cnt ORDER BY platform) AS platform_counts
- FROM (
- SELECT place_natural_key, platform, count(*) AS cnt
- FROM filtered
- GROUP BY place_natural_key, platform
- ) p
- GROUP BY place_natural_key
- )
- SELECT g.place_natural_key, g.evidence_entity_name,
- g.evidence_count, g.latest_captured_at,
- coalesce(pc.platform_counts, '{{}}'::jsonb) AS platform_counts
- FROM grouped g
- LEFT JOIN platform_counts pc USING (place_natural_key)
- ORDER BY g.evidence_count DESC, g.place_natural_key
- LIMIT %s OFFSET %s""",
- [*params, limit, offset],
- )
- return await cur.fetchall(), total
-
-
-def _graph_places(keys: list[str]) -> dict[str, dict[str, Any]]:
- if not keys:
- return {}
- g = _get_graph()
- result = g.query(
- "MATCH (p:Place) WHERE p.element_id IN $keys RETURN p",
- {"keys": keys},
- )
- out: dict[str, dict[str, Any]] = {}
- for row in result.result_set:
- props = _node_props(row[0])
- key = str(props.get("element_id") or "")
- if key:
- out[key] = props
- return out
-
-
-def _graph_tags(keys: list[str]) -> dict[str, list[dict[str, str]]]:
- if not keys:
- return {}
- g = _get_graph()
- result = g.query(
- "MATCH (p:Place)-[:HAS_TAG]->(t:ExperienceTag) "
- "WHERE p.element_id IN $keys "
- "RETURN p.element_id, t.name, coalesce(t.source,'seed')",
- {"keys": keys},
- )
- out: dict[str, list[dict[str, str]]] = {key: [] for key in keys}
- seen: set[tuple[str, str, str]] = set()
- for row in result.result_set:
- key = str(row[0] or "")
- name = str(row[1] or "")
- source = _tag_source_label(str(row[2] or "seed"))
- sig = (key, name, source)
- if key and name and sig not in seen:
- seen.add(sig)
- out.setdefault(key, []).append({"name": name, "source": source})
- return out
-
-
-def _place_from_graph_or_evidence(
- key: str,
- graph: dict[str, Any] | None,
- row: dict[str, Any] | None = None,
-) -> dict[str, Any]:
- graph = graph or {}
- row = row or {}
- return {
- "element_id": key,
- "name": graph.get("name") or row.get("evidence_entity_name") or key,
- "place_type": graph.get("place_type") or "",
- "station_type": graph.get("station_type") or "",
- "district": graph.get("district") or graph.get("area_name") or "",
- "address": graph.get("address") or "",
- "location": graph.get("location") or "",
- "longitude": graph.get("longitude") or graph.get("lng"),
- "latitude": graph.get("latitude") or graph.get("lat"),
- "tel": graph.get("tel") or graph.get("phone") or "",
- "rating": graph.get("rating") or graph.get("biz_ext_rating") or "",
- "raw_graph": graph,
- }
-
-
-@router.get("/evidence/poi-quality")
-async def list_poi_quality(
- keyword: str | None = None,
- platform: str | None = None,
- limit: int = Query(50, ge=1, le=100),
- offset: int = Query(0, ge=0),
- _user: CurrentUser = None,
-):
- """List POIs with social evidence and the tags written back to the graph."""
- try:
- rows, total = await _evidence_summary(
- keyword=keyword,
- platform=platform,
- limit=limit,
- offset=offset,
- )
- keys = [str(r["place_natural_key"]) for r in rows]
- places = _graph_places(keys)
- tags = _graph_tags(keys)
- items = []
- for r in rows:
- key = str(r["place_natural_key"])
- tag_list = tags.get(key, [])
- tag_sources = sorted({t["source"] for t in tag_list if t.get("source")})
- items.append({
- **_place_from_graph_or_evidence(key, places.get(key), r),
- "evidence_count": _row_int(r, "evidence_count"),
- "platform_counts": r.get("platform_counts") or {},
- "latest_captured_at": r.get("latest_captured_at"),
- "tags": tag_list[:16],
- "tag_count": len(tag_list),
- "tag_sources": tag_sources,
- })
- return {
- "items": items,
- "total": total,
- "limit": limit,
- "offset": offset,
- "platform": _normalize_platform(platform) or "all",
- }
- except Exception as exc:
- raise HTTPException(500, f"Failed to load POI quality data: {str(exc)[:300]}")
-
-
-@router.get("/evidence/poi-quality/{place_key:path}")
-async def get_poi_quality(
- place_key: str,
- platform: str | None = None,
- limit: int = Query(80, ge=1, le=300),
- _user: CurrentUser = None,
-):
- """Return one POI, graph tags, and raw social evidence rows."""
- key = place_key.strip()
- if not key:
- raise HTTPException(400, "place_key is required")
-
- try:
- places = _graph_places([key])
- tag_map = _graph_tags([key])
- plat = _normalize_platform(platform)
- s = settings.db_schema
- where = ["place_natural_key = %s"]
- params: list[Any] = [key]
- if plat:
- where.append("platform = %s")
- params.append(plat)
- where_sql = " AND ".join(where)
-
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""SELECT platform, count(*) AS count
- FROM {s}.social_evidence
- WHERE {where_sql}
- GROUP BY platform
- ORDER BY platform""",
- params,
- )
- platform_counts = {
- str(r["platform"]): _row_int(r, "count")
- for r in await cur.fetchall()
- }
- await cur.execute(
- f"""SELECT id, platform, kind, source_id, url, entity_name,
- place_natural_key, keyword, title, content, author,
- author_id, likes, comments, collects, shares,
- publish_time, location, tags, image_urls, raw_jsonb,
- captured_at
- FROM {s}.social_evidence
- WHERE {where_sql}
- ORDER BY likes DESC NULLS LAST,
- comments DESC NULLS LAST,
- captured_at DESC
- LIMIT %s""",
- [*params, limit],
- )
- evidence = await cur.fetchall()
-
- return {
- "place": _place_from_graph_or_evidence(key, places.get(key)),
- "tags": tag_map.get(key, []),
- "tag_count": len(tag_map.get(key, [])),
- "platform_counts": platform_counts,
- "evidence": evidence,
- "evidence_count": sum(platform_counts.values()),
- }
- except Exception as exc:
- raise HTTPException(500, f"Failed to load POI evidence: {str(exc)[:300]}")
diff --git a/app/api/interface_center.py b/app/api/interface_center.py
new file mode 100644
index 0000000..c68f2dd
--- /dev/null
+++ b/app/api/interface_center.py
@@ -0,0 +1,461 @@
+"""Administrative Interface Center and restricted public Data APIs."""
+from __future__ import annotations
+
+import time
+from typing import Any, Awaitable, Callable
+import uuid
+
+from fastapi import APIRouter, Header, HTTPException, Query, Request
+
+from app.auth import CurrentUser
+from app.config import settings
+from app.data_platform.interface_service import (
+ authenticate_api_key,
+ client_catalog,
+ create_api_client,
+ create_api_policy,
+ delete_api_client,
+ delete_api_policy,
+ interface_catalog,
+ interface_summary,
+ issue_api_credential,
+ list_api_call_logs,
+ list_api_clients,
+ list_api_credentials,
+ list_api_policies,
+ resolve_api_policy,
+ revoke_api_credential,
+ update_api_client,
+ update_api_policy,
+ write_api_call_log,
+)
+from app.security_baseline import security_readiness
+from app.data_platform.mysql_service import (
+ create_record,
+ delete_record,
+ inspect_table,
+ list_records,
+ update_record,
+)
+from app.rate_limit import FixedWindowLimiter
+
+
+router = APIRouter(prefix="/interface-center")
+public_router = APIRouter(prefix="/data")
+_public_api_limiter = FixedWindowLimiter(
+ limit=settings.interface_api_rate_limit_per_minute,
+ window_seconds=60,
+)
+
+
+def _require_admin(user: dict[str, Any]) -> None:
+ if "admin" not in user.get("roles", []):
+ raise HTTPException(403, "只有系统管理员可以管理接口中心")
+
+
+def _bad_request(exc: ValueError) -> HTTPException:
+ return HTTPException(400, str(exc))
+
+
+@router.get("/summary")
+async def summary(user: CurrentUser):
+ _require_admin(user)
+ return await interface_summary()
+
+
+@router.get("/security-readiness")
+async def readiness(user: CurrentUser):
+ _require_admin(user)
+ return security_readiness()
+
+
+@router.get("/catalog")
+async def catalog(user: CurrentUser):
+ _require_admin(user)
+ return await interface_catalog()
+
+
+@router.get("/clients")
+async def clients(user: CurrentUser):
+ _require_admin(user)
+ return await list_api_clients()
+
+
+@router.post("/clients")
+async def add_client(body: dict[str, Any], user: CurrentUser):
+ _require_admin(user)
+ try:
+ return await create_api_client(body, user["username"])
+ except ValueError as exc:
+ raise _bad_request(exc) from exc
+
+
+@router.patch("/clients/{client_id}")
+async def edit_client(client_id: str, body: dict[str, Any], user: CurrentUser):
+ _require_admin(user)
+ try:
+ return await update_api_client(client_id, body)
+ except ValueError as exc:
+ raise _bad_request(exc) from exc
+
+
+@router.delete("/clients/{client_id}")
+async def remove_client(client_id: str, user: CurrentUser):
+ _require_admin(user)
+ try:
+ return await delete_api_client(client_id)
+ except ValueError as exc:
+ raise _bad_request(exc) from exc
+
+
+@router.get("/credentials")
+async def credentials(user: CurrentUser, client_id: str | None = None):
+ _require_admin(user)
+ return await list_api_credentials(client_id)
+
+
+@router.post("/clients/{client_id}/credentials")
+async def add_credential(client_id: str, body: dict[str, Any], user: CurrentUser):
+ _require_admin(user)
+ try:
+ return await issue_api_credential(client_id, body)
+ except ValueError as exc:
+ raise _bad_request(exc) from exc
+
+
+@router.post("/credentials/{credential_id}/revoke")
+async def revoke_credential(credential_id: str, user: CurrentUser):
+ _require_admin(user)
+ try:
+ return await revoke_api_credential(credential_id)
+ except ValueError as exc:
+ raise _bad_request(exc) from exc
+
+
+@router.get("/policies")
+async def policies(user: CurrentUser, client_id: str | None = None):
+ _require_admin(user)
+ return await list_api_policies(client_id)
+
+
+@router.post("/policies")
+async def add_policy(body: dict[str, Any], user: CurrentUser):
+ _require_admin(user)
+ try:
+ return await create_api_policy(body)
+ except ValueError as exc:
+ raise _bad_request(exc) from exc
+
+
+@router.patch("/policies/{policy_id}")
+async def edit_policy(policy_id: str, body: dict[str, Any], user: CurrentUser):
+ _require_admin(user)
+ try:
+ return await update_api_policy(policy_id, body)
+ except ValueError as exc:
+ raise _bad_request(exc) from exc
+
+
+@router.delete("/policies/{policy_id}")
+async def remove_policy(policy_id: str, user: CurrentUser):
+ _require_admin(user)
+ try:
+ return await delete_api_policy(policy_id)
+ except ValueError as exc:
+ raise _bad_request(exc) from exc
+
+
+@router.get("/logs")
+async def logs(user: CurrentUser, limit: int = Query(default=200, ge=1, le=1000)):
+ _require_admin(user)
+ return await list_api_call_logs(limit)
+
+
+def _extract_api_key(authorization: str | None, x_api_key: str | None) -> str:
+ if authorization and authorization.lower().startswith("bearer "):
+ return authorization[7:].strip()
+ return str(x_api_key or "").strip()
+
+
+async def _log_safely(**kwargs: Any) -> None:
+ try:
+ await write_api_call_log(**kwargs)
+ except Exception:
+ # Logging must never turn a successful, authorized data operation into
+ # an application error. Infrastructure monitoring handles log failures.
+ return
+
+
+async def _public_operation(
+ *,
+ request: Request,
+ authorization: str | None,
+ x_api_key: str | None,
+ database_id: str | None,
+ table_code: str | None,
+ action: str,
+ operation: Callable[[dict[str, Any], dict[str, Any] | None], Awaitable[Any]],
+ resolve_policy: bool = True,
+) -> Any:
+ started = time.perf_counter()
+ request_id = str(uuid.uuid4())
+ identity: dict[str, Any] | None = None
+ status_code = 200
+ error_message: str | None = None
+ try:
+ identity = await authenticate_api_key(_extract_api_key(authorization, x_api_key))
+ retry_after = await _public_api_limiter.consume(
+ f"credential:{identity['credential_id']}"
+ )
+ if retry_after:
+ raise HTTPException(
+ 429,
+ "接口调用过于频繁,请稍后重试",
+ headers={"Retry-After": str(retry_after)},
+ )
+ policy = None
+ if resolve_policy:
+ if not database_id or not table_code:
+ raise HTTPException(400, "缺少数据库或数据表范围")
+ policy = await resolve_api_policy(
+ str(identity["client_id"]),
+ database_id,
+ table_code,
+ action,
+ )
+ result = await operation(identity, policy)
+ if isinstance(result, dict):
+ result.setdefault("request_id", request_id)
+ return result
+ except HTTPException as exc:
+ status_code = exc.status_code
+ error_message = str(exc.detail)
+ raise
+ except Exception as exc:
+ status_code = 500
+ error_message = str(exc)
+ raise
+ finally:
+ await _log_safely(
+ request_id=request_id,
+ identity=identity,
+ method=request.method,
+ path=request.url.path,
+ database_id=database_id,
+ table_code=table_code,
+ action=action,
+ status_code=status_code,
+ duration_ms=round((time.perf_counter() - started) * 1000, 2),
+ source_ip=request.client.host if request.client else None,
+ error_message=error_message,
+ )
+
+
+@public_router.get("/catalog")
+async def public_catalog(
+ request: Request,
+ authorization: str | None = Header(default=None),
+ x_api_key: str | None = Header(default=None, alias="X-API-Key"),
+):
+ async def operation(identity: dict[str, Any], _policy: dict[str, Any] | None):
+ return {
+ "client": {"id": identity["client_id"], "name": identity["client_name"]},
+ "databases": await client_catalog(str(identity["client_id"])),
+ }
+
+ return await _public_operation(
+ request=request,
+ authorization=authorization,
+ x_api_key=x_api_key,
+ database_id=None,
+ table_code=None,
+ action="metadata",
+ operation=operation,
+ resolve_policy=False,
+ )
+
+
+@public_router.get("/databases/{database_id}/tables/{table_code}/schema")
+async def public_table_schema(
+ database_id: str,
+ table_code: str,
+ request: Request,
+ authorization: str | None = Header(default=None),
+ x_api_key: str | None = Header(default=None, alias="X-API-Key"),
+):
+ async def operation(_identity: dict[str, Any], policy: dict[str, Any] | None):
+ inspection = await inspect_table(database_id, table_code)
+ readable = set((policy or {}).get("readable_fields") or [])
+ if "*" not in readable:
+ visible = {"id", "created_at", "updated_at", *readable}
+ inspection["columns"] = [
+ column
+ for column in inspection["columns"]
+ if column["name"] in visible
+ ]
+ inspection["constraints"] = [
+ constraint
+ for constraint in inspection["constraints"]
+ if set(constraint.get("columns") or []).issubset(visible)
+ ]
+ # Index definitions and relationships can contain restricted field
+ # names. Only expose them to whole-table metadata grants.
+ inspection["indexes"] = []
+ inspection["relationships"] = []
+ inspection["table"]["column_count"] = len(inspection["columns"])
+ inspection["table"]["index_count"] = 0
+ inspection["table"]["constraint_count"] = len(inspection["constraints"])
+ return inspection
+
+ return await _public_operation(
+ request=request,
+ authorization=authorization,
+ x_api_key=x_api_key,
+ database_id=database_id,
+ table_code=table_code,
+ action="metadata",
+ operation=operation,
+ )
+
+
+@public_router.get("/databases/{database_id}/tables/{table_code}/records")
+async def public_records(
+ database_id: str,
+ table_code: str,
+ request: Request,
+ page: int = Query(default=1, ge=1),
+ page_size: int = Query(default=50, ge=1, le=500),
+ search: str | None = None,
+ sort_field: str | None = None,
+ sort_order: str = Query(default="desc", pattern="^(asc|desc)$"),
+ fields: str | None = None,
+ authorization: str | None = Header(default=None),
+ x_api_key: str | None = Header(default=None, alias="X-API-Key"),
+):
+ async def operation(_identity: dict[str, Any], policy: dict[str, Any] | None):
+ readable = set((policy or {}).get("readable_fields") or [])
+ if fields:
+ requested = {value.strip() for value in fields.split(",") if value.strip()}
+ if "*" not in readable and not requested.issubset(readable):
+ raise HTTPException(403, "请求包含未授权读取的字段")
+ readable = requested
+ return await list_records(
+ database_id,
+ table_code,
+ page=page,
+ page_size=page_size,
+ search=search,
+ sort_field=sort_field,
+ sort_order=sort_order,
+ allowed_fields=readable,
+ row_filter=(policy or {}).get("row_filter") or {},
+ )
+
+ return await _public_operation(
+ request=request,
+ authorization=authorization,
+ x_api_key=x_api_key,
+ database_id=database_id,
+ table_code=table_code,
+ action="read",
+ operation=operation,
+ )
+
+
+def _check_writable_fields(policy: dict[str, Any] | None, body: dict[str, Any]) -> None:
+ writable = set((policy or {}).get("writable_fields") or [])
+ if "*" not in writable:
+ unknown = sorted(set(body) - writable)
+ if unknown:
+ raise HTTPException(403, f"包含未授权写入的字段:{', '.join(unknown)}")
+
+
+@public_router.post("/databases/{database_id}/tables/{table_code}/records")
+async def public_add_record(
+ database_id: str,
+ table_code: str,
+ body: dict[str, Any],
+ request: Request,
+ authorization: str | None = Header(default=None),
+ x_api_key: str | None = Header(default=None, alias="X-API-Key"),
+):
+ async def operation(identity: dict[str, Any], policy: dict[str, Any] | None):
+ _check_writable_fields(policy, body)
+ return await create_record(
+ database_id,
+ table_code,
+ body,
+ f"api:{identity['client_id']}",
+ row_filter=(policy or {}).get("row_filter") or {},
+ )
+
+ return await _public_operation(
+ request=request,
+ authorization=authorization,
+ x_api_key=x_api_key,
+ database_id=database_id,
+ table_code=table_code,
+ action="create",
+ operation=operation,
+ )
+
+
+@public_router.patch("/databases/{database_id}/tables/{table_code}/records/{record_id}")
+async def public_edit_record(
+ database_id: str,
+ table_code: str,
+ record_id: str,
+ body: dict[str, Any],
+ request: Request,
+ authorization: str | None = Header(default=None),
+ x_api_key: str | None = Header(default=None, alias="X-API-Key"),
+):
+ async def operation(identity: dict[str, Any], policy: dict[str, Any] | None):
+ _check_writable_fields(policy, body)
+ return await update_record(
+ database_id,
+ table_code,
+ record_id,
+ body,
+ f"api:{identity['client_id']}",
+ row_filter=(policy or {}).get("row_filter") or {},
+ )
+
+ return await _public_operation(
+ request=request,
+ authorization=authorization,
+ x_api_key=x_api_key,
+ database_id=database_id,
+ table_code=table_code,
+ action="update",
+ operation=operation,
+ )
+
+
+@public_router.delete("/databases/{database_id}/tables/{table_code}/records/{record_id}")
+async def public_remove_record(
+ database_id: str,
+ table_code: str,
+ record_id: str,
+ request: Request,
+ authorization: str | None = Header(default=None),
+ x_api_key: str | None = Header(default=None, alias="X-API-Key"),
+):
+ async def operation(identity: dict[str, Any], policy: dict[str, Any] | None):
+ return await delete_record(
+ database_id,
+ table_code,
+ record_id,
+ f"api:{identity['client_id']}",
+ row_filter=(policy or {}).get("row_filter") or {},
+ )
+
+ return await _public_operation(
+ request=request,
+ authorization=authorization,
+ x_api_key=x_api_key,
+ database_id=database_id,
+ table_code=table_code,
+ action="delete",
+ operation=operation,
+ )
diff --git a/app/api/inventory.py b/app/api/inventory.py
deleted file mode 100644
index 3eda243..0000000
--- a/app/api/inventory.py
+++ /dev/null
@@ -1,110 +0,0 @@
-"""STEP 04 — Inventory Issues (知识盘点)."""
-from fastapi import APIRouter, Depends, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.contracts import IssueResolve
-from app.db import (
- list_inventory_issues,
- create_inventory_issue,
- resolve_inventory_issue,
- get_conn,
-)
-from app.project_context import ProjectContext, get_project_context
-
-router = APIRouter()
-
-
-@router.get("/inventory/issues")
-async def _list(
- status: str | None = None,
- severity: str | None = None,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- return await list_inventory_issues(
- context.tenant_id, context.project_id, status, severity
- )
-
-
-@router.post("/inventory/scan")
-async def scan(
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- """Run a daily inventory scan — detect PG-side data quality issues."""
- s = settings.db_schema
- tenant = context.tenant_id
- project = context.project_id
- issues_created = []
-
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- # Check for orphan nodes (candidates that point to non-existent batches)
- await cur.execute(
- f"SELECT COUNT(*) AS cnt FROM {s}.candidate_entities ce "
- "LEFT JOIN {s}.import_batches ib ON ce.batch_id=ib.batch_id "
- "WHERE ce.tenant_id=%s AND ce.project_id=%s AND ce.batch_id IS NOT NULL AND ib.batch_id IS NULL",
- (tenant, project),
- )
- orphan_count = (await cur.fetchone())["cnt"]
- if orphan_count > 0:
- issue = await create_inventory_issue({
- "tenant_id": tenant,
- "project_id": project,
- "issue_type": "orphan_node",
- "severity": "warning",
- "description": f"Found {orphan_count} candidate entities referencing non-existent batches",
- })
- issues_created.append(issue)
-
- # Check for stale data (not updated in 180 days)
- await cur.execute(
- f"SELECT COUNT(*) AS cnt FROM {s}.candidate_entities "
- "WHERE tenant_id=%s AND project_id=%s AND status='published' "
- "AND updated_at < now() - INTERVAL '180 days'",
- (tenant, project),
- )
- stale_count = (await cur.fetchone())["cnt"]
- if stale_count > 0:
- issue = await create_inventory_issue({
- "tenant_id": tenant,
- "project_id": project,
- "issue_type": "stale_data",
- "severity": "info",
- "description": f"Found {stale_count} entities not updated in 180+ days",
- })
- issues_created.append(issue)
-
- # Check for missing required fields (empty core fields in published entities)
- await cur.execute(
- f"SELECT COUNT(*) AS cnt FROM {s}.candidate_entities "
- "WHERE tenant_id=%s AND project_id=%s AND status='published' "
- "AND (payload->>'name' IS NULL OR payload->>'name'='')",
- (tenant, project),
- )
- missing_name = (await cur.fetchone())["cnt"]
- if missing_name > 0:
- issue = await create_inventory_issue({
- "tenant_id": tenant,
- "project_id": project,
- "issue_type": "missing_field",
- "severity": "blocker",
- "target_field": "name",
- "description": f"Found {missing_name} published entities with empty name",
- })
- issues_created.append(issue)
-
- return {"scanned": True, "issues_created": len(issues_created), "issues": issues_created}
-
-
-@router.post("/inventory/issues/{issue_id}/resolve")
-async def _resolve(issue_id: int, body: IssueResolve | None = None, user: CurrentUser = None):
- row = await resolve_inventory_issue(
- issue_id,
- resolved_by=user["username"] if user else "system",
- note=body.resolution_note if body else None,
- )
- if not row:
- raise HTTPException(404, "Issue not found")
- return row
diff --git a/app/api/manual_ingest.py b/app/api/manual_ingest.py
deleted file mode 100644
index 3724c70..0000000
--- a/app/api/manual_ingest.py
+++ /dev/null
@@ -1,4216 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import csv
-import hashlib
-import io
-import json
-import re
-import shutil
-import subprocess
-import time
-import uuid
-from difflib import SequenceMatcher
-from pathlib import Path
-from typing import Any
-
-from fastapi import APIRouter, File, HTTPException, UploadFile
-
-from app.agents.multi_extract import build_extract_pool, decide, fan_out
-from app.db import get_agent_settings
-from app.llm_client import LlmClient
-
-router = APIRouter()
-
-PROJECT_ROOT = Path(__file__).resolve().parents[2]
-MARKITDOWN_UPLOAD_ROOT = PROJECT_ROOT / "tmp" / "manual_ingest" / "markitdown"
-MARKITDOWN_MAX_UPLOAD_BYTES = 40 * 1024 * 1024
-MARKITDOWN_PREVIEW_MAX_BYTES = 5 * 1024 * 1024
-MARKITDOWN_SUPPORTED_EXTENSIONS = {
- ".txt",
- ".md",
- ".markdown",
- ".csv",
- ".json",
- ".xml",
- ".html",
- ".htm",
- ".pdf",
- ".docx",
- ".doc",
- ".pptx",
- ".ppt",
- ".xlsx",
- ".xls",
- ".zip",
- ".epub",
- ".jpg",
- ".jpeg",
- ".png",
- ".gif",
- ".wav",
- ".mp3",
-}
-
-SCENIC_SCHEMA_PATH = (
- Path(__file__).resolve().parents[2]
- / "docs"
- / "kg-redesign"
- / "scenic_spot_schema_v0_2.json"
-)
-_SCENIC_SCHEMA_CACHE: dict[str, Any] | None = None
-
-
-def _safe_upload_stem(filename: str) -> str:
- stem = Path(filename or "upload").stem
- cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", stem).strip("._-")
- return cleaned[:80] or "upload"
-
-
-def _relative_project_path(path: Path) -> str:
- try:
- return str(path.relative_to(PROJECT_ROOT))
- except ValueError:
- return str(path)
-
-
-def _resolve_markitdown_preview_path(path_value: str) -> Path:
- raw = str(path_value or "").strip()
- if not raw:
- raise HTTPException(status_code=400, detail="缺少 MarkItDown 文件路径")
- candidate = Path(raw)
- if not candidate.is_absolute():
- candidate = PROJECT_ROOT / candidate
- resolved = candidate.resolve()
- root = MARKITDOWN_UPLOAD_ROOT.resolve()
- if not resolved.is_relative_to(root):
- raise HTTPException(status_code=400, detail="只允许预览 MarkItDown 转换目录内的文件")
- if resolved.suffix.lower() not in {".md", ".markdown"}:
- raise HTTPException(status_code=400, detail="只能预览 Markdown 文件")
- if not resolved.exists() or not resolved.is_file():
- raise HTTPException(status_code=404, detail="MarkItDown 文件不存在")
- if resolved.stat().st_size > MARKITDOWN_PREVIEW_MAX_BYTES:
- raise HTTPException(status_code=413, detail="Markdown 文件超过 5MB,暂不在线预览")
- return resolved
-
-
-async def _save_markitdown_upload(upload: UploadFile) -> tuple[Path, int]:
- filename = Path(upload.filename or "").name
- ext = Path(filename).suffix.lower()
- if not filename:
- raise HTTPException(status_code=400, detail="请上传一个文件")
- if ext not in MARKITDOWN_SUPPORTED_EXTENSIONS:
- supported = "、".join(sorted(MARKITDOWN_SUPPORTED_EXTENSIONS))
- raise HTTPException(status_code=400, detail=f"暂不支持该文件格式:{ext or '无扩展名'}。支持:{supported}")
-
- MARKITDOWN_UPLOAD_ROOT.mkdir(parents=True, exist_ok=True)
- token = f"{int(time.time())}_{uuid.uuid4().hex[:10]}"
- input_path = MARKITDOWN_UPLOAD_ROOT / f"{token}_{_safe_upload_stem(filename)}{ext}"
- size = 0
- try:
- with input_path.open("wb") as out:
- while True:
- chunk = await upload.read(1024 * 1024)
- if not chunk:
- break
- size += len(chunk)
- if size > MARKITDOWN_MAX_UPLOAD_BYTES:
- raise HTTPException(status_code=413, detail="文件超过 40MB,无法转换")
- out.write(chunk)
- except Exception:
- input_path.unlink(missing_ok=True)
- raise
- finally:
- await upload.close()
-
- if size <= 0:
- input_path.unlink(missing_ok=True)
- raise HTTPException(status_code=400, detail="上传文件为空")
- return input_path, size
-
-
-def _write_markdown_artifact(input_path: Path, markdown: str) -> Path:
- markdown_path = input_path.with_name(f"{input_path.stem}.markitdown.md")
- markdown_path.write_text(markdown + "\n", encoding="utf-8")
- return markdown_path
-
-
-def _convert_pdf_with_pymupdf4llm(input_path: Path) -> str | None:
- """版式/表格感知的 PDF→Markdown;未安装、失败或为空时返回 None 以回退 MarkItDown。"""
- try:
- import pymupdf4llm
- except ImportError:
- return None
- try:
- markdown = pymupdf4llm.to_markdown(str(input_path))
- except Exception: # noqa: BLE001
- return None
- markdown = (markdown or "").strip()
- return markdown or None
-
-
-def _convert_xlsx_smart(input_path: Path) -> str | None:
- """合并单元格感知的 Excel→Markdown:
- - 普通合并单元格:左上角值向整块填充(消除 NaN,使每行自带上下文);
- - 整行合并(标题/须知):单独渲染成段落,不铺满所有列;
- 失败时返回 None 以回退 MarkItDown。"""
- try:
- import openpyxl
-
- workbook = openpyxl.load_workbook(str(input_path), data_only=True)
- except Exception: # noqa: BLE001
- return None
-
- def _cell(value: Any) -> str:
- if value is None:
- return ""
- return str(value).strip().replace("|", "\\|").replace("\n", "
")
-
- try:
- blocks: list[str] = []
- for ws in workbook.worksheets:
- max_row, max_col = ws.max_row or 0, ws.max_column or 0
- if not max_row or not max_col:
- continue
- grid = [[ws.cell(row=r, column=c).value for c in range(1, max_col + 1)]
- for r in range(1, max_row + 1)]
- # 所有合并单元格:左上角值填满整块(纵向合并→每行自带上下文;横向合并→整行同值)
- for mr in ws.merged_cells.ranges:
- value = ws.cell(row=mr.min_row, column=mr.min_col).value
- for r in range(mr.min_row, mr.max_row + 1):
- for c in range(mr.min_col, mr.max_col + 1):
- grid[r - 1][c - 1] = value
- rows = [[_cell(v) for v in row] for row in grid]
- keep = [c for c in range(max_col) if any(row[c] for row in rows)]
- rows = [[row[c] for c in keep] for row in rows]
-
- notes: list[str] = []
- body: list[list[str]] = []
- for row in rows:
- if not any(row):
- continue
- # 整行被同一个值铺满(横向合并的标题/须知)→ 收成一段,避免数十列重复
- if len(set(row)) == 1 and row[0]:
- if not notes or notes[-1] != row[0]: # 纵向合并的备注会跨多行 → 连续去重
- notes.append(row[0])
- else:
- body.append(row)
-
- lines = [f"## {ws.title}", ""]
- for text in notes:
- lines.append(text.replace("
", " ").strip())
- lines.append("")
- if body:
- width = max(len(r) for r in body)
- body = [r + [""] * (width - len(r)) for r in body]
- lines.append("| " + " | ".join(body[0]) + " |")
- lines.append("| " + " | ".join(["---"] * width) + " |")
- for row in body[1:]:
- lines.append("| " + " | ".join(row) + " |")
- lines.append("")
- blocks.append("\n".join(lines))
- except Exception: # noqa: BLE001
- return None
-
- result = "\n".join(blocks).strip()
- return result or None
-
-
-def _pdf_is_scanned(input_path: Path) -> bool:
- """扫描/图片 PDF 判别:取样多页仍几乎无可抽文字 → 扫描件(规则转换无效,需 VLM)。"""
- try:
- import fitz
-
- doc = fitz.open(str(input_path))
- text = "".join(doc[i].get_text() for i in range(min(doc.page_count, 12)))
- return len(text.strip()) < 200
- except Exception: # noqa: BLE001
- return False
-
-
-def _convert_docx_structured(input_path: Path) -> str | None:
- """DOCX→Markdown:按正文顺序保留 标题/段落/表格(比 MarkItDown 更完整保留表格)。失败返回 None。"""
- try:
- import docx
- from docx.oxml.ns import qn
- from docx.table import Table
- from docx.text.paragraph import Paragraph
-
- document = docx.Document(str(input_path))
- except Exception: # noqa: BLE001
- return None
- try:
- lines: list[str] = []
- for child in document.element.body.iterchildren():
- if child.tag == qn("w:p"):
- para = Paragraph(child, document)
- text = para.text.strip()
- if not text:
- continue
- style = (para.style.name or "").lower() if para.style else ""
- if style.startswith("heading"):
- match = re.search(r"(\d+)", style)
- lines.append("#" * min(int(match.group(1)) if match else 1, 6) + " " + text)
- else:
- lines.append(text)
- lines.append("")
- elif child.tag == qn("w:tbl"):
- table_rows = [[c.text.strip().replace("|", "\\|").replace("\n", "
") for c in row.cells]
- for row in Table(child, document).rows]
- table_rows = [r for r in table_rows if any(r)]
- if table_rows:
- width = max(len(r) for r in table_rows)
- table_rows = [r + [""] * (width - len(r)) for r in table_rows]
- lines.append("| " + " | ".join(table_rows[0]) + " |")
- lines.append("| " + " | ".join(["---"] * width) + " |")
- lines += ["| " + " | ".join(r) + " |" for r in table_rows[1:]]
- lines.append("")
- except Exception: # noqa: BLE001
- return None
- result = "\n".join(lines).strip()
- return result or None
-
-
-def _doc_via_olefile(input_path: Path) -> str | None:
- """textutil 读不了的损坏 .doc:从 WordDocument 流(UTF-16)捞中文正文,滤掉超链字段码。失败返回 None。"""
- try:
- import olefile
-
- if not olefile.isOleFile(str(input_path)):
- return None
- ole = olefile.OleFileIO(str(input_path))
- if not ole.exists("WordDocument"):
- return None
- text = ole.openstream("WordDocument").read().decode("utf-16-le", errors="ignore")
- except Exception: # noqa: BLE001
- return None
- out: list[str] = []
- seen: set[str] = set()
- for run in re.findall(r"[一-鿿 -〿-0-9A-Za-z,。:;、()·\-/~:%¥]{6,}", text):
- run = run.strip()
- if "HYPERLINK" in run or "ShowInnerLink" in run or "com/" in run or run.startswith("com"):
- continue
- if re.search(r"[一-鿿]", run) and run not in seen:
- seen.add(run)
- out.append(run)
- result = "\n".join(out).strip()
- return result or None
-
-
-def _convert_doc_robust(input_path: Path) -> str | None:
- """老 .doc:textutil 转 docx→python-docx(保表格);textutil 读不了 → olefile 直捞正文。"""
- conv = input_path.with_name(f"{input_path.stem}.__doc2docx.docx")
- try:
- completed = subprocess.run(
- ["textutil", "-convert", "docx", "-output", str(conv), str(input_path)],
- capture_output=True, text=True, timeout=120,
- )
- if completed.returncode == 0 and conv.exists():
- md = _convert_docx_structured(conv)
- if md and len(md.strip()) >= 30:
- return md
- except Exception: # noqa: BLE001
- pass
- finally:
- conv.unlink(missing_ok=True)
- return _doc_via_olefile(input_path)
-
-
-def _convert_path_with_markitdown(input_path: Path) -> tuple[str, Path, str]:
- ext = input_path.suffix.lower()
- # 结构化优先:PDF→pymupdf4llm、Excel→openpyxl、Word→python-docx(保表格)、老doc→textutil/olefile;失败回退 MarkItDown
- if ext == ".pdf":
- if _pdf_is_scanned(input_path):
- raise HTTPException(
- status_code=422,
- detail="此 PDF 为扫描/图片件,规则转换提取不到文字,需视觉模型(VLM)处理",
- )
- better = _convert_pdf_with_pymupdf4llm(input_path)
- if better:
- return better, _write_markdown_artifact(input_path, better), "pymupdf4llm"
- elif ext in {".xlsx", ".xlsm"}:
- better = _convert_xlsx_smart(input_path)
- if better:
- return better, _write_markdown_artifact(input_path, better), "openpyxl"
- elif ext == ".docx":
- better = _convert_docx_structured(input_path)
- if better:
- return better, _write_markdown_artifact(input_path, better), "python-docx"
- elif ext == ".doc":
- better = _convert_doc_robust(input_path)
- if better:
- return better, _write_markdown_artifact(input_path, better), "doc-robust"
-
- try:
- from markitdown import MarkItDown
- except ImportError as exc:
- raise HTTPException(
- status_code=503,
- detail="MarkItDown 未安装,请先执行:pip install -r requirements.txt",
- ) from exc
-
- converter = MarkItDown(enable_plugins=False)
- try:
- result = converter.convert(str(input_path))
- except Exception as exc: # noqa: BLE001
- if input_path.suffix.lower() == ".doc":
- markdown = _convert_legacy_doc_with_textutil(input_path, converter)
- markdown_path = input_path.with_name(f"{input_path.stem}.markitdown.md")
- markdown_path.write_text(markdown + "\n", encoding="utf-8")
- return markdown, markdown_path, "markitdown"
- raise
- markdown = str(
- getattr(result, "text_content", None)
- or getattr(result, "markdown", None)
- or ""
- ).strip()
- if not markdown and input_path.suffix.lower() == ".doc":
- markdown = _convert_legacy_doc_with_textutil(input_path, converter)
- if not markdown:
- raise HTTPException(status_code=422, detail="MarkItDown 未能从该文件中提取到可抽取文本")
-
- markdown_path = input_path.with_name(f"{input_path.stem}.markitdown.md")
- markdown_path.write_text(markdown + "\n", encoding="utf-8")
- return markdown, markdown_path, "markitdown"
-
-
-def _convert_legacy_doc_with_textutil(input_path: Path, converter: Any) -> str:
- textutil = shutil.which("textutil")
- if textutil:
- html_path = input_path.with_name(f"{input_path.stem}.textutil.html")
- try:
- completed = subprocess.run(
- [textutil, "-convert", "html", "-output", str(html_path), str(input_path)],
- check=False,
- capture_output=True,
- text=True,
- timeout=60,
- )
- except subprocess.TimeoutExpired as exc:
- raise HTTPException(status_code=422, detail=".doc 文件转换超时,请先另存为 .docx 后上传") from exc
-
- if completed.returncode != 0 or not html_path.exists():
- detail = (completed.stderr or completed.stdout or "").strip()
- raise HTTPException(
- status_code=422,
- detail=f".doc 文件转换失败,请先另存为 .docx 后上传。{detail[:160]}",
- )
- return _convert_intermediate_html_to_markdown(html_path, converter)
-
- office = shutil.which("soffice") or shutil.which("libreoffice")
- if office:
- out_dir = input_path.with_name(f"{input_path.stem}.doc-convert-{uuid.uuid4().hex[:8]}")
- out_dir.mkdir(parents=True, exist_ok=True)
- try:
- completed = subprocess.run(
- [office, "--headless", "--convert-to", "html", "--outdir", str(out_dir), str(input_path)],
- check=False,
- capture_output=True,
- text=True,
- timeout=90,
- )
- html_files = list(out_dir.glob("*.html")) + list(out_dir.glob("*.htm"))
- if completed.returncode != 0 or not html_files:
- detail = (completed.stderr or completed.stdout or "").strip()
- raise HTTPException(
- status_code=422,
- detail=f".doc 文件转换失败,请先另存为 .docx 后上传。{detail[:160]}",
- )
- return _convert_intermediate_html_to_markdown(html_files[0], converter, cleanup=False)
- except subprocess.TimeoutExpired as exc:
- raise HTTPException(status_code=422, detail=".doc 文件转换超时,请先另存为 .docx 后上传") from exc
- finally:
- shutil.rmtree(out_dir, ignore_errors=True)
-
- raise HTTPException(
- status_code=422,
- detail="老式 .doc 文件需要先转换为 .docx,或在服务器安装 LibreOffice/textutil 后再上传",
- )
-
-
-def _convert_intermediate_html_to_markdown(html_path: Path, converter: Any, cleanup: bool = True) -> str:
- try:
- result = converter.convert(str(html_path))
- markdown = str(
- getattr(result, "text_content", None)
- or getattr(result, "markdown", None)
- or ""
- ).strip()
- finally:
- if cleanup:
- html_path.unlink(missing_ok=True)
- if not markdown:
- raise HTTPException(
- status_code=422,
- detail=".doc 文件转换后未提取到文本,请先另存为 .docx 后上传",
- )
- return markdown
-
-KG_EXTRACT_SYS = """
-你是一个通用知识图谱深度抽取器。服务对象不是单一城市景点,而是任意行业材料:
-人工粘贴网页正文、CSV、表格、专家笔记、教育/城市/商业等业务文本。
-
-只基于输入原文抽取知识,不要编造。必须输出统一 KG Schema:
-{
- "entities": [],
- "events": [],
- "concepts": [],
- "relations": [],
- "statements": [],
- "schema_proposals": [],
- "evidence_links": [],
- "quality": {}
-}
-
-核心原则:
-1. 只抽取原文明确支持的事实,不能凭常识补充。
-2. 忽略目录编号、页码、脚注序号、纯数字、导航文字、无语义碎片。
-3. 目标是高召回知识图谱构建,不是摘要。不能只抽主干;必须尽量覆盖细粒度实体、子事件、属性、业务动作关系和跨句关系。
-4. Entity 是现实对象或业务对象:地点、区域、人物、机构、企业、系统、模块、设施、设备、传感器、岗位、数据集、指标、产品、课程等。附属机构、经办人、设备、企业、地点不能因为“次要”而漏掉。
-5. Event 必须是有时间或可被事件化描述的发生事实;不仅抽一级大事件,还要抽生命周期子事件,例如立项、内测、公测、终验、上线、运维启动、核查、整改、复核、验收、扣分、抢修、调度、结案、排查。
-6. Concept 是主题/类型/体验/领域概念,如“精细化治理”“闭环处置”“数据清洗”“实体消歧”“隐患分级管理”“信用考核机制”等,业务流程/治理规则/数据能力/模型能力都应抽取。
-7. Relation 是可复用的关系类型,优先用 LOCATED_IN、HAS_PART、HAS_EVENT、HAS_CONCEPT、NEARBY_ATTRACTION、ASSOCIATED_WITH_PERSON、FORMER_NAME、HAS_ADDRESS 等;也允许根据业务证据提出 proposal,如 LEAD_BY、CONSTRUCTED_BY、SUPPORTED_BY、DEVELOPED_BY、SUPERVISED_BY、MAINTAINED_BY、OWNED_BY、REPORTED_BY、TRIGGERED_BY、PARTICIPATED_IN、HANDLED_BY、CHECKED_BY、RECTIFIED_BY、VERIFIED_BY、SCORE_DEDUCT、HAS_FAULT、COOPERATED_WITH、BEFORE、AFTER、HAS_STATUS、HAS_METRIC、HAS_ATTR。
-8. Statement 是事实三元组,subject_ref/object_ref 应引用 temp_id;时间、数值、等级、状态、覆盖率、扣分、处置时长、电话、地址、指标等字面属性也要作为 object_kind="literal" 的 Statement 抽出。
-9. 每个候选必须带 source_spans,至少包含 quote;quote 必须能在原文中找到或高度近似。没有 quote 的候选视为无效。
-10. 不允许输出 HAS_00、HAS_13、HAS_、HAS_600 这类由编号生成的关系。
-11. temp_id 使用稳定简短英文前缀:ent_*, evt_*, cpt_*。
-12. 置信度要校准:1.0 只给原文直接明确且无歧义的事实;0.85-0.95 给清晰事实;0.70-0.84 给跨句或轻推理事实;不要所有候选都写 1.0。
-13. 业务字段必须优先写入 Entity.attributes,不要把产品ID、成人价、预算档、渠道、追单时间等字段升级为 relation 或 schema_proposals。
-14. 字段名必须固定:
- - Entity: temp_id/name/entity_type/aliases/description/attributes/spatial/source_spans/confidence
- - Event: temp_id/title/event_type/time_text/time_norm/description/participants/location_ref/source_spans/confidence
- - Concept: temp_id/name/concept_type/description/parent_concepts/source_spans/confidence
- - Relation: relation_type/source_type/target_type/description/inverse_relation/confidence
- - Statement: subject_ref/predicate/object_ref/object_kind/qualifiers/source_spans/confidence
-15. 数量策略:当前输入若包含 chunk_id/task_type,请对当前片段高召回抽取,建议 entities≤45、events≤35、concepts≤30、statements≤120;若是整篇短文,可到 entities≤80、events≤60、concepts≤45、statements≤180。不要为了整洁只保留主干。
-16. 每条 description 控制在 80 字以内;每条 source_spans.quote 控制在 90 字以内。
-17. source_spans 只需要输出 [{"quote":"原文证据"}],不要输出 start_char/end_char,后端会自动定位。
-18. quality 给出 coverage/main_risks/suggested_next_step,并明确是否可能漏掉细粒度事实。
-
-只输出合法 JSON,不要 markdown,不要解释。
-"""
-
-KG_DECIDE_SYS = """
-你是知识图谱抽取结果裁判。给你多个模型对同一人工材料的抽取结果。
-请合并去重并裁剪噪声,输出同一个 KG Schema JSON:
-entities/events/concepts/relations/statements/schema_proposals/evidence_links/quality。
-
-裁决规则:
-1. 保留证据 quote 能支撑的事实;删除目录编号、页码、纯数字、无语义碎片。
-2. 高召回优先,不要把细粒度实体、子事件、属性 Statement 裁掉;只删除重复、无证据、关系名脏或明显不属于知识图谱的候选。
-3. 多模型一致或证据清晰的候选优先;单模型但 quote 清晰的候选也保留,置信度可调低。
-4. Relation 必须语义明确并可复用,不得保留 HAS_00/HAS_13/HAS_ 这种脏关系。业务动作类关系可进入 schema_proposals。
-5. Event 节点必须保留 time_text;能规范化则写 time_norm。宏观事件和子事件都要保留,并用 BEFORE/AFTER 或 HAS_EVENT 组织。
-6. 不确定但有价值的新类型放 schema_proposals,不直接污染 core relations。
-7. 最终不设摘要式小上限;若结果过多,优先保留有明确时间、主体、动作、数值、地点、证据的候选。
-8. quality 写明 coverage/main_risks/suggested_next_step。
-只输出合法 JSON。
-"""
-
-KG_EXTRACT_COMPACT_SYS = """
-你是通用知识图谱信息抽取器。只根据原文抽取,不编造。
-必须只输出合法 JSON 对象,顶层键固定为:
-entities, events, concepts, relations, statements, schema_proposals, evidence_links, quality。
-
-抽取目标:
-- Entity:地点、区域、人物、机构、企业、系统、模块、设施、设备、指标、数据集等现实/业务对象。
-- Event:有时间或可事件化的事实,如建设、成立、上线、公布、列入、整改、核查、抢修、验收等。
-- Concept:主题、类型、体验、技术、流程、治理规则等抽象概念。
-- Relation:可复用关系类型,如 LOCATED_IN、HAS_PART、HAS_EVENT、HAS_CONCEPT、FORMER_NAME、HAS_ADDRESS、HAS_METRIC、HAS_STATUS、BEFORE、AFTER;新关系可放 proposal。
-- Statement:事实三元组。实体/事件/概念引用 temp_id;数值、面积、长度、等级、时间、别名等属性用 object_kind="literal"。
-- 如果输入 business_schema 是旅行社业务,优先使用 TourProduct、TourVariant、ItineraryDay、ScenicAttraction、HotelResource、RestaurantResource、VehicleService、TransferQuote、PolicyRule、SalesScript、CustomerLead、SalesChannel 等实体类型;优先使用 HAS_VARIANT、HAS_DAY、VISITS、STAYS_AT、MEALS_AT、USES_VEHICLE、HAS_POLICY、HAS_SCRIPT、MATCHES_PRODUCT、FROM_AREA、TO_AREA、FROM_SOURCE、HAS_OBJECTION、CAN_UPGRADE_TO、REPLACEMENT_FOR 等关系。
-- 旅行社产品ID、报价、预算档、渠道、出发日期、人数、酒店偏好、车辆偏好、追单时间等业务字段必须写入 Entity.attributes,不要升级成 relation 或 schema_proposals。
-
-字段要求:
-- entity: temp_id,name,entity_type,description,source_spans,confidence
-- event: temp_id,title,event_type,time_text,time_norm,description,participants,location_ref,source_spans,confidence
-- concept: temp_id,name,concept_type,description,source_spans,confidence
-- relation: relation_type,source_type,target_type,description,confidence
-- statement: subject_ref,predicate,object_ref,object_kind,qualifiers,source_spans,confidence
-- source_spans 只写 [{"quote":"原文证据"}],quote 必须来自原文,少而准。
-
-质量规则:
-1. 不输出 HAS_00、HAS_13、HAS_、纯数字、目录编号、脚注序号。
-2. 置信度要校准,不要全部 1.0。
-3. 对短文本要完整覆盖关键事实,但 JSON 必须紧凑;不要输出空 aliases/attributes/spatial。
-4. 输出中必须包含 Entity、Event、Concept、Relation、Statement 五类;确实没有时用空数组。
-"""
-
-
-def _schema_context_from_body(body: dict[str, Any]) -> dict[str, Any]:
- scene = str(body.get("business_scene") or "").strip()
- schema_model = str(body.get("schema_model") or "").strip()
- fields = body.get("schema_fields") if isinstance(body.get("schema_fields"), list) else []
- entity_types = body.get("schema_entity_types") if isinstance(body.get("schema_entity_types"), list) else []
- relations = body.get("schema_relations") if isinstance(body.get("schema_relations"), list) else []
- if not any([scene, schema_model, fields, entity_types, relations]):
- return {}
- return {
- "business_scene": scene,
- "schema_model": schema_model,
- "preferred_entity_types": [str(x) for x in entity_types if str(x).strip()],
- "preferred_fields": [str(x) for x in fields if str(x).strip()],
- "preferred_relations": [str(x) for x in relations if str(x).strip()],
- "schema_rule": (
- "优先按该业务场景 Schema 抽取字段、实体类型和关系。"
- "原文明确支持但不在 Schema 内的重要事实进入 schema_proposals,不要直接污染正式关系。"
- ),
- }
-
-
-def _select_manual_extract_model(
- extractors: list[tuple[str, LlmClient]],
- agg: tuple[str, LlmClient] | None,
- status_msg: str,
- selected_model_key: str,
-) -> tuple[list[tuple[str, LlmClient]], tuple[str, LlmClient] | None, str]:
- selected = (selected_model_key or "").strip()
- if not selected:
- raise HTTPException(status_code=400, detail="请选择一个知识大模型")
-
- for key, client in extractors:
- if key == selected:
- return [(key, client)], None, f"人工录入知识大模型={key}"
-
- if agg and agg[0] == selected:
- return [], agg, f"人工录入知识大模型={selected}"
-
- available = [key for key, _ in extractors]
- if agg:
- available.append(agg[0])
- raise HTTPException(
- status_code=400,
- detail=f"指定知识大模型不可用:{selected}。可用模型:{','.join(available) or '无'}",
- )
-
-
-KNOWN_FIELD_MAP: dict[str, tuple[str, str]] = {
- "中文名": ("HAS_CHINESE_NAME", "中文名"),
- "名称": ("HAS_NAME", "名称"),
- "地理位置": ("HAS_ADDRESS", "地理位置"),
- "地址": ("HAS_ADDRESS", "地址"),
- "气候条件": ("HAS_CLIMATE", "气候条件"),
- "开放时间": ("HAS_OPENING_HOURS", "开放时间"),
- "营业时间": ("HAS_OPENING_HOURS", "营业时间"),
- "景点级别": ("HAS_SCENIC_LEVEL", "景点级别"),
- "门票价格": ("HAS_TICKET_PRICE", "门票价格"),
- "票价": ("HAS_TICKET_PRICE", "票价"),
- "占地面积": ("HAS_AREA", "占地面积"),
- "美誉": ("HAS_REPUTATION", "美誉"),
- "所属国家": ("LOCATED_IN", "所属国家"),
- "所属城市": ("LOCATED_IN", "所属城市"),
- "所属地区": ("LOCATED_IN", "所属地区"),
- "行政区": ("LOCATED_IN", "行政区"),
- "建议游玩时长": ("HAS_SUGGESTED_DURATION", "建议游玩时长"),
- "适宜游玩季节": ("HAS_BEST_SEASON", "适宜游玩季节"),
- "著名景点": ("HAS_PART", "著名景点"),
- "主要景点": ("HAS_PART", "主要景点"),
- "邻近景点": ("NEARBY_ATTRACTION", "邻近景点"),
- "附近景点": ("NEARBY_ATTRACTION", "附近景点"),
-}
-
-CONCEPT_RULES: list[tuple[str, str, str, list[str]]] = [
- ("cpt_history_culture", "历史文化", "Topic", ["历史", "文化", "名人", "纪念", "古镇", "遗址"]),
- ("cpt_night_tour", "夜游", "Experience", ["夜游", "夜间", "灯光", "夜景", "晚上"]),
- ("cpt_natural_ecology", "自然生态", "Scene", ["生态", "自然", "植物", "湿地", "山水", "溪", "河"]),
- ("cpt_transport_access", "交通可达", "Utility", ["公交", "地铁", "站", "线路", "交通", "停车"]),
- ("cpt_family_leisure", "亲子休闲", "Audience", ["亲子", "家庭", "休闲", "露营", "游乐"]),
- ("cpt_food_experience", "美食体验", "Experience", ["餐饮", "美食", "小吃", "酸汤", "老店"]),
-]
-
-RELATION_ALIASES: dict[str, str] = {
- "BELONGS_TO": "LOCATED_IN",
- "BELONGS_TO_CITY": "LOCATED_IN",
- "BELONGS_TO_COUNTRY": "LOCATED_IN",
- "HAS_LOCATION": "LOCATED_IN",
- "HAS_ATTRACTION": "HAS_PART",
- "HAS_SCENIC_SPOT": "HAS_PART",
- "HAS_FAMOUS_SPOT": "HAS_PART",
- "HAS_OPEN_TIME": "HAS_OPENING_HOURS",
- "HAS_OPEN_HOURS": "HAS_OPENING_HOURS",
- "HAS_SUGGEST_PLAY_TIME": "HAS_SUGGESTED_DURATION",
- "HAS_SUITABLE_PLAY_SEASON": "HAS_BEST_SEASON",
- "HAS_SEASON": "HAS_BEST_SEASON",
- "HAS_LEVEL": "HAS_SCENIC_LEVEL",
- "HAS_RATING_LEVEL": "HAS_SCENIC_LEVEL",
- "HAS_PRICE": "HAS_TICKET_PRICE",
- "HAS_TICKET": "HAS_TICKET_PRICE",
- "MANAGED_BY": "SUPERVISED_BY",
- "OPERATED_BY": "MAINTAINED_BY",
- "MAINTAINED_BY_UNIT": "MAINTAINED_BY",
- "PARTICIPATES_IN": "PARTICIPATED_IN",
- "BELONGS_TO_ORG": "AFFILIATED_WITH",
- "MEMBER_OF": "AFFILIATED_WITH",
- "OCCURS_AT": "OCCURRED_AT",
- "HAS_LOCATION_EVENT": "OCCURRED_AT",
- "REPORTS_TO": "REPORTED_TO",
- "INCLUDES_ACTION": "HAS_ACTION",
- "INVOLVES_DAMAGE": "HAS_DAMAGE",
- "INVOLVES_VIOLATION": "HAS_VIOLATION",
- "INCLUDES_ATTRACTION": "VISITS",
- "VISITS_ATTRACTION": "VISITS",
- "MATCH_PRODUCT": "MATCHES_PRODUCT",
- "MATCHED_PRODUCT": "MATCHES_PRODUCT",
- "SOURCE_CHANNEL": "FROM_SOURCE",
- "LEAD_SOURCE_CHANNEL": "FROM_SOURCE",
- "OBJECTION": "HAS_OBJECTION",
- "UPGRADES_TO": "CAN_UPGRADE_TO",
- "UPGRADE_TO": "CAN_UPGRADE_TO",
-}
-
-CORE_RELATIONS: set[str] = {
- "LOCATED_IN",
- "HAS_PART",
- "HAS_EVENT",
- "HAS_CONCEPT",
- "NEARBY_ATTRACTION",
- "NEAR_TRANSIT",
- "ASSOCIATED_WITH_PERSON",
- "FORMER_NAME",
- "HAS_ADDRESS",
- "HAS_CLIMATE",
- "HAS_OPENING_HOURS",
- "HAS_SCENIC_LEVEL",
- "HAS_TICKET_PRICE",
- "HAS_AREA",
- "HAS_REPUTATION",
- "HAS_SUGGESTED_DURATION",
- "HAS_BEST_SEASON",
- "HAS_NAME",
- "HAS_CHINESE_NAME",
- # Generic operational KG relations used by city-governance extraction.
- "HAS_ID",
- "HAS_TIME",
- "HAS_STATUS",
- "HAS_METRIC",
- "HAS_METRIC_LIMIT",
- "HAS_DURATION",
- "HAS_RESULT",
- "HAS_RISK_LEVEL",
- "HAS_DEADLINE",
- "HAS_ACTION",
- "HAS_DAMAGE",
- "HAS_VIOLATION",
- "OCCURRED_AT",
- "DETECTED_BY",
- "DISPATCHED_TO",
- "PARTICIPATED_IN",
- "HANDLED_BY",
- "CHECKED_BY",
- "RECTIFIED_BY",
- "VERIFIED_BY",
- "BEFORE",
- "AFTER",
- "SCORE_DEDUCT",
- "HAS_FAULT",
- "RESPONSIBLE_FOR",
- "MAINTAINED_BY",
- "SUPERVISED_BY",
- "OWNED_BY",
- "REPORTED_TO",
- "TRIGGERED_BY",
- "ISSUED_BY",
- "AFFILIATED_WITH",
- # Travel agency business scene relations.
- "HAS_VARIANT",
- "HAS_DAY",
- "VISITS",
- "STAYS_AT",
- "MEALS_AT",
- "USES_VEHICLE",
- "HAS_POLICY",
- "HAS_SCRIPT",
- "MATCHES_PRODUCT",
- "FROM_AREA",
- "TO_AREA",
- "FROM_SOURCE",
- "HAS_OBJECTION",
- "CAN_UPGRADE_TO",
- "REPLACEMENT_FOR",
-}
-
-TRAVEL_CORE_RELATIONS: set[str] = {
- "HAS_VARIANT",
- "HAS_DAY",
- "VISITS",
- "STAYS_AT",
- "MEALS_AT",
- "USES_VEHICLE",
- "HAS_POLICY",
- "HAS_SCRIPT",
- "MATCHES_PRODUCT",
- "FROM_AREA",
- "TO_AREA",
- "FROM_SOURCE",
- "HAS_OBJECTION",
- "CAN_UPGRADE_TO",
- "REPLACEMENT_FOR",
-}
-
-TRAVEL_FALLBACK_MODEL_PRIORITY: tuple[str, ...] = (
- "deepseek",
- "qwen",
- "doubao",
- "custom1",
- "custom4",
- "custom3",
-)
-
-TRAVEL_CORE_ENTITY_TYPES: set[str] = {
- "TourProduct",
- "TourVariant",
- "ItineraryDay",
- "ScenicAttraction",
- "HotelResource",
- "RestaurantResource",
- "VehicleService",
- "TransferQuote",
- "PolicyRule",
- "SalesScript",
- "CustomerLead",
- "SalesChannel",
- "Area",
-}
-
-TRAVEL_LIST_FIELDS: set[str] = {
- "optional_items",
- "season_tags",
- "sales_channel",
- "self_pay_items",
- "selling_points",
- "features",
- "applicable_products",
- "signature_dishes",
- "meal_scene",
- "service_scope",
- "intent_tags",
- "required_customer_fields",
- "must_visit",
- "avoid_notes",
- "objection_tags",
-}
-
-TRAVEL_NUMBER_FIELDS: set[str] = {
- "duration_days",
- "duration_nights",
- "capacity_min",
- "capacity_max",
- "adult_price",
- "child_price",
- "ticket_exempt_price",
- "single_room_supplement",
- "inner_transport_fee",
- "day_index",
- "seat_count",
- "price_per_trip",
- "party_size",
- "adult_count",
- "child_count",
- "senior_count",
- "room_count",
-}
-
-TRAVEL_FIELD_ALIASES: dict[str, dict[str, str]] = {
- "PRODUCT_ID": {"TourProduct": "product_id"},
- "HAS_PRODUCT_ID": {"TourProduct": "product_id"},
- "SHORT_NAME": {"TourProduct": "short_name"},
- "SERIES": {"TourProduct": "product_family"},
- "HAS_SERIES": {"TourProduct": "product_family"},
- "PRODUCT_SERIES": {"TourProduct": "product_family"},
- "PRODUCT_TYPE": {"TourProduct": "product_type"},
- "HAS_TYPE": {"TourProduct": "product_type"},
- "TYPE": {"TourProduct": "product_type", "ScenicAttraction": "attraction_type", "PolicyRule": "rule_type"},
- "TOUR_DAYS": {"TourProduct": "duration_days"},
- "DURATION_DAYS": {"TourProduct": "duration_days", "CustomerLead": "duration_days"},
- "HOTEL_NIGHTS": {"TourProduct": "duration_nights"},
- "GROUP_TYPE": {"TourProduct": "group_mode"},
- "HAS_GROUP_TYPE": {"TourProduct": "group_mode"},
- "GROUP_MODE": {"TourProduct": "group_mode"},
- "MIN_PAX": {"TourProduct": "capacity_min"},
- "MAX_PAX": {"TourProduct": "capacity_max"},
- "DEFAULT_HOTEL_GRADE": {"TourProduct": "hotel_grade"},
- "MEAL_STANDARD": {"TourProduct": "meal_standard"},
- "SERVICE_COMMITMENT": {"TourProduct": "service_promise"},
- "SERVICE_PROMISE": {"TourProduct": "service_promise"},
- "NO_SHOPPING": {"TourProduct": "service_promise"},
- "INCLUDES_SUMMARY": {"TourProduct": "included_summary"},
- "INCLUDED_SUMMARY": {"TourProduct": "included_summary"},
- "EXCLUDES_SUMMARY": {"TourProduct": "excluded_summary"},
- "EXCLUDED_SUMMARY": {"TourProduct": "excluded_summary"},
- "VOLUNTARY_ITEMS": {"TourProduct": "optional_items"},
- "OPTIONAL_ITEMS": {"TourProduct": "optional_items", "ScenicAttraction": "optional_items"},
- "BOOKING_NOTES": {"TourProduct": "booking_notes"},
- "REFUND_POLICY": {"TourProduct": "refund_policy"},
- "RISK_TIPS": {"TourProduct": "risk_notes"},
- "RISK_NOTES": {"TourProduct": "risk_notes"},
- "SEASON_TAG": {"TourProduct": "season_tags"},
- "SEASON_TAGS": {"TourProduct": "season_tags"},
- "SALES_CHANNELS": {"TourProduct": "sales_channel"},
- "SOURCE_FILE": {"*": "source_file"},
- "VARIANT_ID": {"TourVariant": "variant_id"},
- "HAS_VARIANT_ID": {"TourVariant": "variant_id"},
- "VARIANT_NAME": {"TourVariant": "variant_name"},
- "SEASON": {"TourVariant": "season"},
- "DATE_RANGE": {"TourVariant": "date_range"},
- "PAX_RANGE": {"TourVariant": "group_size_band"},
- "GROUP_SIZE_BAND": {"TourVariant": "group_size_band"},
- "ROOM_TYPE": {"TourVariant": "room_type"},
- "HOTEL_GRADE": {"TourVariant": "hotel_grade", "HotelResource": "hotel_grade"},
- "CAR_TYPE": {"TourVariant": "vehicle_type", "TransferQuote": "vehicle_type"},
- "VEHICLE_TYPE": {"TourVariant": "vehicle_type", "TransferQuote": "vehicle_type", "VehicleService": "vehicle_type"},
- "ADULT_PRICE": {"TourVariant": "adult_price"},
- "CHILD_PRICE": {"TourVariant": "child_price"},
- "FREE_TICKET_PRICE": {"TourVariant": "ticket_exempt_price"},
- "TICKET_EXEMPT_PRICE": {"TourVariant": "ticket_exempt_price"},
- "SINGLE_ROOM_DIFF": {"TourVariant": "single_room_supplement"},
- "SINGLE_ROOM_SUPPLEMENT": {"TourVariant": "single_room_supplement"},
- "SIGHTSEEING_FEE": {"TourVariant": "inner_transport_fee"},
- "INNER_TRANSPORT_FEE": {"TourVariant": "inner_transport_fee"},
- "CURRENCY": {"TourVariant": "currency", "TransferQuote": "currency"},
- "SELL_RULE": {"TourVariant": "availability_rule"},
- "AVAILABILITY_RULE": {"TourVariant": "availability_rule"},
- "DAY_NUMBER": {"ItineraryDay": "day_index"},
- "DAY_INDEX": {"ItineraryDay": "day_index"},
- "DEPARTURE_CITY": {"ItineraryDay": "start_city"},
- "START_CITY": {"ItineraryDay": "start_city"},
- "ARRIVAL_CITY": {"ItineraryDay": "end_city"},
- "END_CITY": {"ItineraryDay": "end_city"},
- "SUMMARY": {"ItineraryDay": "route_summary"},
- "ROUTE_SUMMARY": {"ItineraryDay": "route_summary"},
- "TRANSPORT_SUMMARY": {"ItineraryDay": "transport_summary"},
- "MEALS": {"ItineraryDay": "meals"},
- "ACCOMMODATION": {"ItineraryDay": "accommodation"},
- "TRAVEL_TIME_HINT": {"ItineraryDay": "travel_time_hint"},
- "SELF_PAY_ITEMS": {"ItineraryDay": "self_pay_items"},
- "TIPS": {"ItineraryDay": "tips"},
- "LOCATION": {"ScenicAttraction": "city", "HotelResource": "region", "RestaurantResource": "region"},
- "CITY": {"ScenicAttraction": "city"},
- "ATTRACTION_TYPE": {"ScenicAttraction": "attraction_type"},
- "TICKET_POLICY": {"ScenicAttraction": "ticket_policy"},
- "WEATHER_RISK": {"ScenicAttraction": "weather_risk", "TourProduct": "risk_notes"},
- "CABLE_CAR_UNCERTAIN": {"ScenicAttraction": "weather_risk"},
- "REPLACEMENT_ATTRACTION": {"ScenicAttraction": "fallback_attraction"},
- "SELLING_POINTS": {"ScenicAttraction": "selling_points"},
- "GRADE": {"ScenicAttraction": "rating_level", "HotelResource": "hotel_grade"},
- "RATING_LEVEL": {"ScenicAttraction": "rating_level"},
- "REGION": {"HotelResource": "region", "RestaurantResource": "region"},
- "FEATURES": {"HotelResource": "features"},
- "LISTED_PRICE": {"HotelResource": "listed_price_text"},
- "OFF_SEASON_PRICE": {"HotelResource": "off_season_price_text"},
- "PEAK_SEASON_PRICE": {"HotelResource": "peak_season_price_text"},
- "APPLICABLE_PRODUCTS": {"HotelResource": "applicable_products"},
- "CONTACT_NAME": {"HotelResource": "contact_name", "RestaurantResource": "contact_name"},
- "CONTACT_PHONE": {"HotelResource": "contact_phone", "RestaurantResource": "contact_phone"},
- "PER_CAPITA": {"RestaurantResource": "per_capita_price_text"},
- "SIGNATURE_DISHES": {"RestaurantResource": "signature_dishes"},
- "MEAL_SCENE": {"RestaurantResource": "meal_scene"},
- "SEAT_COUNT": {"VehicleService": "seat_count"},
- "COMFORT_LEVEL": {"VehicleService": "comfort_level"},
- "SEAT_LAYOUT": {"VehicleService": "seat_layout"},
- "LUGGAGE_LIMIT": {"VehicleService": "luggage_limit"},
- "SERVICE_SCOPE": {"VehicleService": "service_scope"},
- "SUPPLIER_NAME": {"VehicleService": "supplier_name"},
- "NOTES": {"VehicleService": "notes"},
- "ORIGIN": {"TransferQuote": "origin_text"},
- "ORIGIN_TEXT": {"TransferQuote": "origin_text"},
- "DESTINATION": {"TransferQuote": "destination_text"},
- "DESTINATION_TEXT": {"TransferQuote": "destination_text"},
- "PRICE_PER_TRIP": {"TransferQuote": "price_per_trip"},
- "QUOTE_UNIT": {"TransferQuote": "quote_unit"},
- "SERVICE_NOTE": {"TransferQuote": "quote_notes"},
- "QUOTE_NOTES": {"TransferQuote": "quote_notes"},
- "HAS_DESCRIPTION": {"TransferQuote": "quote_notes", "PolicyRule": "rule_text"},
- "HAS_UNIT": {"TransferQuote": "quote_unit"},
- "RULE_ID": {"PolicyRule": "rule_id"},
- "RULE_TYPE": {"PolicyRule": "rule_type"},
- "APPLIES_TO": {"PolicyRule": "applies_to"},
- "RULE_TEXT": {"PolicyRule": "rule_text"},
- "HAS_RULE_CONTENT": {"PolicyRule": "rule_text"},
- "TRIGGER_CONDITION": {"PolicyRule": "rule_text"},
- "REPLACEMENT_ACTION": {"PolicyRule": "rule_text"},
- "SEVERITY": {"PolicyRule": "severity"},
- "CHANNEL": {"SalesScript": "channel"},
- "FUNNEL_STAGE": {"SalesScript": "funnel_stage"},
- "TRIGGER_SCENARIO": {"SalesScript": "trigger_scenario"},
- "MESSAGE_TEMPLATE": {"SalesScript": "message_template"},
- "INTENT_TAGS": {"SalesScript": "intent_tags"},
- "REQUIRED_CUSTOMER_FIELDS": {"SalesScript": "required_customer_fields"},
- "COMPLIANCE_NOTES": {"SalesScript": "compliance_notes"},
- "FOLLOW_UP_TIMING": {"SalesScript": "follow_up_timing"},
- "LEAD_ID": {"CustomerLead": "lead_id"},
- "LEAD_SOURCE": {"CustomerLead": "lead_source"},
- "TRAVEL_MONTH": {"CustomerLead": "travel_month"},
- "START_DATE": {"CustomerLead": "start_date"},
- "HAS_DEPARTURE_DATE": {"CustomerLead": "start_date"},
- "HAS_DEPARTURE_MONTH": {"CustomerLead": "travel_month"},
- "TRAVEL_DATE": {"CustomerLead": "start_date"},
- "TRAVEL_DAYS": {"CustomerLead": "duration_days", "TourProduct": "duration_days"},
- "PARTY_SIZE": {"CustomerLead": "party_size"},
- "ADULT_COUNT": {"CustomerLead": "adult_count"},
- "CHILD_COUNT": {"CustomerLead": "child_count"},
- "SENIOR_COUNT": {"CustomerLead": "senior_count"},
- "ROOM_COUNT": {"CustomerLead": "room_count"},
- "HOTEL_PREFERENCE": {"CustomerLead": "hotel_preference"},
- "HAS_HOTEL_GRADE": {"CustomerLead": "hotel_preference", "TourProduct": "hotel_grade"},
- "VEHICLE_PREFERENCE": {"CustomerLead": "vehicle_preference"},
- "BUDGET_LEVEL": {"CustomerLead": "budget_level"},
- "HAS_BUDGET_PER_PERSON": {"CustomerLead": "budget_level"},
- "MUST_VISIT": {"CustomerLead": "must_visit"},
- "HAS_MUST_GO": {"CustomerLead": "must_visit"},
- "AVOID_NOTES": {"CustomerLead": "avoid_notes"},
- "HAS_AVOID": {"CustomerLead": "avoid_notes"},
- "OBJECTION_TAGS": {"CustomerLead": "objection_tags"},
- "LEAD_STATUS": {"CustomerLead": "lead_status"},
- "HAS_PRICE_ADULT": {"TourVariant": "adult_price"},
- "HAS_PRICE_CHILD": {"TourVariant": "child_price"},
- "HAS_SINGLE_ROOM_SUPPLEMENT": {"TourVariant": "single_room_supplement"},
- "HAS_SINGLE_ROOM_DIFFERENCE": {"TourVariant": "single_room_supplement"},
- "HAS_DATE_RANGE": {"TourVariant": "date_range"},
-}
-
-TRAVEL_IGNORED_PROPOSALS: set[str] = {
- "ProductSeries",
- "RoomType",
- "HotelGrade",
- "PolicyExclusion",
- "ConfirmationRule",
- "AdvanceConfirmation",
- "ReservationRule",
- "DepartureRule",
- "ExclusionRule",
- "HAS_FEATURE",
- "HAS_SERVICE_SCOPE",
- "HAS_SUPPLIER",
- "PRODUCT_HAS_CONCEPT",
- "VARIANT_HAS_CONCEPT",
- "HAS_DATE_RANGE",
- "INCLUDES",
- "HAS_MUST_GO",
- "HAS_BUDGET_PER_PERSON",
- "HAS_AVOID",
- "HAS_PRICE_ADULT",
- "HAS_PRICE_CHILD",
- "HAS_DEPARTURE_DATE",
- "HAS_DEPARTURE_MONTH",
- "HAS_HOTEL_GRADE",
- "TRIGGER_CONDITION",
- "REPLACEMENT_ACTION",
- "REPLACEMENT_FOR",
-}
-
-EVENT_VERBS = (
- "建", "建成", "辟建", "开放", "成立", "更名", "改名", "划为",
- "到访", "游", "举办", "重建", "修建", "竣工", "开通", "迁入",
-)
-
-EVENT_TYPE_RULES: list[tuple[str, str]] = [
- ("成立|开放|开馆|开园", "OpeningEvent"),
- ("改名|更名|命名", "RenamingEvent"),
- ("重建|修建|建成|辟建|建设|竣工", "ConstructionEvent"),
- ("到访|游|参观|考察", "VisitEvent"),
- ("划为|划归|管理|管辖", "ManagementChangeEvent"),
- ("开通|通车|运营", "TransitOpeningEvent"),
- ("举办|活动|节", "ActivityEvent"),
-]
-
-
-def _stable_id(prefix: str, value: str) -> str:
- token = re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "_", value or "").strip("_")
- token = token[:24] or hashlib.md5(value.encode("utf-8")).hexdigest()[:10]
- digest = hashlib.md5(value.encode("utf-8")).hexdigest()[:6]
- return f"{prefix}_{token}_{digest}"
-
-
-def _span(text: str, quote: str, evidence_id: str = "manual_input_1") -> list[dict[str, Any]]:
- quote = (quote or "").strip()
- if not quote:
- return []
- start = text.find(quote)
- if start < 0:
- compact = re.sub(r"\s+", "", text)
- compact_quote = re.sub(r"\s+", "", quote)
- compact_start = compact.find(compact_quote)
- start = None if compact_start < 0 else compact_start
- else:
- start = int(start)
- end = None if start is None else start + len(quote)
- return [{"evidence_id": evidence_id, "quote": quote, "start_char": start, "end_char": end}]
-
-
-def _split_items(value: str) -> list[str]:
- return [x.strip(" ;;,,、") for x in re.split(r"[、,,;;/]+", value or "") if x.strip(" ;;,,、")]
-
-
-def _valid_relation_name(value: str) -> bool:
- rel = (value or "").strip().upper()
- if not re.match(r"^[A-Z][A-Z0-9_]{2,}$", rel):
- return False
- if re.fullmatch(r"HAS_?\d.*", rel) or rel in {"HAS", "HAS_"}:
- return False
- return True
-
-
-def _canonical_relation(value: str) -> str:
- rel = (value or "").strip().upper()
- return RELATION_ALIASES.get(rel, rel)
-
-
-def _relation_schema_status(relation: str) -> str:
- return "core" if relation in CORE_RELATIONS else "proposal"
-
-
-def _travel_field_for_predicate(predicate: str, subject_type: str) -> str | None:
- aliases = TRAVEL_FIELD_ALIASES.get((predicate or "").strip().upper())
- if not aliases:
- return None
- return aliases.get(subject_type) or aliases.get("*")
-
-
-def _is_travel_field_predicate(predicate: str) -> bool:
- return (predicate or "").strip().upper() in TRAVEL_FIELD_ALIASES
-
-
-def _travel_relation_alias(predicate: str, source_type: str, target_type: str, target_kind: str = "entity") -> str:
- rel = (predicate or "").strip().upper()
- source_type = source_type or "Entity"
- target_type = target_type or ("Literal" if target_kind == "literal" else "Entity")
- if rel == "INCLUDES" and target_type == "ScenicAttraction":
- return "VISITS"
- if rel in {"DEFAULT_VEHICLE", "AVAILABLE_UPGRADE"} and target_type == "VehicleService":
- if source_type in {"TourProduct", "TourVariant", "TransferQuote"}:
- return "USES_VEHICLE"
- if source_type == "VehicleService":
- return "CAN_UPGRADE_TO"
- return rel
-
-
-def _coerce_travel_attr(field_name: str, value: Any) -> Any:
- if value is None:
- return ""
- if isinstance(value, (list, dict)):
- return value
- text = str(value).strip()
- if not text:
- return ""
- if field_name in TRAVEL_LIST_FIELDS:
- return _split_items(text) or [text]
- if field_name in TRAVEL_NUMBER_FIELDS:
- match = re.search(r"-?\d+(?:\.\d+)?", text.replace(",", ""))
- if match:
- number = float(match.group(0))
- return int(number) if number.is_integer() else number
- return text
-
-
-def _merge_travel_attr(existing: Any, incoming: Any) -> Any:
- if incoming in ("", None, [], {}):
- return existing
- if existing in ("", None, [], {}):
- return incoming
- if existing == incoming:
- return existing
- if isinstance(existing, list):
- next_values = incoming if isinstance(incoming, list) else [incoming]
- merged = list(existing)
- for item in next_values:
- if item not in merged:
- merged.append(item)
- return merged
- if isinstance(incoming, list):
- return _merge_travel_attr([existing], incoming)
- return existing
-
-
-def _set_travel_attribute(entity: dict[str, Any] | None, field_name: str | None, value: Any) -> None:
- if not entity or not field_name:
- return
- attrs = entity.get("attributes")
- if not isinstance(attrs, dict):
- attrs = {}
- entity["attributes"] = attrs
- coerced = _coerce_travel_attr(field_name, value)
- attrs[field_name] = _merge_travel_attr(attrs.get(field_name), coerced)
-
-
-def _statement_object_value(stmt: dict[str, Any], id_label: dict[str, str]) -> str:
- obj = str(stmt.get("object_ref") or "").strip()
- if obj in id_label:
- return str(id_label[obj])
- return obj
-
-
-def _augment_meals_at_from_restaurant_quotes(
- entities: list[dict[str, Any]],
- statements: list[dict[str, Any]],
-) -> None:
- day_by_name = {
- str(ent.get("name") or "").strip(): ent
- for ent in entities
- if ent.get("entity_type") == "ItineraryDay" and str(ent.get("name") or "").strip()
- }
- if not day_by_name:
- return
- seen = {(s.get("subject_ref"), s.get("predicate"), s.get("object_ref")) for s in statements}
- for restaurant in entities:
- if restaurant.get("entity_type") != "RestaurantResource":
- continue
- quote = ""
- if restaurant.get("source_spans"):
- quote = str((restaurant.get("source_spans") or [{}])[0].get("quote") or "")
- for day_name, day in day_by_name.items():
- if day_name and day_name in quote:
- sig = (day["temp_id"], "MEALS_AT", restaurant["temp_id"])
- if sig in seen:
- break
- statements.append({
- "subject_ref": day["temp_id"],
- "predicate": "MEALS_AT",
- "object_ref": restaurant["temp_id"],
- "object_kind": "entity",
- "qualifiers": {"schema_status": "core", "raw_predicate": "MEALS_AT", "derived_from": "restaurant_quote"},
- "source_spans": restaurant.get("source_spans") or [],
- "confidence": _safe_confidence(restaurant.get("confidence"), 0.82),
- })
- seen.add(sig)
- break
-
-
-TRAVEL_HEADER_FIELDS: dict[str, dict[str, str]] = {
- "TourProduct": {
- "产品ID": "product_id", "product_id": "product_id",
- "产品名称": "name", "短名称": "short_name", "产品系列": "product_family", "产品类型": "product_type",
- "行程天数": "duration_days", "住宿晚数": "duration_nights", "成团方式": "group_mode",
- "capacity_min": "capacity_min", "最小人数": "capacity_min", "capacity_max": "capacity_max", "最大人数": "capacity_max",
- "车型/座椅布局": "vehicle_layout", "默认酒店等级": "hotel_grade", "餐标": "meal_standard",
- "服务承诺": "service_promise", "费用包含摘要": "included_summary", "费用不含摘要": "excluded_summary",
- "自愿消费项目": "optional_items", "报名须知": "booking_notes", "退费政策": "refund_policy",
- "风险提示": "risk_notes", "季节标签": "season_tags", "销售渠道": "sales_channel", "来源文件": "source_file",
- },
- "TourVariant": {
- "variant_id": "variant_id", "变体ID": "variant_id", "变体名称": "variant_name",
- "适用季节/团期": "season", "适用日期范围": "date_range", "人数档": "group_size_band",
- "房型/客栈类型": "room_type", "酒店等级": "hotel_grade", "车辆类型": "vehicle_type",
- "成人价/结算价": "adult_price", "成人价": "adult_price", "儿童价": "child_price",
- "免票人群价": "ticket_exempt_price", "单房差": "single_room_supplement",
- "景交/小交通费": "inner_transport_fee", "币种": "currency", "可售规则": "availability_rule",
- },
- "HotelResource": {
- "resource_id": "hotel_id", "酒店ID": "hotel_id", "名称": "name", "酒店名称": "name",
- "等级/类型": "hotel_grade", "酒店等级": "hotel_grade", "区域": "region", "地址": "address",
- "特点/特色": "features", "特点": "features", "挂牌价": "listed_price_text",
- "淡季价": "off_season_price_text", "旺季价": "peak_season_price_text",
- "适用产品": "applicable_products", "联系人": "contact_name", "联系电话": "contact_phone", "备注": "features",
- },
- "RestaurantResource": {
- "resource_id": "restaurant_id", "餐厅ID": "restaurant_id", "名称": "name", "餐厅名称": "name",
- "区域": "region", "地址": "address", "人均/座位数": "per_capita_price_text", "人均": "per_capita_price_text",
- "特点/特色": "signature_dishes", "特色菜品": "signature_dishes", "适用产品": "meal_scene",
- "适用场景": "meal_scene", "联系人": "contact_name", "联系电话": "contact_phone", "备注": "meal_scene",
- },
- "VehicleService": {
- "resource_id": "vehicle_service_id", "车辆服务ID": "vehicle_service_id", "名称": "vehicle_type",
- "等级/类型": "comfort_level", "车型": "vehicle_type", "车辆类型": "vehicle_type",
- "人均/座位数": "seat_count", "座位数": "seat_count", "特点/特色": "seat_layout",
- "座椅布局": "seat_layout", "行李限制": "luggage_limit", "服务范围": "service_scope",
- "供应方": "supplier_name", "备注": "notes",
- },
- "TransferQuote": {
- "transfer_quote_id": "transfer_quote_id", "接送报价ID": "transfer_quote_id",
- "出发地": "origin_text", "目的地": "destination_text", "车型": "vehicle_type",
- "车辆类型": "vehicle_type", "每趟价格": "price_per_trip", "币种": "currency",
- "计价单位": "quote_unit", "报价说明": "quote_notes",
- },
- "PolicyRule": {
- "rule_id": "rule_id", "规则ID": "rule_id", "规则类型": "rule_type",
- "适用对象": "applies_to", "规则内容": "rule_text", "重要性": "severity", "来源文件": "source_file",
- },
- "SalesScript": {
- "script_id": "script_id", "话术ID": "script_id", "channel": "channel", "渠道": "channel",
- "funnel_stage": "funnel_stage", "转化阶段": "funnel_stage", "trigger_scenario": "trigger_scenario",
- "触发场景": "trigger_scenario", "message_template": "message_template", "话术模板": "message_template",
- "intent_tags": "intent_tags", "意图标签": "intent_tags", "required_customer_fields": "required_customer_fields",
- "需确认客资字段": "required_customer_fields", "compliance_notes": "compliance_notes",
- "合规提示": "compliance_notes", "follow_up_timing": "follow_up_timing", "追单时间": "follow_up_timing",
- },
- "CustomerLead": {
- "lead_id": "lead_id", "线索ID": "lead_id", "lead_source": "lead_source", "来源渠道": "lead_source",
- "travel_month": "travel_month", "出行月份": "travel_month", "start_date": "start_date", "出发日期": "start_date",
- "duration_days": "duration_days", "计划天数": "duration_days", "party_size": "party_size", "总人数": "party_size",
- "adult_count": "adult_count", "成人数": "adult_count", "child_count": "child_count", "儿童数": "child_count",
- "senior_count": "senior_count", "老人数": "senior_count", "room_count": "room_count", "房间数": "room_count",
- "hotel_preference": "hotel_preference", "住宿偏好": "hotel_preference",
- "vehicle_preference": "vehicle_preference", "用车偏好": "vehicle_preference",
- "budget_level": "budget_level", "预算档": "budget_level", "must_visit": "must_visit", "必去景点": "must_visit",
- "avoid_notes": "avoid_notes", "避让需求": "avoid_notes", "objection_tags": "objection_tags",
- "异议标签": "objection_tags", "lead_status": "lead_status", "线索状态": "lead_status",
- },
-}
-
-
-def _header_key(value: str) -> str:
- return re.sub(r"[\s:://()()]+", "", str(value or "")).lower()
-
-
-TRAVEL_HEADER_LOOKUP: dict[str, dict[str, str]] = {
- etype: {_header_key(label): field for label, field in mapping.items()}
- for etype, mapping in TRAVEL_HEADER_FIELDS.items()
-}
-
-
-def _row_match_score(entity: dict[str, Any], row: dict[str, Any]) -> int:
- name = str(entity.get("name") or "").strip()
- etype = str(entity.get("entity_type") or "")
- attrs = entity.get("attributes") if isinstance(entity.get("attributes"), dict) else {}
- candidates = [name, str(entity.get("temp_id") or ""), *[str(v) for v in attrs.values() if isinstance(v, (str, int, float))]]
- row_values = [str(v or "").strip() for v in row.values()]
- score = 0
- for candidate in candidates:
- if not candidate:
- continue
- if candidate in row_values:
- score += 4
- elif any(len(candidate) >= 4 and candidate in value for value in row_values):
- score += 2
- if etype == "TourProduct" and any(row.get(k) == name for k in ("产品名称", "name")):
- score += 6
- if etype == "TourVariant" and any(row.get(k) == name for k in ("变体名称", "variant_name", "variant_id")):
- score += 6
- if etype in {"HotelResource", "RestaurantResource", "VehicleService"} and any(row.get(k) == name for k in ("名称", "resource_id")):
- score += 6
- return score
-
-
-def _apply_row_attributes(entity: dict[str, Any], row: dict[str, Any]) -> None:
- etype = str(entity.get("entity_type") or "")
- lookup = TRAVEL_HEADER_LOOKUP.get(etype) or {}
- for header, raw_value in row.items():
- value = str(raw_value or "").strip()
- if not value:
- continue
- field = lookup.get(_header_key(header))
- if not field or field == "name":
- continue
- _set_travel_attribute(entity, field, value)
-
-
-def _enrich_travel_attributes_from_csv(text: str, entities: list[dict[str, Any]]) -> None:
- if "," not in text or "\n" not in text:
- return
- try:
- rows = list(csv.DictReader(io.StringIO(text)))
- except Exception:
- return
- if not rows or not rows[0]:
- return
- for entity in entities:
- etype = entity.get("entity_type")
- if etype not in TRAVEL_HEADER_LOOKUP:
- continue
- best = max(rows, key=lambda row: _row_match_score(entity, row), default=None)
- if best and _row_match_score(entity, best) >= 2:
- _apply_row_attributes(entity, best)
-
-
-def _enrich_travel_attributes_from_json(text: str, entities: list[dict[str, Any]]) -> None:
- try:
- data = json.loads(text)
- except Exception:
- return
- if not isinstance(data, dict):
- return
- rows_by_type = {
- "SalesScript": data.get("sales_scripts") if isinstance(data.get("sales_scripts"), list) else [],
- "CustomerLead": data.get("customer_leads") if isinstance(data.get("customer_leads"), list) else [],
- }
- for entity in entities:
- rows = rows_by_type.get(str(entity.get("entity_type") or ""))
- if not rows:
- continue
- best = max((row for row in rows if isinstance(row, dict)), key=lambda row: _row_match_score(entity, row), default=None)
- if best and _row_match_score(entity, best) >= 2:
- _apply_row_attributes(entity, best)
-
-
-def _travel_code_from_entity(entity: dict[str, Any], fields: tuple[str, ...], pattern: str) -> str:
- attrs = entity.get("attributes") if isinstance(entity.get("attributes"), dict) else {}
- for field in fields:
- value = str(attrs.get(field) or "").strip()
- if value:
- return value
- name = str(entity.get("name") or "")
- match = re.search(pattern, name)
- return match.group(0) if match else name.strip()
-
-
-def _enrich_travel_attributes_from_label_lines(text: str, entities: list[dict[str, Any]]) -> None:
- product_entities = [e for e in entities if e.get("entity_type") == "TourProduct"]
- if len(product_entities) == 1:
- product = product_entities[0]
- for line in text.splitlines()[:80]:
- if ":" not in line:
- continue
- label, value = line.split(":", 1)
- label = label.strip(" #*-")
- field = TRAVEL_HEADER_LOOKUP["TourProduct"].get(_header_key(label))
- if field and field != "name":
- _set_travel_attribute(product, field, value.strip())
-
- for entity in entities:
- etype = str(entity.get("entity_type") or "")
- if etype == "TourProduct":
- product_id = _travel_code_from_entity(entity, ("product_id",), r"TAP-[A-Z0-9-]+")
- if product_id:
- _set_travel_attribute(entity, "product_id", product_id)
- continue
-
- if etype == "TourVariant":
- variant_id = _travel_code_from_entity(entity, ("variant_id",), r"VAR-[A-Z0-9-]+")
- if variant_id:
- _set_travel_attribute(entity, "variant_id", variant_id)
- match = re.search(rf"{re.escape(variant_id)}[^。]*单房差为([0-9.]+元/人)", text) if variant_id else None
- if match:
- _set_travel_attribute(entity, "single_room_supplement", match.group(1))
- continue
-
- if etype == "VehicleService":
- vehicle_id = _travel_code_from_entity(entity, ("vehicle_service_id",), r"VEH-[A-Z0-9-]+")
- if vehicle_id:
- _set_travel_attribute(entity, "vehicle_service_id", vehicle_id)
- continue
-
- if etype == "PolicyRule":
- rule_id = _travel_code_from_entity(entity, ("rule_id",), r"RULE-[A-Z0-9-]+")
- if rule_id:
- _set_travel_attribute(entity, "rule_id", rule_id)
- pattern = rf"业务规则\s*{re.escape(rule_id)}:(?P.*?)(?:\n\n业务规则|\Z)" if rule_id else ""
- match = re.search(pattern, text, flags=re.S) if pattern else None
- if not match:
- continue
- body = re.sub(r"\s+", " ", match.group("body"))
- for field, regex in {
- "rule_type": r"规则类型为([^,。]+)",
- "applies_to": r"适用对象为([^。]+)",
- "rule_text": r"规则内容[::](.*?)(?:重要性为|产品\s*TAP-|变体\s*VAR-|$)",
- "severity": r"重要性为([^,。]+)",
- }.items():
- hit = re.search(regex, body)
- if hit:
- _set_travel_attribute(entity, field, hit.group(1).strip(" 。;;"))
- continue
-
- if etype != "TransferQuote":
- continue
- quote_id = _travel_code_from_entity(entity, ("transfer_quote_id",), r"TQ-[A-Z0-9-]+")
- if not quote_id:
- continue
- _set_travel_attribute(entity, "transfer_quote_id", quote_id)
- pattern = rf"接送报价\s*{re.escape(quote_id)}:(?P.*?)(?:\n\n接送报价|\n\n业务规则|\Z)"
- match = re.search(pattern, text, flags=re.S)
- if not match:
- continue
- body = re.sub(r"\s+", " ", match.group("body"))
- for field, regex in {
- "origin_text": r"出发地为([^,。]+)",
- "destination_text": r"到达地为([^,。]+)",
- "vehicle_type": r"车辆类型为([^,。]+)",
- "price_per_trip": r"每趟价格为([0-9.]+)",
- "currency": r"币种为([^,。]+)",
- "quote_unit": r"计价单位为([^,。]+)",
- "quote_notes": r"报价说明:([^。]+)",
- }.items():
- hit = re.search(regex, body)
- if hit:
- _set_travel_attribute(entity, field, hit.group(1).strip())
-
-
-def _enrich_travel_attributes(text: str, entities: list[dict[str, Any]]) -> None:
- _enrich_travel_attributes_from_csv(text, entities)
- _enrich_travel_attributes_from_json(text, entities)
- _enrich_travel_attributes_from_label_lines(text, entities)
-
-
-def _entity_match_terms(entity: dict[str, Any], id_fields: tuple[str, ...] = ()) -> list[str]:
- terms = [str(entity.get("name") or "").strip()]
- attrs = entity.get("attributes") if isinstance(entity.get("attributes"), dict) else {}
- for field in id_fields:
- value = attrs.get(field)
- if value:
- terms.append(str(value).strip())
- return [term for term in terms if term]
-
-
-def _find_nearby_quote(text: str, terms: list[str], radius: int = 700) -> str:
- positions = [text.find(term) for term in terms if term and text.find(term) >= 0]
- if not positions:
- return ""
- start = max(0, min(positions) - radius // 2)
- end = min(len(text), max(positions) + radius)
- return re.sub(r"\s+", " ", text[start:end])[:120]
-
-
-def _add_travel_statement(
- statements: list[dict[str, Any]],
- source_text: str,
- subject: str,
- predicate: str,
- obj: str,
- kind: str,
- quote: str,
- confidence: float = 0.82,
-) -> None:
- if not subject or not obj:
- return
- sig = (subject, predicate, obj)
- if any((s.get("subject_ref"), s.get("predicate"), s.get("object_ref")) == sig for s in statements):
- return
- statements.append({
- "subject_ref": subject,
- "predicate": predicate,
- "object_ref": obj,
- "object_kind": kind,
- "qualifiers": {"schema_status": _relation_schema_status(predicate), "raw_predicate": predicate, "derived_from": "travel_context"},
- "source_spans": _span(source_text, quote) if quote else [],
- "confidence": confidence,
- })
-
-
-def _augment_travel_context_relations(text: str, entities: list[dict[str, Any]], statements: list[dict[str, Any]]) -> None:
- products = [e for e in entities if e.get("entity_type") == "TourProduct"]
- leads = [e for e in entities if e.get("entity_type") == "CustomerLead"]
- channels = [e for e in entities if e.get("entity_type") == "SalesChannel"]
-
- for lead in leads:
- lead_terms = _entity_match_terms(lead, ("lead_id",))
- lead_positions = [text.find(term) for term in lead_terms if term and text.find(term) >= 0]
- if not lead_positions:
- continue
- center = min(lead_positions)
- window = text[max(0, center - 900): min(len(text), center + 900)]
- if "抖音" in window:
- _set_travel_attribute(lead, "lead_source", "抖音线索")
- elif "微信" in window:
- _set_travel_attribute(lead, "lead_source", "微信私域")
-
- for product in products:
- product_terms = _entity_match_terms(product, ("product_id",))
- if any(term and term in window for term in product_terms):
- quote = _find_nearby_quote(text, lead_terms + product_terms)
- _add_travel_statement(statements, text, lead["temp_id"], "MATCHES_PRODUCT", product["temp_id"], "entity", quote, 0.84)
-
- source_value = str((lead.get("attributes") or {}).get("lead_source") or "")
- for channel in channels:
- channel_terms = _entity_match_terms(channel, ("channel_id",))
- if source_value and any(term and term in source_value for term in channel_terms):
- quote = _find_nearby_quote(text, lead_terms + channel_terms)
- _add_travel_statement(statements, text, lead["temp_id"], "FROM_SOURCE", channel["temp_id"], "entity", quote, 0.8)
-
-
-def _is_travel_schema_context(schema_context: dict[str, Any] | None) -> bool:
- if not isinstance(schema_context, dict) or not schema_context:
- return False
- scene = str(schema_context.get("business_scene") or "").lower()
- model = str(schema_context.get("schema_model") or "").lower()
- preferred_types = {
- str(item)
- for item in schema_context.get("preferred_entity_types") or []
- if str(item).strip()
- }
- if any(token in scene or token in model for token in ("旅行社", "旅游", "travel_agency", "travel")):
- return True
- return bool(preferred_types & TRAVEL_CORE_ENTITY_TYPES)
-
-
-def _travel_expected_relations(text: str) -> set[str]:
- clean = text or ""
- expected: set[str] = set()
- checks: list[tuple[str, str]] = [
- (r"报价变体|拥有报价变体|VAR-[A-Z0-9-]+|成人价|儿童价|单房差", "HAS_VARIANT"),
- (r"每日行程|包含每日行程|DAY-[A-Z0-9-]+|第\s*\d+\s*天", "HAS_DAY"),
- (r"行程\s*`?DAY-[^`,。]*`?\s*游览景点|当天游览|游览景点", "VISITS"),
- (r"入住酒店|住宿:|住宿为", "STAYS_AT"),
- (r"用餐餐厅|餐厅\s*`|适用场景为.*餐|特色菜品", "MEALS_AT"),
- (r"使用车辆服务|车辆服务是|车辆服务\s*VEH-|使用车辆服务\s*`?VEH-", "USES_VEHICLE"),
- (r"适用业务规则|适用规则|业务规则\s*RULE-|RULE-[A-Z0-9-]+", "HAS_POLICY"),
- (r"使用销售话术|sales_scripts|话术模板|话术\s*`?SCRIPT-", "HAS_SCRIPT"),
- (r"匹配产品|matched_product_id|更适合\s*TAP-|产品ID是|适合.*产品", "MATCHES_PRODUCT"),
- (r"来源渠道|lead_source|来源抖音|抖音线索|微信私域|OTA咨询|门店咨询", "FROM_SOURCE"),
- (r"异议标签|预算异议|担心|预算敏感|不想|疑虑", "HAS_OBJECTION"),
- (r"可升级|升级\s*2\+1|升级保姆车|CAN_UPGRADE", "CAN_UPGRADE_TO"),
- (r"出发地为|从区域|出发区域", "FROM_AREA"),
- (r"到达地为|到达区域|目的地", "TO_AREA"),
- (r"替换为|替换景点|REPLACEMENT", "REPLACEMENT_FOR"),
- ]
- for pattern, relation in checks:
- if re.search(pattern, clean, flags=re.I):
- expected.add(relation)
- return expected
-
-
-def _travel_result_signals(result: dict[str, Any] | None) -> dict[str, Any]:
- if not result:
- return {
- "entity_types": set(),
- "relations": set(),
- "attribute_count": 0,
- "core_statement_count": 0,
- }
- entity_types = {
- str(entity.get("entity_type") or "")
- for entity in result.get("entities") or []
- if isinstance(entity, dict)
- }
- attribute_count = 0
- for entity in result.get("entities") or []:
- if not isinstance(entity, dict) or not isinstance(entity.get("attributes"), dict):
- continue
- attribute_count += sum(
- 1 for value in entity["attributes"].values()
- if value not in ("", None, [], {})
- )
- statement_relations = {
- str(stmt.get("predicate") or "").upper()
- for stmt in result.get("statements") or []
- if isinstance(stmt, dict)
- }
- schema_relations = {
- str(rel.get("relation_type") or "").upper()
- for rel in result.get("relations") or []
- if isinstance(rel, dict)
- }
- relations = statement_relations | schema_relations
- return {
- "entity_types": entity_types,
- "relations": relations,
- "attribute_count": attribute_count,
- "core_statement_count": sum(1 for rel in statement_relations if rel in TRAVEL_CORE_RELATIONS),
- }
-
-
-def _travel_quality_issues(
- result: dict[str, Any] | None,
- text: str,
- schema_context: dict[str, Any] | None,
-) -> list[str]:
- if not _is_travel_schema_context(schema_context):
- return []
- expected = _travel_expected_relations(text)
- if not expected:
- return []
- signals = _travel_result_signals(result)
- present_expected = expected & signals["relations"]
- issues: list[str] = []
- if not (signals["entity_types"] & TRAVEL_CORE_ENTITY_TYPES):
- issues.append("未识别旅行社核心实体类型")
- required = 1 if len(expected) <= 2 else max(2, (len(expected) + 2) // 3)
- if len(present_expected) < required:
- issues.append(
- "旅行社核心关系覆盖不足:"
- f"expected={','.join(sorted(expected))}; "
- f"present={','.join(sorted(present_expected)) or '-'}"
- )
- field_markers = (
- "产品ID", "成人价", "儿童价", "预算", "来源渠道", "出发日期",
- "总人数", "房间数", "车辆偏好", "酒店偏好", "每趟价格",
- )
- if any(marker in (text or "") for marker in field_markers) and int(signals["attribute_count"] or 0) < 3:
- issues.append(f"旅行社字段属性覆盖不足:attributes={signals['attribute_count']}")
- if int(signals["core_statement_count"] or 0) == 0:
- issues.append("未产出旅行社核心 Statement")
- return issues
-
-
-def _needs_travel_quality_retry(
- result: dict[str, Any] | None,
- text: str,
- schema_context: dict[str, Any] | None,
-) -> bool:
- return bool(_travel_quality_issues(result, text, schema_context))
-
-
-def _annotate_travel_quality(
- result: dict[str, Any] | None,
- text: str,
- schema_context: dict[str, Any] | None,
-) -> None:
- if not result:
- return
- issues = _travel_quality_issues(result, text, schema_context)
- if not issues:
- return
- summary = result.setdefault("summary", {})
- summary["travel_quality_warnings"] = issues
-
-
-def _choose_travel_fallback_model_key(
- selected_model_key: str,
- extractors: list[tuple[str, LlmClient]],
- agg: tuple[str, LlmClient] | None,
-) -> str:
- selected = (selected_model_key or "").strip()
- available = {key for key, _ in extractors}
- if agg:
- available.add(agg[0])
- for key in TRAVEL_FALLBACK_MODEL_PRIORITY:
- if key in available and key != selected:
- return key
- return next((key for key in sorted(available) if key != selected), "")
-
-
-def _http_exception_text(exc: HTTPException | None) -> str:
- if not exc:
- return ""
- detail = exc.detail
- if isinstance(detail, (dict, list)):
- return json.dumps(detail, ensure_ascii=False)
- return str(detail or "")
-
-
-def _is_transient_llm_exception(exc: HTTPException | None) -> bool:
- text = _http_exception_text(exc).lower()
- return any(token in text for token in (
- "ssl",
- "eof",
- "timeout",
- "timed out",
- "connection",
- "reset",
- "temporarily",
- "502",
- "503",
- "504",
- ))
-
-
-def _safe_confidence(value: Any, default: float = 0.78) -> float:
- try:
- score = float(value)
- except Exception:
- score = default
- return round(max(0.0, min(1.0, score)), 2)
-
-
-def _normalize_time(text: str) -> str:
- text = text or ""
- date = re.search(r"(\d{3,4})年(?:(\d{1,2})月)?(?:(\d{1,2})日)?", text)
- if date:
- year, month, day = date.groups()
- if month and day:
- return f"{year}-{int(month):02d}-{int(day):02d}"
- if month:
- return f"{year}-{int(month):02d}"
- return year
- return ""
-
-
-def _event_type(sentence: str) -> str:
- for pattern, event_type in EVENT_TYPE_RULES:
- if re.search(pattern, sentence):
- return event_type
- return "Event"
-
-
-def _sentence_candidates(text: str) -> list[str]:
- parts = re.split(r"(?<=[。!?!?;;])|\n+", text)
- return [p.strip() for p in parts if len(p.strip()) >= 8]
-
-
-def _parse_key_values(text: str) -> list[tuple[str, str, str]]:
- rows: list[tuple[str, str, str]] = []
- seen: set[tuple[str, str]] = set()
- for raw_line in text.splitlines():
- line = raw_line.strip().strip("|")
- if not line:
- continue
- if "\t" in line:
- cells = [c.strip() for c in line.split("\t") if c.strip()]
- if len(cells) >= 2:
- key, value = cells[0], " ".join(cells[1:])
- else:
- continue
- elif ":" in line or ":" in line:
- key, value = re.split(r"[::]", line, maxsplit=1)
- key, value = key.strip(), value.strip()
- elif "|" in line:
- cells = [c.strip() for c in line.split("|") if c.strip()]
- if len(cells) >= 2:
- key, value = cells[0], " ".join(cells[1:])
- else:
- continue
- else:
- continue
- if not key or not value or (key, value) in seen:
- continue
- seen.add((key, value))
- rows.append((key, value, raw_line.strip()))
-
- if rows:
- return rows
-
- sample = text[:4096]
- try:
- dialect = csv.Sniffer().sniff(sample)
- reader = csv.DictReader(io.StringIO(text), dialect=dialect)
- for idx, row in enumerate(reader):
- for key, value in row.items():
- if key and value:
- raw = f"第{idx + 1}行 {key}: {value}"
- rows.append((str(key).strip(), str(value).strip(), raw))
- except Exception:
- pass
- return rows
-
-
-def _root_name(text: str, kvs: list[tuple[str, str, str]], explicit: str = "") -> str:
- if explicit.strip():
- return explicit.strip()
- for key, value, _ in kvs:
- if key in {"中文名", "名称", "name", "Name", "标题", "景点名称"} and value.strip():
- return _split_items(value)[0] if _split_items(value) else value.strip()
- first = next((line.strip() for line in text.splitlines() if line.strip()), "")
- return first[:32] or "人工录入知识"
-
-
-def _entity_type_for_key(key: str, value: str) -> str:
- if key in {"所属城市", "所属地区", "行政区", "所属国家"}:
- return "Area"
- if "景点" in key:
- return "ScenicSpot"
- if any(word in value for word in ("山", "河", "湖", "溪", "湿地", "洲")):
- return "NaturalFeature"
- if any(word in value for word in ("站", "公交", "地铁")):
- return "TransitStop"
- return "Place"
-
-
-def _confidence_from_span(spans: list[dict[str, Any]], base: float = 0.82) -> float:
- if not spans:
- return round(base - 0.08, 2)
- start = spans[0].get("start_char")
- return round(base if isinstance(start, int) else base - 0.04, 2)
-
-
-def extract_manual_kg(text: str, source_name: str, source_url: str, root_entity: str = "") -> dict[str, Any]:
- clean_text = (text or "").strip()
- if len(clean_text) < 4:
- raise HTTPException(status_code=400, detail="请输入或上传需要抽取的人工材料")
-
- kvs = _parse_key_values(clean_text)
- root_name = _root_name(clean_text, kvs, root_entity)
- root_id = _stable_id("ent", root_name)
- source_name = source_name.strip() or "人工录入"
- source_url = source_url.strip()
-
- entities: list[dict[str, Any]] = [{
- "temp_id": root_id,
- "name": root_name,
- "entity_type": "Place",
- "aliases": [],
- "description": f"人工材料中识别出的核心对象:{root_name}",
- "attributes": {},
- "spatial": {},
- "source_spans": _span(clean_text, root_name),
- "confidence": 0.9,
- }]
- entity_by_name = {root_name: root_id}
- concepts: list[dict[str, Any]] = []
- events: list[dict[str, Any]] = []
- statements: list[dict[str, Any]] = []
- relation_types: dict[str, tuple[str, str, str]] = {}
- evidence_links: list[dict[str, Any]] = []
-
- def add_entity(name: str, entity_type: str, quote: str, description: str = "") -> str:
- name = name.strip()
- if not name:
- return root_id
- if name in entity_by_name:
- return entity_by_name[name]
- eid = _stable_id("ent", name)
- spans = _span(clean_text, quote or name)
- entity_by_name[name] = eid
- entities.append({
- "temp_id": eid,
- "name": name,
- "entity_type": entity_type,
- "aliases": [],
- "description": description or f"从人工材料中识别出的{entity_type}实体。",
- "attributes": {},
- "spatial": {},
- "source_spans": spans,
- "confidence": _confidence_from_span(spans, 0.84),
- })
- evidence_links.append({
- "target_ref": eid,
- "evidence_id": "manual_input_1",
- "support_type": "supports",
- "quote": spans[0]["quote"] if spans else name,
- })
- return eid
-
- def add_statement(subject: str, predicate: str, obj: str, kind: str, quote: str, confidence: float = 0.84) -> None:
- predicate = _canonical_relation(predicate)
- spans = _span(clean_text, quote)
- statements.append({
- "subject_ref": subject,
- "predicate": predicate,
- "object_ref": obj,
- "object_kind": kind,
- "qualifiers": {"source_name": source_name, "source_url": source_url},
- "source_spans": spans,
- "confidence": _confidence_from_span(spans, confidence),
- })
- relation_types.setdefault(predicate, ("Entity", kind.title(), f"人工材料中抽取的 {predicate} 关系"))
- evidence_links.append({
- "target_ref": f"{subject}:{predicate}:{obj}",
- "evidence_id": "manual_input_1",
- "support_type": "supports",
- "quote": spans[0]["quote"] if spans else quote,
- })
-
- for key, value, raw in kvs:
- normalized_key = key.replace(" ", "")
- field = KNOWN_FIELD_MAP.get(normalized_key)
- if not field:
- continue
- predicate, _ = field
- if predicate in {"HAS_PART", "NEARBY_ATTRACTION", "LOCATED_IN"}:
- for item in _split_items(value):
- etype = _entity_type_for_key(normalized_key, item)
- target_id = add_entity(item, etype, raw, f"{root_name} 相关实体:{item}")
- add_statement(root_id, predicate, target_id, "entity", raw)
- else:
- add_statement(root_id, predicate, value, "literal", raw)
-
- for temp_id, name, concept_type, words in CONCEPT_RULES:
- matched = next((w for w in words if w in clean_text), "")
- if not matched:
- continue
- spans = _span(clean_text, matched)
- cid = temp_id
- concepts.append({
- "temp_id": cid,
- "name": name,
- "concept_type": concept_type,
- "description": f"材料中出现“{matched}”等表达,可归入{name}概念。",
- "parent_concepts": [],
- "source_spans": spans,
- "confidence": _confidence_from_span(spans, 0.78),
- })
- add_statement(root_id, "HAS_CONCEPT", cid, "concept", spans[0]["quote"] if spans else matched, 0.78)
- relation_types.setdefault("HAS_CONCEPT", ("Entity", "Concept", "地点或对象具有某类主题、体验或业务概念"))
-
- seen_events: set[str] = set()
- for sentence in _sentence_candidates(clean_text):
- if not re.search(r"\d{3,4}年", sentence):
- continue
- if not any(v in sentence for v in EVENT_VERBS):
- continue
- title = re.sub(r"^[,。;;、\s]+|[,。;;、\s]+$", "", sentence)
- title = re.sub(r"\[[^\]]+\]", "", title)
- title = title[:42]
- if title in seen_events:
- continue
- seen_events.add(title)
- eid = _stable_id("evt", title)
- time_text_match = re.search(r"\d{3,4}年(?:\d{1,2}月)?(?:\d{1,2}日)?", sentence)
- time_text = time_text_match.group(0) if time_text_match else ""
- spans = _span(clean_text, sentence)
- event_type = _event_type(sentence)
- events.append({
- "temp_id": eid,
- "title": title,
- "event_type": event_type,
- "time_text": time_text,
- "time_norm": _normalize_time(time_text),
- "description": sentence,
- "participants": [],
- "location_ref": root_id,
- "source_spans": spans,
- "confidence": _confidence_from_span(spans, 0.82),
- })
- add_statement(root_id, "HAS_EVENT", eid, "event", sentence, 0.82)
- relation_types.setdefault("HAS_EVENT", ("Entity", "Event", "地点或对象关联历史、建设、运营、管理等事件"))
-
- relations = [
- {
- "relation_type": rel,
- "source_type": source_type,
- "target_type": target_type,
- "description": desc,
- "inverse_relation": None,
- "schema_status": _relation_schema_status(rel),
- "confidence": 0.82,
- }
- for rel, (source_type, target_type, desc) in sorted(relation_types.items())
- ]
-
- schema_proposals = []
- for rel, (_, target_type, desc) in sorted(relation_types.items()):
- if _relation_schema_status(rel) == "proposal":
- schema_proposals.append({
- "proposal_type": "relation_type",
- "name": rel,
- "source_type": "Entity",
- "target_type": target_type,
- "value_type": None,
- "reason": desc,
- "examples": [s["source_spans"][0]["quote"] for s in statements if s["predicate"] == rel and s.get("source_spans")][:2],
- "confidence": 0.78,
- })
-
- result = {
- "source": {
- "source_name": source_name,
- "source_url": source_url,
- "input_chars": len(clean_text),
- "input_format": "manual",
- },
- "summary": {
- "entities": len(entities),
- "events": len(events),
- "concepts": len(concepts),
- "relations": len(relations),
- "statements": len(statements),
- "evidence_links": len(evidence_links),
- "method": "fast_rule_kg_preview_v1",
- "pipeline_status": "规则兜底预览:只抽已知字段、明确事件和少量概念,不代表最终多模型抽取质量",
- },
- "entities": entities,
- "events": events,
- "concepts": concepts,
- "relations": relations,
- "statements": statements,
- "schema_proposals": schema_proposals,
- "evidence_links": evidence_links,
- }
- result["graph_preview"] = _build_graph_preview(result, root_name)
- return result
-
-
-def _normalize_spans(text: str, spans: Any, fallback_quote: str = "") -> list[dict[str, Any]]:
- out: list[dict[str, Any]] = []
- if isinstance(spans, list):
- for raw in spans[:3]:
- if not isinstance(raw, dict):
- continue
- quote = str(raw.get("quote") or "").strip()
- if not quote:
- continue
- located = _span(text, quote, str(raw.get("evidence_id") or "manual_input_1"))
- if located:
- out.append(located[0])
- if not out and fallback_quote:
- out = _span(text, fallback_quote)
- return out
-
-
-def _find_statement_quote(text: str, labels: list[str]) -> str:
- terms = [
- term.strip()
- for term in labels
- if term and term.strip() and not re.match(r"^(ent|evt|cpt)_", term.strip())
- ]
- if not terms:
- return ""
- best = ""
- best_score = 0
- for sentence in _sentence_candidates(text):
- score = sum(1 for term in terms if term in sentence)
- if score > best_score:
- best_score = score
- best = sentence
- if score >= 2:
- break
- if best_score > 0:
- return best[:120]
- return terms[-1][:90]
-
-
-def _graph_key(value: Any) -> str:
- return re.sub(r"[\s\[\]()()《》“”\"'·,,、::;;//\\|-]+", "", str(value or "")).lower()
-
-
-def _graph_item_id(item: dict[str, Any]) -> str:
- return str(item.get("temp_id") or item.get("id") or "").strip()
-
-
-def _graph_item_name(item: dict[str, Any]) -> str:
- return str(item.get("name") or item.get("title") or "").strip()
-
-
-def _register_graph_alias(name_to_id: dict[str, str], label: Any, node_id: str) -> None:
- key = _graph_key(label)
- if key and node_id and key not in name_to_id:
- name_to_id[key] = node_id
-
-
-def _build_ref_lookup(*groups: list[dict[str, Any]]) -> dict[str, str]:
- name_to_id: dict[str, str] = {}
- for group in groups:
- for item in group:
- node_id = _graph_item_id(item)
- if not node_id:
- continue
- _register_graph_alias(name_to_id, node_id, node_id)
- _register_graph_alias(name_to_id, _graph_item_name(item), node_id)
- for alias in item.get("aliases") or []:
- _register_graph_alias(name_to_id, alias, node_id)
- return name_to_id
-
-
-def _resolve_graph_ref(ref: Any, nodes_by_id: dict[str, dict[str, Any]], name_to_id: dict[str, str]) -> str:
- raw = str(ref or "").strip()
- if not raw:
- return ""
- if raw in nodes_by_id:
- return raw
- key = _graph_key(raw)
- if key in name_to_id:
- return name_to_id[key]
- return ""
-
-
-def _looks_like_scenic_part(name: str, entity_type: str) -> bool:
- etype = (entity_type or "").lower()
- if etype in {"attraction", "scenicspot", "naturalfeature", "facility", "transitfacility"}:
- return True
- if etype not in {"place", "entity"}:
- return False
- return bool(re.search(r"(桥|亭|山|河|湖|洲|阁|舍|故居|衣冠冢|广场|大道|步道|码头|门|园|瀑布|洞|峰|谷|坡|池|泉|塔|楼)", name))
-
-
-def _graph_attr_text(item: dict[str, Any]) -> str:
- attrs = item.get("attributes") if isinstance(item.get("attributes"), dict) else {}
- spatial = item.get("spatial") if isinstance(item.get("spatial"), dict) else {}
- parts = [
- _graph_item_name(item),
- str(item.get("description") or ""),
- json.dumps(attrs, ensure_ascii=False, default=str),
- json.dumps(spatial, ensure_ascii=False, default=str),
- ]
- return " ".join(parts)
-
-
-def _build_graph_preview(result: dict[str, Any], root_entity: str = "") -> dict[str, Any]:
- """Build a relationship-first preview graph.
-
- LLM outputs often mix temp_id references and Chinese names. This resolver
- restores those references, hides isolated candidates, and adds conservative
- scenic-domain structure edges so the preview looks like a KG, not a scatter.
- """
- entities = result.get("entities") or []
- events = result.get("events") or []
- concepts = result.get("concepts") or []
- statements = result.get("statements") or []
-
- nodes_by_id: dict[str, dict[str, Any]] = {}
- name_to_id = _build_ref_lookup(entities, events, concepts)
-
- def remember_node(node_id: str, name: str, group: str, value: int, root: bool = False) -> None:
- if not node_id or node_id in nodes_by_id:
- return
- nodes_by_id[node_id] = {
- "id": node_id,
- "name": name or node_id,
- "group": group or "Entity",
- "value": value,
- "root": root,
- }
-
- root_id = ""
- if root_entity:
- root_id = name_to_id.get(_graph_key(root_entity), "")
- if not root_id:
- for ent in entities:
- if str(ent.get("entity_type") or "").lower() in {"scenicarea", "place", "entity"}:
- root_id = _graph_item_id(ent)
- break
- if not root_id and entities:
- root_id = _graph_item_id(entities[0])
-
- for ent in entities:
- node_id = _graph_item_id(ent)
- remember_node(
- node_id,
- str(ent.get("name") or node_id),
- str(ent.get("entity_type") or "Entity"),
- 38 if node_id == root_id else 18,
- node_id == root_id,
- )
- for event in events:
- node_id = _graph_item_id(event)
- remember_node(node_id, str(event.get("title") or node_id), str(event.get("event_type") or "Event"), 18)
- for concept in concepts:
- node_id = _graph_item_id(concept)
- remember_node(node_id, str(concept.get("name") or node_id), str(concept.get("concept_type") or "Concept"), 16)
-
- edges: list[dict[str, Any]] = []
- edge_seen: set[tuple[str, str, str]] = set()
-
- def add_edge(source_ref: Any, target_ref: Any, label: str, inferred: bool = False, confidence: float | None = None) -> None:
- label = _canonical_relation(str(label or "RELATED_TO"))
- source = _resolve_graph_ref(source_ref, nodes_by_id, name_to_id)
- target = _resolve_graph_ref(target_ref, nodes_by_id, name_to_id)
- if not source or not target or source == target:
- return
- sig = (source, target, label)
- if sig in edge_seen:
- return
- edge_seen.add(sig)
- edge = {"source": source, "target": target, "label": label}
- if inferred:
- edge["inferred"] = True
- if confidence is not None:
- edge["confidence"] = confidence
- edges.append(edge)
-
- for stmt in statements:
- if str(stmt.get("object_kind") or "").lower() not in {"entity", "event", "concept"}:
- continue
- add_edge(stmt.get("subject_ref"), stmt.get("object_ref"), str(stmt.get("predicate") or "RELATED_TO"), False, stmt.get("confidence"))
-
- # Event common fields are KG edges in preview, even when the model emitted
- # them only as structured event fields.
- for event in events:
- event_id = _graph_item_id(event)
- loc = event.get("location_ref")
- if loc:
- loc_id = _resolve_graph_ref(loc, nodes_by_id, name_to_id)
- if loc_id == root_id:
- add_edge(root_id, event_id, "HAS_EVENT", True, event.get("confidence"))
- elif loc_id:
- add_edge(event_id, loc_id, "OCCURRED_AT", True, event.get("confidence"))
- elif root_id:
- add_edge(root_id, event_id, "HAS_EVENT", True, event.get("confidence"))
- for participant in event.get("participants") or []:
- add_edge(participant, event_id, "PARTICIPATED_IN", True, event.get("confidence"))
-
- if root_id:
- for concept in concepts:
- add_edge(root_id, _graph_item_id(concept), "HAS_CONCEPT", True, concept.get("confidence"))
-
- entity_by_id = {_graph_item_id(ent): ent for ent in entities}
- for ent in entities:
- ent_id = _graph_item_id(ent)
- if not ent_id or ent_id == root_id:
- continue
- name = str(ent.get("name") or "")
- etype = str(ent.get("entity_type") or "Entity")
- etype_key = etype.lower()
- attr_text = _graph_attr_text(ent)
- if _looks_like_scenic_part(name, etype):
- add_edge(root_id, ent_id, "HAS_PART", True, ent.get("confidence"))
- elif etype_key in {"busline", "metroline"}:
- add_edge(root_id, ent_id, "ACCESSIBLE_BY", True, ent.get("confidence"))
- elif etype_key in {"routetemplate"}:
- add_edge(root_id, ent_id, "HAS_ROUTE", True, ent.get("confidence"))
- elif etype_key in {"routesegment"}:
- add_edge(root_id, ent_id, "HAS_ROUTE_SEGMENT", True, ent.get("confidence"))
- attrs = ent.get("attributes") if isinstance(ent.get("attributes"), dict) else {}
- for key, rel in (
- ("from_ref", "STARTS_AT"),
- ("from", "STARTS_AT"),
- ("from_name", "STARTS_AT"),
- ("to_ref", "ENDS_AT"),
- ("to", "ENDS_AT"),
- ("to_name", "ENDS_AT"),
- ("transport_ref", "USES_TRANSPORT_MODE"),
- ("transport_mode", "USES_TRANSPORT_MODE"),
- ):
- if key in attrs:
- add_edge(ent_id, attrs.get(key), rel, True, ent.get("confidence"))
- elif etype_key in {"pathschedule"}:
- add_edge(root_id, ent_id, "HAS_SCHEDULE", True, ent.get("confidence"))
- elif etype_key in {"mediaasset"}:
- target_id = ""
- attrs = ent.get("attributes") if isinstance(ent.get("attributes"), dict) else {}
- for key in ("target_ref", "target_id", "entity_ref", "entity_id", "belongs_to", "spot_ref"):
- target_id = _resolve_graph_ref(attrs.get(key), nodes_by_id, name_to_id)
- if target_id:
- break
- if not target_id:
- for other_id, other in entity_by_id.items():
- other_name = str(other.get("name") or "")
- if other_id != ent_id and other_name and other_name in attr_text:
- target_id = other_id
- break
- if not target_id and re.search(r"(导览图|导游图|全景图|景区图|地图|路线图|游览图)", attr_text):
- target_id = root_id
- if target_id:
- add_edge(target_id, ent_id, "HAS_MEDIA", True, ent.get("confidence"))
-
- connected: set[str] = set()
- for edge in edges:
- connected.add(str(edge["source"]))
- connected.add(str(edge["target"]))
- if root_id and root_id in nodes_by_id:
- connected.add(root_id)
-
- filtered_nodes = [node for node_id, node in nodes_by_id.items() if node_id in connected]
- if not edges and nodes_by_id:
- filtered_nodes = [nodes_by_id[root_id]] if root_id in nodes_by_id else list(nodes_by_id.values())[:1]
-
- return {
- "nodes": filtered_nodes,
- "edges": edges,
- "isolated_count": max(0, len(nodes_by_id) - len(filtered_nodes)),
- }
-
-
-def _normalize_llm_result(
- data: dict[str, Any],
- text: str,
- source_name: str,
- source_url: str,
- root_entity: str,
- method: str,
- pipeline_status: str,
-) -> dict[str, Any]:
- source_name = source_name.strip() or "人工录入"
- source_url = source_url.strip()
- root_entity = root_entity.strip()
-
- entities: list[dict[str, Any]] = []
- events: list[dict[str, Any]] = []
- concepts: list[dict[str, Any]] = []
- relations: list[dict[str, Any]] = []
- statements: list[dict[str, Any]] = []
- schema_proposals: list[dict[str, Any]] = []
- evidence_links: list[dict[str, Any]] = []
-
- id_map: dict[str, str] = {}
-
- for idx, raw in enumerate(data.get("entities") or [], start=1):
- if not isinstance(raw, dict):
- continue
- name = str(raw.get("name") or "").strip()
- if not name or re.fullmatch(r"\d+", name):
- continue
- old_id = str(raw.get("temp_id") or f"ent_{idx}")
- temp_id = old_id if old_id.startswith("ent_") else _stable_id("ent", name)
- id_map[old_id] = temp_id
- spans = _normalize_spans(text, raw.get("source_spans"), str(raw.get("evidence") or name))
- entities.append({
- "temp_id": temp_id,
- "name": name,
- "entity_type": str(raw.get("entity_type") or raw.get("type") or raw.get("category") or "Entity").strip() or "Entity",
- "aliases": raw.get("aliases") if isinstance(raw.get("aliases"), list) else [],
- "description": str(raw.get("description") or ""),
- "attributes": raw.get("attributes") if isinstance(raw.get("attributes"), dict) else {},
- "spatial": raw.get("spatial") if isinstance(raw.get("spatial"), dict) else {},
- "source_spans": spans,
- "confidence": _safe_confidence(raw.get("confidence"), 0.82),
- })
-
- if root_entity and not any(e["name"] == root_entity for e in entities):
- rid = _stable_id("ent", root_entity)
- entities.insert(0, {
- "temp_id": rid,
- "name": root_entity,
- "entity_type": "Entity",
- "aliases": [],
- "description": "用户指定的核心实体。",
- "attributes": {},
- "spatial": {},
- "source_spans": _span(text, root_entity),
- "confidence": 0.9,
- })
-
- for idx, raw in enumerate(data.get("events") or [], start=1):
- if not isinstance(raw, dict):
- continue
- title = str(raw.get("title") or "").strip()
- if not title or re.fullmatch(r"\d+", title):
- continue
- old_id = str(raw.get("temp_id") or f"evt_{idx}")
- temp_id = old_id if old_id.startswith("evt_") else _stable_id("evt", title)
- id_map[old_id] = temp_id
- time_text = str(raw.get("time_text") or raw.get("event_date") or "")
- spans = _normalize_spans(text, raw.get("source_spans"), str(raw.get("evidence") or raw.get("description") or title))
- events.append({
- "temp_id": temp_id,
- "title": title,
- "event_type": str(raw.get("event_type") or raw.get("type") or "Event").strip() or "Event",
- "time_text": time_text,
- "time_norm": str(raw.get("time_norm") or _normalize_time(time_text)),
- "description": str(raw.get("description") or ""),
- "participants": raw.get("participants") if isinstance(raw.get("participants"), list) else [],
- "location_ref": id_map.get(str(raw.get("location_ref") or ""), raw.get("location_ref")),
- "source_spans": spans,
- "confidence": _safe_confidence(raw.get("confidence"), 0.8),
- })
-
- for idx, raw in enumerate(data.get("concepts") or [], start=1):
- if not isinstance(raw, dict):
- continue
- name = str(raw.get("name") or "").strip()
- if not name or re.fullmatch(r"\d+", name):
- continue
- old_id = str(raw.get("temp_id") or f"cpt_{idx}")
- temp_id = old_id if old_id.startswith("cpt_") else _stable_id("cpt", name)
- id_map[old_id] = temp_id
- spans = _normalize_spans(text, raw.get("source_spans"), str(raw.get("evidence") or name))
- concepts.append({
- "temp_id": temp_id,
- "name": name,
- "concept_type": str(raw.get("concept_type") or raw.get("type") or "Concept").strip() or "Concept",
- "description": str(raw.get("description") or ""),
- "parent_concepts": raw.get("parent_concepts") if isinstance(raw.get("parent_concepts"), list) else [],
- "source_spans": spans,
- "confidence": _safe_confidence(raw.get("confidence"), 0.78),
- })
-
- known_ids = {x["temp_id"] for x in entities + events + concepts}
- name_to_id = _build_ref_lookup(entities, events, concepts)
- id_label = {
- x["temp_id"]: x.get("name") or x.get("title") or x.get("temp_id")
- for x in entities + events + concepts
- }
- type_by_id = {
- x["temp_id"]: x.get("entity_type") or x.get("event_type") or x.get("concept_type") or "Entity"
- for x in entities + events + concepts
- }
- entity_by_id = {x["temp_id"]: x for x in entities}
- for raw in data.get("statements") or []:
- if not isinstance(raw, dict):
- continue
- raw_predicate = str(raw.get("predicate") or raw.get("relation") or raw.get("relation_type") or "")
- predicate = _canonical_relation(raw_predicate)
- if not _valid_relation_name(predicate):
- continue
- subject_raw = str(raw.get("subject_ref") or raw.get("subject") or "")
- object_raw = str(raw.get("object_ref") or raw.get("object") or raw.get("target_ref") or "")
- subject = id_map.get(subject_raw, name_to_id.get(_graph_key(subject_raw), subject_raw))
- obj = id_map.get(object_raw, name_to_id.get(_graph_key(object_raw), object_raw))
- kind = str(raw.get("object_kind") or "literal").strip().lower()
- if not subject or not obj or re.fullmatch(r"\d+", obj):
- continue
- if kind in {"entity", "event", "concept"} and subject not in known_ids:
- subject = name_to_id.get(_graph_key(subject_raw), subject)
- if kind in {"entity", "event", "concept"} and obj not in known_ids:
- obj = name_to_id.get(_graph_key(object_raw), obj)
- if obj not in known_ids:
- kind = "literal"
- subject_type = type_by_id.get(subject, "Entity")
- object_type = type_by_id.get(obj, kind.title())
- field_name = _travel_field_for_predicate(predicate, subject_type)
- if field_name:
- _set_travel_attribute(entity_by_id.get(subject), field_name, _statement_object_value({
- "object_ref": obj,
- }, id_label))
- continue
- predicate = _travel_relation_alias(predicate, subject_type, object_type, kind)
- fallback_quote = str(
- raw.get("evidence")
- or raw.get("evidence_quote")
- or raw.get("quote")
- or raw.get("source_quote")
- or ""
- )
- if not fallback_quote:
- fallback_quote = _find_statement_quote(text, [
- id_label.get(subject, subject),
- id_label.get(obj, obj) if kind in {"entity", "event", "concept"} else obj,
- ])
- spans = _normalize_spans(text, raw.get("source_spans"), fallback_quote)
- qualifiers = raw.get("qualifiers") if isinstance(raw.get("qualifiers"), dict) else {}
- qualifiers = {
- **qualifiers,
- "schema_status": _relation_schema_status(predicate),
- "raw_predicate": raw_predicate.strip().upper() or predicate,
- }
- statements.append({
- "subject_ref": subject,
- "predicate": predicate,
- "object_ref": obj,
- "object_kind": kind if kind in {"entity", "event", "concept", "literal"} else "literal",
- "qualifiers": qualifiers,
- "source_spans": spans,
- "confidence": _safe_confidence(raw.get("confidence"), 0.78),
- })
-
- _augment_meals_at_from_restaurant_quotes(entities, statements)
- _enrich_travel_attributes(text, entities)
- _augment_travel_context_relations(text, entities, statements)
-
- relation_seen: set[str] = set()
- for raw in data.get("relations") or []:
- if not isinstance(raw, dict):
- continue
- rel = _canonical_relation(str(raw.get("relation_type") or ""))
- source_type = str(raw.get("source_type") or "Entity")
- target_type = str(raw.get("target_type") or "Entity")
- if _is_travel_field_predicate(rel):
- continue
- rel = _travel_relation_alias(rel, source_type, target_type)
- if not _valid_relation_name(rel) or rel in relation_seen:
- continue
- relation_seen.add(rel)
- relations.append({
- "relation_type": rel,
- "source_type": source_type,
- "target_type": target_type,
- "description": str(raw.get("description") or ""),
- "inverse_relation": raw.get("inverse_relation"),
- "schema_status": _relation_schema_status(rel),
- "confidence": _safe_confidence(raw.get("confidence"), 0.78),
- })
-
- for s in statements:
- if s["predicate"] not in relation_seen:
- relation_seen.add(s["predicate"])
- relations.append({
- "relation_type": s["predicate"],
- "source_type": "Entity",
- "target_type": s["object_kind"].title(),
- "description": "由 Statement 候选归纳出的关系类型。",
- "inverse_relation": None,
- "schema_status": _relation_schema_status(s["predicate"]),
- "confidence": 0.78,
- })
-
- for raw in data.get("schema_proposals") or []:
- if not isinstance(raw, dict):
- continue
- proposal_name = str(raw.get("name") or raw.get("relation_type") or raw.get("relation_name") or raw.get("entity_type") or "").strip()
- proposal_key = proposal_name.upper()
- if _is_travel_field_predicate(proposal_key):
- continue
- if (
- proposal_key in CORE_RELATIONS
- or proposal_name in TRAVEL_CORE_ENTITY_TYPES
- or proposal_name in TRAVEL_IGNORED_PROPOSALS
- or proposal_key in {x.upper() for x in TRAVEL_IGNORED_PROPOSALS}
- ):
- continue
- normalized_rel = _travel_relation_alias(proposal_key, str(raw.get("source_type") or "Entity"), str(raw.get("target_type") or "Entity"))
- if normalized_rel in CORE_RELATIONS:
- continue
- schema_proposals.append(raw)
- proposed = {str(x.get("name") or "").strip().upper() for x in schema_proposals if isinstance(x, dict)}
- for rel in sorted(r["relation_type"] for r in relations if r.get("schema_status") == "proposal"):
- if rel in proposed:
- continue
- proposed.add(rel)
- examples = [
- span[0]["quote"]
- for s in statements
- if s["predicate"] == rel and (span := s.get("source_spans") or [])
- ][:3]
- schema_proposals.append({
- "proposal_type": "relation_type",
- "name": rel,
- "source_type": "Entity",
- "target_type": "Entity",
- "value_type": None,
- "reason": "LLM 抽取到的非核心关系,需经过 Schema 审核后才能进入正式图谱。",
- "examples": examples,
- "confidence": 0.72,
- "schema_status": "proposal",
- })
-
- for raw in data.get("evidence_links") or []:
- if isinstance(raw, dict):
- evidence_links.append(raw)
- if not evidence_links:
- for row in entities + events + concepts + statements:
- quote = span[0]["quote"] if (span := row.get("source_spans") or []) else ""
- if quote:
- evidence_links.append({
- "target_ref": row.get("temp_id") or f"{row.get('subject_ref')}:{row.get('predicate')}:{row.get('object_ref')}",
- "evidence_id": "manual_input_1",
- "support_type": "supports",
- "quote": quote,
- })
-
- result = {
- "source": {
- "source_name": source_name,
- "source_url": source_url,
- "input_chars": len(text),
- "input_format": "manual",
- },
- "summary": {
- "entities": len(entities),
- "events": len(events),
- "concepts": len(concepts),
- "relations": len(relations),
- "statements": len(statements),
- "evidence_links": len(evidence_links),
- "method": method,
- "pipeline_status": pipeline_status,
- },
- "entities": entities,
- "events": events,
- "concepts": concepts,
- "relations": relations,
- "statements": statements,
- "schema_proposals": schema_proposals,
- "evidence_links": evidence_links,
- }
- result["graph_preview"] = _build_graph_preview(result, root_entity)
- return result
-
-
-def _merge_candidate_results(candidates: list[dict[str, Any]]) -> dict[str, Any]:
- """Deterministically merge valid extractor outputs when the judge LLM fails."""
- merged: dict[str, Any] = {
- "entities": [],
- "events": [],
- "concepts": [],
- "relations": [],
- "statements": [],
- "schema_proposals": [],
- "evidence_links": [],
- "quality": {
- "coverage": "multi_extractor_python_merge",
- "main_risks": ["决策器 JSON 未闭合,暂由确定性合并兜底"],
- "suggested_next_step": "修复决策器输出后重新裁决,低置信候选进入人工审核",
- },
- }
- id_map: dict[str, str] = {}
- seen: dict[str, set[tuple[str, ...]]] = {
- "entities": set(),
- "events": set(),
- "concepts": set(),
- "relations": set(),
- "statements": set(),
- "schema_proposals": set(),
- }
-
- def remember(old_id: Any, new_id: str) -> None:
- old = str(old_id or "").strip()
- if old and new_id:
- id_map[old] = new_id
-
- for data in candidates:
- if not isinstance(data, dict):
- continue
- for raw in data.get("entities") or []:
- if not isinstance(raw, dict):
- continue
- name = str(raw.get("name") or "").strip()
- etype = str(raw.get("entity_type") or raw.get("type") or raw.get("category") or "Entity").strip()
- if not name:
- continue
- sig = (name, etype)
- old_id = raw.get("temp_id") or raw.get("id")
- if sig in seen["entities"]:
- remember(old_id, next((x.get("temp_id") for x in merged["entities"] if x.get("name") == name), ""))
- continue
- seen["entities"].add(sig)
- item = dict(raw)
- item["entity_type"] = etype or "Entity"
- item["temp_id"] = str(item.get("temp_id") or item.get("id") or _stable_id("ent", name))
- remember(old_id, item["temp_id"])
- merged["entities"].append(item)
-
- for raw in data.get("events") or []:
- if not isinstance(raw, dict):
- continue
- title = str(raw.get("title") or "").strip()
- time_text = str(raw.get("time_text") or raw.get("event_date") or raw.get("time") or "").strip()
- if not title:
- continue
- sig = (title, time_text or str(raw.get("time_norm") or ""))
- old_id = raw.get("temp_id") or raw.get("id")
- if sig in seen["events"]:
- remember(old_id, next((x.get("temp_id") for x in merged["events"] if x.get("title") == title), ""))
- continue
- seen["events"].add(sig)
- item = dict(raw)
- item["time_text"] = time_text
- item["event_type"] = str(item.get("event_type") or item.get("type") or "Event")
- item["temp_id"] = str(item.get("temp_id") or item.get("id") or _stable_id("evt", title))
- remember(old_id, item["temp_id"])
- merged["events"].append(item)
-
- for raw in data.get("concepts") or []:
- if not isinstance(raw, dict):
- continue
- name = str(raw.get("name") or "").strip()
- ctype = str(raw.get("concept_type") or raw.get("type") or "Concept").strip()
- if not name:
- continue
- sig = (name, ctype)
- old_id = raw.get("temp_id") or raw.get("id")
- if sig in seen["concepts"]:
- remember(old_id, next((x.get("temp_id") for x in merged["concepts"] if x.get("name") == name), ""))
- continue
- seen["concepts"].add(sig)
- item = dict(raw)
- item["concept_type"] = ctype or "Concept"
- item["temp_id"] = str(item.get("temp_id") or item.get("id") or _stable_id("cpt", name))
- remember(old_id, item["temp_id"])
- merged["concepts"].append(item)
-
- for raw in data.get("relations") or []:
- if not isinstance(raw, dict):
- continue
- rel = _canonical_relation(str(raw.get("relation_type") or raw.get("predicate") or ""))
- if not _valid_relation_name(rel):
- continue
- sig = (rel,)
- if sig in seen["relations"]:
- continue
- seen["relations"].add(sig)
- item = dict(raw)
- item["relation_type"] = rel
- item["schema_status"] = _relation_schema_status(rel)
- merged["relations"].append(item)
-
- for raw in data.get("statements") or []:
- if not isinstance(raw, dict):
- continue
- raw_predicate = str(raw.get("predicate") or raw.get("relation") or raw.get("relation_type") or "")
- predicate = _canonical_relation(raw_predicate)
- if not _valid_relation_name(predicate):
- continue
- subject = id_map.get(str(raw.get("subject_ref") or raw.get("subject") or ""), str(raw.get("subject_ref") or raw.get("subject") or ""))
- obj = id_map.get(str(raw.get("object_ref") or raw.get("object") or raw.get("target_ref") or ""), str(raw.get("object_ref") or raw.get("object") or raw.get("target_ref") or ""))
- if not subject or not obj:
- continue
- sig = (subject, predicate, obj)
- if sig in seen["statements"]:
- continue
- seen["statements"].add(sig)
- item = dict(raw)
- item["subject_ref"] = subject
- item["predicate"] = predicate
- item["object_ref"] = obj
- qualifiers = item.get("qualifiers") if isinstance(item.get("qualifiers"), dict) else {}
- item["qualifiers"] = {
- **qualifiers,
- "schema_status": _relation_schema_status(predicate),
- "raw_predicate": raw_predicate.strip().upper() or predicate,
- }
- merged["statements"].append(item)
-
- for raw in data.get("schema_proposals") or []:
- if not isinstance(raw, dict):
- continue
- name = str(raw.get("name") or raw.get("relation_type") or raw.get("field") or "").strip()
- sig = (str(raw.get("proposal_type") or ""), name)
- if sig in seen["schema_proposals"]:
- continue
- seen["schema_proposals"].add(sig)
- merged["schema_proposals"].append(raw)
- for raw in data.get("evidence_links") or []:
- if isinstance(raw, dict):
- merged["evidence_links"].append(raw)
-
- return merged
-
-
-def _split_long_text(text: str, max_chars: int = 4200, overlap: int = 280) -> list[str]:
- """Split long/manual materials into evidence-preserving chunks."""
- clean = (text or "").strip()
- if len(clean) <= max_chars:
- return [clean] if clean else []
-
- paragraphs = [p.strip() for p in re.split(r"\n{2,}|(?<=。)\s*", clean) if p.strip()]
- chunks: list[str] = []
- current = ""
- for para in paragraphs:
- if len(para) > max_chars:
- if current:
- chunks.append(current.strip())
- current = ""
- start = 0
- while start < len(para):
- chunks.append(para[start:start + max_chars].strip())
- start += max_chars - overlap
- continue
- if current and len(current) + len(para) + 1 > max_chars:
- chunks.append(current.strip())
- tail = current[-overlap:] if overlap and len(current) > overlap else ""
- current = (tail + "\n" + para).strip()
- else:
- current = (current + "\n" + para).strip()
- if current:
- chunks.append(current.strip())
- return [c for c in chunks if c]
-
-
-def _build_deep_extract_chunks(text: str) -> list[dict[str, str]]:
- """Build small extraction tasks that avoid one huge JSON response."""
- clean = (text or "").strip()
- rows = _parse_key_values(clean)
- tasks: list[dict[str, str]] = []
-
- if rows:
- structured = "\n".join(raw for _, _, raw in rows)
- for idx, chunk in enumerate(_split_long_text(structured, max_chars=3600, overlap=120), start=1):
- tasks.append({
- "chunk_id": f"structured_{idx}",
- "task_type": "structured_fields",
- "instruction": "优先抽取表格/键值对中的属性、地点组成、邻近地点、行政归属;不要遗漏字段值。",
- "text": chunk,
- })
-
- for idx, chunk in enumerate(_split_long_text(clean, max_chars=4200, overlap=300), start=1):
- tasks.append({
- "chunk_id": f"narrative_{idx}",
- "task_type": "open_text",
- "instruction": (
- "高召回抽取本片段中的实体、子事件、概念、业务动作关系、属性 Statement;"
- "不要只抽标题式主干,人员、机构、设备、企业、地点、指标、状态、处置动作都要覆盖;只基于本片段证据。"
- ),
- "text": chunk,
- })
-
- deduped: list[dict[str, str]] = []
- seen: set[str] = set()
- for task in tasks:
- sig = hashlib.md5(task["text"].encode("utf-8")).hexdigest()
- if sig in seen:
- continue
- seen.add(sig)
- deduped.append(task)
- return deduped[:24]
-
-
-async def _extract_chunk_candidate(
- task: dict[str, str],
- source_name: str,
- source_url: str,
- root_entity: str,
- extractors: list[tuple[str, LlmClient]],
- agg: tuple[str, LlmClient] | None,
- schema_context: dict[str, Any] | None = None,
-) -> tuple[dict[str, Any] | None, dict[str, Any]]:
- payload = json.dumps({
- "source": {"source_name": source_name or "人工录入", "source_url": source_url or ""},
- "root_entity_hint": root_entity or "",
- "business_schema": schema_context or {},
- "chunk_id": task["chunk_id"],
- "task_type": task["task_type"],
- "task_instruction": task["instruction"],
- "input_text": task["text"],
- "output_note": (
- "只输出本 chunk 的 JSON;跨 chunk 去重和合并由后端完成。"
- "当前片段内高召回优先,必须输出 Entity/Event/Concept/Relation/Statement 五类,"
- "尤其不要漏掉子事件、属性数值、参与人、设备故障、整改核查、协同处置和时序关系。"
- "若 business_schema 非空,优先按其中的业务场景 Schema 约束抽取;Schema 外新事实进入 schema_proposals。"
- + _length_budget_instruction(len(task["text"]), chunk=True)
- ),
- }, ensure_ascii=False)
- wait_seconds = _pool_wait_seconds(
- _llm_wait_seconds(len(task["text"]), chunk=True),
- extractors,
- agg,
- )
- responses = await fan_out(
- _extract_system_for(len(task["text"]), chunk=True),
- payload,
- extractors,
- min_valid=_llm_quorum(extractors),
- max_wait_seconds=wait_seconds,
- )
- valid = [r for r in responses if isinstance(r.get("data"), dict)]
- meta = {
- "chunk_id": task["chunk_id"],
- "task_type": task["task_type"],
- "valid_models": [r["model"] for r in valid],
- "model_timings": [
- {
- "model": r.get("model"),
- "seconds": r.get("seconds"),
- "status": (
- "ok"
- if isinstance(r.get("data"), dict)
- else ("skipped_after_quorum" if r.get("error") == "skipped_after_quorum" else "error")
- ),
- "error": "" if r.get("error") == "skipped_after_quorum" else r.get("error", ""),
- }
- for r in responses
- ],
- "errors": [
- f"{r.get('model')}({r.get('seconds', '-')}s):{r.get('error')}"
- for r in responses
- if r.get("error") and r.get("error") != "skipped_after_quorum"
- ],
- }
- if not valid:
- return None, meta
- if len(valid) == 1:
- meta["decision"] = "single_valid_model_no_decider"
- return valid[0]["data"], meta
- if agg:
- decide_wait = _decide_wait_seconds(len(task["text"]), len(valid))
- agg[1].timeout = max(decide_wait, 20)
- decided, err = await decide(
- KG_DECIDE_SYS,
- json.dumps({
- "chunk_id": task["chunk_id"],
- "task_type": task["task_type"],
- "多模型抽取": valid,
- "原文片段": task["text"],
- }, ensure_ascii=False),
- agg,
- max_wait_seconds=decide_wait,
- )
- if decided:
- meta["decision"] = "llm"
- return decided, meta
- meta["decision"] = f"python_merge_after_decider_error:{err}"
- else:
- meta["decision"] = "python_merge_no_decider"
- return _merge_candidate_results([r["data"] for r in valid]), meta
-
-
-async def _extract_manual_kg_chunked(
- clean_text: str,
- source_name: str,
- source_url: str,
- root_entity: str,
- extractors: list[tuple[str, LlmClient]],
- agg: tuple[str, LlmClient] | None,
- status_msg: str,
- schema_context: dict[str, Any] | None = None,
-) -> dict[str, Any] | None:
- started = time.perf_counter()
- tasks = _build_deep_extract_chunks(clean_text)
- if not tasks or not extractors:
- return None
- max_task_chars = max(len(task["text"]) for task in tasks)
- wait_seconds = _pool_wait_seconds(
- _llm_wait_seconds(max_task_chars, chunk=True),
- extractors,
- agg,
- )
- for _, client in extractors:
- client.timeout = wait_seconds
- if agg:
- agg[1].timeout = wait_seconds
-
- semaphore = asyncio.Semaphore(2)
-
- async def run_task(task: dict[str, str]) -> tuple[dict[str, Any] | None, dict[str, Any]]:
- async with semaphore:
- return await _extract_chunk_candidate(
- task, source_name, source_url, root_entity, extractors, agg, schema_context
- )
-
- chunk_results = await asyncio.gather(*(run_task(task) for task in tasks))
- candidates = [candidate for candidate, _ in chunk_results if candidate]
- metas = [meta for _, meta in chunk_results]
-
- if not candidates:
- return None
- merged = _merge_candidate_results(candidates)
- valid_chunks = sum(1 for m in metas if m.get("valid_models"))
- result = _normalize_llm_result(
- merged,
- clean_text,
- source_name,
- source_url,
- root_entity,
- "multi_llm_chunked_kg_extraction_v1",
- f"{valid_chunks}/{len(tasks)} 个分块产生有效候选;{status_msg};后端已合并去重并标记 Schema 状态",
- )
- result["summary"]["chunks"] = len(tasks)
- result["summary"]["valid_chunks"] = valid_chunks
- result["summary"]["llm_stage"] = "multi_model_chunked_ran"
- result["summary"]["llm_elapsed_seconds"] = round(time.perf_counter() - started, 2)
- result["summary"]["valid_models"] = ",".join(sorted({
- model
- for meta in metas
- for model in (meta.get("valid_models") or [])
- })) or "-"
- result["summary"]["chunk_model_timings"] = [
- {
- "chunk_id": meta.get("chunk_id"),
- "task_type": meta.get("task_type"),
- "models": meta.get("model_timings") or [],
- "decision": meta.get("decision"),
- }
- for meta in metas
- ]
- result["summary"]["model_errors"] = ";".join(
- err
- for meta in metas
- for err in (meta.get("errors") or [])
- )[:900]
- return result
-
-
-def _result_score(result: dict[str, Any] | None) -> int:
- if not result:
- return -1
- summary = result.get("summary") or {}
- method = str(summary.get("method") or "")
- score = 0
- score += int(summary.get("entities") or 0) * 3
- score += int(summary.get("events") or 0) * 4
- score += int(summary.get("concepts") or 0) * 2
- score += int(summary.get("statements") or 0) * 2
- score += int(summary.get("evidence_links") or 0)
- if "fast_rule" in method:
- score -= 50
- if "fallback_no_llm" in method:
- score -= 80
- if "multi_llm" in method:
- score += 30
- if "chunked" in method:
- score -= 5
- return score
-
-
-def _usable_extraction_result(
- result: dict[str, Any] | None,
- text: str = "",
- schema_context: dict[str, Any] | None = None,
-) -> bool:
- if not result:
- return False
- summary = result.get("summary") or {}
- method = str(summary.get("method") or "")
- if "fallback_no_llm" in method or "fast_rule" in method:
- return False
- entities = int(summary.get("entities") or len(result.get("entities") or []))
- statements = int(summary.get("statements") or len(result.get("statements") or []))
- evidence = int(summary.get("evidence_links") or len(result.get("evidence_links") or []))
- if not (entities >= 5 and statements >= 5 and evidence >= max(3, statements // 3)):
- return False
- return not _needs_travel_quality_retry(result, text, schema_context)
-
-
-def _result_brief(result: dict[str, Any] | None) -> dict[str, Any]:
- if not result:
- return {"ok": False, "score": -1}
- summary = result.get("summary") or {}
- return {
- "ok": True,
- "method": summary.get("method"),
- "pipeline_status": summary.get("pipeline_status"),
- "entities": summary.get("entities"),
- "events": summary.get("events"),
- "concepts": summary.get("concepts"),
- "relations": summary.get("relations"),
- "statements": summary.get("statements"),
- "evidence_links": summary.get("evidence_links"),
- "chunks": summary.get("chunks"),
- "valid_chunks": summary.get("valid_chunks"),
- "score": _result_score(result),
- }
-
-
-ADAPTIVE_LONG_TEXT_THRESHOLD = 8000
-
-
-def _llm_wait_seconds(chars: int, chunk: bool = False) -> int:
- """Keep manual extraction responsive while still allowing real LLM reasoning."""
- if chunk:
- if chars <= 2500:
- return 75
- if chars <= 4500:
- return 90
- return 120
- if chars <= 2500:
- return 85
- if chars <= 6000:
- return 110
- if chars <= ADAPTIVE_LONG_TEXT_THRESHOLD:
- return 150
- return 120
-
-
-def _pool_wait_seconds(
- floor_seconds: int,
- extractors: list[tuple[str, LlmClient]],
- agg: tuple[str, LlmClient] | None = None,
-) -> int:
- """Respect the timeout configured in /admin/system/settings."""
- waits = [floor_seconds]
- waits.extend(int(client.timeout or 0) for _, client in extractors)
- if agg:
- waits.append(int(agg[1].timeout or 0))
- return max(waits)
-
-
-def _llm_max_tokens(chars: int, chunk: bool = False) -> int:
- """Minimum token budget guarantee for KG extraction JSON output.
-
- These values are a *floor* applied via ``max(client.max_tokens, budget)``.
- If the model client already carries a higher cap (set automatically by
- ``_model_max_output_tokens`` in multi_extract.py), that higher value wins.
-
- Background: KG JSON is verbose — even a 1 200-char input can produce
- 15 000–20 000 tokens of entities / events / concepts / statements.
- The old values (4 500–9 500) were the real truncation bottleneck for
- doubao / GLM-4 classic / custom proxy models.
- """
- if chunk:
- if chars <= 2500:
- return 16000
- return 20000
- if chars <= 2500:
- return 20000
- if chars <= 6000:
- return 28000
- return 32000
-
-
-def _decide_wait_seconds(chars: int, valid_count: int) -> int:
- if valid_count <= 1:
- return 0
- if chars <= 2500:
- return 35
- if chars <= 6000:
- return 50
- return 70
-
-
-def _llm_quorum(extractors: list[tuple[str, LlmClient]]) -> int:
- """生产抽取优先响应速度:拿到首个有效 JSON 即可进入后处理。"""
- return 1
-
-
-def _length_budget_instruction(chars: int, chunk: bool = False) -> str:
- if chunk:
- return (
- "当前是分块抽取:优先完整覆盖本片段事实,但不要撑满上限。"
- "建议 entities 15-40、events 5-25、concepts 5-20、statements 30-90。"
- )
- if chars <= 2500:
- return (
- "当前输入是短文本:必须深度抽取,但不要生成冗长 JSON。"
- "建议 entities 10-28、events 3-12、concepts 4-12、statements 20-60;"
- "source_spans 每条只保留一个 quote,不要输出 start_char/end_char。"
- )
- if chars <= 6000:
- return (
- "当前输入是中等文本:建议 entities 25-70、events 8-40、concepts 8-30、statements 60-150;"
- "source_spans 每条只保留一个 quote,不要输出 start_char/end_char。"
- )
- return (
- "当前输入较长:高召回优先,必要时覆盖更多候选;"
- "source_spans 每条只保留一个 quote,不要输出 start_char/end_char。"
- )
-
-
-def _extract_system_for(chars: int, chunk: bool = False) -> str:
- """Short/manual inputs need compact instructions; huge system prompts slow JSON IE."""
- if chunk or chars <= 6000:
- return KG_EXTRACT_COMPACT_SYS
- return KG_EXTRACT_SYS
-
-
-def _adaptive_root_name(text: str, kvs: list[tuple[str, str, str]], explicit: str = "") -> str:
- explicit = (explicit or "").strip()
- if explicit:
- return explicit
- for pattern in (
- r"核心实体[::]\s*([^\n]+)",
- r"主题[::]\s*([^\n]+)",
- ):
- match = re.search(pattern, text)
- if match:
- return _split_items(match.group(1).strip())[0]
- return _root_name(text, kvs, explicit)
-
-
-def _adaptive_key_values(text: str) -> list[tuple[str, str, str]]:
- rows = _parse_key_values(text)
- keys = [
- "词条名", "中文名", "名称", "别名", "地理位置", "地址", "气候条件",
- "开放状态", "开放时间", "景点级别", "门票价格", "票价", "占地面积",
- "著名景点", "主要景点", "邻近景点", "附近景点", "美誉", "所属国家",
- "所属城市", "所属行政区", "行政区", "建议游玩时长", "适宜游玩季节",
- "核心地标", "主要体验", "空间标签", "风险提示", "交通站点", "正文",
- ]
- key_pattern = "|".join(re.escape(key) for key in sorted(keys, key=len, reverse=True))
- inline_rows: list[tuple[str, str, str]] = []
- head = text[:5000].replace("美 誉", "美誉")
- for match in re.finditer(rf"({key_pattern})\s*[::]\s*(.*?)(?=\s+(?:{key_pattern})\s*[::]|$)", head, re.S):
- key = match.group(1).strip()
- if key == "正文":
- continue
- value = re.sub(r"\[[^\]]+\]", "", match.group(2)).strip()
- value = re.sub(r"\s+", " ", value)
- if not value or len(value) > 300:
- continue
- raw = f"{key}:{value}"
- inline_rows.append((key, value, raw))
- seen = {(key, value) for key, value, _ in rows}
- for row in inline_rows:
- sig = (row[0], row[1])
- if sig not in seen:
- seen.add(sig)
- rows.append(row)
- return rows
-
-
-def _adaptive_text_units(text: str, source_id: str) -> list[dict[str, Any]]:
- headings: list[tuple[int, str]] = []
- for match in re.finditer(r"(?m)^(?:[一二三四五六七八九十]+[、..-]|八-\d+、|第\d+[章节])\s*([^\n]+)", text):
- headings.append((match.start(), match.group(0).strip()))
- if not headings:
- chunks = _split_long_text(text, max_chars=4200, overlap=260)
- cursor = 0
- units: list[dict[str, Any]] = []
- for idx, chunk in enumerate(chunks, start=1):
- start = text.find(chunk[:80], cursor)
- if start < 0:
- start = cursor
- end = min(len(text), start + len(chunk))
- cursor = end
- units.append({
- "text_unit_id": f"{source_id}_tu_{idx:03d}",
- "section_title": f"自动分段 {idx}",
- "char_start": start,
- "char_end": end,
- "unit_type": "chunk",
- "text": chunk,
- "chars": len(chunk),
- })
- return units
- if headings[0][0] > 0:
- headings.insert(0, (0, "导言"))
- units = []
- for idx, (start, title) in enumerate(headings, start=1):
- end = headings[idx][0] if idx < len(headings) else len(text)
- unit_text = text[start:end].strip()
- if unit_text:
- units.append({
- "text_unit_id": f"{source_id}_tu_{idx:03d}",
- "section_title": title[:80],
- "char_start": start,
- "char_end": end,
- "unit_type": "section",
- "text": unit_text,
- "chars": len(unit_text),
- })
- return units
-
-
-def _adaptive_entity_type(name: str, category: str = "") -> str:
- label = category or name
- if "古镇" in name:
- return "Place"
- if name in {"中国"} or name.endswith(("省", "市", "区", "县", "镇")):
- return "Area"
- if any(word in label for word in ("地铁", "公交", "站")):
- return "TransitStop"
- if any(word in label for word in ("市集", "商业", "美食", "夜市")):
- return "CommercialArea"
- if any(word in label for word in ("桥", "步道", "公园", "景点", "园", "广场", "阁", "楼", "博物馆", "旧址", "古镇")):
- return "Place"
- if any(word in label for word in ("河", "湖", "山", "湿地", "潭")):
- return "NaturalFeature"
- return "Entity"
-
-
-def _adaptive_event_type(sentence: str) -> str:
- if re.search(r"升级开放|开放", sentence):
- return "OpeningEvent"
- if re.search(r"修缮|重建|兴建|始建|建成|建设|竣工|辟建", sentence):
- return "ConstructionEvent"
- if re.search(r"治理|工程", sentence):
- return "GovernanceProjectEvent"
- if re.search(r"主题讲解|灯光互动|民乐演出|非遗展示|活动|节", sentence):
- return "CulturalActivityEvent"
- if re.search(r"商业活动|消费业态|形成|发展|改观", sentence):
- return "DevelopmentEvent"
- if re.search(r"交通组织调整|调整|划为|管理|管辖", sentence):
- return "ManagementChangeEvent"
- if re.search(r"列入|文物保护|名录", sentence):
- return "HeritageListingEvent"
- if re.search(r"改名|更名|命名", sentence):
- return "RenamingEvent"
- if re.search(r"到访|参观", sentence) or (re.search(r"游", sentence) and "游客" not in sentence):
- return "VisitEvent"
- return _event_type(sentence)
-
-
-def _adaptive_add_unique(items: list[dict[str, Any]], item: dict[str, Any], keys: tuple[str, ...]) -> None:
- sig = tuple(str(item.get(k) or "") for k in keys)
- for old in items:
- if tuple(str(old.get(k) or "") for k in keys) == sig:
- return
- items.append(item)
-
-
-def _adaptive_relation(relation_type: str, target_kind: str = "Entity", confidence: float = 0.86) -> dict[str, Any]:
- rel = _canonical_relation(relation_type)
- return {
- "relation_type": rel,
- "source_type": "Entity",
- "target_type": target_kind,
- "description": f"{rel} 关系候选",
- "inverse_relation": None,
- "schema_status": _relation_schema_status(rel),
- "confidence": confidence,
- }
-
-
-def _adaptive_graph(result: dict[str, Any]) -> dict[str, Any]:
- root_name = ""
- entities = result.get("entities") or []
- if entities:
- root_name = str(entities[0].get("name") or "")
- return _build_graph_preview(result, root_name)
-
-
-def _adaptive_efdge_extract(text: str, source_name: str, source_url: str, root_entity: str = "") -> dict[str, Any]:
- clean_text = (text or "").strip()
- if len(clean_text) < 4:
- raise HTTPException(status_code=400, detail="请输入或上传需要抽取的人工材料")
- source_name = source_name.strip() or "人工录入"
- source_url = source_url.strip()
- source_id = _stable_id("src", source_url or source_name)
- kvs = _adaptive_key_values(clean_text)
- root_name = _adaptive_root_name(clean_text, kvs, root_entity)
- root_id = _stable_id("ent", root_name)
- units = _adaptive_text_units(clean_text, source_id)
-
- entities: list[dict[str, Any]] = [{
- "temp_id": root_id,
- "name": root_name,
- "entity_type": "Place",
- "aliases": [],
- "description": f"文档级核心实体:{root_name}",
- "attributes": {},
- "spatial": {},
- "source_spans": _span(clean_text, root_name, source_id),
- "confidence": 0.92,
- }]
- events: list[dict[str, Any]] = []
- concepts: list[dict[str, Any]] = []
- relations: list[dict[str, Any]] = []
- statements: list[dict[str, Any]] = []
-
- def add_relation(predicate: str, target_kind: str = "Entity") -> None:
- _adaptive_add_unique(relations, _adaptive_relation(predicate, target_kind), ("relation_type", "source_type", "target_type"))
-
- def add_entity(name: str, quote: str, entity_type: str | None = None, confidence: float = 0.86,
- attrs: dict[str, Any] | None = None) -> str:
- name = name.strip()
- if not name:
- return root_id
- item = {
- "temp_id": _stable_id("ent", name),
- "name": name,
- "entity_type": entity_type or _adaptive_entity_type(name),
- "aliases": [],
- "description": f"从材料中识别出的实体:{name}",
- "attributes": attrs or {},
- "spatial": {},
- "source_spans": _span(clean_text, quote or name, source_id),
- "confidence": confidence,
- }
- _adaptive_add_unique(entities, item, ("name", "entity_type"))
- return item["temp_id"]
-
- def add_concept(name: str, concept_type: str, quote: str, confidence: float = 0.82) -> str:
- item = {
- "temp_id": _stable_id("cpt", name),
- "name": name,
- "concept_type": concept_type,
- "description": f"材料证据支持的概念:{name}",
- "parent_concepts": [],
- "source_spans": _span(clean_text, quote or name, source_id),
- "confidence": confidence,
- }
- _adaptive_add_unique(concepts, item, ("name",))
- return item["temp_id"]
-
- def add_statement(subject: str, predicate: str, obj: str, kind: str, quote: str,
- confidence: float = 0.84, qualifiers: dict[str, Any] | None = None) -> None:
- rel = _canonical_relation(predicate)
- stmt = {
- "subject_ref": subject,
- "predicate": rel,
- "object_ref": obj,
- "object_kind": kind,
- "qualifiers": {"source_name": source_name, "source_url": source_url, **(qualifiers or {})},
- "source_spans": _span(clean_text, quote, source_id),
- "confidence": confidence,
- "schema_status": _relation_schema_status(rel),
- }
- _adaptive_add_unique(statements, stmt, ("subject_ref", "predicate", "object_ref"))
- add_relation(rel, kind.title())
-
- relation_map = {
- "地理位置": "HAS_ADDRESS",
- "地址": "HAS_ADDRESS",
- "所属城市": "LOCATED_IN",
- "所属行政区": "LOCATED_IN",
- "行政区": "LOCATED_IN",
- "所属国家": "LOCATED_IN",
- "开放状态": "HAS_OPENING_HOURS",
- "开放时间": "HAS_OPENING_HOURS",
- "门票价格": "HAS_TICKET_PRICE",
- "票价": "HAS_TICKET_PRICE",
- "美誉": "HAS_REPUTATION",
- "建议游玩时长": "HAS_SUGGESTED_DURATION",
- "适宜游玩季节": "HAS_BEST_SEASON",
- "著名景点": "HAS_PART",
- "主要景点": "HAS_PART",
- "邻近景点": "NEARBY_ATTRACTION",
- "附近景点": "NEARBY_ATTRACTION",
- "交通站点": "NEAR_TRANSIT",
- "主要体验": "HAS_CONCEPT",
- "空间标签": "HAS_CONCEPT",
- "风险提示": "HAS_RISK_NOTICE",
- }
- for key, value, raw in kvs:
- key_clean = key.replace(" ", "")
- if key_clean == "别名":
- aliases = _split_items(value)
- entities[0]["aliases"] = sorted(set(entities[0].get("aliases", []) + aliases))
- continue
- predicate = relation_map.get(key_clean)
- if not predicate:
- continue
- if predicate in {"HAS_PART", "NEARBY_ATTRACTION", "NEAR_TRANSIT", "LOCATED_IN"}:
- for item_name in _split_items(value):
- target_id = add_entity(item_name, raw)
- add_statement(root_id, predicate, target_id, "entity", raw, 0.86)
- elif predicate == "HAS_CONCEPT":
- for item_name in _split_items(value):
- concept_id = add_concept(item_name, "Experience", raw, 0.82)
- add_statement(root_id, "HAS_CONCEPT", concept_id, "concept", raw, 0.82)
- else:
- add_statement(root_id, predicate, value, "literal", raw, 0.86)
-
- for _, name, concept_type, words in CONCEPT_RULES:
- hit = next((word for word in words if word in clean_text), "")
- if not hit:
- continue
- concept_id = add_concept(name, concept_type, hit, 0.8)
- add_statement(root_id, "HAS_CONCEPT", concept_id, "concept", hit, 0.8)
-
- if "POI_ID,名称,类别,地址,行政区,经度,纬度" in clean_text:
- for line in clean_text.splitlines():
- if not line.startswith("amap:"):
- continue
- cells = [cell.strip() for cell in line.split(",")]
- if len(cells) < 12:
- continue
- poi_id, name, category, address, district, lng, lat, rating, cost, hours, photos, note = cells[:12]
- ent_type = _adaptive_entity_type(name, category)
- target_id = add_entity(
- name, line, ent_type, 0.9,
- {
- "place_id": poi_id,
- "category": category,
- "address": address,
- "district": district,
- "lng": lng,
- "lat": lat,
- "rating": rating,
- "cost": cost,
- "open_time": hours,
- "photo_urls": photos,
- },
- )
- if ent_type == "TransitStop":
- add_statement(root_id, "NEAR_TRANSIT", target_id, "entity", line, 0.86)
- elif category in {"商业街区"}:
- add_statement(root_id, "NEARBY_ATTRACTION", target_id, "entity", line, 0.86)
- elif name in {root_name, "甲秀楼", "浮玉桥", "翠微园", "南明河滨水步道"}:
- add_statement(root_id, "HAS_PART", target_id, "entity", line, 0.86)
- else:
- add_statement(root_id, "NEARBY_ATTRACTION", target_id, "entity", line, 0.84)
- if address:
- add_statement(target_id, "HAS_ADDRESS", address, "literal", line, 0.86)
- if hours:
- add_statement(target_id, "HAS_OPENING_HOURS", hours, "literal", line, 0.84)
-
- for unit in units:
- for sentence in _sentence_candidates(unit["text"]):
- if any(noise in sentence for noise in (
- "可抽取为", "事件类型可", "不应误抽", "应抽取事件候选",
- "不一定进入正式图谱", "不能把", "是否进入核心 Schema",
- )):
- continue
- if not re.search(r"\d{3,4}年", sentence):
- continue
- if not any(word in sentence for word in ("建", "修缮", "开放", "治理", "升级", "活动", "形成", "调整", "列入", "出现", "划为")):
- continue
- time_match = re.search(r"\d{3,4}年(?:\d{1,2}月)?(?:\d{1,2}日)?|2024年春节期间|2025年国庆假期", sentence)
- time_text = time_match.group(0) if time_match else ""
- title = re.sub(r"^[,。;;、\s]+|[,。;;、\s]+$", "", sentence)[:60]
- event_item = {
- "temp_id": _stable_id("evt", f"{time_text}_{title}"),
- "title": title,
- "event_type": _adaptive_event_type(sentence),
- "time_text": time_text,
- "time_norm": _normalize_time(time_text),
- "description": sentence[:140],
- "participants": [],
- "location_ref": root_id,
- "source_spans": _span(clean_text, sentence, source_id),
- "confidence": 0.82,
- }
- _adaptive_add_unique(events, event_item, ("title", "time_text"))
- add_statement(root_id, "HAS_EVENT", event_item["temp_id"], "event", sentence, 0.82)
-
- schema_proposals = []
- for rel in sorted({stmt["predicate"] for stmt in statements if stmt.get("schema_status") == "proposal"}):
- examples = [
- stmt["source_spans"][0]["quote"]
- for stmt in statements
- if stmt.get("predicate") == rel and stmt.get("source_spans")
- ][:3]
- schema_proposals.append({
- "proposal_type": "relation_type",
- "name": rel,
- "source_type": "Entity",
- "target_type": "Literal",
- "value_type": None,
- "reason": f"{rel} 不在当前 Core Schema 中,需人工判断是否提升为领域 Schema。",
- "examples": examples,
- "confidence": 0.74,
- })
-
- result = {
- "source": {
- "source_name": source_name,
- "source_url": source_url,
- "source_id": source_id,
- "input_chars": len(clean_text),
- "input_format": "manual",
- },
- "summary": {
- "entities": len(entities),
- "events": len(events),
- "concepts": len(concepts),
- "relations": len(relations),
- "statements": len(statements),
- "schema_proposals": len(schema_proposals),
- "evidence_links": len(statements),
- "text_units": len(units),
- "method": "unified_adaptive_kg_extraction_v2",
- "strategy": "adaptive_efdge" if len(clean_text) > ADAPTIVE_LONG_TEXT_THRESHOLD else "adaptive_whole",
- "adaptive_reason": (
- f"input_chars > {ADAPTIVE_LONG_TEXT_THRESHOLD},自动使用 TextUnit + Anchor + Statement + Evidence"
- if len(clean_text) > ADAPTIVE_LONG_TEXT_THRESHOLD
- else f"input_chars <= {ADAPTIVE_LONG_TEXT_THRESHOLD},自动使用整篇证据驱动抽取"
- ),
- "pipeline_status": "统一抽取输出:Entity / Event / Concept / Relation / Statement;新关系进入 Schema Proposal,不直接污染正式图谱。",
- "evidence_coverage": 1.0 if statements else 0,
- },
- "text_units": [{k: v for k, v in unit.items() if k != "text"} for unit in units],
- "entities": entities[:80],
- "events": events[:40],
- "concepts": concepts[:30],
- "relations": relations,
- "statements": statements[:160],
- "schema_proposals": schema_proposals,
- "evidence_links": [
- {
- "target_ref": f"{stmt.get('subject_ref')}:{stmt.get('predicate')}:{stmt.get('object_ref')}",
- "evidence_id": stmt["source_spans"][0]["evidence_id"] if stmt.get("source_spans") else source_id,
- "support_type": "supports",
- "quote": stmt["source_spans"][0]["quote"] if stmt.get("source_spans") else "",
- }
- for stmt in statements[:160]
- ],
- "quality": {
- "auto_publish_ready": False,
- "main_risks": ["同步抽取阶段先生成候选,最终发布前仍需 Entity Alignment、冲突检测和低置信审核。"],
- "suggested_next_step": "进入人工审核或后台 LLM verifier,对低置信、新 Schema 和跨段关系进行裁判。",
- },
- }
- result["graph_preview"] = _adaptive_graph(result)
- return result
-
-
-def _compact_adaptive_anchor(base: dict[str, Any]) -> dict[str, Any]:
- """Keep deterministic evidence anchors small enough for the LLM prompt."""
- return {
- "summary": base.get("summary") or {},
- "entities": (base.get("entities") or [])[:30],
- "events": (base.get("events") or [])[:24],
- "concepts": (base.get("concepts") or [])[:18],
- "relations": (base.get("relations") or [])[:24],
- "statements": (base.get("statements") or [])[:60],
- "text_units": (base.get("text_units") or [])[:18],
- }
-
-
-async def _extract_manual_kg_adaptive_deep(
- clean_text: str,
- source_name: str,
- source_url: str,
- root_entity: str,
- extractors: list[tuple[str, LlmClient]],
- agg: tuple[str, LlmClient] | None,
- status_msg: str,
- schema_context: dict[str, Any] | None = None,
-) -> dict[str, Any]:
- started = time.perf_counter()
- source_name = source_name.strip() or "人工录入"
- source_url = source_url.strip()
- source_id = _stable_id("src", source_url or source_name)
- text_units = _adaptive_text_units(clean_text, source_id)
- llm_errors: list[str] = []
- valid_models: list[str] = []
- model_timings: list[dict[str, Any]] = []
- decision_status = "未运行"
- llm_candidate: dict[str, Any] | None = None
-
- if len(clean_text) > ADAPTIVE_LONG_TEXT_THRESHOLD:
- whole, whole_errors = await _extract_manual_kg_whole(
- clean_text,
- source_name,
- source_url,
- root_entity,
- extractors,
- agg,
- status_msg,
- schema_context,
- )
- if _usable_extraction_result(whole, clean_text, schema_context):
- elapsed_ms = int((time.perf_counter() - started) * 1000)
- whole["summary"]["strategy"] = "auto_whole_first"
- whole["summary"]["adaptive_reason"] = (
- f"input_chars > {ADAPTIVE_LONG_TEXT_THRESHOLD},但生产策略优先整篇长上下文;"
- "整篇结果已达标,跳过分块以降低耗时和重复合并风险。"
- )
- whole["summary"]["llm_elapsed_ms"] = elapsed_ms
- whole["summary"]["llm_elapsed_seconds"] = round(elapsed_ms / 1000, 2)
- whole["text_units"] = [{k: v for k, v in unit.items() if k != "text"} for unit in text_units]
- whole["summary"]["text_units"] = len(text_units)
- whole["graph_preview"] = _adaptive_graph(whole)
- return whole
-
- llm_errors.extend(whole_errors)
- chunked = await _extract_manual_kg_chunked(
- clean_text,
- source_name,
- source_url,
- root_entity,
- extractors,
- agg,
- status_msg,
- schema_context,
- )
- if chunked:
- elapsed_ms = int((time.perf_counter() - started) * 1000)
- chunked["summary"]["method"] = "unified_llm_only_chunked_fallback_kg_extraction_v2"
- chunked["summary"]["strategy"] = "auto_chunked_fallback"
- chunked["summary"]["pipeline_status"] = (
- "统一深度抽取已运行:先尝试整篇长上下文;整篇未达质量门槛后,"
- "才按 TextUnit/Chunk 分块抽取并合并去重。"
- )
- chunked["summary"]["adaptive_reason"] = (
- f"input_chars > {ADAPTIVE_LONG_TEXT_THRESHOLD};整篇结果未达标,fallback 到分块。"
- )
- chunked["summary"]["llm_elapsed_ms"] = elapsed_ms
- chunked["summary"]["llm_elapsed_seconds"] = round(elapsed_ms / 1000, 2)
- chunked["text_units"] = [{k: v for k, v in unit.items() if k != "text"} for unit in text_units]
- chunked["summary"]["text_units"] = len(text_units)
- chunked["graph_preview"] = _adaptive_graph(chunked)
- return chunked
-
- if whole:
- elapsed_ms = int((time.perf_counter() - started) * 1000)
- whole["summary"]["strategy"] = "auto_whole_low_quality_chunk_failed"
- whole["summary"]["adaptive_reason"] = (
- f"input_chars > {ADAPTIVE_LONG_TEXT_THRESHOLD};整篇结果未达质量门槛,"
- "但分块 fallback 没有产生有效候选,因此返回整篇候选供人工审核。"
- )
- whole["summary"]["llm_elapsed_ms"] = elapsed_ms
- whole["summary"]["llm_elapsed_seconds"] = round(elapsed_ms / 1000, 2)
- whole["summary"]["model_errors"] = ";".join((whole_errors or [])[:5])
- whole["text_units"] = [{k: v for k, v in unit.items() if k != "text"} for unit in text_units]
- whole["summary"]["text_units"] = len(text_units)
- whole["graph_preview"] = _adaptive_graph(whole)
- return whole
-
- user_payload = json.dumps({
- "source": {"source_name": source_name or "人工录入", "source_url": source_url or ""},
- "root_entity_hint": root_entity or "",
- "business_schema": schema_context or {},
- "strategy": "adaptive_deep_llm_only_full_document",
- "input_chars": len(clean_text),
- "input_text": clean_text[:100000],
- "text_units": [{k: v for k, v in unit.items() if k != "text"} for unit in text_units[:24]],
- "important_instruction": (
- "最终候选必须完全来自你对 input_text 的大模型深度阅读与抽取。"
- "后端不会使用规则候选补结果;如果你漏掉 Entity/Event/Concept/Relation/Statement,最终结果就会缺失。"
- "必须删除目录编号、碎片数字、无语义 HAS_XX,并保留可定位 quote。"
- "若 business_schema 非空,优先按其中的业务场景 Schema 约束抽取;Schema 外新事实进入 schema_proposals。"
- + _length_budget_instruction(len(clean_text))
- ),
- }, ensure_ascii=False)
-
- if extractors:
- wait_seconds = _pool_wait_seconds(_llm_wait_seconds(len(clean_text)), extractors, agg)
- for _, client in extractors:
- client.timeout = wait_seconds
- if agg:
- agg[1].timeout = wait_seconds
-
- responses = await fan_out(
- _extract_system_for(len(clean_text)),
- user_payload,
- extractors,
- min_valid=_llm_quorum(extractors),
- max_wait_seconds=wait_seconds,
- )
- model_timings = [
- {
- "model": r.get("model"),
- "seconds": r.get("seconds"),
- "status": (
- "ok"
- if isinstance(r.get("data"), dict)
- else ("skipped_after_quorum" if r.get("error") == "skipped_after_quorum" else "error")
- ),
- "error": "" if r.get("error") == "skipped_after_quorum" else r.get("error", ""),
- }
- for r in responses
- ]
- valid = [r for r in responses if isinstance(r.get("data"), dict)]
- valid_models = [str(r.get("model")) for r in valid]
- llm_errors.extend(
- f"{r.get('model')}({r.get('seconds', '-')}s):{r.get('error')}"
- for r in responses
- if r.get("error") and r.get("error") != "skipped_after_quorum"
- )
- if len(valid) == 1:
- llm_candidate = valid[0]["data"]
- decision_status = f"只有 1 个有效知识大模型({valid_models[0]}),已直接采用单模型结果"
- elif valid and agg:
- decide_wait = _decide_wait_seconds(len(clean_text), len(valid))
- agg[1].timeout = max(decide_wait, 20)
- decided, err = await decide(
- KG_DECIDE_SYS,
- json.dumps({
- "多模型抽取": valid,
- "原文": clean_text[:30000],
- "裁决要求": "合并多模型候选,保留 Entity/Event/Concept/Relation/Statement 五类;低证据和脏关系删除。",
- }, ensure_ascii=False),
- agg,
- max_wait_seconds=decide_wait,
- )
- if decided:
- llm_candidate = decided
- decision_status = f"LLM 合并模型 {agg[0]} 已合并裁剪"
- else:
- llm_candidate = _merge_candidate_results([r["data"] for r in valid])
- decision_status = f"LLM 合并失败后 Python 兜底合并:{err}"
- if err:
- llm_errors.append(f"{agg[0]}:{err}")
- elif valid:
- llm_candidate = _merge_candidate_results([r["data"] for r in valid])
- decision_status = "无 LLM 合并模型,Python 兜底合并多模型候选"
- else:
- decision_status = "知识大模型均未返回有效 JSON"
- else:
- llm_errors.append(f"无可用知识大模型:{status_msg}")
-
- if not llm_candidate and not extractors and agg:
- try:
- wait_seconds = _llm_wait_seconds(len(clean_text))
- agg[1].timeout = max(wait_seconds, 120)
- data = await asyncio.to_thread(agg[1].chat_json, _extract_system_for(len(clean_text)), user_payload)
- if isinstance(data, dict):
- llm_candidate = data
- valid_models = [agg[0]]
- decision_status = f"已用知识大模型 {agg[0]} 单模型深抽"
- else:
- llm_errors.append(f"{agg[0]}:返回非 JSON dict(type={type(data).__name__})")
- except Exception as exc: # noqa: BLE001
- llm_errors.append(f"{agg[0]}:{str(exc)[:180]}")
-
- if not llm_candidate and not extractors:
- try:
- client = LlmClient.from_settings()
- if client.available():
- wait_seconds = _llm_wait_seconds(len(clean_text))
- client.timeout = max(wait_seconds, 120)
- data = await asyncio.to_thread(client.chat_json, _extract_system_for(len(clean_text)), user_payload)
- if isinstance(data, dict):
- llm_candidate = data
- valid_models = ["global_llm"]
- decision_status = "抽取池不可用,已用全局 LLM 单模型深抽"
- else:
- llm_errors.append(f"global_llm:返回非 JSON dict(type={type(data).__name__})")
- except Exception as exc: # noqa: BLE001
- llm_errors.append(f"global_llm:{str(exc)[:180]}")
-
- elapsed_ms = int((time.perf_counter() - started) * 1000)
- if llm_candidate:
- result = _normalize_llm_result(
- llm_candidate,
- clean_text,
- source_name,
- source_url,
- root_entity,
- "unified_llm_only_multi_model_kg_extraction_v1",
- (
- "统一深度抽取已运行:最终候选只来自多模型 LLM 深度抽取与裁决/合并;"
- "输出 Entity / Event / Concept / Relation / Statement,发布前仍需 Entity Alignment 与低置信审核。"
- ),
- )
- result["text_units"] = [{k: v for k, v in unit.items() if k != "text"} for unit in text_units]
- result["summary"]["text_units"] = len(text_units)
- result["summary"]["strategy"] = (
- "llm_only_textunit_context" if len(clean_text) > ADAPTIVE_LONG_TEXT_THRESHOLD else "llm_only_whole_context"
- )
- result["summary"]["adaptive_reason"] = (
- f"input_chars > {ADAPTIVE_LONG_TEXT_THRESHOLD},文本会附带 TextUnit 元数据辅助大模型理解长文档结构"
- if len(clean_text) > ADAPTIVE_LONG_TEXT_THRESHOLD
- else f"input_chars <= {ADAPTIVE_LONG_TEXT_THRESHOLD},使用整篇长上下文 LLM 深度抽取"
- )
- result["summary"]["llm_stage"] = "multi_model_ran" if len(valid_models) > 1 else "single_model_ran"
- result["summary"]["valid_models"] = ",".join(valid_models) or "-"
- result["summary"]["llm_elapsed_ms"] = elapsed_ms
- result["summary"]["llm_elapsed_seconds"] = round(elapsed_ms / 1000, 2)
- result["summary"]["multi_model_status"] = f"{status_msg};有效模型={','.join(valid_models) or '无'};{decision_status}"
- result["summary"]["model_timings"] = model_timings
- result["summary"]["model_errors"] = ";".join(llm_errors[:5])
- result["graph_preview"] = _adaptive_graph(result)
- return result
-
- error_msg = ";".join(llm_errors[:8]) or status_msg
- raise HTTPException(
- status_code=502,
- detail=(
- "统一 LLM 深度抽取失败,没有返回有效 JSON。"
- f"耗时 {round(elapsed_ms / 1000, 2)} 秒;模型状态:{status_msg};错误:{error_msg}"
- ),
- )
-
-
-async def _extract_manual_kg_whole(
- clean_text: str,
- source_name: str,
- source_url: str,
- root_entity: str,
- extractors: list[tuple[str, LlmClient]],
- agg: tuple[str, LlmClient] | None,
- status_msg: str,
- schema_context: dict[str, Any] | None = None,
-) -> tuple[dict[str, Any] | None, list[str]]:
- user_payload = json.dumps({
- "source": {"source_name": source_name or "人工录入", "source_url": source_url or ""},
- "root_entity_hint": root_entity or "",
- "business_schema": schema_context or {},
- "strategy": "whole_document_long_context",
- "input_chars": len(clean_text),
- "input_text": clean_text[:100000],
- "important_instruction": (
- "若 business_schema 非空,优先按其中的业务场景 Schema 约束抽取;"
- "Schema 外新事实进入 schema_proposals。"
- + _length_budget_instruction(len(clean_text))
- ),
- }, ensure_ascii=False)
-
- llm_errors: list[str] = []
- if extractors:
- wait_seconds = _pool_wait_seconds(_llm_wait_seconds(len(clean_text)), extractors, agg)
- for _, client in extractors:
- client.timeout = wait_seconds
- if agg:
- agg[1].timeout = wait_seconds
- responses = await fan_out(
- _extract_system_for(len(clean_text)),
- user_payload,
- extractors,
- min_valid=_llm_quorum(extractors),
- max_wait_seconds=wait_seconds,
- )
- valid = [r for r in responses if isinstance(r.get("data"), dict)]
- if len(valid) == 1:
- return _normalize_llm_result(
- valid[0]["data"],
- clean_text,
- source_name,
- source_url,
- root_entity,
- "single_valid_llm_whole_no_decider_v1",
- f"整篇长上下文:1/{len(extractors)} 个知识大模型有效,已直接采用单模型结果",
- ), llm_errors
- if valid and agg:
- decide_wait = _decide_wait_seconds(len(clean_text), len(valid))
- agg[1].timeout = max(decide_wait, 20)
- decided, err = await decide(
- KG_DECIDE_SYS,
- json.dumps({"多模型抽取": valid, "原文": clean_text[:30000]}, ensure_ascii=False),
- agg,
- max_wait_seconds=decide_wait,
- )
- if decided:
- return _normalize_llm_result(
- decided,
- clean_text,
- source_name,
- source_url,
- root_entity,
- "multi_llm_whole_kg_extraction_v1",
- f"整篇长上下文:{len(valid)}/{len(extractors)} 个知识大模型有效,LLM 合并模型 {agg[0]} 已合并裁剪",
- ), llm_errors
- merged = _merge_candidate_results([r["data"] for r in valid])
- return _normalize_llm_result(
- merged,
- clean_text,
- source_name,
- source_url,
- root_entity,
- "multi_llm_whole_python_merge_fallback_v1",
- f"整篇长上下文:LLM 合并失败:{err};已确定性合并 {len(valid)}/{len(extractors)} 个知识大模型候选",
- ), llm_errors
- if valid:
- merged = _merge_candidate_results([r["data"] for r in valid])
- return _normalize_llm_result(
- merged,
- clean_text,
- source_name,
- source_url,
- root_entity,
- "multi_llm_whole_python_merge_no_decider_v1",
- f"整篇长上下文:已确定性合并 {len(valid)}/{len(extractors)} 个知识大模型候选",
- ), llm_errors
- failed = [
- f"{r.get('model')}({r.get('seconds', '-')}s):{r.get('error')}"
- for r in responses
- if r.get("error") and r.get("error") != "skipped_after_quorum"
- ]
- if failed:
- llm_errors.append("整篇知识大模型失败:" + ";".join(failed[:3]))
-
- if not extractors and agg:
- try:
- wait_seconds = _llm_wait_seconds(len(clean_text))
- agg[1].timeout = max(wait_seconds, 120)
- data = await asyncio.to_thread(agg[1].chat_json, _extract_system_for(len(clean_text)), user_payload)
- if isinstance(data, dict):
- return _normalize_llm_result(
- data,
- clean_text,
- source_name,
- source_url,
- root_entity,
- "single_llm_whole_aggregator_v1",
- f"整篇长上下文:{status_msg};已使用知识大模型 {agg[0]} 单模型深度抽取",
- ), llm_errors
- llm_errors.append(f"整篇知识大模型返回非 JSON dict(type={type(data).__name__})")
- except Exception as exc: # noqa: BLE001
- llm_errors.append(f"整篇知识大模型调用失败:{str(exc)[:180]}")
-
- if extractors:
- return None, llm_errors
-
- try:
- client = LlmClient.from_settings()
- if client.available():
- wait_seconds = _llm_wait_seconds(len(clean_text))
- client.timeout = max(wait_seconds, 120)
- data = await asyncio.to_thread(client.chat_json, _extract_system_for(len(clean_text)), user_payload)
- if isinstance(data, dict):
- return _normalize_llm_result(
- data,
- clean_text,
- source_name,
- source_url,
- root_entity,
- "single_llm_whole_global_v1",
- f"整篇长上下文:抽取池不可用:{status_msg};已使用全局 LLM 单模型通用抽取",
- ), llm_errors
- llm_errors.append(f"整篇全局 LLM 返回非 JSON dict(type={type(data).__name__})")
- except Exception as exc: # noqa: BLE001
- llm_errors.append(f"整篇全局 LLM 调用失败:{str(exc)[:180]}")
-
- return None, llm_errors
-
-
-async def extract_manual_kg_llm(
- text: str,
- source_name: str,
- source_url: str,
- root_entity: str = "",
- strategy: str = "auto",
- schema_context: dict[str, Any] | None = None,
- selected_model_key: str = "",
-) -> dict[str, Any]:
- clean_text = (text or "").strip()
- if len(clean_text) < 4:
- raise HTTPException(status_code=400, detail="请输入或上传需要抽取的人工材料")
-
- cfg = await get_agent_settings()
- all_extractors, all_agg, all_status_msg = build_extract_pool(
- cfg,
- skip_duplicate_aggregator=not bool((selected_model_key or "").strip()),
- )
- extractors, agg, status_msg = _select_manual_extract_model(
- all_extractors,
- all_agg,
- all_status_msg,
- selected_model_key,
- )
- strategy = (strategy or "auto").strip().lower()
- if strategy not in {"adaptive", "auto", "whole", "chunked", "compare"}:
- strategy = "auto"
-
- async def run_once(
- run_extractors: list[tuple[str, LlmClient]],
- run_agg: tuple[str, LlmClient] | None,
- run_status_msg: str,
- ) -> dict[str, Any]:
- if strategy in {"adaptive", "auto"}:
- return await _extract_manual_kg_adaptive_deep(
- clean_text,
- source_name,
- source_url,
- root_entity,
- run_extractors,
- run_agg,
- run_status_msg,
- schema_context,
- )
-
- use_chunked = strategy == "chunked" or (strategy == "auto" and len(clean_text) > 8000)
- if use_chunked:
- chunked = await _extract_manual_kg_chunked(
- clean_text,
- source_name,
- source_url,
- root_entity,
- run_extractors,
- run_agg,
- run_status_msg,
- schema_context,
- )
- if chunked:
- chunked["summary"]["strategy"] = "chunked"
- return chunked
-
- if strategy == "compare":
- whole, whole_errors = await _extract_manual_kg_whole(
- clean_text,
- source_name,
- source_url,
- root_entity,
- run_extractors,
- run_agg,
- run_status_msg,
- schema_context,
- )
- chunked = await _extract_manual_kg_chunked(
- clean_text,
- source_name,
- source_url,
- root_entity,
- run_extractors,
- run_agg,
- run_status_msg,
- schema_context,
- )
- whole_score = _result_score(whole)
- chunked_score = _result_score(chunked)
- selected = chunked if chunked_score >= whole_score else whole
- if selected:
- selected["summary"]["strategy"] = "compare"
- selected["comparison"] = {
- "selected": "chunked" if selected is chunked else "whole",
- "whole": _result_brief(whole),
- "chunked": _result_brief(chunked),
- "whole_errors": whole_errors[:3],
- "decision_rule": "优先选择非规则兜底、证据更多、Statement/Event 覆盖更高的结果",
- }
- return selected
-
- whole, llm_errors = await _extract_manual_kg_whole(
- clean_text,
- source_name,
- source_url,
- root_entity,
- run_extractors,
- run_agg,
- run_status_msg,
- schema_context,
- )
- if whole:
- whole["summary"]["strategy"] = "whole" if strategy == "whole" else "auto_whole"
- return whole
-
- error_msg = ";".join(x for x in llm_errors if x)
- raise HTTPException(
- status_code=502,
- detail=(
- f"通用 LLM 深度抽取失败:{run_status_msg}"
- f"{';' + error_msg if error_msg else ''}。系统不会再用快速规则结果冒充深度抽取。"
- ),
- )
-
- primary_error: HTTPException | None = None
- result: dict[str, Any] | None = None
- try:
- result = await run_once(extractors, agg, status_msg)
- except HTTPException as exc:
- primary_error = exc
-
- if (
- result is None
- and primary_error is not None
- and _is_travel_schema_context(schema_context)
- and _is_transient_llm_exception(primary_error)
- ):
- retry_error = primary_error
- try:
- result = await run_once(extractors, agg, status_msg)
- primary_error = None
- result.setdefault("summary", {})["travel_quality_retry"] = {
- "model": (selected_model_key or "").strip(),
- "reason": "旅行社抽取遇到模型接口瞬时错误,已同模型重试成功",
- "first_error": _http_exception_text(retry_error)[:240],
- }
- except HTTPException as exc:
- primary_error = exc
-
- if _is_travel_schema_context(schema_context):
- should_retry = result is None or _needs_travel_quality_retry(result, clean_text, schema_context)
- fallback_key = (
- _choose_travel_fallback_model_key(selected_model_key, all_extractors, all_agg)
- if should_retry
- else ""
- )
- if fallback_key:
- try:
- fallback_extractors, fallback_agg, fallback_status = _select_manual_extract_model(
- all_extractors,
- all_agg,
- all_status_msg,
- fallback_key,
- )
- fallback_result = await run_once(fallback_extractors, fallback_agg, fallback_status)
- _annotate_travel_quality(fallback_result, clean_text, schema_context)
- fallback_ok = not _needs_travel_quality_retry(fallback_result, clean_text, schema_context)
- primary_score = _result_score(result)
- fallback_score = _result_score(fallback_result)
- if fallback_ok or result is None or fallback_score >= primary_score:
- summary = fallback_result.setdefault("summary", {})
- summary["travel_quality_fallback"] = {
- "from_model": (selected_model_key or "").strip(),
- "to_model": fallback_key,
- "reason": "旅行社核心关系/属性质量门槛未达标或主模型失败",
- "primary_error": str(primary_error.detail)[:240] if primary_error else "",
- }
- return fallback_result
- except HTTPException as exc:
- if result is None and primary_error is None:
- primary_error = exc
-
- if result is not None:
- _annotate_travel_quality(result, clean_text, schema_context)
- return result
-
- if result is not None:
- return result
- if primary_error:
- raise primary_error
- raise HTTPException(status_code=502, detail="通用 LLM 深度抽取失败:未知错误")
-
-
-@router.post("/manual-ingest/convert-file")
-async def manual_ingest_convert_file(file: UploadFile = File(...)) -> dict[str, Any]:
- input_path, size = await _save_markitdown_upload(file)
- try:
- markdown, markdown_path, converter_name = await asyncio.to_thread(
- _convert_path_with_markitdown,
- input_path,
- )
- except HTTPException:
- raise
- except Exception as exc: # noqa: BLE001
- raise HTTPException(status_code=422, detail=f"MarkItDown 转换失败:{str(exc)[:240]}") from exc
-
- # ②结构化(自动路由:价格表用免费确定性、行程单/复杂件上 LLM)+ ③校验(失败不影响基础转换)
- structured_markdown, validation, structured_mode = "", None, ""
- try:
- from app.api.doc_restructure import build_structured
- structured_markdown, validation, structured_mode = await asyncio.to_thread(
- build_structured, markdown, Path(file.filename or input_path.name).name,
- )
- except Exception: # noqa: BLE001
- structured_markdown, validation, structured_mode = "", None, ""
-
- return {
- "filename": Path(file.filename or input_path.name).name,
- "content_type": file.content_type or "",
- "size": size,
- "markdown": markdown,
- "markdown_chars": len(markdown),
- "markdown_path": _relative_project_path(markdown_path),
- "source_path": _relative_project_path(input_path),
- "input_format": "markdown",
- "converter": converter_name,
- "structured_markdown": structured_markdown,
- "structured_chars": len(structured_markdown),
- "structured_mode": structured_mode,
- "validation": validation,
- }
-
-
-@router.get("/manual-ingest/markdown-preview")
-def manual_ingest_markdown_preview(path: str) -> dict[str, Any]:
- markdown_path = _resolve_markitdown_preview_path(path)
- markdown = markdown_path.read_text(encoding="utf-8", errors="replace")
- return {
- "filename": markdown_path.name,
- "markdown_path": _relative_project_path(markdown_path),
- "markdown": markdown,
- "markdown_chars": len(markdown),
- "size": markdown_path.stat().st_size,
- }
-
-
-@router.post("/manual-ingest/extract")
-async def manual_ingest_extract(body: dict[str, Any]) -> dict[str, Any]:
- text = str(body.get("text") or "")
- source_name = str(body.get("source_name") or "人工录入")
- source_url = str(body.get("source_url") or "")
- root_entity = str(body.get("root_entity") or "")
- mode = str(body.get("extract_mode") or "llm")
- strategy = str(body.get("extract_strategy") or "auto")
- selected_model_key = str(body.get("extract_model_key") or "")
- schema_context = _schema_context_from_body(body)
- if mode == "fast":
- raise HTTPException(status_code=400, detail="人工录入已统一为 LLM 深度抽取,不再支持快速规则抽取")
- return await extract_manual_kg_llm(
- text,
- source_name,
- source_url,
- root_entity,
- strategy,
- schema_context,
- selected_model_key,
- )
-
-
-@router.post("/manual-ingest/evidence-locate")
-def manual_ingest_evidence_locate(body: dict[str, Any]) -> dict[str, Any]:
- text = str(body.get("text") or "")
- quote = str(body.get("quote") or "").strip()
- if not text or not quote:
- raise HTTPException(status_code=400, detail="缺少原文或证据片段")
- start = text.find(quote)
- if start >= 0:
- end = start + len(quote)
- return {
- "start_char": start,
- "end_char": end,
- "score": 1.0,
- "quote": quote,
- "context": text[max(0, start - 80): min(len(text), end + 80)],
- "method": "exact_quote",
- }
- candidates = _sentence_candidates(text) or [text[i:i + 160] for i in range(0, len(text), 120)]
- best = max(candidates, key=lambda s: SequenceMatcher(None, quote, s).ratio())
- score = SequenceMatcher(None, quote, best).ratio()
- start = text.find(best)
- end = start + len(best) if start >= 0 else None
- return {
- "start_char": start if start >= 0 else None,
- "end_char": end,
- "score": round(score, 3),
- "quote": best,
- "context": best,
- "method": "semantic_similarity_fallback",
- }
diff --git a/app/api/notifications.py b/app/api/notifications.py
deleted file mode 100644
index 6bc289d..0000000
--- a/app/api/notifications.py
+++ /dev/null
@@ -1,42 +0,0 @@
-"""In-app notifications / inbox (P4)."""
-from fastapi import APIRouter, HTTPException
-
-from app.auth import CurrentUser
-from app.db import (
- get_user_id_by_username,
- list_notifications,
- mark_all_notifications_read,
- mark_notification_read,
- unread_notification_count,
-)
-
-router = APIRouter()
-
-
-async def _uid(user: dict) -> int:
- uid = await get_user_id_by_username(user["username"])
- if uid is None:
- raise HTTPException(404, "用户不存在")
- return uid
-
-
-@router.get("/notifications")
-async def _list(user: CurrentUser, only_unread: bool = False):
- return await list_notifications(await _uid(user), only_unread)
-
-
-@router.get("/notifications/unread-count")
-async def _count(user: CurrentUser):
- return {"count": await unread_notification_count(await _uid(user))}
-
-
-@router.post("/notifications/{notif_id}/read")
-async def _read(notif_id: int, user: CurrentUser):
- await mark_notification_read(notif_id, await _uid(user))
- return {"ok": True}
-
-
-@router.post("/notifications/read-all")
-async def _read_all(user: CurrentUser):
- n = await mark_all_notifications_read(await _uid(user))
- return {"ok": True, "marked": n}
diff --git a/app/api/plaza.py b/app/api/plaza.py
index 49f779e..e841eda 100644
--- a/app/api/plaza.py
+++ b/app/api/plaza.py
@@ -14,7 +14,7 @@ from fastapi import APIRouter, Depends, HTTPException
from app.auth import CurrentUser
from app.config import settings
-from app.db import get_agent_settings, get_plaza_overview, get_plaza_alerts, get_conn
+from app.db import get_agent_settings, get_plaza_overview, get_conn
from app.graph_qa_engine import answer_graph_question
from app.llm_client import LlmClient
from app.project_context import ProjectContext, get_project_context
@@ -373,44 +373,6 @@ async def overview(
return await get_plaza_overview(context.tenant_id, context.project_id)
-@router.get("/plaza/usage")
-async def usage(
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- """Return usage statistics — top hot and cold entities."""
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"SELECT entity_type, COUNT(*) AS cnt FROM {s}.candidate_entities "
- "WHERE tenant_id=%s AND project_id=%s AND status='published' "
- "GROUP BY entity_type ORDER BY cnt DESC",
- (context.tenant_id, context.project_id),
- )
- by_type = await cur.fetchall()
-
- await cur.execute(
- f"SELECT COUNT(*) AS cnt FROM {s}.candidate_entities "
- "WHERE tenant_id=%s AND project_id=%s AND status='pending_review'",
- (context.tenant_id, context.project_id),
- )
- pending = (await cur.fetchone())["cnt"]
-
- return {
- "entities_by_type": by_type,
- "pending_review": pending,
- }
-
-
-@router.get("/plaza/alerts")
-async def alerts(
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- return await get_plaza_alerts(context.tenant_id, context.project_id)
-
-
@router.get("/plaza/amap-config")
async def amap_config(_user: CurrentUser = None):
"""Return browser-side AMap JS API configuration for admin map canvases."""
diff --git a/app/api/publish_jobs.py b/app/api/publish_jobs.py
deleted file mode 100644
index b77d8e4..0000000
--- a/app/api/publish_jobs.py
+++ /dev/null
@@ -1,122 +0,0 @@
-"""STEP 05 — Publish Jobs + Rollback + Diff."""
-import json
-
-from fastapi import APIRouter, Depends, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.contracts import RollbackRequest
-from app.db import (
- get_publish_job,
- list_publish_jobs,
- update_publish_job,
- update_candidate_entity,
- get_conn,
-)
-from app.project_context import ProjectContext, get_project_context
-
-router = APIRouter()
-
-
-@router.get("/publish-jobs")
-async def _list(
- limit: int = 50,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- return await list_publish_jobs(context.tenant_id, context.project_id, limit)
-
-
-@router.get("/publish-jobs/{job_id}")
-async def _get(job_id: int, _user: CurrentUser = None):
- job = await get_publish_job(job_id)
- if not job:
- raise HTTPException(404, "Publish job not found")
- return job
-
-
-@router.post("/publish-jobs")
-async def _create(
- body: dict,
- user: CurrentUser,
- context: ProjectContext = Depends(get_project_context),
-):
- """Create a publish job for approved candidates."""
- s = settings.db_schema
- candidate_ids = body.get("candidate_ids", [])
-
- diff = {
- "entities_added": len(candidate_ids),
- "entities_updated": 0,
- "relations_added": 0,
- "field_changes": {},
- }
-
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""INSERT INTO {s}.publish_jobs
- (tenant_id, project_id, candidate_ids, status, actor, diff_summary_jsonb)
- VALUES (%s, %s, %s, 'pending', %s, %s)
- RETURNING *""",
- (
- context.tenant_id,
- context.project_id,
- json.dumps(candidate_ids),
- user["username"],
- json.dumps(diff),
- ),
- )
- job = await cur.fetchone()
- await conn.commit()
-
- # Mark candidates as published
- for cid in candidate_ids:
- await update_candidate_entity(cid, {"status": "published"})
-
- # Mark job as completed
- await update_publish_job(job["id"], {"status": "completed"})
-
- return {**job, "status": "completed"}
-
-
-@router.get("/publish-jobs/{job_id}/diff")
-async def _diff(job_id: int, _user: CurrentUser = None):
- """Return the diff summary before/after a publish."""
- job = await get_publish_job(job_id)
- if not job:
- raise HTTPException(404, "Publish job not found")
- diff = job.get("diff_summary_jsonb")
- if isinstance(diff, str):
- diff = json.loads(diff)
- return {"job_id": job_id, "diff": diff}
-
-
-@router.post("/publish-jobs/{job_id}/rollback")
-async def _rollback(job_id: int, body: RollbackRequest | None = None, user: CurrentUser = None):
- """Rollback a publish — revert candidate statuses."""
- job = await get_publish_job(job_id)
- if not job:
- raise HTTPException(404, "Publish job not found")
- if job["status"] != "completed":
- raise HTTPException(400, "Can only rollback completed jobs")
-
- cids = job.get("candidate_ids")
- if isinstance(cids, str):
- cids = json.loads(cids)
-
- # Revert candidates back to approved
- for cid in (cids or []):
- await update_candidate_entity(cid, {"status": "approved"})
-
- await update_publish_job(job_id, {
- "status": "rolled_back",
- "rollback_target_release_id": job_id,
- })
-
- return {
- "rolled_back": True,
- "job_id": job_id,
- "reason": body.reason if body else None,
- "reverted_candidates": len(cids or []),
- }
diff --git a/app/api/question_traces.py b/app/api/question_traces.py
deleted file mode 100644
index 4eeaf3c..0000000
--- a/app/api/question_traces.py
+++ /dev/null
@@ -1,105 +0,0 @@
-"""STEP 02 — Question Traces endpoints."""
-from fastapi import APIRouter, Depends, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.contracts import QuestionTraceCreate, QuestionTraceBatch
-from app.db import (
- create_question_trace,
- list_question_traces,
- get_question_trace,
- update_question_trace,
-)
-from app.project_context import ProjectContext, get_project_context
-
-router = APIRouter()
-
-
-@router.post("/question-traces")
-async def _create(
- body: QuestionTraceCreate,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- data = body.model_dump()
- data.setdefault("tenant_id", context.tenant_id)
- data.setdefault("project_id", context.project_id)
- return await create_question_trace(data)
-
-
-@router.post("/question-traces/batch")
-async def _batch(
- body: QuestionTraceBatch,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- results = []
- for q in body.questions:
- data = q.model_dump()
- data.setdefault("tenant_id", context.tenant_id)
- data.setdefault("project_id", context.project_id)
- results.append(await create_question_trace(data))
- return {"count": len(results), "traces": results}
-
-
-@router.post("/question-traces/simulate")
-async def _simulate(
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- """Create traces from enabled sim questions, audit them in the background."""
- from app.db import list_simulation_questions, create_audit_run
- from app.agents.auditor import schedule_audit
-
- tenant = context.tenant_id
- project = context.project_id
-
- sim_questions = await list_simulation_questions(tenant, project)
- enabled = [q for q in sim_questions if q.get("enabled")]
-
- created = []
- for sq in enabled:
- trace = await create_question_trace({
- "tenant_id": tenant,
- "project_id": project,
- "source": "simulated",
- "origin": "panel",
- "question_text": sq["question_text"],
- "scenario_tags": sq.get("scenario_tags", []),
- })
- created.append(trace)
-
- trace_ids = [t["id"] for t in created]
- if not trace_ids:
- return {"created": 0, "run_id": None, "total": 0}
-
- run_id = await create_audit_run("simulate", len(trace_ids))
- schedule_audit(trace_ids, run_id)
- return {"created": len(created), "run_id": run_id, "total": len(trace_ids)}
-
-
-@router.get("/question-traces")
-async def _list(
- source: str | None = None,
- suggested_action: str | None = None,
- limit: int = 50,
- offset: int = 0,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- return await list_question_traces(
- context.tenant_id,
- context.project_id,
- source,
- suggested_action,
- limit,
- offset,
- )
-
-
-@router.get("/question-traces/{trace_id}")
-async def _get(trace_id: int, _user: CurrentUser = None):
- trace = await get_question_trace(trace_id)
- if not trace:
- raise HTTPException(404, "Question trace not found")
- return trace
diff --git a/app/api/rbac.py b/app/api/rbac.py
index 901fc7b..1ba99d6 100644
--- a/app/api/rbac.py
+++ b/app/api/rbac.py
@@ -1,13 +1,12 @@
"""RBAC & Accounts (P1) — roles, capability matrix, users CRUD."""
from fastapi import APIRouter, HTTPException
-from app.auth import CurrentUser
+from app.auth import AdminUser
from app.contracts import (
CapabilityCreate,
RoleCapCell,
RoleCreate,
RoleUpdate,
- UserAreasSet,
UserCreate,
UserUpdate,
)
@@ -21,13 +20,10 @@ from app.db import (
get_permission_matrix,
list_capabilities,
list_roles,
- list_user_areas,
list_users,
set_role_cap,
- set_user_areas,
update_role,
update_user,
- upsert_custom_area,
)
from app.security import hash_password
@@ -37,12 +33,12 @@ router = APIRouter()
# ── Roles ────────────────────────────────────────────────────────────────────
@router.get("/roles")
-async def _list_roles(_user: CurrentUser = None):
+async def _list_roles(_user: AdminUser):
return await list_roles()
@router.post("/roles")
-async def _create_role(body: RoleCreate, _user: CurrentUser):
+async def _create_role(body: RoleCreate, _user: AdminUser):
try:
return await create_role(body.model_dump())
except Exception as e: # unique violation etc.
@@ -50,7 +46,7 @@ async def _create_role(body: RoleCreate, _user: CurrentUser):
@router.patch("/roles/{role_key}")
-async def _update_role(role_key: str, body: RoleUpdate, _user: CurrentUser):
+async def _update_role(role_key: str, body: RoleUpdate, _user: AdminUser):
row = await update_role(role_key, body.model_dump(exclude_none=True))
if not row:
raise HTTPException(404, "角色不存在")
@@ -58,7 +54,7 @@ async def _update_role(role_key: str, body: RoleUpdate, _user: CurrentUser):
@router.delete("/roles/{role_key}")
-async def _delete_role(role_key: str, _user: CurrentUser):
+async def _delete_role(role_key: str, _user: AdminUser):
try:
ok = await delete_role(role_key)
except ValueError as e:
@@ -71,12 +67,12 @@ async def _delete_role(role_key: str, _user: CurrentUser):
# ── Capabilities ─────────────────────────────────────────────────────────────
@router.get("/capabilities")
-async def _list_caps(_user: CurrentUser = None):
+async def _list_caps(_user: AdminUser):
return await list_capabilities()
@router.post("/capabilities")
-async def _create_cap(body: CapabilityCreate, _user: CurrentUser):
+async def _create_cap(body: CapabilityCreate, _user: AdminUser):
try:
return await create_capability(body.model_dump())
except Exception as e:
@@ -84,7 +80,7 @@ async def _create_cap(body: CapabilityCreate, _user: CurrentUser):
@router.delete("/capabilities/{cap_key}")
-async def _delete_cap(cap_key: str, _user: CurrentUser):
+async def _delete_cap(cap_key: str, _user: AdminUser):
await delete_capability(cap_key)
return {"deleted": cap_key}
@@ -92,12 +88,12 @@ async def _delete_cap(cap_key: str, _user: CurrentUser):
# ── Permission matrix ────────────────────────────────────────────────────────
@router.get("/permission-matrix")
-async def _matrix(_user: CurrentUser = None):
+async def _matrix(_user: AdminUser):
return await get_permission_matrix()
@router.put("/permission-matrix")
-async def _set_cell(body: RoleCapCell, _user: CurrentUser):
+async def _set_cell(body: RoleCapCell, _user: AdminUser):
await set_role_cap(body.role_key, body.cap_key, body.value)
return {"ok": True, "role_key": body.role_key, "cap_key": body.cap_key, "value": body.value}
@@ -105,12 +101,12 @@ async def _set_cell(body: RoleCapCell, _user: CurrentUser):
# ── Users ────────────────────────────────────────────────────────────────────
@router.get("/users")
-async def _list_users(_user: CurrentUser = None):
+async def _list_users(_user: AdminUser):
return await list_users()
@router.post("/users")
-async def _create_user(body: UserCreate, _user: CurrentUser):
+async def _create_user(body: UserCreate, _user: AdminUser):
data = body.model_dump()
roles = data.pop("roles", [])
pw = data.pop("password")
@@ -122,7 +118,7 @@ async def _create_user(body: UserCreate, _user: CurrentUser):
@router.patch("/users/{user_id}")
-async def _update_user(user_id: int, body: UserUpdate, _user: CurrentUser):
+async def _update_user(user_id: int, body: UserUpdate, _user: AdminUser):
data = body.model_dump(exclude_none=True)
roles = data.pop("roles", None)
if "password" in data:
@@ -134,22 +130,6 @@ async def _update_user(user_id: int, body: UserUpdate, _user: CurrentUser):
@router.delete("/users/{user_id}")
-async def _delete_user(user_id: int, _user: CurrentUser):
+async def _delete_user(user_id: int, _user: AdminUser):
await delete_user(user_id)
return {"deleted": user_id}
-
-
-@router.get("/users/{user_id}/areas")
-async def _get_user_areas(user_id: int, _user: CurrentUser = None):
- return await list_user_areas(user_id)
-
-
-@router.put("/users/{user_id}/areas")
-async def _set_user_areas(user_id: int, body: UserAreasSet, _user: CurrentUser):
- """Set the areas this user is responsible for (existing + free-text)."""
- area_ids = list(body.area_ids)
- for name in body.custom_areas:
- if name and name.strip():
- area_ids.append(await upsert_custom_area(name))
- await set_user_areas(user_id, area_ids)
- return await list_user_areas(user_id)
diff --git a/app/api/simulation_panel.py b/app/api/simulation_panel.py
deleted file mode 100644
index a643078..0000000
--- a/app/api/simulation_panel.py
+++ /dev/null
@@ -1,98 +0,0 @@
-"""STEP 02 — Simulation Panel CRUD + multi-source question intake.
-
-Question bank for the AI quality audit. Sources:
- manual 手工添加
- imported 批量导入的真实用户问题
- resolved 从"已答好"的 question_traces 采纳的回归用例
- ai_gen AI 生成的覆盖性问题
-"""
-from fastapi import APIRouter, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.contracts import SimulationQuestion, SimulationQuestionUpdate
-from app.db import (
- list_simulation_questions,
- create_simulation_question,
- update_simulation_question,
- delete_simulation_question,
- bulk_create_simulation_questions,
- adopt_hit_question_traces,
-)
-
-router = APIRouter()
-_T = settings.default_tenant
-_P = settings.default_project
-
-
-@router.get("/simulation-panel")
-async def _list(_user: CurrentUser = None):
- return await list_simulation_questions(_T, _P)
-
-
-@router.post("/simulation-panel")
-async def _create(body: SimulationQuestion, _user: CurrentUser):
- data = body.model_dump()
- data["tenant_id"] = _T
- data["project_id"] = _P
- data["source"] = "manual"
- return await create_simulation_question(data)
-
-
-@router.post("/simulation-panel/import")
-async def _import(body: dict, _user: CurrentUser):
- """Bulk import real user questions — one per line, or a list."""
- raw = body.get("text") or body.get("questions") or ""
- texts = raw if isinstance(raw, list) else str(raw).splitlines()
- added = await bulk_create_simulation_questions(_T, _P, texts, "imported")
- return {"added": added}
-
-
-@router.post("/simulation-panel/adopt-hits")
-async def _adopt_hits(_user: CurrentUser):
- """Adopt well-answered questions as regression cases (source=resolved)."""
- added = await adopt_hit_question_traces(_T, _P)
- return {"added": added}
-
-
-@router.post("/simulation-panel/ai-generate")
-async def _ai_generate(body: dict, _user: CurrentUser):
- """Let the LLM propose representative coverage questions."""
- from app.llm_client import LlmClient
-
- llm = LlmClient.from_settings() if settings.llm_api_key else None
- if not llm or not llm.available():
- raise HTTPException(400, "未配置 LLM(Agent 设置里填好 API Key 后可用)")
-
- count = max(1, min(int(body.get("count", 10)), 30))
- topic = (body.get("topic") or "城市本地生活知识图谱(地点、区域、路线、夜间体验、标签)").strip()
- try:
- out = llm.chat_json(
- system=(
- "你是知识图谱质量稽查助手。请基于给定主题,生成有代表性、"
- "覆盖不同场景的真实用户中文问题,用于检验知识图谱数据的完善度。"
- '严格返回 JSON:{"questions": ["...", "..."]}'
- ),
- user=f"主题:{topic}。生成 {count} 个不同角度、不重复的问题。",
- )
- qs = out.get("questions") or []
- qs = [str(q).strip() for q in qs if str(q).strip()][:count]
- except Exception as e:
- raise HTTPException(400, f"AI 生成失败:{str(e)[:160]}")
-
- added = await bulk_create_simulation_questions(_T, _P, qs, "ai_gen")
- return {"added": added, "generated": len(qs)}
-
-
-@router.patch("/simulation-panel/{q_id}")
-async def _update(q_id: int, body: SimulationQuestionUpdate, _user: CurrentUser):
- row = await update_simulation_question(q_id, body.model_dump(exclude_none=True))
- if not row:
- raise HTTPException(404, "Simulation question not found")
- return row
-
-
-@router.delete("/simulation-panel/{q_id}")
-async def _delete(q_id: int, _user: CurrentUser):
- await delete_simulation_question(q_id)
- return {"ok": True}
diff --git a/app/api/source_profiles.py b/app/api/source_profiles.py
deleted file mode 100644
index a1bca8d..0000000
--- a/app/api/source_profiles.py
+++ /dev/null
@@ -1,43 +0,0 @@
-"""STEP 01 — Source Profiles CRUD."""
-from fastapi import APIRouter, Depends, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.contracts import SourceProfileCreate, SourceProfileUpdate
-from app.db import list_source_profiles, create_source_profile, update_source_profile
-from app.project_context import ProjectContext, get_project_context
-
-router = APIRouter()
-
-
-@router.get("/source-profiles")
-async def _list(
- tenant_id: str | None = None,
- project_id: str | None = None,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- return await list_source_profiles(
- tenant_id or context.tenant_id,
- project_id or context.project_id,
- )
-
-
-@router.post("/source-profiles")
-async def _create(
- body: SourceProfileCreate,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- data = body.model_dump()
- data.setdefault("tenant_id", context.tenant_id)
- data.setdefault("project_id", context.project_id)
- return await create_source_profile(data)
-
-
-@router.patch("/source-profiles/{profile_id}")
-async def _update(profile_id: int, body: SourceProfileUpdate, _user: CurrentUser):
- row = await update_source_profile(profile_id, body.model_dump(exclude_none=True))
- if not row:
- raise HTTPException(404, "Source profile not found")
- return row
diff --git a/app/api/treatment_history.py b/app/api/treatment_history.py
deleted file mode 100644
index c7e15a4..0000000
--- a/app/api/treatment_history.py
+++ /dev/null
@@ -1,27 +0,0 @@
-"""STEP 04 — Treatment History (历史治理记录)."""
-from fastapi import APIRouter, Depends
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.db import get_conn
-
-router = APIRouter()
-
-
-@router.get("/treatment-history")
-async def _list(limit: int = 50, _user: CurrentUser = None):
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""SELECT
- ra.id, ra.candidate_id, ra.candidate_type,
- ra.action, ra.actor, ra.note, ra.field_decisions_jsonb,
- ra.created_at,
- ce.entity_type, ce.natural_key
- FROM {s}.review_actions ra
- LEFT JOIN {s}.candidate_entities ce ON ra.candidate_id=ce.id
- ORDER BY ra.created_at DESC LIMIT %s""",
- (limit,),
- )
- return await cur.fetchall()
diff --git a/app/api/vocabulary.py b/app/api/vocabulary.py
deleted file mode 100644
index d02f31d..0000000
--- a/app/api/vocabulary.py
+++ /dev/null
@@ -1,94 +0,0 @@
-"""STEP 05 — Vocabulary Terms (权威词表) CRUD + lookup + merge."""
-from fastapi import APIRouter, Depends, HTTPException
-
-from app.auth import CurrentUser
-from app.config import settings
-from app.contracts import VocabularyTermCreate, VocabularyTermUpdate
-from app.db import (
- list_vocabulary_terms,
- create_vocabulary_term,
- update_vocabulary_term,
- lookup_vocabulary,
- get_conn,
-)
-from app.project_context import ProjectContext, get_project_context
-
-router = APIRouter()
-
-
-@router.get("/vocabulary")
-async def _list(
- entity_type: str | None = None,
- search: str | None = None,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- return await list_vocabulary_terms(
- context.tenant_id, context.project_id, entity_type, search
- )
-
-
-@router.post("/vocabulary")
-async def _create(
- body: VocabularyTermCreate,
- user: CurrentUser,
- context: ProjectContext = Depends(get_project_context),
-):
- data = body.model_dump()
- data["tenant_id"] = context.tenant_id
- data["project_id"] = context.project_id
- data["created_by"] = user["username"]
- return await create_vocabulary_term(data)
-
-
-@router.patch("/vocabulary/{term_id}")
-async def _update(term_id: int, body: VocabularyTermUpdate, _user: CurrentUser):
- row = await update_vocabulary_term(term_id, body.model_dump(exclude_none=True))
- if not row:
- raise HTTPException(404, "Term not found")
- return row
-
-
-@router.get("/vocabulary/lookup")
-async def _lookup(
- name: str,
- context: ProjectContext = Depends(get_project_context),
- _user: CurrentUser = None,
-):
- """Normalize a name against the vocabulary."""
- result = await lookup_vocabulary(context.tenant_id, context.project_id, name)
- if not result:
- return {"found": False, "name": name}
- return {"found": True, "canonical_name": result["canonical_name"], "term": result}
-
-
-@router.post("/vocabulary/{term_id}/merge-into/{target_id}")
-async def _merge(term_id: int, target_id: int, _user: CurrentUser):
- """Merge aliases from term_id into target_id, then delete term_id."""
- import json
- s = settings.db_schema
-
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(f"SELECT * FROM {s}.vocabulary_terms WHERE id=%s", (term_id,))
- source = await cur.fetchone()
- await cur.execute(f"SELECT * FROM {s}.vocabulary_terms WHERE id=%s", (target_id,))
- target = await cur.fetchone()
-
- if not source or not target:
- raise HTTPException(404, "Term not found")
-
- source_aliases = source["aliases"] if isinstance(source["aliases"], list) else json.loads(source["aliases"] or "[]")
- target_aliases = target["aliases"] if isinstance(target["aliases"], list) else json.loads(target["aliases"] or "[]")
-
- merged = list(set(target_aliases + [source["canonical_name"]] + source_aliases))
- if target["canonical_name"] in merged:
- merged.remove(target["canonical_name"])
-
- await update_vocabulary_term(target_id, {"aliases": json.dumps(merged)})
-
- async with conn.cursor() as cur:
- await cur.execute(f"DELETE FROM {s}.vocabulary_terms WHERE id=%s", (term_id,))
- await conn.commit()
-
- return {"merged": term_id, "into": target_id, "aliases": merged}
diff --git a/app/auth.py b/app/auth.py
index 2864dff..9e770d0 100644
--- a/app/auth.py
+++ b/app/auth.py
@@ -3,6 +3,7 @@ from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Annotated
+import uuid
from fastapi import Depends, HTTPException, status
from fastapi.security import OAuth2PasswordBearer
@@ -32,27 +33,67 @@ async def authenticate(username: str, password: str) -> dict | None:
def create_access_token(data: dict) -> str:
+ now = datetime.now(timezone.utc)
payload = data.copy()
- payload["exp"] = datetime.now(timezone.utc) + timedelta(minutes=settings.auth_token_expire_minutes)
+ payload.update(
+ {
+ "iss": settings.auth_issuer,
+ "aud": settings.auth_audience,
+ "iat": now,
+ "nbf": now,
+ "jti": str(uuid.uuid4()),
+ "exp": now + timedelta(minutes=settings.auth_token_expire_minutes),
+ }
+ )
return jwt.encode(payload, settings.auth_secret, algorithm=settings.auth_algorithm)
-def get_current_user(token: Annotated[str, Depends(oauth2_scheme)]) -> dict:
- """Identity is reconstructed from JWT claims (sub + roles) — no per-request DB hit."""
+async def get_current_user(token: Annotated[str, Depends(oauth2_scheme)]) -> dict:
+ """Validate the token and reload status/roles so revocation is immediate."""
credentials_exc = HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Could not validate credentials",
headers={"WWW-Authenticate": "Bearer"},
)
try:
- payload = jwt.decode(token, settings.auth_secret, algorithms=[settings.auth_algorithm])
+ payload = jwt.decode(
+ token,
+ settings.auth_secret,
+ algorithms=[settings.auth_algorithm],
+ issuer=settings.auth_issuer,
+ audience=settings.auth_audience,
+ )
username: str = payload.get("sub", "")
if not username:
raise credentials_exc
except JWTError:
raise credentials_exc
- return {"username": username, "roles": list(payload.get("roles") or [])}
+ user = await db.get_user_auth(username)
+ if not user or user.get("status") != "active":
+ raise credentials_exc
+ return {
+ "username": user["username"],
+ "full_name": user.get("full_name"),
+ "roles": list(user.get("roles") or []),
+ }
+
+
+def require_admin(user: "CurrentUser") -> dict:
+ if "admin" not in set(user.get("roles") or []):
+ raise HTTPException(status_code=403, detail="只有系统管理员可以执行此操作")
+ return user
+
+
+def require_data_operator(user: "CurrentUser") -> dict:
+ if not ({"admin", "operator"} & set(user.get("roles") or [])):
+ raise HTTPException(
+ status_code=403,
+ detail="当前账号没有数据中心管理权限",
+ )
+ return user
CurrentUser = Annotated[dict, Depends(get_current_user)]
+AdminUser = Annotated[dict, Depends(require_admin)]
+DataOperatorUser = Annotated[dict, Depends(require_data_operator)]
diff --git a/app/config.py b/app/config.py
index 361819c..da82768 100644
--- a/app/config.py
+++ b/app/config.py
@@ -4,11 +4,57 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+ # Runtime security profile. Development remains convenient, while the
+ # server overlay enables strict mode so placeholder secrets or unsafe
+ # network settings stop the process instead of producing a warning.
+ app_environment: str = "development"
+ security_strict_mode: bool = False
+ cors_allowed_origins: str = (
+ "http://localhost:8102,http://127.0.0.1:8102,"
+ "http://localhost:5173,http://127.0.0.1:5173"
+ )
+ trusted_hosts: str = "*"
+ security_headers_enabled: bool = True
+
# Database
database_url: str = "postgresql://postgres:postgres@localhost:5432/kg_db"
db_schema: str = "kg_admin"
db_migrations_enabled: bool = True
+ # Data Center (independent MySQL service; PostgreSQL above remains the
+ # system/graph-metadata store and is intentionally not replaced here).
+ data_mysql_url: str = (
+ "mysql://data_center:data_center@localhost:3306/platform_control"
+ )
+ data_mysql_required: bool = False
+ data_mysql_pool_min_size: int = 1
+ data_mysql_pool_max_size: int = 10
+ data_sql_console_write_enabled: bool = False
+ interface_api_secret: str = ""
+ # Optional connection information shown to trusted data administrators.
+ # The secure production path is two-stage: DBeaver reaches the server over
+ # an SSH tunnel, then connects to the MySQL host port on 127.0.0.1. Keep
+ # the legacy public host/port fields for older deployments, but do not use
+ # them to imply that the MySQL port should be exposed publicly.
+ data_mysql_direct_access_enabled: bool = False
+ data_mysql_public_host: str = ""
+ data_mysql_public_port: int = 0
+ data_mysql_direct_transport: str = "SSH 隧道(强制)"
+ data_mysql_ssh_tunnel_required: bool = True
+ data_mysql_ssh_host: str = ""
+ data_mysql_ssh_port: int = 22
+ data_mysql_ssh_auth_method: str = "SSH 私钥 / SSH Agent"
+ data_mysql_admin_host: str = "127.0.0.1"
+ data_mysql_admin_port: int = 0
+ data_mysql_admin_account_policy: str = "一人一号 · 单库授权 · 禁止 root"
+ data_mysql_audit_enabled: bool = False
+ # Mirrors the host-side Docker port binding so the Interface Center can
+ # report an unsafe deployment instead of merely showing reassuring copy.
+ mysql_host_bind: str = "127.0.0.1"
+ data_backup_enabled: bool = False
+ data_backup_encryption_required: bool = True
+ data_backup_retention_days: int = 30
+
# FalkorDB
falkordb_host: str = "localhost"
falkordb_port: int = 6379
@@ -18,9 +64,17 @@ class Settings(BaseSettings):
# Auth
auth_secret: str = "change-me-at-least-32-chars-long-secret"
auth_algorithm: str = "HS256"
- auth_token_expire_minutes: int = 480
+ auth_issuer: str = "zn-kg-admin"
+ auth_audience: str = "zn-kg-admin-web"
+ auth_token_expire_minutes: int = 60
auth_default_username: str = "admin@example.com"
auth_default_password: str = "admin"
+ auth_login_max_attempts: int = 5
+ auth_login_window_seconds: int = 300
+ auth_login_lock_seconds: int = 900
+ interface_api_default_expiry_days: int = 30
+ interface_api_max_expiry_days: int = 90
+ interface_api_rate_limit_per_minute: int = 120
# LLM
llm_api_base: str = ""
diff --git a/app/contracts.py b/app/contracts.py
index 2250162..d175cf4 100644
--- a/app/contracts.py
+++ b/app/contracts.py
@@ -1,21 +1,14 @@
"""Pydantic request/response models."""
from __future__ import annotations
-from datetime import datetime
-from typing import Any, Literal
-
-from pydantic import BaseModel, ConfigDict, Field
-
-
-class _Base(BaseModel):
- model_config = ConfigDict(from_attributes=True)
+from pydantic import BaseModel, Field
# ── Auth ─────────────────────────────────────────────────────────────────────
class LoginRequest(BaseModel):
- username: str
- password: str
+ username: str = Field(min_length=3, max_length=120)
+ password: str = Field(min_length=1, max_length=72)
class TokenResponse(BaseModel):
@@ -23,136 +16,6 @@ class TokenResponse(BaseModel):
token_type: str = "bearer"
-# ── Source Profiles ──────────────────────────────────────────────────────────
-
-class SourceProfileCreate(BaseModel):
- source_code: str
- source_name: str
- source_type: Literal["manual", "python_crawler", "api", "csv", "external_system"]
- description: str | None = None
- api_endpoint: str | None = None
- auth_method: str | None = None
- update_frequency: Literal["daily", "weekly", "monthly", "on_demand"] | None = None
- authority_level: int = Field(default=3, ge=1, le=5)
- enabled: bool = True
- metadata_jsonb: dict = Field(default_factory=dict)
-
-
-class SourceProfileUpdate(BaseModel):
- source_name: str | None = None
- description: str | None = None
- authority_level: int | None = Field(default=None, ge=1, le=5)
- enabled: bool | None = None
- update_frequency: str | None = None
-
-
-# ── Question Traces ──────────────────────────────────────────────────────────
-
-class QuestionTraceCreate(BaseModel):
- question_text: str
- source: Literal["real", "simulated"] = "real"
- origin: str | None = None
- user_session: str | None = None
- asked_at: datetime | None = None
-
-
-class QuestionTraceBatch(BaseModel):
- questions: list[QuestionTraceCreate]
-
-
-class SimulationQuestion(BaseModel):
- question_text: str
- scenario_tags: list[str] = Field(default_factory=list)
- enabled: bool = True
-
-
-class SimulationQuestionUpdate(BaseModel):
- question_text: str | None = None
- scenario_tags: list[str] | None = None
- enabled: bool | None = None
-
-
-# ── Field Decisions ──────────────────────────────────────────────────────────
-
-class FieldDecisionUpdate(BaseModel):
- field_decisions: dict[str, str]
- note: str | None = None
-
-
-# ── Acquisition Tasks ────────────────────────────────────────────────────────
-
-class AcquisitionTaskCreate(BaseModel):
- title: str
- description: str | None = None
- scenario_tags: list[str] = Field(default_factory=list)
- target_entity_types: list[str] = Field(default_factory=list)
- target_fields: list[str] = Field(default_factory=list)
- suggested_collection_method: str | None = None
- priority: int = Field(default=3, ge=1, le=5)
- due_at: datetime | None = None
-
-
-class TaskFromGap(BaseModel):
- trace_id: int
- title: str | None = None
- priority: int = Field(default=3, ge=1, le=5)
-
-
-class TaskAssign(BaseModel):
- assignee: str
-
-
-class TaskComplete(BaseModel):
- result_summary: str | None = None
-
-
-# ── Inventory ────────────────────────────────────────────────────────────────
-
-class IssueResolve(BaseModel):
- resolution_note: str | None = None
-
-
-# ── Vocabulary ───────────────────────────────────────────────────────────────
-
-class VocabularyTermCreate(BaseModel):
- entity_type: str
- canonical_name: str
- aliases: list[str] = Field(default_factory=list)
- forbidden_aliases: list[str] = Field(default_factory=list)
- notes: str | None = None
-
-
-class VocabularyTermUpdate(BaseModel):
- canonical_name: str | None = None
- aliases: list[str] | None = None
- forbidden_aliases: list[str] | None = None
- notes: str | None = None
-
-
-# ── Conflicts ────────────────────────────────────────────────────────────────
-
-class ConflictResolve(BaseModel):
- resolution: str
- chosen_value: Any | None = None
- note: str | None = None
-
-
-# ── Aligner ──────────────────────────────────────────────────────────────────
-
-class AlignSuggestRequest(BaseModel):
- candidate_ids: list[int]
-
-
-class MergeEntities(BaseModel):
- note: str | None = None
-
-
-# ── Rollback ─────────────────────────────────────────────────────────────────
-
-class RollbackRequest(BaseModel):
- reason: str | None = None
-
-
# ── RBAC & Accounts (P1) ─────────────────────────────────────────────────────
class RoleCreate(BaseModel):
@@ -182,7 +45,7 @@ class RoleCapCell(BaseModel):
class UserCreate(BaseModel):
username: str = Field(min_length=3, max_length=120)
- password: str = Field(min_length=4, max_length=72)
+ password: str = Field(min_length=12, max_length=72)
full_name: str | None = None
phone: str | None = None
status: str | None = None
@@ -193,16 +56,5 @@ class UserUpdate(BaseModel):
full_name: str | None = None
phone: str | None = None
status: str | None = None
- password: str | None = Field(default=None, min_length=4, max_length=72)
+ password: str | None = Field(default=None, min_length=12, max_length=72)
roles: list[str] | None = None
-
-
-class AreaUpdate(BaseModel):
- name: str | None = None
- note: str | None = None
- responsible_user_id: int | None = None
-
-
-class UserAreasSet(BaseModel):
- area_ids: list[str] = [] # existing area ids picked from the tree
- custom_areas: list[str] = [] # free-text area names typed by admin
diff --git a/app/data_platform/interface_service.py b/app/data_platform/interface_service.py
new file mode 100644
index 0000000..8664ab6
--- /dev/null
+++ b/app/data_platform/interface_service.py
@@ -0,0 +1,704 @@
+"""API client, credential and least-privilege policy management."""
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+import hashlib
+import hmac
+import json
+import secrets
+from typing import Any
+from urllib.parse import urlsplit
+import uuid
+
+from fastapi import HTTPException
+
+from app.config import settings
+from app.data_platform.mysql_db import data_pool_available, get_data_conn
+from app.data_platform.mysql_service import (
+ ensure_platform_registry,
+ list_project_table_entries,
+)
+
+
+VALID_ACTIONS = {"metadata", "read", "create", "update", "delete"}
+
+
+async def _ready() -> None:
+ if not data_pool_available():
+ raise HTTPException(
+ 503,
+ "接口中心依赖的数据中心 MySQL 服务未连接",
+ )
+ await ensure_platform_registry()
+
+
+def _decode_json(value: Any, fallback: Any) -> Any:
+ if value is None:
+ return fallback
+ if isinstance(value, (list, dict)):
+ return value
+ if isinstance(value, (bytes, bytearray)):
+ value = value.decode("utf-8")
+ try:
+ return json.loads(str(value))
+ except (TypeError, ValueError):
+ return fallback
+
+
+def _safe(value: Any) -> Any:
+ if isinstance(value, datetime):
+ return value.isoformat()
+ if isinstance(value, dict):
+ return {key: _safe(item) for key, item in value.items()}
+ if isinstance(value, list):
+ return [_safe(item) for item in value]
+ return value
+
+
+def _row(row: dict[str, Any] | None) -> dict[str, Any] | None:
+ if row is None:
+ return None
+ result = {key: _safe(value) for key, value in dict(row).items()}
+ for key in (
+ "actions_json",
+ "readable_fields_json",
+ "writable_fields_json",
+ "row_filter_json",
+ ):
+ if key in result:
+ result[key.removesuffix("_json")] = _decode_json(result.pop(key), [] if key != "row_filter_json" else {})
+ return result
+
+
+def _api_key_hash(api_key: str) -> str:
+ pepper = settings.interface_api_secret or settings.auth_secret
+ return hmac.new(
+ pepper.encode("utf-8"),
+ api_key.encode("utf-8"),
+ hashlib.sha256,
+ ).hexdigest()
+
+
+def _parse_expiry(value: Any) -> datetime:
+ now = datetime.now(timezone.utc)
+ if value in (None, ""):
+ parsed = now + timedelta(days=settings.interface_api_default_expiry_days)
+ elif isinstance(value, datetime):
+ parsed = value
+ else:
+ try:
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ except ValueError as exc:
+ raise ValueError("凭证过期时间格式不正确") from exc
+ if parsed.tzinfo is None:
+ parsed_utc = parsed.replace(tzinfo=timezone.utc)
+ else:
+ parsed_utc = parsed.astimezone(timezone.utc)
+ if parsed_utc <= now:
+ raise ValueError("凭证过期时间必须晚于当前时间")
+ maximum = now + timedelta(days=settings.interface_api_max_expiry_days)
+ if parsed_utc > maximum:
+ raise ValueError(
+ f"接口密钥最长只能签发 {settings.interface_api_max_expiry_days} 天"
+ )
+ return parsed_utc.replace(tzinfo=None)
+
+
+async def interface_summary() -> dict[str, Any]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT
+ (SELECT COUNT(*) FROM api_clients WHERE status='active') AS active_clients,
+ (SELECT COUNT(*) FROM api_credentials
+ WHERE revoked_at IS NULL AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP(6))) AS active_credentials,
+ (SELECT COUNT(*) FROM api_policies WHERE status='active') AS active_policies,
+ (SELECT COUNT(*) FROM api_call_logs
+ WHERE created_at >= CURRENT_TIMESTAMP(6) - INTERVAL 24 HOUR) AS calls_24h,
+ (SELECT COUNT(*) FROM api_call_logs
+ WHERE created_at >= CURRENT_TIMESTAMP(6) - INTERVAL 24 HOUR
+ AND status_code >= 400) AS errors_24h
+ """
+ )
+ summary = dict(await cur.fetchone())
+ mysql_url = urlsplit(settings.data_mysql_url)
+ database_port = (
+ settings.data_mysql_admin_port
+ or settings.data_mysql_public_port
+ or mysql_url.port
+ or 3306
+ )
+ ssh_host = (
+ settings.data_mysql_ssh_host.strip()
+ or settings.data_mysql_public_host.strip()
+ )
+ host_bind = settings.mysql_host_bind.strip() or "127.0.0.1"
+ loopback_hosts = {"127.0.0.1", "localhost", "::1"}
+ publicly_bound = host_bind not in loopback_hosts
+ return {
+ **{key: int(value or 0) for key, value in summary.items()},
+ "data_engine": "MySQL",
+ "public_base_path": "/v1/openapi/data",
+ "direct_database_access": "DBeaver / MySQL account (separate authorization)",
+ "direct_access_enabled": settings.data_mysql_direct_access_enabled,
+ # Legacy aliases retained for existing clients. New clients should
+ # use the explicit SSH and database endpoint fields below.
+ "direct_access_host": ssh_host,
+ "direct_access_port": database_port,
+ "direct_access_transport": settings.data_mysql_direct_transport,
+ "ssh_tunnel_required": settings.data_mysql_ssh_tunnel_required,
+ "ssh_host": ssh_host,
+ "ssh_port": settings.data_mysql_ssh_port,
+ "ssh_auth_method": settings.data_mysql_ssh_auth_method,
+ "database_host": settings.data_mysql_admin_host.strip() or "127.0.0.1",
+ "database_port": database_port,
+ "database_account_policy": settings.data_mysql_admin_account_policy,
+ "mysql_host_bind": host_bind,
+ "mysql_publicly_bound": publicly_bound,
+ "database_audit_enabled": settings.data_mysql_audit_enabled,
+ "application_audit_enabled": True,
+ "sql_console_write_enabled": settings.data_sql_console_write_enabled,
+ "backup_enabled": settings.data_backup_enabled,
+ "backup_encryption_required": settings.data_backup_encryption_required,
+ "backup_retention_days": settings.data_backup_retention_days,
+ }
+
+
+async def interface_catalog() -> list[dict[str, Any]]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT project_id, display_name, database_name
+ FROM project_databases WHERE status='ready'
+ ORDER BY display_name, project_id
+ """
+ )
+ databases = [dict(item) for item in await cur.fetchall()]
+ result = []
+ for database in databases:
+ entries = await list_project_table_entries(str(database["project_id"]))
+ result.append(
+ {
+ **_safe(database),
+ "tables": [
+ {
+ "code": definition.code,
+ "label": definition.label,
+ "group": definition.group,
+ "fields": [field.as_dict() for field in definition.fields],
+ }
+ for definition, _origin, _source in entries
+ ],
+ }
+ )
+ return result
+
+
+async def list_api_clients() -> list[dict[str, Any]]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT c.*,
+ COUNT(DISTINCT CASE WHEN k.revoked_at IS NULL THEN k.id END) AS credential_count,
+ COUNT(DISTINCT CASE WHEN p.status='active' THEN p.id END) AS policy_count
+ FROM api_clients c
+ LEFT JOIN api_credentials k ON k.client_id=c.id
+ LEFT JOIN api_policies p ON p.client_id=c.id
+ GROUP BY c.id
+ ORDER BY c.created_at DESC
+ """
+ )
+ return [_row(dict(item)) or {} for item in await cur.fetchall()]
+
+
+async def create_api_client(body: dict[str, Any], actor: str) -> dict[str, Any]:
+ await _ready()
+ name = str(body.get("name") or "").strip()
+ if not name:
+ raise ValueError("请输入应用或设备名称")
+ if len(name) > 100:
+ raise ValueError("应用或设备名称不能超过 100 个字符")
+ client_id = str(uuid.uuid4())
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ INSERT INTO api_clients (id, name, description, owner, status)
+ VALUES (%s, %s, %s, %s, 'active')
+ """,
+ (client_id, name, str(body.get("description") or "")[:500], actor),
+ )
+ await cur.execute("SELECT * FROM api_clients WHERE id=%s", (client_id,))
+ result = _row(await cur.fetchone()) or {}
+ await conn.commit()
+ return result
+
+
+async def update_api_client(client_id: str, body: dict[str, Any]) -> dict[str, Any]:
+ await _ready()
+ updates: list[str] = []
+ params: list[Any] = []
+ if "name" in body:
+ name = str(body.get("name") or "").strip()
+ if not name:
+ raise ValueError("请输入应用或设备名称")
+ updates.append("name=%s")
+ params.append(name[:100])
+ if "description" in body:
+ updates.append("description=%s")
+ params.append(str(body.get("description") or "")[:500])
+ if "status" in body:
+ status = str(body.get("status") or "")
+ if status not in {"active", "disabled"}:
+ raise ValueError("客户端状态不正确")
+ updates.append("status=%s")
+ params.append(status)
+ if not updates:
+ raise ValueError("没有可修改内容")
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"UPDATE api_clients SET {', '.join(updates)} WHERE id=%s",
+ [*params, client_id],
+ )
+ if cur.rowcount != 1:
+ raise ValueError("接口客户端不存在")
+ if body.get("status") == "disabled":
+ await cur.execute(
+ "UPDATE api_credentials SET revoked_at=CURRENT_TIMESTAMP(6) WHERE client_id=%s AND revoked_at IS NULL",
+ (client_id,),
+ )
+ await cur.execute(
+ "UPDATE api_policies SET status='disabled' WHERE client_id=%s",
+ (client_id,),
+ )
+ await cur.execute("SELECT * FROM api_clients WHERE id=%s", (client_id,))
+ result = _row(await cur.fetchone()) or {}
+ await conn.commit()
+ return result
+
+
+async def delete_api_client(client_id: str) -> dict[str, bool]:
+ await update_api_client(client_id, {"status": "disabled"})
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "UPDATE api_policies SET status='disabled' WHERE client_id=%s",
+ (client_id,),
+ )
+ await conn.commit()
+ return {"ok": True}
+
+
+async def list_api_credentials(client_id: str | None = None) -> list[dict[str, Any]]:
+ await _ready()
+ where = "WHERE k.client_id=%s" if client_id else ""
+ params = (client_id,) if client_id else ()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"""
+ SELECT k.*, c.name AS client_name
+ FROM api_credentials k
+ JOIN api_clients c ON c.id=k.client_id
+ {where}
+ ORDER BY k.created_at DESC
+ """,
+ params,
+ )
+ result = [_row(dict(item)) or {} for item in await cur.fetchall()]
+ for item in result:
+ item.pop("key_hash", None)
+ return result
+
+
+async def issue_api_credential(client_id: str, body: dict[str, Any]) -> dict[str, Any]:
+ await _ready()
+ name = str(body.get("name") or "默认密钥").strip() or "默认密钥"
+ expires_at = _parse_expiry(body.get("expires_at"))
+ credential_id = str(uuid.uuid4())
+ prefix = f"dc_live_{secrets.token_hex(4)}"
+ api_key = f"{prefix}.{secrets.token_urlsafe(32)}"
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "SELECT 1 FROM api_clients WHERE id=%s AND status='active'",
+ (client_id,),
+ )
+ if not await cur.fetchone():
+ raise ValueError("接口客户端不存在或未启用")
+ await cur.execute(
+ """
+ INSERT INTO api_credentials (
+ id, client_id, name, key_prefix, key_hash, expires_at
+ ) VALUES (%s, %s, %s, %s, %s, %s)
+ """,
+ (credential_id, client_id, name[:100], prefix, _api_key_hash(api_key), expires_at),
+ )
+ await cur.execute("SELECT * FROM api_credentials WHERE id=%s", (credential_id,))
+ result = _row(await cur.fetchone()) or {}
+ await conn.commit()
+ result.pop("key_hash", None)
+ result["api_key"] = api_key
+ result["shown_once"] = True
+ return result
+
+
+async def revoke_api_credential(credential_id: str) -> dict[str, bool]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ UPDATE api_credentials SET revoked_at=CURRENT_TIMESTAMP(6)
+ WHERE id=%s AND revoked_at IS NULL
+ """,
+ (credential_id,),
+ )
+ if cur.rowcount != 1:
+ raise ValueError("接口密钥不存在或已撤销")
+ await conn.commit()
+ return {"ok": True}
+
+
+async def list_api_policies(client_id: str | None = None) -> list[dict[str, Any]]:
+ await _ready()
+ where = "WHERE p.client_id=%s" if client_id else ""
+ params = (client_id,) if client_id else ()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"""
+ SELECT p.*, c.name AS client_name, d.display_name AS database_name
+ FROM api_policies p
+ JOIN api_clients c ON c.id=p.client_id
+ LEFT JOIN project_databases d ON d.project_id=p.database_id
+ {where}
+ ORDER BY p.created_at DESC
+ """,
+ params,
+ )
+ return [_row(dict(item)) or {} for item in await cur.fetchall()]
+
+
+async def _validate_policy(body: dict[str, Any]) -> dict[str, Any]:
+ client_id = str(body.get("client_id") or "").strip()
+ database_id = str(body.get("database_id") or "").strip()
+ table_code = str(body.get("table_code") or "*").strip() or "*"
+ actions = sorted({str(value) for value in body.get("actions") or []})
+ invalid = sorted(set(actions) - VALID_ACTIONS)
+ if not client_id or not database_id:
+ raise ValueError("请选择接口客户端和数据库")
+ if not actions:
+ raise ValueError("请至少授予一个接口动作")
+ if invalid:
+ raise ValueError(f"不支持的接口动作:{', '.join(invalid)}")
+ readable = [str(value) for value in body.get("readable_fields") or ["*"]]
+ writable = [str(value) for value in body.get("writable_fields") or ["*"]]
+ row_filter = body.get("row_filter") or {}
+ if not isinstance(row_filter, dict):
+ raise ValueError("行级数据范围必须是 JSON 对象")
+ status = str(body.get("status") or "active")
+ if status not in {"active", "disabled"}:
+ raise ValueError("权限策略状态不正确")
+ for field_code, expected in row_filter.items():
+ values = expected if isinstance(expected, list) else [expected]
+ if any(isinstance(value, (dict, list)) for value in values):
+ raise ValueError(f"行级范围 {field_code} 仅支持标量或标量数组")
+
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute("SELECT 1 FROM api_clients WHERE id=%s", (client_id,))
+ if not await cur.fetchone():
+ raise ValueError("接口客户端不存在")
+ await cur.execute(
+ "SELECT 1 FROM project_databases WHERE project_id=%s AND status='ready'",
+ (database_id,),
+ )
+ if not await cur.fetchone():
+ raise ValueError("数据中心数据库不存在")
+
+ if table_code == "*":
+ if readable != ["*"] or writable != ["*"] or row_filter:
+ raise ValueError("授权全部表时,字段范围必须为 * 且不能设置跨表行过滤")
+ else:
+ entry = next(
+ (
+ item
+ for item in await list_project_table_entries(database_id)
+ if item[0].code == table_code
+ ),
+ None,
+ )
+ if not entry:
+ raise ValueError("授权的数据表不存在")
+ field_codes = {field.code for field in entry[0].fields}
+ for label, fields in (("可读字段", readable), ("可写字段", writable)):
+ unknown = sorted(set(fields) - field_codes - {"*"})
+ if unknown:
+ raise ValueError(f"{label}不存在:{', '.join(unknown)}")
+ unknown_scope = sorted(set(row_filter) - field_codes)
+ if unknown_scope:
+ raise ValueError(f"行级范围字段不存在:{', '.join(unknown_scope)}")
+ return {
+ "client_id": client_id,
+ "database_id": database_id,
+ "table_code": table_code,
+ "actions": actions,
+ "readable_fields": readable,
+ "writable_fields": writable,
+ "row_filter": row_filter,
+ "status": status,
+ }
+
+
+async def create_api_policy(body: dict[str, Any]) -> dict[str, Any]:
+ await _ready()
+ policy = await _validate_policy(body)
+ policy_id = str(uuid.uuid4())
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ try:
+ await cur.execute(
+ """
+ INSERT INTO api_policies (
+ id, client_id, database_id, table_code, actions_json,
+ readable_fields_json, writable_fields_json, row_filter_json, status
+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+ """,
+ (
+ policy_id,
+ policy["client_id"],
+ policy["database_id"],
+ policy["table_code"],
+ json.dumps(policy["actions"], ensure_ascii=False),
+ json.dumps(policy["readable_fields"], ensure_ascii=False),
+ json.dumps(policy["writable_fields"], ensure_ascii=False),
+ json.dumps(policy["row_filter"], ensure_ascii=False),
+ policy["status"],
+ ),
+ )
+ except Exception as exc:
+ raise ValueError("同一客户端、数据库和表只能创建一条权限策略") from exc
+ await cur.execute("SELECT * FROM api_policies WHERE id=%s", (policy_id,))
+ result = _row(await cur.fetchone()) or {}
+ await conn.commit()
+ return result
+
+
+async def update_api_policy(policy_id: str, body: dict[str, Any]) -> dict[str, Any]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute("SELECT * FROM api_policies WHERE id=%s", (policy_id,))
+ current = _row(await cur.fetchone())
+ if not current:
+ raise ValueError("权限策略不存在")
+ merged = {
+ "client_id": body.get("client_id", current["client_id"]),
+ "database_id": body.get("database_id", current["database_id"]),
+ "table_code": body.get("table_code", current["table_code"]),
+ "actions": body.get("actions", current["actions"]),
+ "readable_fields": body.get("readable_fields", current["readable_fields"]),
+ "writable_fields": body.get("writable_fields", current["writable_fields"]),
+ "row_filter": body.get("row_filter", current["row_filter"]),
+ "status": body.get("status", current["status"]),
+ }
+ policy = await _validate_policy(merged)
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ UPDATE api_policies SET
+ client_id=%s, database_id=%s, table_code=%s, actions_json=%s,
+ readable_fields_json=%s, writable_fields_json=%s,
+ row_filter_json=%s, status=%s
+ WHERE id=%s
+ """,
+ (
+ policy["client_id"], policy["database_id"], policy["table_code"],
+ json.dumps(policy["actions"], ensure_ascii=False),
+ json.dumps(policy["readable_fields"], ensure_ascii=False),
+ json.dumps(policy["writable_fields"], ensure_ascii=False),
+ json.dumps(policy["row_filter"], ensure_ascii=False),
+ policy["status"], policy_id,
+ ),
+ )
+ await cur.execute("SELECT * FROM api_policies WHERE id=%s", (policy_id,))
+ result = _row(await cur.fetchone()) or {}
+ await conn.commit()
+ return result
+
+
+async def delete_api_policy(policy_id: str) -> dict[str, bool]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute("DELETE FROM api_policies WHERE id=%s", (policy_id,))
+ if cur.rowcount != 1:
+ raise ValueError("权限策略不存在")
+ await conn.commit()
+ return {"ok": True}
+
+
+async def list_api_call_logs(limit: int = 200) -> list[dict[str, Any]]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT l.*, c.name AS client_name
+ FROM api_call_logs l
+ LEFT JOIN api_clients c ON c.id=l.client_id
+ ORDER BY l.created_at DESC LIMIT %s
+ """,
+ (max(1, min(1000, limit)),),
+ )
+ return [_row(dict(item)) or {} for item in await cur.fetchall()]
+
+
+async def authenticate_api_key(api_key: str) -> dict[str, Any]:
+ await _ready()
+ if not api_key or "." not in api_key:
+ raise HTTPException(401, "缺少或无效的接口密钥")
+ key_hash = _api_key_hash(api_key)
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT k.id AS credential_id, k.client_id, k.key_prefix,
+ k.expires_at, c.name AS client_name
+ FROM api_credentials k
+ JOIN api_clients c ON c.id=k.client_id
+ WHERE k.key_hash=%s AND k.revoked_at IS NULL
+ AND (k.expires_at IS NULL OR k.expires_at > CURRENT_TIMESTAMP(6))
+ AND c.status='active'
+ """,
+ (key_hash,),
+ )
+ identity = _row(await cur.fetchone())
+ if not identity:
+ raise HTTPException(401, "接口密钥无效、已撤销或已过期")
+ await cur.execute(
+ "UPDATE api_credentials SET last_used_at=CURRENT_TIMESTAMP(6) WHERE id=%s",
+ (identity["credential_id"],),
+ )
+ await conn.commit()
+ return identity
+
+
+async def resolve_api_policy(
+ client_id: str,
+ database_id: str,
+ table_code: str,
+ action: str,
+) -> dict[str, Any]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT * FROM api_policies
+ WHERE client_id=%s AND database_id=%s
+ AND table_code IN (%s, '*') AND status='active'
+ ORDER BY (table_code=%s) DESC, updated_at DESC
+ """,
+ (client_id, database_id, table_code, table_code),
+ )
+ policies = [_row(dict(item)) or {} for item in await cur.fetchall()]
+ policy = next((item for item in policies if action in item["actions"]), None)
+ if not policy:
+ raise HTTPException(403, f"当前接口客户端没有 {action} 权限")
+ return policy
+
+
+async def client_catalog(client_id: str) -> list[dict[str, Any]]:
+ policies = [
+ item for item in await list_api_policies(client_id) if item["status"] == "active"
+ ]
+ catalog = await interface_catalog()
+ result: list[dict[str, Any]] = []
+ for database in catalog:
+ db_policies = [
+ item
+ for item in policies
+ if item["database_id"] == database["project_id"]
+ and ({"metadata", "read"} & set(item["actions"]))
+ ]
+ if not db_policies:
+ continue
+ tables = []
+ for table in database["tables"]:
+ policy = next(
+ (item for item in db_policies if item["table_code"] == table["code"]),
+ next((item for item in db_policies if item["table_code"] == "*"), None),
+ )
+ if not policy:
+ continue
+ readable = set(policy["readable_fields"])
+ fields = table["fields"] if "*" in readable else [
+ field for field in table["fields"] if field["code"] in readable
+ ]
+ tables.append({
+ **table,
+ "fields": fields,
+ "actions": policy["actions"],
+ })
+ if tables:
+ result.append({
+ "project_id": database["project_id"],
+ "display_name": database["display_name"],
+ "tables": tables,
+ })
+ return result
+
+
+async def write_api_call_log(
+ *,
+ request_id: str,
+ identity: dict[str, Any] | None,
+ method: str,
+ path: str,
+ database_id: str | None,
+ table_code: str | None,
+ action: str | None,
+ status_code: int,
+ duration_ms: float,
+ source_ip: str | None,
+ error_message: str | None = None,
+) -> None:
+ if not data_pool_available():
+ return
+ await ensure_platform_registry()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ INSERT INTO api_call_logs (
+ request_id, client_id, credential_id, method, path,
+ database_id, table_code, action_name, status_code,
+ duration_ms, source_ip, error_message
+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+ """,
+ (
+ request_id,
+ identity.get("client_id") if identity else None,
+ identity.get("credential_id") if identity else None,
+ method,
+ path[:500],
+ database_id,
+ table_code,
+ action,
+ status_code,
+ duration_ms,
+ source_ip,
+ (error_message or "")[:500] or None,
+ ),
+ )
+ await conn.commit()
diff --git a/app/data_platform/mysql_db.py b/app/data_platform/mysql_db.py
new file mode 100644
index 0000000..500362a
--- /dev/null
+++ b/app/data_platform/mysql_db.py
@@ -0,0 +1,155 @@
+"""Async MySQL connection management for the independent Data Center.
+
+The application still uses PostgreSQL for accounts and graph metadata. This
+module owns a separate MySQL pool whose default database is the protected
+``platform_control`` catalog. Business databases are selected explicitly for
+the duration of a request and reset before the pooled connection is returned.
+"""
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+import logging
+import re
+from typing import AsyncIterator
+from urllib.parse import unquote, urlparse
+
+import aiomysql
+
+from app.config import settings
+
+
+logger = logging.getLogger(__name__)
+_DATABASE_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
+_pool: aiomysql.Pool | None = None
+_control_database = "platform_control"
+_pool_generation = 0
+
+
+class DataCenterUnavailable(RuntimeError):
+ """Raised when the independent MySQL service is not connected."""
+
+
+def _connection_options(*, include_database: bool) -> dict[str, object]:
+ parsed = urlparse(settings.data_mysql_url)
+ if parsed.scheme not in {"mysql", "mysql+aiomysql"}:
+ raise ValueError("DATA_MYSQL_URL 必须使用 mysql:// 协议")
+ database = unquote(parsed.path.lstrip("/")) or "platform_control"
+ if not _DATABASE_PATTERN.fullmatch(database):
+ raise ValueError("DATA_MYSQL_URL 中的控制数据库名称不合法")
+
+ global _control_database
+ _control_database = database
+ options: dict[str, object] = {
+ "host": parsed.hostname or "localhost",
+ "port": parsed.port or 3306,
+ "user": unquote(parsed.username or "root"),
+ "password": unquote(parsed.password or ""),
+ "charset": "utf8mb4",
+ "autocommit": False,
+ "cursorclass": aiomysql.DictCursor,
+ }
+ if include_database:
+ options["db"] = database
+ return options
+
+
+def control_database_name() -> str:
+ return _control_database
+
+
+def data_pool_available() -> bool:
+ return _pool is not None
+
+
+def data_pool_generation() -> int:
+ return _pool_generation
+
+
+async def init_data_pool() -> bool:
+ """Create the Data Center catalog and pool.
+
+ Development can start without MySQL when ``DATA_MYSQL_REQUIRED=false`` so
+ graph-only pages remain available. Data Center endpoints then return 503
+ instead of silently falling back to PostgreSQL.
+ """
+
+ global _pool, _pool_generation
+ if _pool is not None:
+ return True
+ try:
+ bootstrap = _connection_options(include_database=False)
+ bootstrap["autocommit"] = True
+ conn = await aiomysql.connect(**bootstrap)
+ try:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "SELECT SCHEMA_NAME FROM information_schema.SCHEMATA "
+ "WHERE SCHEMA_NAME=%s",
+ (_control_database,),
+ )
+ if not await cur.fetchone():
+ await cur.execute(
+ f"CREATE DATABASE `{_control_database}` "
+ "CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"
+ )
+ finally:
+ conn.close()
+
+ options = _connection_options(include_database=True)
+ _pool = await aiomysql.create_pool(
+ **options,
+ minsize=max(1, settings.data_mysql_pool_min_size),
+ maxsize=max(
+ settings.data_mysql_pool_min_size,
+ settings.data_mysql_pool_max_size,
+ ),
+ pool_recycle=1800,
+ )
+ _pool_generation += 1
+ logger.info("Data Center MySQL pool connected")
+ return True
+ except Exception:
+ _pool = None
+ if settings.data_mysql_required:
+ raise
+ logger.warning(
+ "Data Center MySQL is unavailable; Data Center/API Center routes "
+ "will return 503",
+ exc_info=True,
+ )
+ return False
+
+
+async def close_data_pool() -> None:
+ global _pool, _pool_generation
+ if _pool is None:
+ return
+ _pool.close()
+ await _pool.wait_closed()
+ _pool = None
+ _pool_generation += 1
+
+
+@asynccontextmanager
+async def get_data_conn(database: str | None = None) -> AsyncIterator[aiomysql.Connection]:
+ if _pool is None:
+ raise DataCenterUnavailable("数据中心 MySQL 服务尚未连接")
+ if database is not None and not _DATABASE_PATTERN.fullmatch(database):
+ raise ValueError("MySQL 数据库名称不合法")
+
+ async with _pool.acquire() as conn:
+ selected = database or _control_database
+ await conn.select_db(selected)
+ try:
+ yield conn
+ except Exception:
+ await conn.rollback()
+ raise
+ finally:
+ # Prevent a later request from inheriting a business database.
+ try:
+ await conn.rollback()
+ if selected != _control_database:
+ await conn.select_db(_control_database)
+ except Exception:
+ logger.warning("Failed to reset a Data Center MySQL connection", exc_info=True)
diff --git a/app/data_platform/mysql_service.py b/app/data_platform/mysql_service.py
new file mode 100644
index 0000000..f0cca49
--- /dev/null
+++ b/app/data_platform/mysql_service.py
@@ -0,0 +1,2317 @@
+"""MySQL implementation of the Data Center service contract.
+
+The React Data Center keeps using the existing ``/data-platform`` endpoints.
+Only their persistence implementation changes: every logical database now maps
+to a real MySQL database, while catalog, API-permission and audit metadata live
+in the protected MySQL control database.
+"""
+from __future__ import annotations
+
+import csv
+from datetime import date, datetime, timezone
+from decimal import Decimal
+import hashlib
+import io
+import json
+import re
+import time
+from typing import Any, Iterable
+import uuid
+import warnings
+
+from fastapi import HTTPException
+from pymysql import MySQLError
+
+from app.config import settings
+from app.data_platform.mysql_db import (
+ control_database_name,
+ data_pool_available,
+ data_pool_generation,
+ get_data_conn,
+)
+from app.data_platform.registry import TABLE_REGISTRY, FieldDefinition, TableDefinition
+from app.data_platform.schema import (
+ IDENTIFIER_PATTERN,
+ SYSTEM_COLUMN_CODES,
+ _custom_definition_from_payload,
+ _field_row,
+ _structure_field_from_payload,
+ custom_definition_from_create_sql,
+)
+
+
+MAX_CSV_BYTES = 50 * 1024 * 1024
+MAX_IMPORT_ROWS = 50_000
+MAX_EXPORT_ROWS = 200_000
+PREVIEW_ROWS = 5
+MAX_REPORTED_ERRORS = 100
+MAX_SQL_LENGTH = 50_000
+MAX_RESULT_ROWS = 500
+STATEMENT_TIMEOUT_MS = 15_000
+
+_registry_generation = -1
+_MYSQL_IDENTIFIER = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
+_MYSQL_SYSTEM_DATABASES = {"information_schema", "mysql", "performance_schema", "sys"}
+_ALLOWED_SQL = {"select", "with", "insert", "update", "delete", "explain", "show", "describe", "desc"}
+_FORBIDDEN_SQL = re.compile(
+ r"\b(?:call|grant|revoke|prepare|execute|deallocate|load|outfile|dumpfile|"
+ r"create|alter|drop|truncate|rename|lock|unlock|handler|install|uninstall|use)\b",
+ re.IGNORECASE,
+)
+
+
+def _require_data_center() -> None:
+ if not data_pool_available():
+ raise HTTPException(
+ 503,
+ "数据中心 MySQL 服务未连接,请检查 DATA_MYSQL_URL 与 MySQL 服务状态",
+ )
+
+
+def _q(identifier: str) -> str:
+ if not _MYSQL_IDENTIFIER.fullmatch(identifier):
+ raise ValueError(f"不合法的 MySQL 标识符:{identifier}")
+ return f"`{identifier}`"
+
+
+def _decode_json(value: Any, fallback: Any) -> Any:
+ if value is None:
+ return fallback
+ if isinstance(value, (dict, list)):
+ return value
+ if isinstance(value, (bytes, bytearray)):
+ value = value.decode("utf-8")
+ try:
+ return json.loads(str(value))
+ except (TypeError, ValueError, json.JSONDecodeError):
+ return fallback
+
+
+def _json_safe(value: Any) -> Any:
+ if isinstance(value, Decimal):
+ return float(value)
+ if isinstance(value, uuid.UUID):
+ return str(value)
+ if isinstance(value, (date, datetime)):
+ return value.isoformat()
+ if isinstance(value, (bytes, bytearray, memoryview)):
+ return bytes(value).hex()
+ if isinstance(value, dict):
+ return {key: _json_safe(item) for key, item in value.items()}
+ if isinstance(value, (list, tuple)):
+ return [_json_safe(item) for item in value]
+ return value
+
+
+def _row_json(row: dict[str, Any] | None) -> dict[str, Any] | None:
+ if row is None:
+ return None
+ return {key: _json_safe(value) for key, value in dict(row).items()}
+
+
+def project_database_name(project_id: str) -> str:
+ """Use the user-entered database code as the physical MySQL name."""
+
+ return project_id
+
+
+def _managed_index_name(prefix: str, column_code: str) -> str:
+ candidate = f"{prefix}_{column_code}"
+ if len(candidate) <= 64:
+ return candidate
+ digest = hashlib.sha1(candidate.encode("utf-8")).hexdigest()[:8]
+ return f"{candidate[:55]}_{digest}"
+
+
+def _mysql_type(sql_type: str, *, unique: bool = False) -> str:
+ value = re.sub(r"\s+", " ", sql_type.strip().upper())
+ if value == "UUID":
+ return "CHAR(36)"
+ if value in {"JSON", "JSONB"}:
+ return "JSON"
+ if value.startswith("TIMESTAMPTZ") or "WITH TIME ZONE" in value:
+ precision = re.search(r"\((\d)\)", value)
+ return f"DATETIME({precision.group(1)})" if precision else "DATETIME(6)"
+ if value.startswith("TIMESTAMP"):
+ precision = re.search(r"\((\d)\)", value)
+ return f"DATETIME({precision.group(1)})" if precision else "DATETIME(6)"
+ if value in {"DOUBLE PRECISION", "FLOAT8"}:
+ return "DOUBLE"
+ if value in {"REAL", "FLOAT4"}:
+ return "FLOAT"
+ if value in {"BOOLEAN", "BOOL"}:
+ return "BOOLEAN"
+ if value in {"INTEGER", "INT", "INT4", "SERIAL"}:
+ return "INT"
+ if value in {"SMALLINT", "INT2", "SMALLSERIAL"}:
+ return "SMALLINT"
+ if value in {"BIGINT", "INT8", "BIGSERIAL"}:
+ return "BIGINT"
+ numeric = re.fullmatch(r"(?:NUMERIC|DECIMAL)(\s*\([^)]*\))?", value)
+ if numeric:
+ return f"DECIMAL{numeric.group(1) or '(18,4)'}".replace(" ", "")
+ varchar = re.fullmatch(r"(?:CHARACTER VARYING|VARCHAR)\s*(\(\d+\))?", value)
+ if varchar:
+ length = int((varchar.group(1) or "(255)")[1:-1])
+ if not 1 <= length <= 16_383:
+ raise ValueError("VARCHAR 长度必须在 1–16383 之间")
+ if unique and length > 512:
+ raise ValueError("唯一文本字段长度不能超过 512,以兼容 utf8mb4 索引")
+ return f"VARCHAR({length})"
+ char = re.fullmatch(r"(?:CHARACTER|CHAR)\s*(\(\d+\))?", value)
+ if char:
+ length = int((char.group(1) or "(1)")[1:-1])
+ if not 1 <= length <= 255:
+ raise ValueError("CHAR 长度必须在 1–255 之间")
+ return f"CHAR({length})"
+ if value == "TEXT":
+ # MySQL cannot create an unrestricted UNIQUE index on TEXT.
+ return "VARCHAR(512)" if unique else "TEXT"
+ if value == "DATE":
+ return "DATE"
+ raise ValueError(f"暂不支持的 MySQL 字段类型:{sql_type}")
+
+
+def _mysql_default(default_sql: str | None, mysql_type: str) -> str | None:
+ if not default_sql:
+ return None
+ value = re.sub(
+ r"\s*::\s*(?:text|varchar|json|jsonb|uuid|date|timestamptz)\s*$",
+ "",
+ default_sql.strip(),
+ flags=re.IGNORECASE,
+ )
+ lowered = value.lower()
+ if lowered in {"now()", "current_timestamp"}:
+ return "CURRENT_TIMESTAMP(6)"
+ if lowered == "gen_random_uuid()":
+ return "(UUID())"
+ if lowered == "true":
+ return "1"
+ if lowered == "false":
+ return "0"
+ if mysql_type == "JSON":
+ if value == "'{}'":
+ return "(JSON_OBJECT())"
+ if value == "'[]'":
+ return "(JSON_ARRAY())"
+ return f"({value})"
+ if mysql_type in {"TEXT", "BLOB"}:
+ return f"({value})"
+ return value
+
+
+def _column_clause(field: FieldDefinition, *, include_name: bool = True) -> str:
+ mysql_type = _mysql_type(field.sql_type, unique=field.unique)
+ parts = [_q(field.code), mysql_type] if include_name else [mysql_type]
+ parts.append("NOT NULL" if field.required else "NULL")
+ default = _mysql_default(field.default_sql, mysql_type)
+ if default is not None:
+ parts.extend(("DEFAULT", default))
+ return " ".join(parts)
+
+
+async def ensure_platform_registry() -> bool:
+ """Create the protected MySQL catalog used by both new centers."""
+
+ global _registry_generation
+ generation = data_pool_generation()
+ if _registry_generation == generation:
+ return True
+ if not data_pool_available():
+ return False
+ statements = (
+ """
+ CREATE TABLE IF NOT EXISTS project_databases (
+ project_id VARCHAR(63) PRIMARY KEY,
+ tenant_id VARCHAR(100) NOT NULL,
+ display_name VARCHAR(100) NOT NULL,
+ database_name VARCHAR(64) NOT NULL UNIQUE,
+ schema_name VARCHAR(64) NOT NULL UNIQUE,
+ engine VARCHAR(16) NOT NULL DEFAULT 'mysql',
+ status VARCHAR(24) NOT NULL DEFAULT 'ready',
+ created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)
+ ON UPDATE CURRENT_TIMESTAMP(6)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+ """,
+ """
+ CREATE TABLE IF NOT EXISTS project_table_definitions (
+ project_id VARCHAR(63) NOT NULL,
+ source_code VARCHAR(63) NOT NULL,
+ table_code VARCHAR(63) NOT NULL,
+ label VARCHAR(100) NOT NULL,
+ group_name VARCHAR(100) NOT NULL DEFAULT '自定义',
+ description TEXT NOT NULL,
+ fields_json JSON NOT NULL,
+ origin VARCHAR(16) NOT NULL DEFAULT 'custom',
+ display_order INT NOT NULL DEFAULT 1000,
+ allow_create BOOLEAN NOT NULL DEFAULT TRUE,
+ allow_update BOOLEAN NOT NULL DEFAULT TRUE,
+ allow_delete BOOLEAN NOT NULL DEFAULT TRUE,
+ status VARCHAR(16) NOT NULL DEFAULT 'active',
+ created_by VARCHAR(191) NOT NULL,
+ created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)
+ ON UPDATE CURRENT_TIMESTAMP(6),
+ PRIMARY KEY (project_id, source_code),
+ UNIQUE KEY uq_project_table_code (project_id, table_code),
+ KEY idx_project_table_status (project_id, status, display_order)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+ """,
+ """
+ CREATE TABLE IF NOT EXISTS data_change_logs (
+ id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ tenant_id VARCHAR(100) NOT NULL,
+ project_id VARCHAR(63) NOT NULL,
+ table_code VARCHAR(63) NOT NULL,
+ record_id CHAR(36) NOT NULL,
+ operation VARCHAR(24) NOT NULL,
+ before_data JSON NULL,
+ after_data JSON NULL,
+ actor VARCHAR(191) NOT NULL,
+ created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ KEY idx_data_change_record (project_id, table_code, record_id),
+ KEY idx_data_change_created (created_at)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+ """,
+ """
+ CREATE TABLE IF NOT EXISTS api_clients (
+ id CHAR(36) PRIMARY KEY,
+ name VARCHAR(100) NOT NULL,
+ description VARCHAR(500) NOT NULL DEFAULT '',
+ owner VARCHAR(191) NOT NULL,
+ status VARCHAR(16) NOT NULL DEFAULT 'active',
+ created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)
+ ON UPDATE CURRENT_TIMESTAMP(6),
+ KEY idx_api_clients_status (status)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+ """,
+ """
+ CREATE TABLE IF NOT EXISTS api_credentials (
+ id CHAR(36) PRIMARY KEY,
+ client_id CHAR(36) NOT NULL,
+ name VARCHAR(100) NOT NULL,
+ key_prefix VARCHAR(24) NOT NULL UNIQUE,
+ key_hash CHAR(64) NOT NULL UNIQUE,
+ expires_at DATETIME(6) NULL,
+ last_used_at DATETIME(6) NULL,
+ revoked_at DATETIME(6) NULL,
+ created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ KEY idx_api_credentials_client (client_id, revoked_at)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+ """,
+ """
+ CREATE TABLE IF NOT EXISTS api_policies (
+ id CHAR(36) PRIMARY KEY,
+ client_id CHAR(36) NOT NULL,
+ database_id VARCHAR(63) NOT NULL,
+ table_code VARCHAR(63) NOT NULL DEFAULT '*',
+ actions_json JSON NOT NULL,
+ readable_fields_json JSON NOT NULL,
+ writable_fields_json JSON NOT NULL,
+ row_filter_json JSON NOT NULL,
+ status VARCHAR(16) NOT NULL DEFAULT 'active',
+ created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)
+ ON UPDATE CURRENT_TIMESTAMP(6),
+ UNIQUE KEY uq_api_policy_scope (client_id, database_id, table_code),
+ KEY idx_api_policy_client (client_id, status)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+ """,
+ """
+ CREATE TABLE IF NOT EXISTS api_call_logs (
+ id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ request_id CHAR(36) NOT NULL,
+ client_id CHAR(36) NULL,
+ credential_id CHAR(36) NULL,
+ method VARCHAR(12) NOT NULL,
+ path VARCHAR(500) NOT NULL,
+ database_id VARCHAR(63) NULL,
+ table_code VARCHAR(63) NULL,
+ action_name VARCHAR(24) NULL,
+ status_code INT NOT NULL,
+ duration_ms DECIMAL(12,2) NOT NULL DEFAULT 0,
+ source_ip VARCHAR(64) NULL,
+ error_message VARCHAR(500) NULL,
+ created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ KEY idx_api_call_created (created_at),
+ KEY idx_api_call_client (client_id, created_at)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+ """,
+ """
+ CREATE TABLE IF NOT EXISTS admin_action_logs (
+ id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ actor VARCHAR(191) NOT NULL,
+ action_name VARCHAR(64) NOT NULL,
+ resource_type VARCHAR(64) NOT NULL,
+ resource_id VARCHAR(191) NOT NULL,
+ outcome VARCHAR(16) NOT NULL,
+ statement_hash CHAR(64) NULL,
+ affected_rows BIGINT NOT NULL DEFAULT 0,
+ source_ip VARCHAR(64) NULL,
+ details_json JSON NOT NULL,
+ created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ KEY idx_admin_action_created (created_at),
+ KEY idx_admin_action_actor (actor, created_at),
+ KEY idx_admin_action_resource (resource_type, resource_id, created_at)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+ """,
+ )
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ for statement in statements:
+ with warnings.catch_warnings():
+ warnings.filterwarnings(
+ "ignore",
+ message=r"Table '.*' already exists",
+ )
+ await cur.execute(statement)
+ await conn.commit()
+ _registry_generation = generation
+ return True
+
+
+async def _ready() -> None:
+ _require_data_center()
+ await ensure_platform_registry()
+
+
+def _table_from_row(row: dict[str, Any]) -> TableDefinition:
+ fields = tuple(
+ FieldDefinition(
+ code=str(item["code"]),
+ label=str(item["label"]),
+ sql_type=str(item["sql_type"]),
+ data_type=str(item.get("data_type") or "text"),
+ required=bool(item.get("required")),
+ searchable=bool(item.get("searchable")),
+ sortable=bool(item.get("sortable", True)),
+ editable=bool(item.get("editable", True)),
+ visible_in_list=bool(item.get("visible_in_list", True)),
+ default_sql=str(item["default_sql"]) if item.get("default_sql") else None,
+ unique=bool(item.get("unique")),
+ options=tuple(str(value) for value in item.get("options") or ()),
+ )
+ for item in _decode_json(row.get("fields_json"), [])
+ )
+ return TableDefinition(
+ code=str(row["table_code"]),
+ label=str(row["label"]),
+ group=str(row.get("group_name") or "自定义"),
+ description=str(row.get("description") or ""),
+ fields=fields,
+ allow_create=bool(row.get("allow_create", True)),
+ allow_update=bool(row.get("allow_update", True)),
+ allow_delete=bool(row.get("allow_delete", True)),
+ )
+
+
+async def get_project_database(project_id: str) -> dict[str, Any] | None:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "SELECT * FROM project_databases WHERE project_id=%s AND status='ready'",
+ (project_id,),
+ )
+ return _row_json(await cur.fetchone())
+
+
+async def _database_or_404(project_id: str) -> dict[str, Any]:
+ database = await get_project_database(project_id)
+ if database:
+ return database
+ raise HTTPException(404, "关系数据库不存在或未就绪")
+
+
+async def list_project_table_entries(
+ project_id: str,
+) -> tuple[tuple[TableDefinition, str, str], ...]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT * FROM project_table_definitions
+ WHERE project_id=%s AND status='active'
+ ORDER BY display_order, created_at, table_code
+ """,
+ (project_id,),
+ )
+ rows = await cur.fetchall()
+ return tuple(
+ (_table_from_row(dict(row)), str(row["origin"]), str(row["source_code"]))
+ for row in rows
+ )
+
+
+async def list_project_table_definitions(project_id: str) -> tuple[TableDefinition, ...]:
+ return tuple(item[0] for item in await list_project_table_entries(project_id))
+
+
+async def resolve_project_table_entry(
+ project_id: str,
+ table_code: str,
+) -> tuple[TableDefinition, str, str] | None:
+ return next(
+ (entry for entry in await list_project_table_entries(project_id) if entry[0].code == table_code),
+ None,
+ )
+
+
+async def _table_or_404(project_id: str, table_code: str) -> TableDefinition:
+ entry = await resolve_project_table_entry(project_id, table_code)
+ if entry:
+ return entry[0]
+ raise HTTPException(404, "数据表未注册")
+
+
+async def _create_physical_table(
+ database_name: str,
+ definition: TableDefinition,
+) -> None:
+ business_columns = [_column_clause(field) for field in definition.fields]
+ columns = [
+ "`id` CHAR(36) NOT NULL PRIMARY KEY",
+ "`tenant_id` VARCHAR(100) NOT NULL",
+ "`project_id` VARCHAR(63) NOT NULL",
+ *business_columns,
+ "`created_at` DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)",
+ "`updated_at` DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6)",
+ "`deleted_at` DATETIME(6) NULL",
+ "`deleted_by` VARCHAR(191) NULL",
+ "KEY `idx_updated_at` (`updated_at`)",
+ "KEY `idx_project_active` (`project_id`, `deleted_at`)",
+ ]
+ for field in definition.fields:
+ if field.unique:
+ columns.append(
+ f"UNIQUE KEY {_q(_managed_index_name('uq', field.code))} ({_q(field.code)})"
+ )
+ async with get_data_conn(database_name) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"CREATE TABLE IF NOT EXISTS {_q(definition.code)} "
+ f"({', '.join(columns)}) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 "
+ "COLLATE=utf8mb4_0900_ai_ci"
+ )
+ await conn.commit()
+
+
+async def ensure_project_database(
+ project_id: str,
+ tenant_id: str,
+ display_name: str,
+) -> dict[str, Any]:
+ await _ready()
+ if not IDENTIFIER_PATTERN.fullmatch(project_id):
+ raise ValueError("数据库编码格式不正确")
+ next_name = display_name.strip()
+ if not next_name:
+ raise ValueError("请输入数据库名称")
+ if len(next_name) > 100:
+ raise ValueError("数据库名称不能超过 100 个字符")
+ database_name = project_database_name(project_id)
+ if database_name.lower() in {
+ *_MYSQL_SYSTEM_DATABASES,
+ control_database_name().lower(),
+ }:
+ raise ValueError("数据库编码不能与 MySQL 系统数据库或数据中心控制库重名")
+
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "SELECT database_name FROM project_databases WHERE project_id=%s",
+ (project_id,),
+ )
+ existing = await cur.fetchone()
+ if existing:
+ # Preserve the catalogued physical name during idempotent
+ # recovery. Legacy prefixed names are migrated separately.
+ database_name = str(existing["database_name"])
+ await cur.execute(
+ f"CREATE DATABASE IF NOT EXISTS {_q(database_name)} "
+ "CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"
+ )
+ await cur.execute(
+ """
+ INSERT INTO project_databases (
+ project_id, tenant_id, display_name, database_name,
+ schema_name, engine, status
+ ) VALUES (%s, %s, %s, %s, %s, 'mysql', 'provisioning')
+ ON DUPLICATE KEY UPDATE
+ tenant_id=%s, display_name=%s,
+ status='provisioning', updated_at=CURRENT_TIMESTAMP(6)
+ """,
+ (
+ project_id,
+ tenant_id,
+ next_name,
+ database_name,
+ database_name,
+ tenant_id,
+ next_name,
+ ),
+ )
+ await conn.commit()
+
+ try:
+ # A newly created Data Center database is intentionally empty. Existing
+ # registered definitions are provisioned only for idempotent recovery
+ # of an already catalogued database; legacy templates are never injected
+ # into a new database here.
+ for definition in await list_project_table_definitions(project_id):
+ await _create_physical_table(database_name, definition)
+ except Exception:
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "UPDATE project_databases SET status='provision_failed' WHERE project_id=%s",
+ (project_id,),
+ )
+ await conn.commit()
+ raise
+
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "UPDATE project_databases SET status='ready', updated_at=CURRENT_TIMESTAMP(6) WHERE project_id=%s",
+ (project_id,),
+ )
+ await conn.commit()
+ database = await get_project_database(project_id)
+ if not database:
+ raise ValueError("MySQL 数据库创建失败")
+ return database
+
+
+async def rename_project_database(project_id: str, display_name: str) -> dict[str, Any]:
+ await _ready()
+ next_name = display_name.strip()
+ if not next_name:
+ raise ValueError("请输入数据库名称")
+ if len(next_name) > 100:
+ raise ValueError("数据库名称不能超过 100 个字符")
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ UPDATE project_databases
+ SET display_name=%s, updated_at=CURRENT_TIMESTAMP(6)
+ WHERE project_id=%s AND status='ready'
+ """,
+ (next_name, project_id),
+ )
+ if cur.rowcount != 1:
+ raise ValueError("关系数据库不存在或未就绪")
+ await conn.commit()
+ database = await get_project_database(project_id)
+ return database or {}
+
+
+async def delete_project_database(project_id: str, confirm_name: str) -> dict[str, str]:
+ await _ready()
+ database = await get_project_database(project_id)
+ if not database:
+ raise ValueError("关系数据库不存在或未就绪")
+ if confirm_name.strip() not in {
+ str(database["display_name"]),
+ str(database["project_id"]),
+ str(database["database_name"]),
+ }:
+ raise ValueError("确认名称不正确,未执行删除")
+
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(f"DROP DATABASE {_q(str(database['database_name']))}")
+ await cur.execute(
+ "DELETE FROM project_table_definitions WHERE project_id=%s",
+ (project_id,),
+ )
+ await cur.execute(
+ "UPDATE api_policies SET status='disabled' WHERE database_id=%s",
+ (project_id,),
+ )
+ await cur.execute("DELETE FROM project_databases WHERE project_id=%s", (project_id,))
+ await conn.commit()
+ return {"project_id": project_id, "status": "deleted"}
+
+
+async def _create_custom_table_definition(
+ project_id: str,
+ definition: TableDefinition,
+ field_rows: list[dict[str, Any]],
+ actor: str,
+) -> TableDefinition:
+ database = await _database_or_404(project_id)
+ if any(item.code == definition.code for item in await list_project_table_definitions(project_id)):
+ raise ValueError("当前数据库中已存在同名数据表")
+ await _create_physical_table(str(database["database_name"]), definition)
+ try:
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ INSERT INTO project_table_definitions (
+ project_id, source_code, table_code, label, group_name,
+ description, fields_json, origin, display_order,
+ allow_create, allow_update, allow_delete, created_by
+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, 'custom', 1000, %s, %s, %s, %s)
+ """,
+ (
+ project_id,
+ definition.code,
+ definition.code,
+ definition.label,
+ definition.group,
+ definition.description,
+ json.dumps(field_rows, ensure_ascii=False),
+ int(definition.allow_create),
+ int(definition.allow_update),
+ int(definition.allow_delete),
+ actor,
+ ),
+ )
+ await conn.commit()
+ except Exception:
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(f"DROP TABLE IF EXISTS {_q(definition.code)}")
+ await conn.commit()
+ raise
+ return definition
+
+
+async def create_custom_table(
+ project_id: str,
+ body: dict[str, Any],
+ actor: str,
+) -> TableDefinition:
+ definition, rows = _custom_definition_from_payload(body)
+ return await _create_custom_table_definition(project_id, definition, rows, actor)
+
+
+async def create_custom_table_from_sql(
+ project_id: str,
+ body: dict[str, Any],
+ actor: str,
+) -> TableDefinition:
+ definition, rows = custom_definition_from_create_sql(body)
+ return await _create_custom_table_definition(project_id, definition, rows, actor)
+
+
+async def _persist_fields(
+ project_id: str,
+ source_code: str,
+ fields: Iterable[FieldDefinition],
+) -> None:
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ UPDATE project_table_definitions
+ SET fields_json=%s, updated_at=CURRENT_TIMESTAMP(6)
+ WHERE project_id=%s AND source_code=%s AND status='active'
+ """,
+ (
+ json.dumps([_field_row(field) for field in fields], ensure_ascii=False),
+ project_id,
+ source_code,
+ ),
+ )
+ if cur.rowcount != 1:
+ raise ValueError("数据表字段元数据不存在")
+ await conn.commit()
+
+
+async def rename_custom_table(
+ project_id: str,
+ table_code: str,
+ new_code: str,
+ new_label: str,
+) -> TableDefinition:
+ database = await _database_or_404(project_id)
+ next_code = new_code.strip()
+ next_label = new_label.strip()
+ if not IDENTIFIER_PATTERN.fullmatch(next_code):
+ raise ValueError("表名必须以小写字母开头,只能包含小写字母、数字和下划线,长度为 2–63 位")
+ if next_code in {"data_change_logs", "graph_sync_queue"}:
+ raise ValueError("该表名已被系统占用")
+ if not next_label:
+ raise ValueError("请输入数据表名称")
+ entry = await resolve_project_table_entry(project_id, table_code)
+ if not entry:
+ raise ValueError("数据表不存在")
+ current, _origin, source_code = entry
+ if next_code in TABLE_REGISTRY and next_code != source_code:
+ raise ValueError("该表名已被系统占用")
+ if next_code != table_code and any(
+ item.code == next_code for item in await list_project_table_definitions(project_id)
+ ):
+ raise ValueError("当前数据库中已存在同名数据表")
+
+ if next_code != table_code:
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"RENAME TABLE {_q(table_code)} TO {_q(next_code)}"
+ )
+ await conn.commit()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ UPDATE project_table_definitions
+ SET table_code=%s, label=%s, updated_at=CURRENT_TIMESTAMP(6)
+ WHERE project_id=%s AND source_code=%s AND status='active'
+ """,
+ (next_code, next_label, project_id, source_code),
+ )
+ await cur.execute(
+ "UPDATE data_change_logs SET table_code=%s WHERE project_id=%s AND table_code=%s",
+ (next_code, project_id, table_code),
+ )
+ await conn.commit()
+ return TableDefinition(
+ code=next_code,
+ label=next_label,
+ group=current.group,
+ description=current.description,
+ fields=current.fields,
+ allow_create=current.allow_create,
+ allow_update=current.allow_update,
+ allow_delete=current.allow_delete,
+ )
+
+
+async def delete_custom_table(
+ project_id: str,
+ table_code: str,
+ confirm_name: str,
+) -> dict[str, str]:
+ database = await _database_or_404(project_id)
+ entry = await resolve_project_table_entry(project_id, table_code)
+ if not entry:
+ raise ValueError("数据表不存在")
+ current, _origin, source_code = entry
+ if confirm_name.strip() not in {table_code, current.label}:
+ raise ValueError("确认名称不正确,未执行删除")
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(f"DROP TABLE {_q(table_code)}")
+ await conn.commit()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ UPDATE project_table_definitions
+ SET status='deleted', updated_at=CURRENT_TIMESTAMP(6)
+ WHERE project_id=%s AND source_code=%s
+ """,
+ (project_id, source_code),
+ )
+ await cur.execute(
+ "UPDATE api_policies SET status='disabled' WHERE database_id=%s AND table_code=%s",
+ (project_id, table_code),
+ )
+ await conn.commit()
+ return {"project_id": project_id, "table_code": table_code, "status": "deleted"}
+
+
+async def create_table_column(
+ project_id: str,
+ table_code: str,
+ body: dict[str, Any],
+) -> TableDefinition:
+ database = await _database_or_404(project_id)
+ entry = await resolve_project_table_entry(project_id, table_code)
+ if not entry:
+ raise ValueError("数据表不存在")
+ current, _origin, source_code = entry
+ definition = _structure_field_from_payload(body)
+ if any(field.code == definition.code for field in current.fields):
+ raise ValueError("当前数据表已存在同名字段")
+ if definition.required and not definition.default_sql:
+ raise ValueError("新增非空字段必须设置默认值,避免现有记录无法补值")
+ try:
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"ALTER TABLE {_q(table_code)} ADD COLUMN {_column_clause(definition)}"
+ )
+ if definition.unique:
+ await cur.execute(
+ f"ALTER TABLE {_q(table_code)} ADD UNIQUE KEY "
+ f"{_q(_managed_index_name('uq', definition.code))} ({_q(definition.code)})"
+ )
+ await conn.commit()
+ except MySQLError as exc:
+ raise ValueError(f"MySQL 拒绝结构修改:{exc}") from exc
+ next_fields = (*current.fields, definition)
+ await _persist_fields(project_id, source_code, next_fields)
+ return TableDefinition(
+ current.code, current.label, current.group, current.description, next_fields,
+ current.allow_create, current.allow_update, current.allow_delete,
+ )
+
+
+async def _single_column_unique_indexes(
+ database_name: str,
+ table_code: str,
+ column_code: str,
+) -> list[str]:
+ async with get_data_conn(database_name) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT index_name AS index_name
+ FROM information_schema.statistics
+ WHERE table_schema=%s AND table_name=%s AND column_name=%s
+ AND non_unique=0 AND index_name <> 'PRIMARY'
+ GROUP BY index_name
+ HAVING COUNT(*)=1
+ """,
+ (database_name, table_code, column_code),
+ )
+ return [str(row["index_name"]) for row in await cur.fetchall()]
+
+
+async def update_table_column(
+ project_id: str,
+ table_code: str,
+ column_code: str,
+ body: dict[str, Any],
+) -> TableDefinition:
+ if column_code in SYSTEM_COLUMN_CODES:
+ raise ValueError("系统字段不可修改")
+ database = await _database_or_404(project_id)
+ entry = await resolve_project_table_entry(project_id, table_code)
+ if not entry:
+ raise ValueError("数据表不存在")
+ current, _origin, source_code = entry
+ current_field = next((field for field in current.fields if field.code == column_code), None)
+ if not current_field:
+ raise ValueError("业务字段不存在或不可修改")
+ definition = _structure_field_from_payload(body, current=current_field)
+ if definition.code != column_code and any(
+ field.code == definition.code for field in current.fields
+ ):
+ raise ValueError("当前数据表已存在同名字段")
+ database_name = str(database["database_name"])
+ unique_indexes = await _single_column_unique_indexes(database_name, table_code, column_code)
+ try:
+ async with get_data_conn(database_name) as conn:
+ async with conn.cursor() as cur:
+ for index_name in unique_indexes:
+ await cur.execute(
+ f"ALTER TABLE {_q(table_code)} DROP INDEX {_q(index_name)}"
+ )
+ await cur.execute(
+ f"ALTER TABLE {_q(table_code)} CHANGE COLUMN {_q(column_code)} {_column_clause(definition)}"
+ )
+ if definition.unique:
+ await cur.execute(
+ f"ALTER TABLE {_q(table_code)} ADD UNIQUE KEY "
+ f"{_q(_managed_index_name('uq', definition.code))} ({_q(definition.code)})"
+ )
+ await conn.commit()
+ except MySQLError as exc:
+ raise ValueError(f"MySQL 拒绝结构修改:{exc}") from exc
+ next_fields = tuple(
+ definition if field.code == column_code else field for field in current.fields
+ )
+ await _persist_fields(project_id, source_code, next_fields)
+ return TableDefinition(
+ current.code, current.label, current.group, current.description, next_fields,
+ current.allow_create, current.allow_update, current.allow_delete,
+ )
+
+
+async def delete_table_column(
+ project_id: str,
+ table_code: str,
+ column_code: str,
+ confirm_name: str,
+) -> TableDefinition:
+ if column_code in SYSTEM_COLUMN_CODES:
+ raise ValueError("系统字段不可删除")
+ if confirm_name.strip() != column_code:
+ raise ValueError("确认字段名不正确,未执行删除")
+ database = await _database_or_404(project_id)
+ entry = await resolve_project_table_entry(project_id, table_code)
+ if not entry:
+ raise ValueError("数据表不存在")
+ current, _origin, source_code = entry
+ if not any(field.code == column_code for field in current.fields):
+ raise ValueError("业务字段不存在或不可删除")
+ next_fields = tuple(field for field in current.fields if field.code != column_code)
+ if not next_fields:
+ raise ValueError("数据表至少需要保留一个业务字段")
+ try:
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"ALTER TABLE {_q(table_code)} DROP COLUMN {_q(column_code)}"
+ )
+ await conn.commit()
+ except MySQLError as exc:
+ raise ValueError(f"MySQL 拒绝结构修改:{exc}") from exc
+ await _persist_fields(project_id, source_code, next_fields)
+ return TableDefinition(
+ current.code, current.label, current.group, current.description, next_fields,
+ current.allow_create, current.allow_update, current.allow_delete,
+ )
+
+
+async def list_databases() -> list[dict[str, Any]]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT * FROM project_databases
+ WHERE status='ready'
+ ORDER BY CASE WHEN project_id='yunyou_libo' THEN 0 ELSE 1 END,
+ created_at DESC
+ """
+ )
+ databases = [dict(row) for row in await cur.fetchall()]
+
+ result: list[dict[str, Any]] = []
+ for database in databases:
+ project_id = str(database["project_id"])
+ database_name = str(database["database_name"])
+ definitions = await list_project_table_definitions(project_id)
+ record_count = 0
+ latest_updates: list[datetime] = []
+ async with get_data_conn(database_name) as conn:
+ async with conn.cursor() as cur:
+ for definition in definitions:
+ await cur.execute(
+ f"SELECT COUNT(*) AS count, MAX(`updated_at`) AS updated_at "
+ f"FROM {_q(definition.code)} WHERE `deleted_at` IS NULL"
+ )
+ summary = await cur.fetchone()
+ record_count += int(summary["count"] or 0)
+ if summary.get("updated_at"):
+ latest_updates.append(summary["updated_at"])
+ row = _row_json(database) or {}
+ row.update(
+ {
+ "table_count": len(definitions),
+ "record_count": record_count,
+ "updated_at": _json_safe(max(latest_updates))
+ if latest_updates
+ else _json_safe(database.get("updated_at")),
+ }
+ )
+ result.append(row)
+ return result
+
+
+async def list_tables(project_id: str) -> dict[str, Any]:
+ database = await _database_or_404(project_id)
+ entries = await list_project_table_entries(project_id)
+ tables: list[dict[str, Any]] = []
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ for definition, origin, _source_code in entries:
+ await cur.execute(
+ f"SELECT COUNT(*) AS count, MAX(`updated_at`) AS updated_at "
+ f"FROM {_q(definition.code)} WHERE `deleted_at` IS NULL"
+ )
+ summary = await cur.fetchone()
+ item = definition.as_dict()
+ item.update(
+ {
+ "record_count": int(summary["count"] or 0),
+ "updated_at": _json_safe(summary.get("updated_at")),
+ "is_custom": origin == "custom",
+ "origin": origin,
+ }
+ )
+ tables.append(item)
+ return {"database": _row_json(database), "tables": tables}
+
+
+async def inspect_table(project_id: str, table_code: str) -> dict[str, Any]:
+ """Return the physical MySQL structure and foreign-key relationships."""
+
+ table = await _table_or_404(project_id, table_code)
+ database = await _database_or_404(project_id)
+ database_name = str(database["database_name"])
+ definitions = await list_project_table_definitions(project_id)
+ table_labels = {definition.code: definition.label for definition in definitions}
+ field_definitions = {field.code: field for field in table.fields}
+ system_labels = {
+ "id": "记录 ID",
+ "tenant_id": "租户 ID",
+ "project_id": "数据库编码",
+ "created_at": "创建时间",
+ "updated_at": "更新时间",
+ "deleted_at": "删除时间",
+ "deleted_by": "删除人",
+ }
+
+ async with get_data_conn(database_name) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT ordinal_position AS ordinal_position,
+ column_name AS name, column_type AS sql_type,
+ is_nullable AS is_nullable, column_default AS default_value,
+ column_comment AS comment, column_key, extra
+ FROM information_schema.columns
+ WHERE table_schema=%s AND table_name=%s
+ ORDER BY ordinal_position
+ """,
+ (database_name, table_code),
+ )
+ physical_columns = [dict(row) for row in await cur.fetchall()]
+
+ await cur.execute(
+ """
+ SELECT tc.constraint_name AS constraint_name,
+ tc.constraint_type AS constraint_type,
+ GROUP_CONCAT(kcu.column_name ORDER BY kcu.ordinal_position) AS columns_csv
+ FROM information_schema.table_constraints tc
+ LEFT JOIN information_schema.key_column_usage kcu
+ ON kcu.constraint_schema=tc.constraint_schema
+ AND kcu.table_name=tc.table_name
+ AND kcu.constraint_name=tc.constraint_name
+ WHERE tc.table_schema=%s AND tc.table_name=%s
+ GROUP BY tc.constraint_name, tc.constraint_type
+ ORDER BY FIELD(tc.constraint_type, 'PRIMARY KEY', 'FOREIGN KEY', 'UNIQUE'),
+ tc.constraint_name
+ """,
+ (database_name, table_code),
+ )
+ constraint_rows = [dict(row) for row in await cur.fetchall()]
+
+ await cur.execute(
+ """
+ SELECT index_name AS index_name, non_unique AS non_unique,
+ GROUP_CONCAT(column_name ORDER BY seq_in_index) AS columns_csv,
+ index_type AS index_type
+ FROM information_schema.statistics
+ WHERE table_schema=%s AND table_name=%s
+ GROUP BY index_name, non_unique, index_type
+ ORDER BY (index_name='PRIMARY') DESC, non_unique, index_name
+ """,
+ (database_name, table_code),
+ )
+ index_rows = [dict(row) for row in await cur.fetchall()]
+
+ await cur.execute(
+ """
+ SELECT kcu.constraint_name AS name,
+ kcu.table_schema AS source_schema,
+ kcu.table_name AS source_table,
+ kcu.column_name AS source_column,
+ kcu.referenced_table_schema AS target_schema,
+ kcu.referenced_table_name AS target_table,
+ kcu.referenced_column_name AS target_column,
+ rc.update_rule AS on_update,
+ rc.delete_rule AS on_delete
+ FROM information_schema.key_column_usage kcu
+ JOIN information_schema.referential_constraints rc
+ ON rc.constraint_schema=kcu.constraint_schema
+ AND rc.constraint_name=kcu.constraint_name
+ WHERE kcu.referenced_table_name IS NOT NULL
+ AND (
+ (kcu.table_schema=%s AND kcu.table_name=%s)
+ OR (kcu.referenced_table_schema=%s AND kcu.referenced_table_name=%s)
+ )
+ ORDER BY kcu.constraint_name, kcu.ordinal_position
+ """,
+ (database_name, table_code, database_name, table_code),
+ )
+ relationship_rows = [dict(row) for row in await cur.fetchall()]
+
+ type_names = {
+ "PRIMARY KEY": "primary_key",
+ "UNIQUE": "unique",
+ "FOREIGN KEY": "foreign_key",
+ "CHECK": "check",
+ }
+ constraints: list[dict[str, Any]] = []
+ for row in constraint_rows:
+ columns = str(row.get("columns_csv") or "").split(",")
+ constraint_type = type_names.get(
+ str(row["constraint_type"]).upper(),
+ str(row["constraint_type"]).lower().replace(" ", "_"),
+ )
+ constraints.append(
+ {
+ "name": str(row["constraint_name"]),
+ "type": constraint_type,
+ "columns": [value for value in columns if value],
+ "definition": f"{row['constraint_type']} ({', '.join(columns)})",
+ }
+ )
+ primary_columns = {
+ column
+ for constraint in constraints
+ if constraint["type"] == "primary_key"
+ for column in constraint["columns"]
+ }
+ unique_columns = {
+ column
+ for constraint in constraints
+ if constraint["type"] in {"primary_key", "unique"}
+ and len(constraint["columns"]) == 1
+ for column in constraint["columns"]
+ }
+ foreign_columns = {
+ column
+ for constraint in constraints
+ if constraint["type"] == "foreign_key"
+ for column in constraint["columns"]
+ }
+
+ columns: list[dict[str, Any]] = []
+ for row in physical_columns:
+ code = str(row["name"])
+ definition = field_definitions.get(code)
+ columns.append(
+ {
+ "ordinal_position": int(row["ordinal_position"]),
+ "name": code,
+ "label": definition.label if definition else system_labels.get(code, code),
+ "data_type": definition.data_type if definition else "system",
+ "sql_type": str(row["sql_type"]),
+ "nullable": str(row["is_nullable"]).upper() == "YES",
+ "default": _json_safe(row.get("default_value")),
+ "primary_key": code in primary_columns,
+ "unique": code in unique_columns,
+ "foreign_key": code in foreign_columns,
+ "editable": bool(definition.editable) if definition else False,
+ "structure_editable": bool(definition) and code not in system_labels,
+ "searchable": bool(definition.searchable) if definition else False,
+ "sortable": bool(definition.sortable) if definition else False,
+ "visible_in_list": bool(definition.visible_in_list) if definition else False,
+ "comment": row.get("comment") or None,
+ }
+ )
+
+ indexes = []
+ for row in index_rows:
+ index_columns = [value for value in str(row.get("columns_csv") or "").split(",") if value]
+ name = str(row["index_name"])
+ indexes.append(
+ {
+ "name": name,
+ "unique": not bool(row["non_unique"]),
+ "primary": name == "PRIMARY",
+ "definition": (
+ f"{'UNIQUE ' if not row['non_unique'] else ''}INDEX {name} "
+ f"({', '.join(index_columns)}) USING {row['index_type']}"
+ ),
+ }
+ )
+
+ grouped_relationships: dict[tuple[str, str, str], dict[str, Any]] = {}
+ for row in relationship_rows:
+ key = (str(row["name"]), str(row["source_table"]), str(row["target_table"]))
+ relationship = grouped_relationships.setdefault(
+ key,
+ {
+ "name": str(row["name"]),
+ "direction": "outgoing" if str(row["source_table"]) == table_code else "incoming",
+ "source_schema": str(row["source_schema"]),
+ "source_table": str(row["source_table"]),
+ "source_table_label": table_labels.get(str(row["source_table"]), str(row["source_table"])),
+ "source_columns": [],
+ "target_schema": str(row["target_schema"]),
+ "target_table": str(row["target_table"]),
+ "target_table_label": table_labels.get(str(row["target_table"]), str(row["target_table"])),
+ "target_columns": [],
+ "on_update": str(row["on_update"]),
+ "on_delete": str(row["on_delete"]),
+ },
+ )
+ relationship["source_columns"].append(str(row["source_column"]))
+ relationship["target_columns"].append(str(row["target_column"]))
+
+ return {
+ "table": {
+ "schema_name": database_name,
+ "table_code": table.code,
+ "label": table.label,
+ "description": table.description,
+ "column_count": len(columns),
+ "index_count": len(indexes),
+ "constraint_count": len(constraints),
+ "primary_key": list(primary_columns),
+ },
+ "columns": columns,
+ "constraints": constraints,
+ "indexes": indexes,
+ "relationships": list(grouped_relationships.values()),
+ }
+
+
+def _coerce_value(field: FieldDefinition, value: Any) -> Any:
+ if value in ("", None):
+ return None
+ if field.data_type == "json":
+ if isinstance(value, str):
+ try:
+ value = json.loads(value)
+ except json.JSONDecodeError as exc:
+ raise HTTPException(400, f"{field.label}不是合法 JSON") from exc
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+ if field.data_type == "boolean":
+ if isinstance(value, bool):
+ return int(value)
+ lowered = str(value).lower()
+ if lowered in {"true", "1", "yes", "是", "启用"}:
+ return 1
+ if lowered in {"false", "0", "no", "否", "禁用"}:
+ return 0
+ raise HTTPException(400, f"{field.label}必须是布尔值")
+ if field.data_type == "number":
+ if any(token in field.sql_type.upper() for token in ("INT", "SERIAL")):
+ try:
+ return int(value)
+ except (TypeError, ValueError) as exc:
+ raise HTTPException(400, f"{field.label}必须是整数") from exc
+ try:
+ return Decimal(str(value))
+ except Exception as exc:
+ raise HTTPException(400, f"{field.label}必须是数字") from exc
+ if field.data_type == "date":
+ try:
+ return date.fromisoformat(str(value)).isoformat()
+ except ValueError as exc:
+ raise HTTPException(400, f"{field.label}必须使用 YYYY-MM-DD 格式") from exc
+ if field.data_type == "datetime":
+ parsed = str(value).replace("Z", "+00:00")
+ try:
+ return datetime.fromisoformat(parsed).replace(tzinfo=None)
+ except ValueError as exc:
+ raise HTTPException(400, f"{field.label}不是合法日期时间") from exc
+ if field.options and str(value) not in field.options:
+ raise HTTPException(400, f"{field.label}可选值为:{', '.join(field.options)}")
+ return value
+
+
+def _validated_payload(
+ table: TableDefinition,
+ body: dict[str, Any],
+ *,
+ create: bool,
+) -> dict[str, Any]:
+ definitions = {field.code: field for field in table.fields}
+ unknown = sorted(set(body) - set(definitions))
+ if unknown:
+ raise HTTPException(400, f"不允许的字段:{', '.join(unknown)}")
+ payload: dict[str, Any] = {}
+ for code, value in body.items():
+ field = definitions[code]
+ if field.editable:
+ payload[code] = _coerce_value(field, value)
+ if create:
+ missing = [
+ field.label
+ for field in table.fields
+ if field.required and payload.get(field.code) in (None, "")
+ ]
+ if missing:
+ raise HTTPException(400, f"缺少必填字段:{', '.join(missing)}")
+ return payload
+
+
+def _where_scope(
+ table: TableDefinition,
+ row_filter: dict[str, Any] | None,
+) -> tuple[list[str], list[Any]]:
+ if not row_filter:
+ return [], []
+ definitions = {field.code: field for field in table.fields}
+ clauses: list[str] = []
+ params: list[Any] = []
+ for code, expected in row_filter.items():
+ field = definitions.get(code)
+ if not field:
+ raise HTTPException(403, f"接口行级范围字段不存在:{code}")
+ if isinstance(expected, list):
+ if not expected:
+ clauses.append("1=0")
+ continue
+ values = [_coerce_value(field, value) for value in expected]
+ clauses.append(f"{_q(code)} IN ({', '.join(['%s'] * len(values))})")
+ params.extend(values)
+ elif expected is None:
+ clauses.append(f"{_q(code)} IS NULL")
+ else:
+ clauses.append(f"{_q(code)}=%s")
+ params.append(_coerce_value(field, expected))
+ return clauses, params
+
+
+def _payload_in_scope(
+ table: TableDefinition,
+ payload: dict[str, Any],
+ row_filter: dict[str, Any] | None,
+ *,
+ inject_missing: bool,
+) -> dict[str, Any]:
+ if not row_filter:
+ return payload
+ definitions = {field.code: field for field in table.fields}
+ scoped = dict(payload)
+ for code, expected in row_filter.items():
+ field = definitions.get(code)
+ if not field:
+ raise HTTPException(403, f"接口行级范围字段不存在:{code}")
+ if isinstance(expected, list):
+ allowed = [_coerce_value(field, value) for value in expected]
+ if code not in scoped or scoped[code] not in allowed:
+ raise HTTPException(403, f"字段 {code} 超出接口授权的数据范围")
+ continue
+ normalized = _coerce_value(field, expected) if expected is not None else None
+ if code not in scoped and inject_missing:
+ scoped[code] = normalized
+ elif code in scoped and scoped[code] != normalized:
+ raise HTTPException(403, f"字段 {code} 超出接口授权的数据范围")
+ return scoped
+
+
+def _selected_columns(
+ table: TableDefinition,
+ allowed_fields: set[str] | None = None,
+) -> list[str]:
+ business = [field.code for field in table.fields]
+ if allowed_fields is not None and "*" not in allowed_fields:
+ business = [code for code in business if code in allowed_fields]
+ return ["id", *business, "created_at", "updated_at"]
+
+
+def _decode_record(table: TableDefinition, row: dict[str, Any] | None) -> dict[str, Any] | None:
+ if row is None:
+ return None
+ result = _row_json(row) or {}
+ for field in table.fields:
+ if field.data_type == "json" and field.code in result and result[field.code] is not None:
+ result[field.code] = _decode_json(result[field.code], result[field.code])
+ if field.data_type == "boolean" and field.code in result and result[field.code] is not None:
+ result[field.code] = bool(result[field.code])
+ return result
+
+
+async def list_records(
+ project_id: str,
+ table_code: str,
+ *,
+ page: int,
+ page_size: int,
+ search: str | None,
+ sort_field: str | None,
+ sort_order: str,
+ allowed_fields: set[str] | None = None,
+ row_filter: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+ table = await _table_or_404(project_id, table_code)
+ database = await _database_or_404(project_id)
+ page = max(1, page)
+ page_size = max(1, min(5000, page_size))
+ offset = (page - 1) * page_size
+ readable = {field.code for field in table.fields}
+ if allowed_fields is not None and "*" not in allowed_fields:
+ readable &= allowed_fields
+ allowed_sort = {
+ "id",
+ "created_at",
+ "updated_at",
+ *(field.code for field in table.fields if field.sortable and field.code in readable),
+ }
+ order_field = sort_field if sort_field in allowed_sort else "updated_at"
+ order_keyword = "ASC" if sort_order.lower() == "asc" else "DESC"
+ where_parts = ["`deleted_at` IS NULL"]
+ params: list[Any] = []
+ scope_parts, scope_params = _where_scope(table, row_filter)
+ where_parts.extend(scope_parts)
+ params.extend(scope_params)
+ searchable = [
+ field for field in table.fields if field.searchable and field.code in readable
+ ]
+ if search and searchable:
+ pattern = f"%{search.strip()}%"
+ where_parts.append(
+ "(" + " OR ".join(
+ f"CAST({_q(field.code)} AS CHAR) LIKE %s" for field in searchable
+ ) + ")"
+ )
+ params.extend(pattern for _ in searchable)
+ where_clause = " AND ".join(where_parts)
+ columns = _selected_columns(table, allowed_fields)
+
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"SELECT COUNT(*) AS count FROM {_q(table.code)} WHERE {where_clause}",
+ params,
+ )
+ total = int((await cur.fetchone())["count"])
+ await cur.execute(
+ f"SELECT {', '.join(_q(column) for column in columns)} "
+ f"FROM {_q(table.code)} WHERE {where_clause} "
+ f"ORDER BY {_q(order_field)} {order_keyword} LIMIT %s OFFSET %s",
+ [*params, page_size, offset],
+ )
+ rows = [_decode_record(table, dict(row)) for row in await cur.fetchall()]
+ table_payload = table.as_dict()
+ if allowed_fields is not None and "*" not in allowed_fields:
+ table_payload["fields"] = [
+ field for field in table_payload["fields"]
+ if field["code"] in {"id", "created_at", "updated_at", *allowed_fields}
+ ]
+ return {
+ "table": table_payload,
+ "items": rows,
+ "page": page,
+ "page_size": page_size,
+ "total": total,
+ }
+
+
+async def _write_audit(
+ cur: Any,
+ *,
+ tenant_id: str,
+ project_id: str,
+ table_code: str,
+ record_id: str,
+ operation: str,
+ before_data: dict[str, Any] | None,
+ after_data: dict[str, Any] | None,
+ actor: str,
+) -> None:
+ await cur.execute(
+ f"""
+ INSERT INTO {_q(control_database_name())}.data_change_logs (
+ tenant_id, project_id, table_code, record_id, operation,
+ before_data, after_data, actor
+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+ """,
+ (
+ tenant_id,
+ project_id,
+ table_code,
+ record_id,
+ operation,
+ json.dumps(before_data, ensure_ascii=False) if before_data is not None else None,
+ json.dumps(after_data, ensure_ascii=False) if after_data is not None else None,
+ actor,
+ ),
+ )
+
+
+async def _write_admin_action_audit(
+ cur: Any,
+ *,
+ actor: str,
+ action_name: str,
+ resource_type: str,
+ resource_id: str,
+ outcome: str,
+ statement_hash: str | None = None,
+ affected_rows: int = 0,
+ source_ip: str | None = None,
+ details: dict[str, Any] | None = None,
+) -> None:
+ await cur.execute(
+ f"""
+ INSERT INTO {_q(control_database_name())}.admin_action_logs (
+ actor, action_name, resource_type, resource_id, outcome,
+ statement_hash, affected_rows, source_ip, details_json
+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+ """,
+ (
+ actor[:191],
+ action_name[:64],
+ resource_type[:64],
+ resource_id[:191],
+ outcome[:16],
+ statement_hash,
+ max(0, int(affected_rows)),
+ (source_ip or "")[:64] or None,
+ json.dumps(details or {}, ensure_ascii=False),
+ ),
+ )
+
+
+async def _record_admin_action_safely(**payload: Any) -> None:
+ try:
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await _write_admin_action_audit(cur, **payload)
+ await conn.commit()
+ except Exception:
+ # The original security decision or SQL error must remain visible even
+ # if the audit store is temporarily unavailable.
+ return
+
+
+async def list_admin_action_logs(limit: int = 200) -> list[dict[str, Any]]:
+ await _ready()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT id, actor, action_name, resource_type, resource_id,
+ outcome, statement_hash, affected_rows, source_ip,
+ details_json, created_at
+ FROM admin_action_logs
+ ORDER BY created_at DESC
+ LIMIT %s
+ """,
+ (max(1, min(1000, limit)),),
+ )
+ rows = []
+ for item in await cur.fetchall():
+ row = _row_json(dict(item)) or {}
+ row["details"] = _decode_json(row.pop("details_json", None), {})
+ rows.append(row)
+ return rows
+
+
+async def _select_record(
+ cur: Any,
+ table: TableDefinition,
+ record_id: str,
+ *,
+ row_filter: dict[str, Any] | None = None,
+) -> dict[str, Any] | None:
+ where = ["`id`=%s", "`deleted_at` IS NULL"]
+ params: list[Any] = [record_id]
+ scope, scope_params = _where_scope(table, row_filter)
+ where.extend(scope)
+ params.extend(scope_params)
+ columns = _selected_columns(table)
+ await cur.execute(
+ f"SELECT {', '.join(_q(column) for column in columns)} "
+ f"FROM {_q(table.code)} WHERE {' AND '.join(where)}",
+ params,
+ )
+ return _decode_record(table, await cur.fetchone())
+
+
+async def create_record(
+ project_id: str,
+ table_code: str,
+ body: dict[str, Any],
+ actor: str,
+ *,
+ row_filter: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+ table = await _table_or_404(project_id, table_code)
+ if not table.allow_create:
+ raise HTTPException(403, "该表不允许新增")
+ database = await _database_or_404(project_id)
+ payload = _payload_in_scope(
+ table,
+ _validated_payload(table, body, create=True),
+ row_filter,
+ inject_missing=True,
+ )
+ record_id = str(uuid.uuid4())
+ columns = ["id", "tenant_id", "project_id", *payload.keys()]
+ values = [record_id, database["tenant_id"], project_id, *payload.values()]
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ try:
+ await cur.execute(
+ f"INSERT INTO {_q(table.code)} "
+ f"({', '.join(_q(column) for column in columns)}) "
+ f"VALUES ({', '.join(['%s'] * len(values))})",
+ values,
+ )
+ row = await _select_record(cur, table, record_id)
+ await _write_audit(
+ cur,
+ tenant_id=str(database["tenant_id"]),
+ project_id=project_id,
+ table_code=table_code,
+ record_id=record_id,
+ operation="create",
+ before_data=None,
+ after_data=row,
+ actor=actor,
+ )
+ await conn.commit()
+ except MySQLError as exc:
+ raise HTTPException(400, f"MySQL 数据约束校验失败:{exc}") from exc
+ return row or {}
+
+
+async def update_record(
+ project_id: str,
+ table_code: str,
+ record_id: str,
+ body: dict[str, Any],
+ actor: str,
+ *,
+ row_filter: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+ table = await _table_or_404(project_id, table_code)
+ if not table.allow_update:
+ raise HTTPException(403, "该表不允许修改")
+ try:
+ str(uuid.UUID(record_id))
+ except ValueError as exc:
+ raise HTTPException(400, "记录 ID 格式错误") from exc
+ database = await _database_or_404(project_id)
+ payload = _validated_payload(table, body, create=False)
+ if not payload:
+ raise HTTPException(400, "没有可修改字段")
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ before = await _select_record(cur, table, record_id, row_filter=row_filter)
+ if not before:
+ raise HTTPException(404, "记录不存在或超出接口授权范围")
+ merged = {**before, **payload}
+ _payload_in_scope(table, merged, row_filter, inject_missing=False)
+ assignments = [f"{_q(column)}=%s" for column in payload]
+ if any(field.code == "version" for field in table.fields):
+ assignments.append("`version`=`version`+1")
+ assignments.append("`updated_at`=CURRENT_TIMESTAMP(6)")
+ try:
+ await cur.execute(
+ f"UPDATE {_q(table.code)} SET {', '.join(assignments)} WHERE `id`=%s",
+ [*payload.values(), record_id],
+ )
+ after = await _select_record(cur, table, record_id)
+ await _write_audit(
+ cur,
+ tenant_id=str(database["tenant_id"]),
+ project_id=project_id,
+ table_code=table_code,
+ record_id=record_id,
+ operation="update",
+ before_data=before,
+ after_data=after,
+ actor=actor,
+ )
+ await conn.commit()
+ except MySQLError as exc:
+ raise HTTPException(400, f"MySQL 数据约束校验失败:{exc}") from exc
+ return after or {}
+
+
+async def delete_record(
+ project_id: str,
+ table_code: str,
+ record_id: str,
+ actor: str,
+ *,
+ row_filter: dict[str, Any] | None = None,
+) -> dict[str, bool]:
+ table = await _table_or_404(project_id, table_code)
+ if not table.allow_delete:
+ raise HTTPException(403, "该表不允许删除")
+ try:
+ str(uuid.UUID(record_id))
+ except ValueError as exc:
+ raise HTTPException(400, "记录 ID 格式错误") from exc
+ database = await _database_or_404(project_id)
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ before = await _select_record(cur, table, record_id, row_filter=row_filter)
+ if not before:
+ raise HTTPException(404, "记录不存在或超出接口授权范围")
+ await cur.execute(
+ f"UPDATE {_q(table.code)} SET `deleted_at`=CURRENT_TIMESTAMP(6), "
+ "`deleted_by`=%s, `updated_at`=CURRENT_TIMESTAMP(6) WHERE `id`=%s",
+ (actor, record_id),
+ )
+ await _write_audit(
+ cur,
+ tenant_id=str(database["tenant_id"]),
+ project_id=project_id,
+ table_code=table_code,
+ record_id=record_id,
+ operation="delete",
+ before_data=before,
+ after_data=None,
+ actor=actor,
+ )
+ await conn.commit()
+ return {"ok": True}
+
+
+def _decode_csv(content: bytes) -> tuple[str, str]:
+ if not content:
+ raise HTTPException(400, "CSV 文件为空")
+ if len(content) > MAX_CSV_BYTES:
+ raise HTTPException(413, "CSV 文件不能超过 50 MB")
+ for encoding in ("utf-8-sig", "utf-8", "gb18030"):
+ try:
+ return content.decode(encoding), encoding
+ except UnicodeDecodeError:
+ continue
+ raise HTTPException(400, "CSV 编码无法识别,请使用 UTF-8 或 GB18030")
+
+
+def _csv_reader(text: str) -> csv.DictReader:
+ try:
+ dialect = csv.Sniffer().sniff(text[:8192], delimiters=",;\t|")
+ except csv.Error:
+ dialect = csv.excel
+ return csv.DictReader(io.StringIO(text, newline=""), dialect=dialect)
+
+
+def _normalize_header(value: str | None) -> str:
+ return str(value or "").replace("\ufeff", "").strip()
+
+
+_SYSTEM_CSV_HEADERS = {
+ "id", "记录 id", "记录id", "tenant_id", "租户 id", "租户id",
+ "project_id", "数据库编码", "created_at", "创建时间", "updated_at", "更新时间",
+ "deleted_at", "deleted_by",
+}
+
+
+def _parse_csv(
+ table: TableDefinition,
+ content: bytes,
+ file_name: str,
+) -> tuple[dict[str, Any], list[dict[str, Any]], list[str]]:
+ text, encoding = _decode_csv(content)
+ reader = _csv_reader(text)
+ headers = [_normalize_header(header) for header in (reader.fieldnames or [])]
+ by_code = {field.code.lower(): field for field in table.fields}
+ by_label = {field.label: field for field in table.fields}
+ mappings: list[dict[str, Any]] = []
+ mapped: dict[str, FieldDefinition] = {}
+ mapped_codes: set[str] = set()
+ schema_errors: list[str] = []
+ if not headers or not any(headers):
+ schema_errors.append("CSV 缺少表头")
+ for header in headers:
+ field = by_code.get(header.lower()) or by_label.get(header)
+ if field and field.code not in mapped_codes:
+ mapped[header] = field
+ mapped_codes.add(field.code)
+ mappings.append({
+ "source_header": header,
+ "field_code": field.code,
+ "field_label": field.label,
+ "status": "mapped",
+ })
+ elif field:
+ mappings.append({
+ "source_header": header,
+ "field_code": field.code,
+ "field_label": field.label,
+ "status": "duplicate",
+ })
+ schema_errors.append(f"字段“{field.label}”在 CSV 中重复映射")
+ elif header.lower() in _SYSTEM_CSV_HEADERS:
+ mappings.append({
+ "source_header": header,
+ "field_code": None,
+ "field_label": "平台自动生成",
+ "status": "ignored",
+ })
+ else:
+ mappings.append({
+ "source_header": header,
+ "field_code": None,
+ "field_label": None,
+ "status": "unknown",
+ })
+ schema_errors.append(f"无法识别 CSV 列“{header}”")
+ missing = [
+ field.label for field in table.fields if field.required and field.code not in mapped_codes
+ ]
+ if missing:
+ schema_errors.append(f"缺少必填列:{', '.join(missing)}")
+ if not mapped:
+ schema_errors.append("CSV 没有可导入的业务字段")
+
+ rows: list[dict[str, Any]] = []
+ errors: list[dict[str, Any]] = []
+ invalid_count = 0
+ total_rows = 0
+ for physical_row, source in enumerate(reader, start=2):
+ normalized = {
+ _normalize_header(key): value for key, value in source.items() if key is not None
+ }
+ if None in source and source[None]:
+ invalid_count += 1
+ if len(errors) < MAX_REPORTED_ERRORS:
+ errors.append({"row": physical_row, "field": "", "kind": "invalid", "message": "该行列数多于表头"})
+ continue
+ if not any(str(value or "").strip() for value in normalized.values()):
+ continue
+ total_rows += 1
+ if total_rows > MAX_IMPORT_ROWS:
+ raise HTTPException(400, f"单次最多导入 {MAX_IMPORT_ROWS:,} 条记录")
+ raw: dict[str, Any] = {}
+ row_error: str | None = None
+ row_field = ""
+ for header, field in mapped.items():
+ value = str(normalized.get(header) or "").strip()
+ if not value:
+ continue
+ try:
+ raw[field.code] = _coerce_value(field, value)
+ except HTTPException as exc:
+ row_error = str(exc.detail)
+ row_field = field.label
+ break
+ if row_error is None:
+ try:
+ payload = _validated_payload(table, raw, create=True)
+ except HTTPException as exc:
+ row_error = str(exc.detail)
+ payload = {}
+ else:
+ payload = {}
+ if row_error or not payload:
+ invalid_count += 1
+ if len(errors) < MAX_REPORTED_ERRORS:
+ errors.append({
+ "row": physical_row,
+ "field": row_field,
+ "kind": "invalid",
+ "message": row_error or "该行没有可导入的业务字段",
+ })
+ continue
+ rows.append({"row": physical_row, "payload": payload})
+ if total_rows == 0:
+ schema_errors.append("CSV 没有可导入的数据行")
+ preview = {
+ "file_name": file_name,
+ "encoding": encoding,
+ "total_rows": total_rows,
+ "valid_rows": len(rows),
+ "invalid_rows": invalid_count,
+ "duplicate_rows": 0,
+ "file_duplicate_rows": 0,
+ "existing_duplicate_rows": 0,
+ "skipped_rows": invalid_count,
+ "columns": mappings,
+ "schema_errors": schema_errors,
+ "errors": errors,
+ "errors_truncated": invalid_count > len(errors),
+ "preview_rows": [],
+ "can_import": not schema_errors and bool(rows),
+ }
+ return preview, rows, [field.code for field in mapped.values()]
+
+
+def _signature_value(value: Any) -> tuple[str, Any]:
+ if value is None:
+ return ("null", None)
+ if isinstance(value, bool):
+ return ("boolean", value)
+ if isinstance(value, (int, float, Decimal)):
+ return ("number", format(Decimal(str(value)).normalize(), "f"))
+ if isinstance(value, str):
+ stripped = value.strip()
+ if stripped.startswith(("{", "[")):
+ try:
+ return ("json", json.dumps(json.loads(stripped), ensure_ascii=False, sort_keys=True, separators=(",", ":")))
+ except json.JSONDecodeError:
+ pass
+ return ("text", stripped.casefold())
+ return ("json", json.dumps(_json_safe(value), ensure_ascii=False, sort_keys=True, separators=(",", ":")))
+
+
+def _payload_signature(payload: dict[str, Any], fields: list[str]) -> tuple[Any, ...]:
+ return tuple((field, _signature_value(payload.get(field))) for field in fields)
+
+
+async def _prepare_csv_import(
+ project_id: str,
+ table: TableDefinition,
+ database: dict[str, Any],
+ content: bytes,
+ file_name: str,
+) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+ preview, candidates, dedupe_fields = _parse_csv(table, content, file_name)
+ if preview["schema_errors"] or not candidates or not dedupe_fields:
+ preview["can_import"] = False
+ return preview, []
+ existing: set[tuple[Any, ...]] = set()
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"SELECT {', '.join(_q(code) for code in dedupe_fields)} "
+ f"FROM {_q(table.code)} WHERE `deleted_at` IS NULL"
+ )
+ for row in await cur.fetchall():
+ existing.add(_payload_signature(dict(row), dedupe_fields))
+ valid: list[dict[str, Any]] = []
+ seen: dict[tuple[Any, ...], int] = {}
+ duplicate_errors: list[dict[str, Any]] = []
+ file_duplicates = 0
+ existing_duplicates = 0
+ for candidate in candidates:
+ signature = _payload_signature(candidate["payload"], dedupe_fields)
+ row_number = int(candidate["row"])
+ if signature in existing:
+ existing_duplicates += 1
+ duplicate_errors.append({"row": row_number, "field": "", "kind": "duplicate", "message": "与当前数据表中的数据重复,已跳过"})
+ elif signature in seen:
+ file_duplicates += 1
+ duplicate_errors.append({"row": row_number, "field": "", "kind": "duplicate", "message": f"与 CSV 第 {seen[signature]} 行重复,已跳过"})
+ else:
+ seen[signature] = row_number
+ valid.append(candidate)
+ duplicate_count = file_duplicates + existing_duplicates
+ slots = max(0, MAX_REPORTED_ERRORS - len(preview["errors"]))
+ preview["errors"].extend(duplicate_errors[:slots])
+ preview["errors"].sort(key=lambda item: int(item["row"]))
+ preview.update({
+ "valid_rows": len(valid),
+ "duplicate_rows": duplicate_count,
+ "file_duplicate_rows": file_duplicates,
+ "existing_duplicate_rows": existing_duplicates,
+ "skipped_rows": int(preview["invalid_rows"]) + duplicate_count,
+ "errors_truncated": int(preview["invalid_rows"]) + duplicate_count > len(preview["errors"]),
+ "preview_rows": [_json_safe(item["payload"]) for item in valid[:PREVIEW_ROWS]],
+ "can_import": bool(valid),
+ })
+ return preview, valid
+
+
+async def preview_csv_import(
+ project_id: str,
+ table_code: str,
+ content: bytes,
+ file_name: str,
+) -> dict[str, Any]:
+ table = await _table_or_404(project_id, table_code)
+ if not table.allow_create:
+ raise HTTPException(403, "该表不允许导入新增记录")
+ database = await _database_or_404(project_id)
+ preview, _ = await _prepare_csv_import(project_id, table, database, content, file_name)
+ return preview
+
+
+async def import_csv_records(
+ project_id: str,
+ table_code: str,
+ content: bytes,
+ file_name: str,
+ actor: str,
+) -> dict[str, Any]:
+ table = await _table_or_404(project_id, table_code)
+ if not table.allow_create:
+ raise HTTPException(403, "该表不允许导入新增记录")
+ database = await _database_or_404(project_id)
+ preview, rows = await _prepare_csv_import(project_id, table, database, content, file_name)
+ if not preview["can_import"]:
+ detail = (preview["schema_errors"] or ["CSV 没有可导入的新数据"])[0]
+ raise HTTPException(400, f"CSV 校验失败:{detail}")
+ imported = 0
+ failed = 0
+ failed_errors: list[dict[str, Any]] = []
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ for candidate in rows:
+ payload = candidate["payload"]
+ record_id = str(uuid.uuid4())
+ columns = ["id", "tenant_id", "project_id", *payload.keys()]
+ values = [record_id, database["tenant_id"], project_id, *payload.values()]
+ try:
+ await cur.execute(
+ f"INSERT INTO {_q(table.code)} ({', '.join(_q(column) for column in columns)}) "
+ f"VALUES ({', '.join(['%s'] * len(values))})",
+ values,
+ )
+ imported += 1
+ await _write_audit(
+ cur,
+ tenant_id=str(database["tenant_id"]),
+ project_id=project_id,
+ table_code=table_code,
+ record_id=record_id,
+ operation="import",
+ before_data=None,
+ after_data={"id": record_id, **_json_safe(payload)},
+ actor=actor,
+ )
+ except MySQLError:
+ failed += 1
+ if len(failed_errors) < MAX_REPORTED_ERRORS:
+ failed_errors.append({"row": int(candidate["row"]), "field": "", "kind": "failed", "message": "数据库约束校验失败,已跳过"})
+ await conn.commit()
+ return {
+ "ok": True,
+ "file_name": file_name,
+ "imported_count": imported,
+ "invalid_count": int(preview["invalid_rows"]),
+ "duplicate_count": int(preview["duplicate_rows"]),
+ "failed_count": failed,
+ "skipped_count": int(preview["skipped_rows"]) + failed,
+ "errors": failed_errors,
+ "table_code": table.code,
+ }
+
+
+def _csv_value(value: Any) -> str:
+ value = _json_safe(value)
+ if value is None:
+ return ""
+ if isinstance(value, bool):
+ return "true" if value else "false"
+ if isinstance(value, (dict, list)):
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+ return str(value)
+
+
+async def export_csv_records(
+ project_id: str,
+ table_code: str,
+ search: str | None = None,
+) -> tuple[bytes, str, int]:
+ table = await _table_or_404(project_id, table_code)
+ database = await _database_or_404(project_id)
+ columns = _selected_columns(table)
+ headers = ["记录 ID", *(field.label for field in table.fields), "创建时间", "更新时间"]
+ where = ["`deleted_at` IS NULL"]
+ params: list[Any] = []
+ searchable = [field for field in table.fields if field.searchable]
+ if search and searchable:
+ where.append("(" + " OR ".join(f"CAST({_q(field.code)} AS CHAR) LIKE %s" for field in searchable) + ")")
+ params.extend(f"%{search.strip()}%" for _ in searchable)
+ where_clause = " AND ".join(where)
+ async with get_data_conn(str(database["database_name"])) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"SELECT COUNT(*) AS count FROM {_q(table.code)} WHERE {where_clause}",
+ params,
+ )
+ total = int((await cur.fetchone())["count"])
+ if total > MAX_EXPORT_ROWS:
+ raise HTTPException(400, f"当前数据量为 {total:,} 条,单次最多导出 {MAX_EXPORT_ROWS:,} 条,请先搜索筛选")
+ await cur.execute(
+ f"SELECT {', '.join(_q(column) for column in columns)} "
+ f"FROM {_q(table.code)} WHERE {where_clause} ORDER BY `created_at` ASC",
+ params,
+ )
+ rows = [dict(row) for row in await cur.fetchall()]
+ output = io.StringIO(newline="")
+ writer = csv.writer(output, lineterminator="\n")
+ writer.writerow(headers)
+ for row in rows:
+ writer.writerow([_csv_value(row.get(column)) for column in columns])
+ content = ("\ufeff" + output.getvalue()).encode("utf-8")
+ timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+ return content, f"{table.code}_{timestamp}.csv", total
+
+
+def validate_console_sql(raw_sql: str, database_name: str) -> tuple[str, str]:
+ statement = raw_sql.strip()
+ if not statement:
+ raise ValueError("请输入要执行的 SQL")
+ if len(statement) > MAX_SQL_LENGTH:
+ raise ValueError("单次 SQL 不能超过 50,000 个字符")
+ if "--" in statement or "/*" in statement or "*/" in statement:
+ raise ValueError("控制台暂不支持 SQL 注释,请删除注释后重试")
+ if statement.endswith(";"):
+ statement = statement[:-1].rstrip()
+ if ";" in statement:
+ raise ValueError("一次只能执行一条 SQL 语句")
+ match = re.match(r"^\s*([a-z]+)\b", statement, re.IGNORECASE)
+ keyword = match.group(1).lower() if match else ""
+ if keyword not in _ALLOWED_SQL:
+ raise ValueError("控制台仅支持受控查询;建表或修改结构请使用页面中的表结构功能")
+ if _FORBIDDEN_SQL.search(statement):
+ raise ValueError("SQL 包含控制台不允许执行的结构、文件或权限操作")
+ if keyword in {"insert", "update", "delete"} and not settings.data_sql_console_write_enabled:
+ raise ValueError("SQL 控制台当前为只读模式;数据修改请使用数据表页面,以保留完整变更记录")
+ if keyword == "delete":
+ raise ValueError("SQL 控制台禁止物理删除;请使用数据表页面执行可审计的软删除")
+ if keyword == "update":
+ if not re.search(r"\bwhere\b", statement, re.IGNORECASE):
+ raise ValueError("UPDATE 必须包含明确的 WHERE 条件")
+ if re.search(r"\bwhere\s+(?:1\s*=\s*1|true)\b", statement, re.IGNORECASE):
+ raise ValueError("UPDATE 禁止使用恒真 WHERE 条件")
+ if keyword == "with" and re.search(
+ r"\b(?:insert|update|delete)\b",
+ statement,
+ re.IGNORECASE,
+ ):
+ raise ValueError("WITH 在 SQL 控制台中仅允许只读查询")
+ if re.search(
+ r"\b(?:sleep|benchmark|get_lock|release_lock|load_file)\s*\(",
+ statement,
+ re.IGNORECASE,
+ ):
+ raise ValueError("SQL 包含控制台禁止调用的高风险函数")
+ if re.search(r"\bfor\s+update\b|\block\s+in\s+share\s+mode\b", statement, re.IGNORECASE):
+ raise ValueError("SQL 控制台禁止执行可能长期持有数据锁的查询")
+ if keyword == "select" and re.search(r"\binto\b", statement, re.IGNORECASE):
+ raise ValueError("SQL 控制台禁止 SELECT INTO")
+ lowered = statement.lower()
+ if keyword == "show" and not re.match(
+ r"^\s*show\s+(?:full\s+)?(?:tables|columns|fields|index|indexes|keys|create\s+table)\b",
+ statement,
+ re.IGNORECASE,
+ ):
+ raise ValueError("SHOW 仅允许查看当前数据库的表、字段、索引或建表语句")
+ qualified_objects = re.findall(
+ r"\b(?:from|join|update|into)\s+`?([a-z][a-z0-9_]*)`?\s*\.",
+ lowered,
+ re.IGNORECASE,
+ )
+ if keyword in {"describe", "desc"}:
+ described = re.match(
+ r"^\s*(?:describe|desc)\s+`?([a-z][a-z0-9_]*)`?\s*\.",
+ lowered,
+ re.IGNORECASE,
+ )
+ if described:
+ qualified_objects.append(described.group(1))
+ for qualifier in qualified_objects:
+ if qualifier.lower() != database_name.lower():
+ raise ValueError("控制台禁止访问其他业务数据库")
+ if keyword == "show":
+ show_tables = re.match(r"^\s*show\s+(?:full\s+)?tables\b", lowered)
+ show_scopes = re.findall(r"\b(?:from|in)\s+`?([a-z][a-z0-9_]*)`?", lowered)
+ explicit_scope = (
+ show_scopes[-1]
+ if show_tables and show_scopes
+ else show_scopes[-1] if len(show_scopes) >= 2 else None
+ )
+ if explicit_scope and explicit_scope.lower() != database_name.lower():
+ raise ValueError("控制台禁止查看其他业务数据库")
+ protected = {
+ "information_schema", "mysql", "performance_schema", "sys",
+ control_database_name().lower(),
+ }
+ for name in protected:
+ if re.search(rf"(? dict[str, Any]:
+ database = await _database_or_404(project_id)
+ database_name = str(database["database_name"])
+ raw_hash = hashlib.sha256(raw_sql.strip().encode("utf-8")).hexdigest()
+ try:
+ statement, keyword = validate_console_sql(raw_sql, database_name)
+ except ValueError as exc:
+ await _record_admin_action_safely(
+ actor=actor,
+ action_name="sql_console",
+ resource_type="database",
+ resource_id=project_id,
+ outcome="rejected",
+ statement_hash=raw_hash,
+ source_ip=source_ip,
+ details={"reason": str(exc)[:300]},
+ )
+ raise
+ started_at = time.perf_counter()
+ async with get_data_conn(database_name) as conn:
+ try:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "SELECT @@SESSION.sql_mode AS sql_mode, "
+ "@@SESSION.MAX_EXECUTION_TIME AS max_execution_time"
+ )
+ session_state = await cur.fetchone()
+ try:
+ # Existing frontend-generated SQL quotes identifiers with
+ # ". ANSI_QUOTES preserves that API contract on MySQL.
+ await cur.execute(
+ "SET SESSION sql_mode=CONCAT_WS(',', @@sql_mode, 'ANSI_QUOTES')"
+ )
+ await cur.execute(
+ "SET SESSION MAX_EXECUTION_TIME=%s",
+ (STATEMENT_TIMEOUT_MS,),
+ )
+ await cur.execute(statement)
+ columns: list[str] = []
+ rows: list[dict[str, Any]] = []
+ truncated = False
+ if cur.description:
+ columns = [str(item[0]) for item in cur.description]
+ fetched = await cur.fetchmany(MAX_RESULT_ROWS + 1)
+ truncated = len(fetched) > MAX_RESULT_ROWS
+ rows = [
+ {key: _json_safe(value) for key, value in dict(row).items()}
+ for row in fetched[:MAX_RESULT_ROWS]
+ ]
+ affected_rows = max(int(cur.rowcount or 0), 0)
+ command = keyword.upper()
+ finally:
+ await cur.execute(
+ "SET SESSION sql_mode=%s",
+ (session_state["sql_mode"],),
+ )
+ await cur.execute(
+ "SET SESSION MAX_EXECUTION_TIME=%s",
+ (session_state["max_execution_time"],),
+ )
+ await _write_admin_action_audit(
+ cur,
+ actor=actor,
+ action_name="sql_console",
+ resource_type="database",
+ resource_id=project_id,
+ outcome="success",
+ statement_hash=raw_hash,
+ affected_rows=affected_rows,
+ source_ip=source_ip,
+ details={
+ "statement_type": keyword,
+ "returned_rows": len(rows),
+ "truncated": truncated,
+ },
+ )
+ await conn.commit()
+ except MySQLError as exc:
+ await conn.rollback()
+ try:
+ async with conn.cursor() as cur:
+ await _write_admin_action_audit(
+ cur,
+ actor=actor,
+ action_name="sql_console",
+ resource_type="database",
+ resource_id=project_id,
+ outcome="failed",
+ statement_hash=raw_hash,
+ source_ip=source_ip,
+ details={"error": str(exc)[:300]},
+ )
+ await conn.commit()
+ except Exception:
+ await conn.rollback()
+ raise ValueError(f"MySQL 执行失败:{exc}") from exc
+ return {
+ "database": {
+ "project_id": project_id,
+ "display_name": database["display_name"],
+ "database_name": database_name,
+ "schema_name": database_name,
+ },
+ "statement_type": keyword,
+ "command": command,
+ "columns": columns,
+ "rows": rows,
+ "returned_rows": len(rows),
+ "affected_rows": affected_rows,
+ "truncated": truncated,
+ "max_result_rows": MAX_RESULT_ROWS,
+ "duration_ms": round((time.perf_counter() - started_at) * 1000, 2),
+ }
diff --git a/app/db.py b/app/db.py
index 97e9677..ee39640 100644
--- a/app/db.py
+++ b/app/db.py
@@ -326,33 +326,31 @@ async def ensure_project_lifecycle_support() -> None:
await conn.commit()
-# 3 roles kept by design: 系统管理员 / 运营审核员 / 采集人员
+# 3 roles kept by design: 系统管理员 / 运营人员 / 采集人员
_SEED_ROLES = [
- ("admin", "系统管理员", "全部权限(含 Schema / 发布 / 用户与权限管理)", True, 10),
- ("operator", "运营审核员", "审核候选、字段级裁决、领治理工单", True, 20),
+ ("admin", "系统管理员", "全部权限(含用户与权限管理)", True, 10),
+ ("operator", "运营人员", "知识广场与日常运营", True, 20),
("collector", "采集人员", "领补藏任务、CSV 导入、Python 采集", True, 30),
]
+_RETIRED_CAPABILITY_KEYS = {
+ "modify_schema",
+ "review_candidate",
+ "review_field",
+ "publish_falkor",
+ "governance",
+}
+
_SEED_CAPS = [
("view_plaza", "查看知识广场", 10),
- ("modify_schema", "修改 Schema", 20),
("import_data", "导入数据", 30),
- ("review_candidate", "审核候选", 40),
- ("review_field", "字段级审核", 50),
- ("publish_falkor", "发布到 Falkor", 60),
- ("governance", "治理工作台", 70),
("system_settings", "系统设置", 80),
]
# matrix[cap_key] = {role_key: value}
_SEED_MATRIX = {
"view_plaza": {"admin": "✓", "operator": "✓", "collector": "部分"},
- "modify_schema": {"admin": "✓", "operator": "—", "collector": "—"},
"import_data": {"admin": "✓", "operator": "✓", "collector": "✓"},
- "review_candidate": {"admin": "✓", "operator": "✓", "collector": "—"},
- "review_field": {"admin": "✓", "operator": "✓", "collector": "—"},
- "publish_falkor": {"admin": "✓", "operator": "—", "collector": "—"},
- "governance": {"admin": "✓", "operator": "✓", "collector": "领任务"},
"system_settings": {"admin": "✓", "operator": "—", "collector": "—"},
}
@@ -1907,34 +1905,6 @@ async def get_plaza_overview(tenant_id: str, project_id: str) -> dict:
}
-async def get_audit_gaps(tenant_id: str, project_id: str) -> list[dict]:
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"SELECT * FROM {s}.question_traces "
- "WHERE tenant_id=%s AND project_id=%s AND suggested_action='gap' "
- "AND acquisition_task_id IS NULL "
- "ORDER BY evaluated_at DESC LIMIT 50",
- (tenant_id, project_id),
- )
- return await cur.fetchall()
-
-
-async def get_plaza_alerts(tenant_id: str, project_id: str) -> list[dict]:
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"SELECT * FROM {s}.inventory_issues "
- "WHERE tenant_id=%s AND project_id=%s AND status='open' "
- "ORDER BY CASE severity WHEN 'blocker' THEN 1 WHEN 'warning' THEN 2 ELSE 3 END, "
- "detected_at DESC LIMIT 20",
- (tenant_id, project_id),
- )
- return await cur.fetchall()
-
-
# ── Agent Call Logs ──────────────────────────────────────────────────────────
async def log_agent_call(data: dict) -> None:
@@ -2022,7 +1992,8 @@ async def list_capabilities() -> list[dict]:
await cur.execute(
f"SELECT * FROM {s}.rbac_capabilities ORDER BY sort_order, cap_key"
)
- return await cur.fetchall()
+ rows = await cur.fetchall()
+ return [row for row in rows if row["cap_key"] not in _RETIRED_CAPABILITY_KEYS]
async def create_capability(data: dict) -> dict:
@@ -2063,11 +2034,17 @@ async def get_permission_matrix() -> dict:
await cur.execute(
f"SELECT * FROM {s}.rbac_capabilities ORDER BY sort_order, cap_key"
)
- caps = await cur.fetchall()
+ caps = [
+ row
+ for row in await cur.fetchall()
+ if row["cap_key"] not in _RETIRED_CAPABILITY_KEYS
+ ]
await cur.execute(f"SELECT role_key, cap_key, value FROM {s}.rbac_role_caps")
cells = await cur.fetchall()
matrix: dict[str, dict[str, str]] = {}
for c in cells:
+ if c["cap_key"] in _RETIRED_CAPABILITY_KEYS:
+ continue
matrix.setdefault(c["cap_key"], {})[c["role_key"]] = c["value"]
return {"roles": roles, "capabilities": caps, "matrix": matrix}
@@ -2096,13 +2073,7 @@ async def list_users() -> list[dict]:
u.created_at, u.updated_at,
COALESCE((SELECT array_agg(ur.role_key)
FROM {s}.user_roles ur
- WHERE ur.user_id=u.id), '{{}}') AS roles,
- COALESCE((SELECT array_agg(a.name ORDER BY a.area_id)
- FROM {s}.areas a
- WHERE a.responsible_user_id=u.id), '{{}}') AS area_names,
- COALESCE((SELECT array_agg(a.area_id ORDER BY a.area_id)
- FROM {s}.areas a
- WHERE a.responsible_user_id=u.id), '{{}}') AS area_ids
+ WHERE ur.user_id=u.id), '{{}}') AS roles
FROM {s}.users u
ORDER BY u.created_at"""
)
@@ -2192,170 +2163,7 @@ async def delete_user(user_id: int) -> bool:
return True
-# ── City areas & responsibility (P3) ─────────────────────────────────────────
-
-async def list_areas() -> list[dict]:
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""SELECT a.area_id, a.name, a.level, a.parent_id,
- a.responsible_user_id, a.note,
- u.username AS responsible_username,
- u.full_name AS responsible_name,
- a.updated_at
- FROM {s}.areas a
- LEFT JOIN {s}.users u ON u.id = a.responsible_user_id
- ORDER BY a.area_id"""
- )
- return await cur.fetchall()
-
-
-async def upsert_area(
- area_id: str, name: str,
- level: str | None = None, parent_id: str | None = None,
-) -> None:
- """Used by sync-from-graph; never overwrites a manual responsible mapping."""
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""INSERT INTO {s}.areas (area_id, name, level, parent_id)
- VALUES (%s, %s, %s, %s)
- ON CONFLICT (area_id) DO UPDATE
- SET name=EXCLUDED.name, level=EXCLUDED.level,
- parent_id=EXCLUDED.parent_id, updated_at=now()""",
- (area_id, name, level, parent_id),
- )
- await conn.commit()
-
-
-async def upsert_custom_area(name: str) -> str:
- """Free-text area typed by an admin; stable id derived from the name."""
- import hashlib
- s = settings.db_schema
- area_id = "c_" + hashlib.md5(name.strip().encode()).hexdigest()[:10]
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""INSERT INTO {s}.areas (area_id, name, level)
- VALUES (%s, %s, 'custom')
- ON CONFLICT (area_id) DO UPDATE SET name=EXCLUDED.name""",
- (area_id, name.strip()),
- )
- await conn.commit()
- return area_id
-
-
-async def list_user_areas(user_id: int) -> list[dict]:
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""SELECT area_id, name, level FROM {s}.areas
- WHERE responsible_user_id=%s ORDER BY area_id""",
- (user_id,),
- )
- return await cur.fetchall()
-
-
-async def set_user_areas(user_id: int, area_ids: list[str]) -> None:
- """Make user the responsible person for exactly `area_ids`
- (clears areas they previously owned but are no longer selected)."""
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""UPDATE {s}.areas SET responsible_user_id=NULL, updated_at=now()
- WHERE responsible_user_id=%s AND NOT (area_id = ANY(%s))""",
- (user_id, area_ids or [""]),
- )
- for aid in area_ids:
- await cur.execute(
- f"""UPDATE {s}.areas SET responsible_user_id=%s, updated_at=now()
- WHERE area_id=%s""",
- (user_id, aid),
- )
- await conn.commit()
-
-
-async def update_area(area_id: str, data: dict) -> dict | None:
- s = settings.db_schema
- if not data:
- return None
- sets = ", ".join(f"{k}=%({k})s" for k in data)
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"UPDATE {s}.areas SET {sets}, updated_at=now() "
- f"WHERE area_id=%(area_id)s RETURNING *",
- {**data, "area_id": area_id},
- )
- row = await cur.fetchone()
- await conn.commit()
- return row
-
-
-async def get_area_responsible(area_id: str) -> dict | None:
- """Resolve the collector responsible for an area (for gap routing)."""
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""SELECT u.id, u.username, u.full_name
- FROM {s}.areas a
- JOIN {s}.users u ON u.id = a.responsible_user_id
- WHERE a.area_id=%s AND u.status='active'""",
- (area_id,),
- )
- return await cur.fetchone()
-
-
-# ── Gap routing + notifications (P4) ─────────────────────────────────────────
-
-async def get_user_id_by_username(username: str) -> int | None:
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(f"SELECT id FROM {s}.users WHERE username=%s", (username,))
- row = await cur.fetchone()
- return row["id"] if row else None
-
-
-async def resolve_area_from_entities(entity_ids: list) -> str | None:
- """Most common area_id among the matched candidate entities' payloads."""
- if not entity_ids:
- return None
- s = settings.db_schema
- ids = [int(x) for x in entity_ids if str(x).isdigit()]
- if not ids:
- return None
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"SELECT payload->>'area_id' AS area_id "
- f"FROM {s}.candidate_entities WHERE id = ANY(%s)",
- (ids,),
- )
- rows = await cur.fetchall()
- from collections import Counter
- vals = [r["area_id"] for r in rows if r.get("area_id")]
- return Counter(vals).most_common(1)[0][0] if vals else None
-
-
-async def set_task_routing(task_id: int, area_id: str | None, assignee: str) -> None:
- """Auto-assign a gap task to the area's responsible collector."""
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""UPDATE {s}.acquisition_tasks
- SET area_id=%s, assignee=%s, status='assigned', assigned_at=now()
- WHERE id=%s""",
- (area_id, assignee, task_id),
- )
- await conn.commit()
-
+# ── Super Agent escalation records ───────────────────────────────────────────
async def create_notification(
user_id: int, title: str, body: str | None = None,
@@ -2376,58 +2184,6 @@ async def create_notification(
return row
-async def list_notifications(user_id: int, only_unread: bool = False) -> list[dict]:
- s = settings.db_schema
- q = f"SELECT * FROM {s}.notifications WHERE user_id=%s"
- if only_unread:
- q += " AND is_read=FALSE"
- q += " ORDER BY created_at DESC LIMIT 100"
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(q, (user_id,))
- return await cur.fetchall()
-
-
-async def unread_notification_count(user_id: int) -> int:
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"SELECT count(*) AS c FROM {s}.notifications "
- f"WHERE user_id=%s AND is_read=FALSE",
- (user_id,),
- )
- row = await cur.fetchone()
- return row["c"] if row else 0
-
-
-async def mark_notification_read(notif_id: int, user_id: int) -> bool:
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"UPDATE {s}.notifications SET is_read=TRUE "
- f"WHERE id=%s AND user_id=%s",
- (notif_id, user_id),
- )
- await conn.commit()
- return True
-
-
-async def mark_all_notifications_read(user_id: int) -> int:
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"UPDATE {s}.notifications SET is_read=TRUE "
- f"WHERE user_id=%s AND is_read=FALSE",
- (user_id,),
- )
- n = cur.rowcount
- await conn.commit()
- return n
-
-
# ── Agent settings (LLM + per-sub-agent config) ──────────────────────────────
def _default_agent(label: str, enabled: bool = True) -> dict:
@@ -2442,8 +2198,17 @@ def _default_agent(label: str, enabled: bool = True) -> dict:
}
+_RETIRED_AGENT_KEYS = {"publisher", "auditor"}
+_RETIRED_THRESHOLD_KEYS = {"audit_hit", "audit_gap"}
+
+
DEFAULT_AGENT_CONFIG: dict = {
- "global": {"base_url": "", "model": "deepseek-chat", "api_key": "", "timeout": 30},
+ "global": {
+ "base_url": settings.llm_api_base,
+ "model": settings.llm_model,
+ "api_key": settings.llm_api_key,
+ "timeout": settings.llm_timeout_seconds,
+ },
"api_access": {
"api_keys": settings.ingest_api_keys,
"default_graph": "baixinghui_travel_agency",
@@ -2457,18 +2222,12 @@ DEFAULT_AGENT_CONFIG: dict = {
"max_tokens": 1600,
},
},
- "thresholds": {
- "audit_hit": 0.7,
- "audit_gap": 0.4,
- "aligner_auto_merge": 0.95,
- },
+ "thresholds": {"aligner_auto_merge": 0.95},
"agents": {
- "auditor": _default_agent("AI 质量稽查"),
"aligner": _default_agent("实体归一"),
"extraction": _default_agent("值归一", enabled=False),
"field_mapping": _default_agent("字段映射"),
"schema_lint": _default_agent("Schema 审计"),
- "publisher": _default_agent("图谱发布"),
"web_agent": _default_agent("联网采集 web_agent"),
"xhs_agent": _default_agent("小红书采集 xhs_agent"),
"douyin_agent": _default_agent("抖音采集 douyin_agent"),
@@ -2547,6 +2306,9 @@ async def get_agent_settings() -> dict:
cfg = row["config"]
merged = {**DEFAULT_AGENT_CONFIG, **cfg}
merged["global"] = {**DEFAULT_AGENT_CONFIG["global"], **cfg.get("global", {})}
+ for key in ("base_url", "model", "api_key", "timeout"):
+ if not merged["global"].get(key):
+ merged["global"][key] = DEFAULT_AGENT_CONFIG["global"][key]
saved_api_access = cfg.get("api_access", {}) or {}
api_access = {**DEFAULT_AGENT_CONFIG["api_access"], **saved_api_access}
api_access["qa_llm"] = {
@@ -2554,9 +2316,18 @@ async def get_agent_settings() -> dict:
**(saved_api_access.get("qa_llm") or {}),
}
merged["api_access"] = api_access
- merged["thresholds"] = {**DEFAULT_AGENT_CONFIG["thresholds"], **cfg.get("thresholds", {})}
+ saved_thresholds = {
+ key: value
+ for key, value in (cfg.get("thresholds", {}) or {}).items()
+ if key not in _RETIRED_THRESHOLD_KEYS
+ }
+ merged["thresholds"] = {**DEFAULT_AGENT_CONFIG["thresholds"], **saved_thresholds}
# deep-merge each agent so newly added per-agent fields always appear
- saved_agents = cfg.get("agents", {})
+ saved_agents = {
+ key: value
+ for key, value in (cfg.get("agents", {}) or {}).items()
+ if key not in _RETIRED_AGENT_KEYS
+ }
agents: dict = {}
for key, dft in DEFAULT_AGENT_CONFIG["agents"].items():
agents[key] = {**dft, **saved_agents.get(key, {})}
@@ -2596,26 +2367,6 @@ async def get_agent_settings() -> dict:
return merged
-async def save_agent_settings(config: dict) -> dict:
- s = settings.db_schema
- async with get_conn() as conn:
- async with conn.cursor() as cur:
- await cur.execute(
- f"""UPDATE {s}.agent_settings
- SET config=%s, updated_at=now()
- WHERE settings_key='agent'""",
- (json.dumps(config),),
- )
- if cur.rowcount == 0:
- await cur.execute(
- f"""INSERT INTO {s}.agent_settings (settings_key, config, updated_at)
- VALUES ('agent', %s, now())""",
- (json.dumps(config),),
- )
- await conn.commit()
- return config
-
-
# ── Audit runs (async progress tracking) ─────────────────────────────────────
_ACTION_COL = {
diff --git a/app/main.py b/app/main.py
index 60dff95..aedd157 100644
--- a/app/main.py
+++ b/app/main.py
@@ -7,11 +7,44 @@ from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from starlette.exceptions import HTTPException as StarletteHTTPException
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.middleware.trustedhost import TrustedHostMiddleware
+from starlette.requests import Request
from app.api import api_router, openapi_router
from app.api.mcp_server import router as mcp_router
from app.db import init_pool, close_pool
-from app.data_platform.schema import ensure_platform_registry
+from app.data_platform.mysql_db import close_data_pool, init_data_pool
+from app.data_platform.mysql_service import ensure_platform_registry
+from app.config import settings
+from app.security_baseline import enforce_security_baseline
+
+
+def _csv_setting(value: str) -> list[str]:
+ return [item.strip() for item in value.split(",") if item.strip()]
+
+
+class SecurityHeadersMiddleware(BaseHTTPMiddleware):
+ async def dispatch(self, request: Request, call_next):
+ response = await call_next(request)
+ if not settings.security_headers_enabled:
+ return response
+ response.headers.setdefault("X-Content-Type-Options", "nosniff")
+ response.headers.setdefault("X-Frame-Options", "DENY")
+ response.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin")
+ response.headers.setdefault(
+ "Permissions-Policy",
+ "camera=(), microphone=(), geolocation=(), payment=(), usb=()",
+ )
+ if request.url.path.startswith("/v1/"):
+ response.headers.setdefault("Cache-Control", "no-store")
+ forwarded_proto = request.headers.get("x-forwarded-proto", "").split(",")[0].strip()
+ if request.url.scheme == "https" or forwarded_proto == "https":
+ response.headers.setdefault(
+ "Strict-Transport-Security",
+ "max-age=31536000; includeSubDomains",
+ )
+ return response
class SPAStaticFiles(StaticFiles):
@@ -69,10 +102,17 @@ class SPAStaticFiles(StaticFiles):
@asynccontextmanager
async def lifespan(_app: FastAPI):
+ enforce_security_baseline()
await init_pool()
- await ensure_platform_registry()
- yield
- await close_pool()
+ try:
+ if await init_data_pool():
+ await ensure_platform_registry()
+ yield
+ finally:
+ try:
+ await close_data_pool()
+ finally:
+ await close_pool()
app = FastAPI(
@@ -83,11 +123,17 @@ app = FastAPI(
app.add_middleware(
CORSMiddleware,
- allow_origins=["*"],
+ allow_origins=_csv_setting(settings.cors_allowed_origins),
allow_credentials=True,
- allow_methods=["*"],
- allow_headers=["*"],
+ allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
+ allow_headers=["Authorization", "Content-Type", "X-API-Key", "X-KG-API-Key"],
+ expose_headers=["Content-Disposition", "X-Exported-Count"],
)
+app.add_middleware(
+ TrustedHostMiddleware,
+ allowed_hosts=_csv_setting(settings.trusted_hosts) or ["localhost", "127.0.0.1"],
+)
+app.add_middleware(SecurityHeadersMiddleware)
app.include_router(api_router)
app.include_router(openapi_router)
diff --git a/app/rate_limit.py b/app/rate_limit.py
new file mode 100644
index 0000000..cd756ab
--- /dev/null
+++ b/app/rate_limit.py
@@ -0,0 +1,88 @@
+"""Small in-process limiters for the single-worker API deployment.
+
+The reverse proxy remains the first rate-limit layer in production. These
+limiters provide a second fail-safe around authentication and API credentials.
+"""
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+import math
+import time
+
+
+@dataclass
+class _Bucket:
+ window_started: float
+ attempts: int = 0
+ blocked_until: float = 0.0
+ last_seen: float = 0.0
+
+
+class FixedWindowLimiter:
+ def __init__(
+ self,
+ *,
+ limit: int,
+ window_seconds: int,
+ block_seconds: int = 0,
+ ) -> None:
+ self.limit = max(1, int(limit))
+ self.window_seconds = max(1, int(window_seconds))
+ self.block_seconds = max(0, int(block_seconds))
+ self._buckets: dict[str, _Bucket] = {}
+ self._lock = asyncio.Lock()
+
+ async def check(self, key: str) -> int:
+ """Return seconds remaining when blocked, otherwise zero."""
+
+ now = time.monotonic()
+ async with self._lock:
+ bucket = self._buckets.get(key)
+ if not bucket:
+ return 0
+ bucket.last_seen = now
+ if bucket.blocked_until > now:
+ return max(1, math.ceil(bucket.blocked_until - now))
+ if now - bucket.window_started >= self.window_seconds:
+ self._buckets.pop(key, None)
+ return 0
+
+ async def record(self, key: str) -> int:
+ """Count one attempt and return a retry delay if the limit is reached."""
+
+ now = time.monotonic()
+ async with self._lock:
+ bucket = self._buckets.get(key)
+ if not bucket or now - bucket.window_started >= self.window_seconds:
+ bucket = _Bucket(window_started=now, last_seen=now)
+ self._buckets[key] = bucket
+ bucket.last_seen = now
+ if bucket.blocked_until > now:
+ return max(1, math.ceil(bucket.blocked_until - now))
+ bucket.attempts += 1
+ if bucket.attempts >= self.limit:
+ delay = self.block_seconds or max(
+ 1,
+ math.ceil(self.window_seconds - (now - bucket.window_started)),
+ )
+ bucket.blocked_until = now + delay
+ return delay
+ self._prune(now)
+ return 0
+
+ async def consume(self, key: str) -> int:
+ blocked = await self.check(key)
+ return blocked or await self.record(key)
+
+ async def reset(self, key: str) -> None:
+ async with self._lock:
+ self._buckets.pop(key, None)
+
+ def _prune(self, now: float) -> None:
+ if len(self._buckets) < 2_000:
+ return
+ cutoff = now - max(self.window_seconds, self.block_seconds, 60) * 2
+ stale = [key for key, value in self._buckets.items() if value.last_seen < cutoff]
+ for key in stale:
+ self._buckets.pop(key, None)
diff --git a/app/security_baseline.py b/app/security_baseline.py
new file mode 100644
index 0000000..2e34907
--- /dev/null
+++ b/app/security_baseline.py
@@ -0,0 +1,194 @@
+"""Fail-closed production security checks without exposing secret values."""
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import unquote, urlsplit
+
+from app.config import Settings, settings
+
+
+_PLACEHOLDER_MARKERS = (
+ "change-me",
+ "change-this",
+ "password",
+ "dev-key",
+ "example",
+)
+_LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"}
+
+
+def _finding(code: str, severity: str, message: str) -> dict[str, str]:
+ return {"code": code, "severity": severity, "message": message}
+
+
+def _looks_insecure_secret(value: str, *, minimum: int = 32) -> bool:
+ normalized = value.strip().lower()
+ return (
+ len(value.strip()) < minimum
+ or any(marker in normalized for marker in _PLACEHOLDER_MARKERS)
+ )
+
+
+def _url_password(value: str) -> str:
+ try:
+ parsed = urlsplit(value)
+ return unquote(parsed.password or "")
+ except ValueError:
+ return ""
+
+
+def _csv_values(value: str) -> list[str]:
+ return [item.strip() for item in value.split(",") if item.strip()]
+
+
+def security_readiness(config: Settings = settings) -> dict[str, Any]:
+ """Return a sanitized security report suitable for an admin endpoint."""
+
+ production = config.app_environment.strip().lower() in {
+ "production",
+ "prod",
+ "server",
+ }
+ findings: list[dict[str, str]] = []
+
+ if _looks_insecure_secret(config.auth_secret):
+ findings.append(
+ _finding("AUTH_SECRET", "critical", "JWT 签名密钥未替换或强度不足")
+ )
+ if _looks_insecure_secret(config.interface_api_secret):
+ findings.append(
+ _finding(
+ "INTERFACE_API_SECRET",
+ "critical",
+ "接口密钥散列密钥未独立配置或强度不足",
+ )
+ )
+ if (
+ config.interface_api_secret.strip()
+ and config.interface_api_secret == config.auth_secret
+ ):
+ findings.append(
+ _finding(
+ "SECRET_REUSE",
+ "critical",
+ "JWT 与接口密钥不能共用同一签名密钥",
+ )
+ )
+ if _looks_insecure_secret(config.auth_default_password, minimum=12):
+ findings.append(
+ _finding(
+ "DEFAULT_ADMIN_PASSWORD",
+ "critical",
+ "默认管理员密码未替换或少于 12 位",
+ )
+ )
+
+ for code, url in (
+ ("POSTGRES_PASSWORD", config.database_url),
+ ("MYSQL_PASSWORD", config.data_mysql_url),
+ ):
+ if _looks_insecure_secret(_url_password(url), minimum=16):
+ findings.append(
+ _finding(code, "critical", f"{code} 未替换或强度不足")
+ )
+
+ if (config.mysql_host_bind.strip() or "127.0.0.1") not in _LOOPBACK_HOSTS:
+ findings.append(
+ _finding("MYSQL_PUBLIC_BIND", "critical", "MySQL 不能绑定公网地址")
+ )
+ if config.data_mysql_direct_access_enabled and not config.data_mysql_ssh_tunnel_required:
+ findings.append(
+ _finding(
+ "DBEAVER_WITHOUT_TUNNEL",
+ "critical",
+ "启用 DBeaver 管理接入时必须强制 SSH 隧道",
+ )
+ )
+
+ origins = _csv_values(config.cors_allowed_origins)
+ if not origins or "*" in origins:
+ findings.append(
+ _finding(
+ "CORS_WILDCARD",
+ "critical" if production else "warning",
+ "生产环境必须明确配置允许访问后台的 HTTPS 来源",
+ )
+ )
+ if production and any(
+ not origin.startswith("https://")
+ and not origin.startswith("http://localhost")
+ and not origin.startswith("http://127.0.0.1")
+ for origin in origins
+ ):
+ findings.append(
+ _finding(
+ "CORS_WITHOUT_HTTPS",
+ "critical",
+ "生产后台来源必须使用 HTTPS",
+ )
+ )
+ trusted_hosts = _csv_values(config.trusted_hosts)
+ if not trusted_hosts or "*" in trusted_hosts:
+ findings.append(
+ _finding(
+ "TRUSTED_HOSTS_WILDCARD",
+ "critical" if production else "warning",
+ "生产环境必须明确配置可信域名",
+ )
+ )
+
+ if config.auth_algorithm not in {"HS256", "HS384", "HS512"}:
+ findings.append(
+ _finding("JWT_ALGORITHM", "critical", "JWT 算法不在允许列表中")
+ )
+ if not 5 <= config.auth_token_expire_minutes <= 120:
+ findings.append(
+ _finding(
+ "JWT_LIFETIME",
+ "critical" if production else "warning",
+ "后台登录令牌有效期必须为 5–120 分钟",
+ )
+ )
+ if any(
+ _looks_insecure_secret(item, minimum=24)
+ for item in _csv_values(config.ingest_api_keys)
+ ):
+ findings.append(
+ _finding(
+ "INGEST_API_KEYS",
+ "critical" if production else "warning",
+ "外部问答接口仍包含开发密钥或弱密钥",
+ )
+ )
+
+ if not config.data_backup_enabled:
+ findings.append(
+ _finding("BACKUP_DISABLED", "warning", "尚未声明已启用自动加密备份")
+ )
+ if not config.data_mysql_audit_enabled:
+ findings.append(
+ _finding("DB_AUDIT_DISABLED", "warning", "DBeaver 数据库审计尚未启用")
+ )
+
+ critical_count = sum(item["severity"] == "critical" for item in findings)
+ warning_count = sum(item["severity"] == "warning" for item in findings)
+ return {
+ "environment": config.app_environment,
+ "strict_mode": config.security_strict_mode,
+ "status": "blocked" if critical_count else "ready",
+ "critical_count": critical_count,
+ "warning_count": warning_count,
+ "findings": findings,
+ }
+
+
+def enforce_security_baseline(config: Settings = settings) -> None:
+ """Abort a strict deployment before it starts with unsafe settings."""
+
+ if not config.security_strict_mode:
+ return
+ report = security_readiness(config)
+ critical = [item for item in report["findings"] if item["severity"] == "critical"]
+ if critical:
+ codes = ", ".join(item["code"] for item in critical)
+ raise RuntimeError(f"生产安全基线检查失败:{codes}")
diff --git a/deploy/nginx/travel-kg.conf.example b/deploy/nginx/travel-kg.conf.example
new file mode 100644
index 0000000..c4ec77a
--- /dev/null
+++ b/deploy/nginx/travel-kg.conf.example
@@ -0,0 +1,49 @@
+# Install inside the nginx http {} context after replacing the domain and
+# certificate paths. The application itself listens only on 127.0.0.1:8102.
+limit_req_zone $binary_remote_addr zone=znkg_login:10m rate=10r/m;
+limit_req_zone $binary_remote_addr zone=znkg_openapi:10m rate=120r/m;
+
+server {
+ listen 80;
+ listen [::]:80;
+ server_name data.example.cn;
+ return 301 https://$host$request_uri;
+}
+
+server {
+ listen 443 ssl;
+ listen [::]:443 ssl;
+ http2 on;
+ server_name data.example.cn;
+
+ ssl_certificate /etc/letsencrypt/live/data.example.cn/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/data.example.cn/privkey.pem;
+ ssl_protocols TLSv1.2 TLSv1.3;
+ ssl_session_timeout 1d;
+ ssl_session_cache shared:ZNKG_SSL:10m;
+ ssl_session_tickets off;
+
+ client_max_body_size 55m;
+ server_tokens off;
+
+ location = /v1/admin/auth/login {
+ limit_req zone=znkg_login burst=5 nodelay;
+ proxy_pass http://127.0.0.1:8102;
+ include proxy_params;
+ proxy_set_header X-Forwarded-Proto https;
+ }
+
+ location /v1/openapi/ {
+ limit_req zone=znkg_openapi burst=40 nodelay;
+ proxy_pass http://127.0.0.1:8102;
+ include proxy_params;
+ proxy_set_header X-Forwarded-Proto https;
+ }
+
+ location / {
+ proxy_pass http://127.0.0.1:8102;
+ include proxy_params;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_read_timeout 120s;
+ }
+}
diff --git a/design-qa.md b/design-qa.md
index 882fed8..7b77937 100644
--- a/design-qa.md
+++ b/design-qa.md
@@ -101,6 +101,109 @@
- 当前没有遗留 P0、P1 或 P2 问题。
- final result: passed
+## 2026-08-24 接口中心方案 1 重构验收
+
+### 设计依据与证据
+
+- Source visual truth: `/Users/xuexue/.codex/generated_images/01a036ab-ef5f-7070-b58c-d97d01d994b8/exec-8747d548-dc3e-4441-9476-2f7366d7c3db.png`
+- Source pixels: 1487 × 1058;归一化为 1488 × 1024。
+- Implementation route: `http://localhost:8102/admin/system/interfaces`.
+- Implementation screenshot: `/Users/xuexue/new2/qa/design/interface-center-mysql-connection.png`.
+- API connection screenshot: `/Users/xuexue/new2/qa/design/interface-center-api-connection.png`.
+- API security screenshot: `/Users/xuexue/new2/qa/design/interface-center-api-security.png`.
+- Responsive screenshot: `/Users/xuexue/new2/qa/design/interface-center-responsive-1024-loaded.png`.
+- Full-view comparison: `/Users/xuexue/new2/qa/design/interface-center-mysql-comparison.jpg`.
+- Implementation pixels / CSS viewport: 1488 × 1024;devicePixelRatio 1。
+- State: MySQL 直连 / 接入信息;服务器被外部 DBeaver 连接,不主动连接外部数据库。
+
+### Full-view comparison evidence
+
+已将归一化参考图和浏览器最终实现按 1488 × 1024 并排对照。实现保留参考方案的双接入方式、两级导航、主表单与右侧权限摘要结构,并基于实际产品壳层补充接口说明、调用日志及“服务器被访问”的方向说明。现有系统没有参考图中的全局管理员顶栏,因此继续使用项目原有侧栏和页面动作区;这是产品壳层约束,不是设计漂移。
+
+### Focused-region comparison evidence
+
+参考图与实现图均为 1:1 桌面截图,模式切换、表单、权限摘要和底部操作在全视图原图中清晰可读,不需要额外裁切。HTTPS API 接入、安全设置和 1024px 响应式状态另有独立浏览器截图验证。
+
+### Findings
+
+- Fonts and typography: 沿用项目现有系统字体栈;25px 页面标题、14–15px 主信息和 11–12px 辅助信息层级与参考方向一致,无异常换行或裁切。
+- Spacing and layout rhythm: 1488px 下主内容和 320px 权限栏比例稳定;1024px 下 `documentWidth = innerWidth = 1024`,无横向溢出。
+- Colors and visual tokens: 沿用 Ant Design `#1677ff` 蓝色、绿色运行/安全状态和中性灰阶;没有引入不一致的渐变或阴影体系。
+- Image and icon fidelity: 目标没有业务图片素材;所有界面图标均使用项目已有 Ant Design 图标库,没有占位图、手绘 SVG 或 CSS 图形替代。
+- Copy and content: 已明确“服务器被访问,不主动连接外部数据库”;MySQL 直连仅面向数据管理员,普通电脑、设备和业务系统统一走 HTTPS API。
+- Primary interactions: MySQL/HTTPS API 切换、接入信息/安全设置切换、新建接入方弹窗打开与取消、接口说明抽屉、调用日志抽屉均通过。
+- Server-interface simulation: MySQL TCP 3308、管理员认证、受限目录、Schema、查询、新增、更新、删除、密钥撤销及审计日志均通过;无效密钥返回 401,只读写入和跨库访问返回 403。
+- Console errors: authenticated implementation route returned no browser console errors or warnings.
+- Remaining P0/P1/P2 findings: none.
+
+### Comparison history
+
+1. Preflight pass: browser was initially unauthenticated, so visual comparison was blocked.
+2. Authenticated pass: captured implementation at the source viewport, compared both images together and found no actionable P0/P1/P2 mismatch.
+
+### Implementation checklist
+
+- Frontend production build: passed.
+- Interface Center source contract tests: 4 passed.
+- Python management/MySQL contract tests: 10 passed.
+- Docker Compose local and server override validation: passed.
+- Temporary interface-test database, client, key, policy, records and audit rows: cleaned; residual count 0.
+
+final result: passed
+
+## 2026-07-30 数据中心标题与数据库树交互验收
+
+### 设计依据与证据
+
+- 原项目节点图标参考:`/var/folders/sl/bt4qhk351ndfm_1x3_4wkw440000gn/T/TemporaryItems/NSIRD_screencaptureui_tW2RJM/截屏2026-07-30 12.23.06.png`
+- 最终数据中心状态:`/Users/xuexue/new2/qa/data-center-collapsible-tree-final.png`
+- 前后对照图:`/Users/xuexue/new2/qa/data-center-reference-comparison.png`
+
+### 修复记录
+
+1. [用户要求] 顶部“数据中心”和左栏“数据库导航器”标题前不再显示图标,只保留文字标题。
+2. [P1 已修复] 项目数据库节点原来只能展开。现在再次点击已展开项目可收起,收起后右侧当前数据表保持可用,再次点击可重新展开。
+3. [P1 已修复] 数据表分类文件夹不再被 `open` 属性强制保持展开;正式实体、公共明细、酒店、美食、景区和交通均可独立展开或收起。
+4. [P2 已修复] 项目节点继续使用 Tabler `IconDatabase` 圆柱形数据库图标,标题去图标后没有影响数据库层级识别。
+
+### 可见质量与交互检查
+
+- 标题区:数据中心、数据库导航器前均无图标,间距保持紧凑。
+- 项目节点:云游荔波保留传统圆柱形数据库图标和表数量。
+- 项目交互:展开、收起、重新展开均通过。
+- 文件夹交互:酒店分类收起后,其 4 个子表隐藏;重新展开状态可恢复。
+- 前端生产构建、容器更新与 API 健康检查:通过。
+
+### 本轮结论
+
+- 当前没有遗留 P0、P1 或 P2 问题。
+- final result: passed
+
+## 2026-07-30 关系数据中心全屏工作台验收
+
+### 设计依据与证据
+
+- DBeaver 参考界面:`/var/folders/sl/bt4qhk351ndfm_1x3_4wkw440000gn/T/TemporaryItems/NSIRD_screencaptureui_Scgbhv/截屏2026-07-30 11.57.02.png`
+- 原拥挤项目卡片界面:`/var/folders/sl/bt4qhk351ndfm_1x3_4wkw440000gn/T/TemporaryItems/NSIRD_screencaptureui_2IFYRg/截屏2026-07-30 11.55.37.png`
+- 第一版全屏实现:`/var/folders/sl/bt4qhk351ndfm_1x3_4wkw440000gn/T/TemporaryItems/NSIRD_screencaptureui_dxfzSu/截屏2026-07-30 12.11.01.png`
+- 同尺寸对照图:`/tmp/data-center-design-compare.png`
+
+### 布局与交互修复
+
+1. [P1 已修复] 移除项目数据库卡片墙、底部统计卡片和页面级滚动,数据中心改为占满内容视口的数据库工作台。
+2. [P1 已修复] 采用 DBeaver 式左右结构:左侧数据库导航树,右侧表签、工具栏、上下文栏和高密度数据网格。
+3. [P1 已修复] 顶部标题固定为“数据中心”,不再把“项目 / 技术数据库名 / 数据表”拼成页面标题。
+4. [P1 已修复] 左侧第一级节点以项目名称展示全部项目数据库;点击项目节点即可切换数据库,仅当前项目展开 Schema、分组与数据表。
+5. [P2 已修复] 表格使用紧凑行高、斑马纹、固定表头、横纵滚动和内部翻页;搜索、刷新、新增、编辑、删除能力保持不变。
+6. [P2 已修复] 技术数据库名、Schema 名和表代码下沉到上下文与元信息区域,不再与项目名称竞争主层级。
+
+### 检查结果
+
+- 2940 × 1912 参考与实现同尺寸对照通过;工作区层级、左树右表结构和网格密度符合参考方向。
+- 多项目数据库切换逻辑、当前项目展开状态及数据表切换状态通过 TypeScript 生产构建检查。
+- 前端生产构建通过;Docker API 镜像重建通过;`/v1/admin/health` 返回 `{"status":"ok"}`。
+- final result: passed
+
## 2026-07-29 酒店周边交通距离单行验收
### 设计依据与证据
diff --git a/docker-compose.server.yml b/docker-compose.server.yml
index 4bd70ae..096ea3d 100644
--- a/docker-compose.server.yml
+++ b/docker-compose.server.yml
@@ -1,9 +1,34 @@
services:
+ postgres:
+ environment:
+ POSTGRES_USER: ${POSTGRES_USER:?必须设置 POSTGRES_USER}
+ POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?必须设置 POSTGRES_PASSWORD}
+ POSTGRES_DB: ${POSTGRES_DB:?必须设置 POSTGRES_DB}
+
+ mysql:
+ environment:
+ MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:?必须设置 MYSQL_ROOT_PASSWORD}
+ MYSQL_DATABASE: ${MYSQL_DATABASE:-platform_control}
+ MYSQL_USER: ${MYSQL_USER:?必须设置 MYSQL_USER}
+ MYSQL_PASSWORD: ${MYSQL_PASSWORD:?必须设置 MYSQL_PASSWORD}
+
api:
network_mode: host
ports: !override []
environment:
- DATABASE_URL: postgresql://${POSTGRES_USER:-admin}:${POSTGRES_PASSWORD:-password}@127.0.0.1:${POSTGRES_PORT:-5433}/${POSTGRES_DB:-kg_admin}
+ APP_ENVIRONMENT: production
+ SECURITY_STRICT_MODE: "true"
+ DATABASE_URL: postgresql://${POSTGRES_USER:?必须设置 POSTGRES_USER}:${POSTGRES_PASSWORD:?必须设置 POSTGRES_PASSWORD}@127.0.0.1:${POSTGRES_PORT:-5433}/${POSTGRES_DB:?必须设置 POSTGRES_DB}
+ DATA_MYSQL_URL: mysql://${MYSQL_USER:?必须设置 MYSQL_USER}:${MYSQL_PASSWORD:?必须设置 MYSQL_PASSWORD}@127.0.0.1:${MYSQL_PORT:-3307}/${MYSQL_DATABASE:-platform_control}
+ AUTH_SECRET: ${AUTH_SECRET:?必须设置至少 32 位 AUTH_SECRET}
+ AUTH_DEFAULT_PASSWORD: ${AUTH_DEFAULT_PASSWORD:?必须设置至少 12 位管理员密码}
+ INTERFACE_API_SECRET: ${INTERFACE_API_SECRET:?必须设置独立的 INTERFACE_API_SECRET}
+ INGEST_API_KEYS: ${INGEST_API_KEYS:?必须设置外部接口密钥}
+ CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:?必须设置后台 HTTPS 来源}
+ TRUSTED_HOSTS: ${TRUSTED_HOSTS:?必须设置服务器域名}
+ MYSQL_HOST_BIND: 127.0.0.1
+ DATA_MYSQL_SSH_TUNNEL_REQUIRED: "true"
+ DATA_SQL_CONSOLE_WRITE_ENABLED: "false"
FALKORDB_HOST: 127.0.0.1
FALKORDB_PORT: ${FALKORDB_PORT:-6380}
command:
@@ -12,7 +37,7 @@ services:
- uvicorn
- app.main:app
- --host
- - 0.0.0.0
+ - ${API_BIND_HOST:-127.0.0.1}
- --port
- ${API_PORT:-8102}
healthcheck:
diff --git a/docker-compose.yml b/docker-compose.yml
index 26c052b..449992d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -21,6 +21,41 @@ services:
timeout: 5s
retries: 10
+ mysql:
+ image: mysql:8.4
+ container_name: travel-kg-mysql
+ restart: unless-stopped
+ environment:
+ MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-change-this-root-password}
+ MYSQL_DATABASE: ${MYSQL_DATABASE:-platform_control}
+ MYSQL_USER: ${MYSQL_USER:-data_center}
+ MYSQL_PASSWORD: ${MYSQL_PASSWORD:-change-this-data-password}
+ command:
+ - --character-set-server=utf8mb4
+ - --collation-server=utf8mb4_0900_ai_ci
+ - --default-time-zone=+00:00
+ - --partial-revokes=ON
+ - --local-infile=OFF
+ - --secure-file-priv=NULL
+ - --skip-name-resolve=ON
+ - --skip-symbolic-links
+ - --server-id=1
+ - --log-bin=mysql-bin
+ - --binlog-format=ROW
+ - --binlog-expire-logs-seconds=604800
+ - --sync-binlog=1
+ - --innodb-flush-log-at-trx-commit=1
+ ports:
+ - "${MYSQL_HOST_BIND:-127.0.0.1}:${MYSQL_PORT:-3307}:3306"
+ volumes:
+ - mysql-data:/var/lib/mysql
+ - ./docker/mysql-init/01-data-center-grants.sh:/docker-entrypoint-initdb.d/01-data-center-grants.sh:ro
+ healthcheck:
+ test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -p\"$${MYSQL_ROOT_PASSWORD}\" --silent"]
+ interval: 10s
+ timeout: 5s
+ retries: 15
+
falkordb-seed:
image: busybox:1.36
restart: "no"
@@ -67,15 +102,50 @@ services:
DATABASE_URL: postgresql://${POSTGRES_USER:-admin}:${POSTGRES_PASSWORD:-password}@postgres:5432/${POSTGRES_DB:-kg_admin}
DB_SCHEMA: ${DB_SCHEMA:-kg_admin_new2}
DB_MIGRATIONS_ENABLED: ${DOCKER_DB_MIGRATIONS_ENABLED:-false}
+ DATA_MYSQL_URL: mysql://${MYSQL_USER:-data_center}:${MYSQL_PASSWORD:-change-this-data-password}@mysql:3306/${MYSQL_DATABASE:-platform_control}
+ DATA_MYSQL_REQUIRED: "true"
+ DATA_MYSQL_POOL_MIN_SIZE: ${DATA_MYSQL_POOL_MIN_SIZE:-2}
+ DATA_MYSQL_POOL_MAX_SIZE: ${DATA_MYSQL_POOL_MAX_SIZE:-20}
+ DATA_SQL_CONSOLE_WRITE_ENABLED: ${DATA_SQL_CONSOLE_WRITE_ENABLED:-false}
+ INTERFACE_API_SECRET: ${INTERFACE_API_SECRET:-change-me-interface-api-secret}
+ DATA_MYSQL_DIRECT_ACCESS_ENABLED: ${DATA_MYSQL_DIRECT_ACCESS_ENABLED:-false}
+ DATA_MYSQL_PUBLIC_HOST: ${DATA_MYSQL_PUBLIC_HOST:-}
+ DATA_MYSQL_PUBLIC_PORT: ${DATA_MYSQL_PUBLIC_PORT:-0}
+ DATA_MYSQL_DIRECT_TRANSPORT: ${DATA_MYSQL_DIRECT_TRANSPORT:-SSH 隧道(强制)}
+ DATA_MYSQL_SSH_TUNNEL_REQUIRED: ${DATA_MYSQL_SSH_TUNNEL_REQUIRED:-true}
+ DATA_MYSQL_SSH_HOST: ${DATA_MYSQL_SSH_HOST:-}
+ DATA_MYSQL_SSH_PORT: ${DATA_MYSQL_SSH_PORT:-22}
+ DATA_MYSQL_SSH_AUTH_METHOD: ${DATA_MYSQL_SSH_AUTH_METHOD:-SSH 私钥 / SSH Agent}
+ DATA_MYSQL_ADMIN_HOST: ${DATA_MYSQL_ADMIN_HOST:-127.0.0.1}
+ DATA_MYSQL_ADMIN_PORT: ${DATA_MYSQL_ADMIN_PORT:-${MYSQL_PORT:-3307}}
+ DATA_MYSQL_ADMIN_ACCOUNT_POLICY: ${DATA_MYSQL_ADMIN_ACCOUNT_POLICY:-一人一号 · 单库授权 · 禁止 root}
+ DATA_MYSQL_AUDIT_ENABLED: ${DATA_MYSQL_AUDIT_ENABLED:-false}
+ MYSQL_HOST_BIND: ${MYSQL_HOST_BIND:-127.0.0.1}
+ DATA_BACKUP_ENABLED: ${DATA_BACKUP_ENABLED:-false}
+ DATA_BACKUP_ENCRYPTION_REQUIRED: ${DATA_BACKUP_ENCRYPTION_REQUIRED:-true}
+ DATA_BACKUP_RETENTION_DAYS: ${DATA_BACKUP_RETENTION_DAYS:-30}
FALKORDB_HOST: falkordb
FALKORDB_PORT: "6379"
FALKORDB_GRAPH: ${FALKORDB_GRAPH:-guiyang_new2}
FALKORDB_PASSWORD: ${FALKORDB_PASSWORD:-}
AUTH_SECRET: ${AUTH_SECRET:-change-me-32-chars-minimum-secret}
AUTH_ALGORITHM: ${AUTH_ALGORITHM:-HS256}
- AUTH_TOKEN_EXPIRE_MINUTES: ${AUTH_TOKEN_EXPIRE_MINUTES:-480}
+ AUTH_ISSUER: ${AUTH_ISSUER:-zn-kg-admin}
+ AUTH_AUDIENCE: ${AUTH_AUDIENCE:-zn-kg-admin-web}
+ AUTH_TOKEN_EXPIRE_MINUTES: ${AUTH_TOKEN_EXPIRE_MINUTES:-60}
AUTH_DEFAULT_USERNAME: ${AUTH_DEFAULT_USERNAME:-admin@example.com}
AUTH_DEFAULT_PASSWORD: ${AUTH_DEFAULT_PASSWORD:-change-me}
+ AUTH_LOGIN_MAX_ATTEMPTS: ${AUTH_LOGIN_MAX_ATTEMPTS:-5}
+ AUTH_LOGIN_WINDOW_SECONDS: ${AUTH_LOGIN_WINDOW_SECONDS:-300}
+ AUTH_LOGIN_LOCK_SECONDS: ${AUTH_LOGIN_LOCK_SECONDS:-900}
+ INTERFACE_API_DEFAULT_EXPIRY_DAYS: ${INTERFACE_API_DEFAULT_EXPIRY_DAYS:-30}
+ INTERFACE_API_MAX_EXPIRY_DAYS: ${INTERFACE_API_MAX_EXPIRY_DAYS:-90}
+ INTERFACE_API_RATE_LIMIT_PER_MINUTE: ${INTERFACE_API_RATE_LIMIT_PER_MINUTE:-120}
+ APP_ENVIRONMENT: ${APP_ENVIRONMENT:-development}
+ SECURITY_STRICT_MODE: ${SECURITY_STRICT_MODE:-false}
+ CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-http://localhost:8102,http://127.0.0.1:8102}
+ TRUSTED_HOSTS: ${TRUSTED_HOSTS:-*}
+ SECURITY_HEADERS_ENABLED: ${SECURITY_HEADERS_ENABLED:-true}
DEFAULT_TENANT: ${DEFAULT_TENANT:-CityGraph-new2}
DEFAULT_PROJECT: ${DEFAULT_PROJECT:-CityGraph-new2}
INGEST_API_KEYS: ${INGEST_API_KEYS:-dev-key-1}
@@ -94,6 +164,8 @@ services:
depends_on:
postgres:
condition: service_healthy
+ mysql:
+ condition: service_healthy
falkordb:
condition: service_healthy
healthcheck:
@@ -111,4 +183,5 @@ services:
volumes:
postgres-data:
+ mysql-data:
falkordb-data:
diff --git a/docker/mysql-init/01-data-center-grants.sh b/docker/mysql-init/01-data-center-grants.sh
new file mode 100755
index 0000000..3bcff82
--- /dev/null
+++ b/docker/mysql-init/01-data-center-grants.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+set -eu
+
+# The database code is the physical MySQL database name, so project databases
+# do not share a technical prefix. Grant the non-administrative DDL/DML set
+# globally, then use MySQL partial revokes to exclude the mutable mysql schema
+# and sys views. information_schema and performance_schema are server-managed
+# read-only schemas, and the SQL console also blocks every system schema. The
+# dedicated platform_control database remains available to the service.
+mysql --protocol=socket -uroot -p"${MYSQL_ROOT_PASSWORD}" -e \
+ "GRANT CREATE, DROP, ALTER, INDEX, REFERENCES, SELECT, INSERT, UPDATE, DELETE, CREATE VIEW, SHOW VIEW, TRIGGER ON *.* TO '${MYSQL_USER}'@'%';
+ REVOKE CREATE, DROP, ALTER, INDEX, REFERENCES, SELECT, INSERT, UPDATE, DELETE, CREATE VIEW, SHOW VIEW, TRIGGER ON \`mysql\`.* FROM '${MYSQL_USER}'@'%';
+ REVOKE CREATE, DROP, ALTER, INDEX, REFERENCES, SELECT, INSERT, UPDATE, DELETE, CREATE VIEW, SHOW VIEW, TRIGGER ON \`sys\`.* FROM '${MYSQL_USER}'@'%';
+ FLUSH PRIVILEGES;"
diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md
index cc6ba88..9ffd136 100644
--- a/docs/API_REFERENCE.md
+++ b/docs/API_REFERENCE.md
@@ -26,24 +26,17 @@ http://localhost:8102/docs
| `/v1/admin/auth/login` | `POST` | 管理员登录 |
| `/v1/admin/auth/me` | `GET` | 当前登录用户 |
| `/v1/admin/projects` | `GET/POST` | 项目管理 |
+| `/v1/admin/ontology-schemas` | `GET` | 当前项目创建的 Schema 列表 |
| `/v1/admin/ontology-schemas/current` | `GET` | 当前 schema |
-| `/v1/admin/source-profiles` | `GET/POST/PATCH` | 数据源管理 |
-| `/v1/admin/batches` | `GET` | 批次管理 |
-| `/v1/admin/entities` | `GET` | 候选实体列表 |
-| `/v1/admin/conflicts` | `GET` | 冲突列表 |
-| `/v1/admin/publish-jobs` | `GET/POST` | 发布任务 |
| `/v1/admin/graph/overview` | `GET` | 图谱概览 |
| `/v1/admin/graph/query` | `POST` | 图谱查询 |
| `/v1/admin/plaza/overview` | `GET` | 图谱广场概览 |
-| `/v1/admin/manual-ingest/extract` | `POST` | 手动抽取 |
| `/v1/admin/travel/assistant-query` | `POST` | 旅行客服问答 |
| `/v1/openapi/knowledge-qa/query` | `POST` | 百姓惠智能客服第三方问答接口 |
| `/v1/admin/travel/customer-service-query` | `POST` | 兼容旧版外部问答接口 |
| `/v1/admin/super-agent/run` | `POST` | Super Agent 任务 |
| `/v1/admin/roles` | `GET/POST` | 角色管理 |
| `/v1/admin/users` | `GET/POST` | 用户管理 |
-| `/v1/admin/areas/tree` | `GET` | 区域树 |
-| `/v1/admin/notifications` | `GET` | 通知列表 |
## 登录示例
@@ -85,7 +78,7 @@ curl http://localhost:8102/v1/admin/travel/assistant-query \
该接口用于对接外部客服系统。调用方传入用户自然语言问题,服务端默认选择百姓惠旅行社知识图谱,按“LLM 生成只读 Cypher -> FalkorDB 图查询 -> LLM 基于证据组织答案”的链路返回客服话术和图谱证据。生产环境需要配置问答环节 LLM。
-认证方式使用接口 Key。Key 可以在后台 `系统 -> Agent 设置 -> 外部图谱问答 API` 维护;同时兼容 `.env` / `docker-compose.yml` 的 `INGEST_API_KEYS`。同一模块底部可以配置“问答环节 LLM 模型”,外部客服话术融合会优先使用该模型,未填写时继承全局 LLM:
+认证方式使用接口 Key,通过 `.env` 或 `docker-compose.yml` 的 `INGEST_API_KEYS` 配置。问答环节使用 `LLM_API_BASE`、`LLM_API_KEY` 和 `LLM_MODEL` 指定的模型:
```bash
curl http://localhost:8102/v1/openapi/knowledge-qa/query \
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 2b663ef..98f1d4b 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -23,7 +23,7 @@ flowchart TB
| 服务 | 职责 |
| --- | --- |
| `api` | 提供 `/v1/admin/*` API,挂载 `/admin` 前端页面,连接 PostgreSQL 与 FalkorDB |
-| `postgres` | 保存后台业务数据、用户权限、项目、候选实体、审核记录和任务 |
+| `postgres` | 保存项目、Schema、结构化业务数据、用户权限和后台流程支撑数据 |
| `falkordb-seed` | 首次启动时把仓库内的 `snapshots/falkordb/dump.rdb` 写入数据卷 |
| `falkordb` | 保存图数据库,支持 Cypher/Redis 协议访问和 FalkorDB Browser |
@@ -35,7 +35,7 @@ flowchart TB
| `app/config.py` | 环境变量配置 |
| `app/db.py` | PostgreSQL 连接池 |
| `app/api/` | 管理后台 API 路由 |
-| `app/agents/` | 采集、抽取、对齐、审计和外部站点 Agent |
+| `app/agents/` | 采集、抽取、对齐和外部站点 Agent |
| `app/kg_core/` | 空间图谱与核心图谱辅助逻辑 |
| `app/schemas/` | 抽取 schema |
| `app/security.py`、`app/auth.py` | 登录、令牌和权限相关逻辑 |
@@ -46,18 +46,16 @@ flowchart TB
| --- | --- |
| `admin-web/src/App.tsx` | 管理后台主应用与路由 |
| `admin-web/src/api.ts` | API 客户端 |
-| `admin-web/src/panels/plaza/` | 图谱广场、用户查询、手动抽取和 Super Agent |
-| `admin-web/src/panels/acquisition/` | 数据源、批次和冲突工作台 |
-| `admin-web/src/panels/review/` | 证据质量、字段审核、专家签核和资产库 |
-| `admin-web/src/panels/modeling/` | Schema、词表和健康检查 |
-| `admin-web/src/panels/publish/` | 发布与回滚 |
-| `admin-web/src/panels/system/` | 用户、权限、区域、通知、Agent 设置和日志 |
+| `admin-web/src/panels/plaza/` | 数据大屏、业务问答、图谱浏览和 Super Agent |
+| `admin-web/src/panels/data-platform/` | 项目数据库、表、记录、CSV 与 SQL 工作区 |
+| `admin-web/src/panels/modeling/` | 当前项目 Schema 的只读展示 |
+| `admin-web/src/panels/system/` | 用户、权限、Agent 任务和调用日志 |
## 数据层
PostgreSQL 和 FalkorDB 承担不同职责:
-- PostgreSQL:结构化管理数据、审核过程数据、用户权限、任务、来源、候选实体和证据。
+- PostgreSQL:项目、Schema、结构化业务数据、用户权限,以及 Agent 所需的内部流程数据。
- FalkorDB:图谱实体、关系、路线、资源、POI、空间索引和面向查询的图结构。
默认配置:
diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md
index 0f2ee2d..72c6fcd 100644
--- a/docs/DEPLOYMENT.md
+++ b/docs/DEPLOYMENT.md
@@ -7,9 +7,9 @@
- Docker Desktop 或 Docker Engine
- Docker Compose v2
- 至少 4 GB 可用内存
-- 至少 3 GB 可用磁盘空间
+- 开发环境至少 10 GB、生产环境至少 50 GB 可用磁盘空间,并单独规划异地备份容量
-## 一键启动
+## 本地开发启动
```bash
docker compose up -d --build
@@ -21,20 +21,18 @@ docker compose up -d --build
http://localhost:8102/admin
```
-服务器部署后把 `localhost` 换成服务器 IP 或域名。例如本项目迁移到 `8.163.40.99` 后:
-
-```text
-http://8.163.40.99:8102/admin
-http://8.163.40.99:8102/docs
-http://8.163.40.99:8102/openapi.json
-```
-
-默认账号:
+以下默认账号只允许本地开发使用:
```text
admin@example.com / change-me
```
+生产环境必须先完成后文中的密钥、域名、HTTPS 和防火墙配置,再使用服务器覆盖文件启动:
+
+```bash
+docker compose -f docker-compose.yml -f docker-compose.server.yml up -d --build
+```
+
## 首次启动会发生什么
1. 构建 API 镜像,并打包 React 管理后台。
@@ -103,6 +101,20 @@ sudo loginctl enable-linger dockerop
这样 rootless Docker 可以由 systemd 用户服务长期托管。
+## 生产安全入口
+
+服务器覆盖配置默认令 API 只监听 `127.0.0.1:8102`。公网只能通过 Nginx 的
+HTTPS 443 端口访问,不再直接开放 8102。可复制
+`deploy/nginx/travel-kg.conf.example`,替换域名和证书路径后启用。
+
+安全组入站规则建议仅保留:
+
+- `443/tcp`:所有合法业务来源;
+- `22/tcp`:仅运维固定 IP 或 VPN 网段;
+- `80/tcp`:仅用于跳转 HTTPS 或证书签发。
+
+严禁向公网开放 `8102`、`3307`、`5433`、`6380` 和 `3002`。
+
## 端口配置
可以在启动时覆盖端口:
@@ -119,10 +131,13 @@ docker compose up -d --build
| 变量 | 默认值 | 说明 |
| --- | --- | --- |
-| `API_HOST_BIND` | `0.0.0.0` | API/后台监听地址 |
+| `API_HOST_BIND` | `0.0.0.0` | 本地 Compose 的 API/后台监听地址 |
+| `API_BIND_HOST` | 服务器覆盖默认 `127.0.0.1` | 生产 Uvicorn 监听地址,只允许 Nginx 转发 |
| `API_PORT` | `8102` | FastAPI 与管理后台 |
| `POSTGRES_HOST_BIND` | `127.0.0.1` | PostgreSQL 只绑定服务器本机 |
| `POSTGRES_PORT` | `5433` | PostgreSQL 映射端口 |
+| `MYSQL_HOST_BIND` | `127.0.0.1` | MySQL 只绑定服务器本机,DBeaver 经 SSH 隧道接入 |
+| `MYSQL_PORT` | `3307` | MySQL 服务器本机映射端口,不向公网开放 |
| `FALKORDB_HOST_BIND` | `127.0.0.1` | FalkorDB Redis 协议只绑定服务器本机 |
| `FALKORDB_PORT` | `6380` | FalkorDB Redis 协议端口 |
| `FALKORDB_BROWSER_HOST_BIND` | `127.0.0.1` | FalkorDB Browser 只绑定服务器本机 |
@@ -130,7 +145,9 @@ docker compose up -d --build
## 环境变量
-Docker Compose 已提供可运行默认值。生产部署时建议改成 `.env` 文件或部署平台的环境变量。
+基础 Docker Compose 提供开发默认值;服务器覆盖配置会对关键变量使用必填校验,
+任何占位密码、弱签名密钥、公开 MySQL 绑定、通配 CORS 或不可信 Host 都会令
+应用拒绝启动。
服务器首次部署建议复制模板后修改:
@@ -142,16 +159,16 @@ cp .env.example .env
```env
AUTH_SECRET=换成32位以上随机字符串
-AUTH_DEFAULT_PASSWORD=换成后台管理员密码
-INGEST_API_KEYS=换成给外部系统的接口Key
+AUTH_DEFAULT_PASSWORD=换成12位以上后台管理员密码
+INTERFACE_API_SECRET=与AUTH_SECRET不同的32位以上随机字符串
+INGEST_API_KEYS=至少24位的外部系统接口Key
+CORS_ALLOWED_ORIGINS=https://你的后台域名
+TRUSTED_HOSTS=你的后台域名
LLM_API_BASE=你的OpenAI兼容模型地址
LLM_API_KEY=你的模型Key
```
-部署后也可以在后台 `系统 -> Agent 设置` 中配置:
-
-- `全局 LLM 配置`:填写 OpenAI 兼容模型地址、模型 ID、LLM API Key。
-- `外部图谱问答 API`:位于页面底部,维护给其他系统调用的 `X-KG-API-Key`,设置默认图谱、默认 LLM 策略和“问答环节 LLM 模型”。该模型优先用于外部客服问答,留空则继承全局 LLM。
+新系统不再提供后台 Agent 设置页面。LLM 与外部接口 Key 统一通过 `.env`、Compose 环境变量或部署平台的密钥配置维护。
| 变量 | 说明 |
| --- | --- |
@@ -159,9 +176,16 @@ LLM_API_KEY=你的模型Key
| `DATABASE_URL` | 后端连接 PostgreSQL 的 URL |
| `DB_SCHEMA` | 默认 `kg_admin_new2` |
| `DB_MIGRATIONS_ENABLED` | 快照部署默认 `false` |
+| `DATA_MYSQL_SSH_HOST`、`DATA_MYSQL_SSH_PORT` | DBeaver SSH 标签页使用的服务器地址和 SSH 端口 |
+| `DATA_MYSQL_ADMIN_HOST`、`DATA_MYSQL_ADMIN_PORT` | DBeaver Main 标签页使用的隧道远端 MySQL 地址,默认 `127.0.0.1:3307` |
+| `DATA_MYSQL_AUDIT_ENABLED` | 只有数据库审计真实启用后才设置为 `true` |
+| `DATA_SQL_CONSOLE_WRITE_ENABLED` | 默认 `false`,生产 SQL 控制台保持只读 |
+| `DATA_BACKUP_ENABLED` | 自动备份任务真实部署后才设置为 `true` |
+| `DATA_BACKUP_PASSPHRASE_FILE` | 仓库外的备份加密口令文件,权限必须为 600 |
| `FALKORDB_HOST` | Docker 内默认 `falkordb` |
| `FALKORDB_GRAPH` | 默认业务图 `guiyang_new2` |
| `AUTH_SECRET` | JWT 签名密钥,生产必须替换 |
+| `INTERFACE_API_SECRET` | 接口密钥散列专用密钥,不得与 JWT 密钥复用 |
| `AUTH_DEFAULT_USERNAME` | 默认管理员用户名 |
| `AUTH_DEFAULT_PASSWORD` | 默认管理员密码 |
| `LLM_API_BASE` | OpenAI 兼容模型服务地址,可选 |
@@ -178,20 +202,21 @@ LLM_API_KEY=你的模型Key
给外部系统对接时,只开放这个接口即可:
```text
-POST http://8.163.40.99:8102/v1/openapi/knowledge-qa/query
+POST https://你的域名/v1/openapi/knowledge-qa/query
```
`/v1/admin/travel/customer-service-query` 会继续保留为兼容旧调用,新接入的第三方系统建议统一使用 `/v1/openapi/knowledge-qa/query`。
-服务器安全组需要放行 TCP `8102`。如果服务器本机 `curl http://127.0.0.1:8102/v1/admin/health` 正常,但外部访问 `http://8.163.40.99:8102` 超时,优先检查云控制台安全组/防火墙入方向规则。
-不要把 `5433`、`6380`、`3002` 暴露到公网;默认 Compose 已把这些数据端口绑定到 `127.0.0.1`。
-生产环境必须在后台 `外部图谱问答 API` 或环境变量 `INGEST_API_KEYS` 中配置至少一个接口 Key;未配置时,对外问答接口会返回 `503`,避免无鉴权开放。
+生产服务器不应放行 TCP `8102`。服务器本机可用
+`curl http://127.0.0.1:8102/v1/admin/health` 检查应用,外部统一访问
+`https://你的域名/`。不要把 `3307`、`5433`、`6380`、`3002` 暴露到公网。
+生产环境必须通过环境变量 `INGEST_API_KEYS` 配置至少一个接口 Key;未配置时,对外问答接口会返回 `503`,避免无鉴权开放。
接口 Key 是给第三方系统调用本接口用的访问凭证;问答环节 LLM 的 API Key 是本系统调用大模型用的凭证,两者不要混用。
请求示例:
```bash
-curl http://8.163.40.99:8102/v1/openapi/knowledge-qa/query \
+curl https://你的域名/v1/openapi/knowledge-qa/query \
-H 'Content-Type: application/json' \
-H 'X-KG-API-Key: 你的INGEST_API_KEYS之一' \
-d '{
@@ -205,6 +230,29 @@ curl http://8.163.40.99:8102/v1/openapi/knowledge-qa/query \
}'
```
+## MySQL 加密备份
+
+先在代码仓库之外创建口令文件:
+
+```bash
+sudo install -d -m 700 /etc/travel-kg/secrets
+openssl rand -base64 48 | sudo tee /etc/travel-kg/secrets/mysql-backup-passphrase >/dev/null
+sudo chmod 600 /etc/travel-kg/secrets/mysql-backup-passphrase
+```
+
+执行备份和校验:
+
+```bash
+export DATA_BACKUP_PASSPHRASE_FILE=/etc/travel-kg/secrets/mysql-backup-passphrase
+./scripts/backup_mysql_encrypted.sh
+./scripts/verify_mysql_backup.sh ./backups/mysql/mysql-all-时间.sql.gz.enc
+```
+
+备份采用 `mysqldump --single-transaction`、gzip 和 AES-256/PBKDF2,并生成
+SHA-256 校验文件。`.enc`、`.sha256`、`.json` 三个文件必须同步到独立账号的
+异地对象存储。校验脚本不等于恢复演练;至少每月在隔离 MySQL 实例执行一次
+完整恢复,确认账号、表结构和业务数据均可恢复。
+
默认问答链路需要配置问答环节 LLM;可在后台 `外部图谱问答 API -> 问答环节 LLM 模型` 单独配置,也可以继承全局 LLM。
主要返回字段:
@@ -279,5 +327,5 @@ API_PORT=18102 docker compose up -d
- 不要把真实 `.env`、LLM key、高德 key 提交到仓库。
- 用反向代理提供 HTTPS。
- 给 PostgreSQL 和 FalkorDB 配置持久化备份。
-- 如果面向公网,限制数据库端口暴露,只暴露 API/前端。
+- MySQL、PostgreSQL 和 FalkorDB 全部只绑定服务器本机;DBeaver 经 SSH 私钥隧道接入,公网只暴露 HTTPS 和受控 SSH。
- 开启日志采集和容器监控。
diff --git a/docs/PROJECT_OVERVIEW.md b/docs/PROJECT_OVERVIEW.md
index 969b6b2..88b0d5b 100644
--- a/docs/PROJECT_OVERVIEW.md
+++ b/docs/PROJECT_OVERVIEW.md
@@ -1,11 +1,10 @@
# 系统介绍
-旅行知识图谱管理系统是 `new2` 版本的城市与旅行领域知识图谱平台,面向景区、旅行社、文旅运营和智能客服场景。系统把采集资料、百科文本、POI 空间数据、线路产品、酒店/餐饮/车辆等资源沉淀为可审核、可发布、可查询的图谱资产。
+旅行知识图谱管理系统是 `new2` 版本的城市与旅行领域知识图谱平台,面向景区、旅行社、文旅运营和智能客服场景。系统把项目 Schema、结构化业务数据、POI 空间数据、线路产品、酒店/餐饮/车辆等资源统一组织为可管理、可查询的业务数据与图谱资产。
## 目标用户
-- 文旅运营人员:查看图谱覆盖、数据质量、缺口和发布状态。
-- 数据标注与审核人员:处理实体字段、证据来源、冲突合并和专家签核。
+- 文旅运营人员:查看数据大屏、图谱覆盖和业务数据。
- 产品和线路人员:维护固定线路、景点组合、报价说明和资源约束。
- 智能客服研发人员:基于图谱接口构建线路问答、周边资源推荐和报价查询。
- 工程维护人员:通过 Docker、快照和脚本复现系统与数据。
@@ -14,25 +13,21 @@
| 能力 | 说明 |
| --- | --- |
-| 数据源与批次管理 | 管理来源、采集批次、原始记录和质量摘要 |
-| 实体审核 | 查看候选实体、字段决策、证据链、审查历史和合并 |
-| 图谱广场 | 汇总图谱规模、使用情况、健康告警和用户查询 |
-| Schema 管理 | 管理 ontology schema、DSL、版本和发布记录 |
-| 证据质量 | 聚合 POI 证据、资源质量和字段可信度 |
-| 发布与回滚 | 创建发布任务、查看 diff、回滚图谱版本 |
+| 图谱中心 | 创建和管理图谱项目及项目 Schema |
+| 数据中心 | 管理项目数据库、数据表、记录、CSV 和 SQL 工作区 |
+| 知识广场 | 提供数据大屏、业务问答和图谱浏览 |
+| 本体 Schema | 只读展示图谱中心当前项目创建的 Schema 与版本 |
| 城市空间图谱 | 使用高德 POI 与空间网格支持周边检索 |
| 旅行客服 Agent | 支持线路清单、线路匹配、线路报价、酒店资源、车辆和附近资源查询 |
-| 权限与组织 | 内置角色、能力矩阵、用户和区域责任管理 |
+| 权限与组织 | 内置角色、能力矩阵和用户管理 |
## 典型业务流程
```mermaid
flowchart LR
- A["采集/导入资料"] --> B["抽取候选实体"]
- B --> C["字段证据与质量检查"]
- C --> D["人工审核/冲突合并"]
- D --> E["发布到 FalkorDB 图谱"]
- E --> F["图谱查询/客服问答/运营分析"]
+ A["图谱中心创建项目与 Schema"] --> B["数据中心管理业务数据"]
+ B --> C["数据大屏与图谱浏览"]
+ C --> D["业务问答与运营分析"]
```
## 随仓库发布的内容
diff --git a/docs/data-center-mysql-interface-center.md b/docs/data-center-mysql-interface-center.md
new file mode 100644
index 0000000..470ffa4
--- /dev/null
+++ b/docs/data-center-mysql-interface-center.md
@@ -0,0 +1,128 @@
+# 数据中心 MySQL 与接口中心部署说明
+
+## 已确定的系统边界
+
+- PostgreSQL:继续保存账号、权限、图谱项目与 Schema 等系统元数据。
+- FalkorDB:继续保存已发布知识图谱节点和关系。
+- MySQL:只负责“数据中心”的关系型业务数据。
+- 接口中心:只通过受控 HTTPS API 访问 MySQL 数据中心,不允许外部用户执行任意 SQL。
+- DBeaver:仅供受信任的数据管理员通过 SSH 隧道管理 MySQL,使用独立 MySQL 用户和 `GRANT` 权限;不复用接口密钥,也不公开 MySQL 端口。
+- SQL 控制台:生产默认只读;写入通过数据表页面执行,以保留记录级变更历史,物理删除始终禁止。
+
+每个数据中心数据库都是真实 MySQL Database:中文名称只用于页面显示,英文数据库编码直接作为物理 MySQL 数据库名。前端数据中心的现有 URL、布局和 `/v1/admin/data-platform/*` 契约保持不变。
+
+## 配置
+
+关键环境变量:
+
+```dotenv
+DATA_MYSQL_URL=mysql://data_center:strong-password@mysql:3306/platform_control
+DATA_MYSQL_REQUIRED=true
+DATA_MYSQL_POOL_MIN_SIZE=2
+DATA_MYSQL_POOL_MAX_SIZE=20
+INTERFACE_API_SECRET=another-long-random-secret
+DATA_MYSQL_DIRECT_ACCESS_ENABLED=false
+MYSQL_HOST_BIND=127.0.0.1
+MYSQL_PORT=3307
+DATA_MYSQL_DIRECT_TRANSPORT=SSH 隧道(强制)
+DATA_MYSQL_SSH_TUNNEL_REQUIRED=true
+DATA_MYSQL_SSH_HOST=server.example.com
+DATA_MYSQL_SSH_PORT=22
+DATA_MYSQL_SSH_AUTH_METHOD=SSH 私钥 / SSH Agent
+DATA_MYSQL_ADMIN_HOST=127.0.0.1
+DATA_MYSQL_ADMIN_PORT=3307
+DATA_MYSQL_ADMIN_ACCOUNT_POLICY=一人一号 · 单库授权 · 禁止 root
+DATA_MYSQL_AUDIT_ENABLED=false
+DATA_SQL_CONSOLE_WRITE_ENABLED=false
+DATA_BACKUP_ENABLED=false
+```
+
+`MYSQL_HOST_BIND` 在生产环境必须保持 `127.0.0.1`。`DATA_MYSQL_DIRECT_ACCESS_ENABLED` 只表示管理员接入是否完成运维配置,不会开放端口;本项目不提供公开 MySQL 直连模式。只有数据库审计真实启用后,才把 `DATA_MYSQL_AUDIT_ENABLED` 改成 `true`。
+
+应用会记录 API 调用、记录新增/修改/软删除以及 SQL 控制台的成功、失败和拒绝
+事件;SQL 审计只保存语句 SHA-256,不保存可能包含敏感值的原文。DBeaver 绕过
+应用,因此仍必须由数据库审计能力单独留痕,未完成前不得开放长期写权限。
+
+生产 MySQL 开启 ROW Binlog、`sync_binlog=1` 与
+`innodb_flush_log_at_trx_commit=1`,用于时间点恢复与事务落盘;加密备份脚本和
+恢复要求见 `docs/DEPLOYMENT.md`。
+
+`platform_control` 保存数据库目录、表定义、数据审计、接口客户端、密钥哈希、权限策略与接口调用日志。完整接口密钥不会保存,只保存带服务端 pepper 的 HMAC-SHA256 哈希。
+
+## 首次部署
+
+```bash
+docker compose up -d mysql
+docker compose up -d --build api
+```
+
+接口中心页面:`/admin/system/interfaces`。
+
+页面以“服务器被访问”为方向:`DBeaver 管理接入`只给受信任的数据管理员展示 SSH 隧道参数,`HTTPS API`负责电脑、设备和业务系统的身份、密钥、最小权限与调用日志。接口中心不会主动连接外部数据库。
+
+## DBeaver 安全连接
+
+DBeaver 不使用 URL 模式直接填写服务器公网地址。按两组参数配置:
+
+1. `SSH` 标签页:`Host/IP=服务器域名`、`Port=22`、认证方式使用私钥或 SSH Agent;
+2. `Main` 标签页:`Server Host=127.0.0.1`、`Port=3307`、`Database=获授权数据库编码`;
+3. 关闭 `Show all databases`;
+4. 使用运维为当前人员签发的独立 MySQL 账号,禁止使用 `root` 和后端服务账号 `data_center`;
+5. 默认只授予 `SELECT`,写入和结构修改权限按需临时授予。
+
+运维人员可在服务器上创建或撤销独立账号:
+
+```bash
+./scripts/mysql_admin_account.sh create zhangsan tourism_data read
+./scripts/mysql_admin_account.sh create lisi tourism_data write
+./scripts/mysql_admin_account.sh revoke zhangsan
+```
+
+脚本只授予指定数据库权限:`read` 为 `SELECT/SHOW VIEW`,`write` 额外包含
+`INSERT/UPDATE/DELETE`,不授予建库、删库、授权、文件或系统库权限;账号连续
+失败 5 次锁定一天,密码 90 天过期。MySQL 端口仍必须绑定回环地址,账号的
+`'%'` 主机范围不能替代网络隔离。
+
+接口中心不会显示或复制 MySQL 密码。服务器安全组只允许受控来源访问 SSH 与 HTTPS,不开放 `3307`。
+
+外部数据接口前缀:`/v1/openapi/data`。调用时使用:
+
+```http
+Authorization: Bearer YOUR_API_KEY
+```
+
+## 迁移旧数据中心
+
+先预览,不写入:
+
+```bash
+python3 scripts/migrate_data_center_postgres_to_mysql.py
+```
+
+确认计划后执行:
+
+```bash
+python3 scripts/migrate_data_center_postgres_to_mysql.py --execute
+```
+
+如果目标 MySQL 中已存在同编码数据库,脚本默认拒绝覆盖。只有确认目标可以重建时才使用:
+
+```bash
+python3 scripts/migrate_data_center_postgres_to_mysql.py --execute --replace-target
+```
+
+迁移会保留记录 UUID、创建/更新时间、软删除状态和数据变更审计,从而避免实体引用失效;不会修改 PostgreSQL 源数据,也不会操作 FalkorDB 或知识图谱。
+
+## 权限模型
+
+一条接口策略包含:
+
+1. 客户端(电脑、应用、合作方或设备);
+2. 数据库;
+3. 数据表或全部表;
+4. `metadata/read/create/update/delete` 动作;
+5. 可读字段与可写字段;
+6. 可选的行级 JSON 范围,例如 `{"district":"荔波县"}`;
+7. 启用/停用状态。
+
+接口客户端无法访问 `platform_control`、MySQL 系统库或其他业务数据库,也不能通过 API 执行 SQL。每次接口调用都会记录客户端、动作、资源范围、状态码、耗时和来源 IP。
diff --git a/docs/markitdown-evaluation.md b/docs/markitdown-evaluation.md
deleted file mode 100644
index 8a8daee..0000000
--- a/docs/markitdown-evaluation.md
+++ /dev/null
@@ -1,82 +0,0 @@
-# MarkItDown 转换评测方法
-
-本项目已经接入 Microsoft MarkItDown,但上线判断不能只看“能不能转”。推荐用一组真实业务样本文档做可重复评测。
-
-## 评测维度
-
-1. 转换成功率:不同格式是否稳定返回 Markdown。
-2. 信息保真度:关键字段、价格、日期、地点、人名、产品 ID 是否还在。
-3. 结构保留度:标题、表格、列表、链接是否保留为 Markdown 结构。
-4. 噪声控制:乱码、超长行、残留 HTML、空文本、重复内容是否明显。
-5. 下游效果:把转换后的 Markdown 送入知识抽取后,实体、关系、证据覆盖是否提升。
-
-## 准备样本
-
-把测试文件放到:
-
-```bash
-data/markitdown_eval/input/
-```
-
-建议每类至少 5 份:
-
-- PDF:普通 PDF、扫描 PDF、复杂表格 PDF
-- Word:合同、行程单、产品说明
-- Excel:价格表、团期表、资源表
-- PPT:介绍资料、图文页
-- HTML/Markdown/CSV/JSON/XML
-- 图片或截图类资料
-
-## Manifest 示例
-
-创建 `data/markitdown_eval/manifest.json`:
-
-```json
-{
- "cases": [
- {
- "case_id": "travel_product_docx_001",
- "file": "travel_product.docx",
- "must_terms": ["产品ID", "成人价", "儿童价", "费用包含", "退费政策"],
- "forbidden_terms": ["�"],
- "expected_headings_min": 2,
- "expected_tables_min": 1,
- "expected_lists_min": 3,
- "expected_links_min": 0,
- "min_chars": 800,
- "notes": "旅行社产品说明 Word"
- }
- ]
-}
-```
-
-## 运行评测
-
-```bash
-python3 scripts/evaluate_markitdown_conversion.py \
- --input-dir data/markitdown_eval/input \
- --manifest data/markitdown_eval/manifest.json \
- --output-dir outputs/markitdown_eval \
- --fail-under 0.70
-```
-
-输出:
-
-- `outputs/markitdown_eval/converted/*.markitdown.md`
-- `outputs/markitdown_eval/markitdown_eval_report.json`
-- `outputs/markitdown_eval/markitdown_eval_report.md`
-
-## 判定建议
-
-- 平均分 `>= 0.85`:可作为默认转换方案,但仍抽检复杂文件。
-- `0.70 - 0.85`:可用,但要看缺失字段和结构损失,必要时加 OCR 或人工校正。
-- `< 0.70`:不建议直接进入自动知识抽取,应启用替代方案。
-
-## 进一步增强
-
-当前接入是 MarkItDown 本地转换。若样本中大量是扫描 PDF、图片文字、复杂表格、音视频,建议再评估:
-
-- MarkItDown OCR plugin
-- Azure Document Intelligence
-- Azure Content Understanding
-- 针对旅行社/城市知识图谱的自定义后处理规则
diff --git a/qa/data-center-collapse-comparison.png b/qa/data-center-collapse-comparison.png
new file mode 100644
index 0000000..87f2ea3
Binary files /dev/null and b/qa/data-center-collapse-comparison.png differ
diff --git a/qa/data-center-collapsed-final.png b/qa/data-center-collapsed-final.png
new file mode 100644
index 0000000..1cd0e5e
Binary files /dev/null and b/qa/data-center-collapsed-final.png differ
diff --git a/qa/data-center-collapsible-tree-final.png b/qa/data-center-collapsible-tree-final.png
new file mode 100644
index 0000000..402e11d
Binary files /dev/null and b/qa/data-center-collapsible-tree-final.png differ
diff --git a/qa/data-center-collapsible-tree.png b/qa/data-center-collapsible-tree.png
new file mode 100644
index 0000000..4f51bfd
Binary files /dev/null and b/qa/data-center-collapsible-tree.png differ
diff --git a/qa/data-center-context-menu-comparison.jpg b/qa/data-center-context-menu-comparison.jpg
new file mode 100644
index 0000000..9ed7947
Binary files /dev/null and b/qa/data-center-context-menu-comparison.jpg differ
diff --git a/qa/data-center-context-menu-final.jpg b/qa/data-center-context-menu-final.jpg
new file mode 100644
index 0000000..bc90983
Binary files /dev/null and b/qa/data-center-context-menu-final.jpg differ
diff --git a/qa/data-center-database-context-menu-final.jpg b/qa/data-center-database-context-menu-final.jpg
new file mode 100644
index 0000000..6fc5fba
Binary files /dev/null and b/qa/data-center-database-context-menu-final.jpg differ
diff --git a/qa/data-center-dbeaver-grid-final.png b/qa/data-center-dbeaver-grid-final.png
new file mode 100644
index 0000000..f4565b4
Binary files /dev/null and b/qa/data-center-dbeaver-grid-final.png differ
diff --git a/qa/data-center-expanded-final.png b/qa/data-center-expanded-final.png
new file mode 100644
index 0000000..55f2778
Binary files /dev/null and b/qa/data-center-expanded-final.png differ
diff --git a/qa/data-center-final.png b/qa/data-center-final.png
new file mode 100644
index 0000000..55f2778
Binary files /dev/null and b/qa/data-center-final.png differ
diff --git a/qa/data-center-no-white-gap.png b/qa/data-center-no-white-gap.png
new file mode 100644
index 0000000..4f979f7
Binary files /dev/null and b/qa/data-center-no-white-gap.png differ
diff --git a/qa/data-center-reference-comparison.png b/qa/data-center-reference-comparison.png
new file mode 100644
index 0000000..1aa4f48
Binary files /dev/null and b/qa/data-center-reference-comparison.png differ
diff --git a/qa/data-center-top-refresh-comparison.jpg b/qa/data-center-top-refresh-comparison.jpg
new file mode 100644
index 0000000..b14d1f5
Binary files /dev/null and b/qa/data-center-top-refresh-comparison.jpg differ
diff --git a/qa/data-center-top-refresh-removed.jpg b/qa/data-center-top-refresh-removed.jpg
new file mode 100644
index 0000000..197b6a1
Binary files /dev/null and b/qa/data-center-top-refresh-removed.jpg differ
diff --git a/qa/design/data-create-sql-group.png b/qa/design/data-create-sql-group.png
new file mode 100644
index 0000000..1e86785
Binary files /dev/null and b/qa/design/data-create-sql-group.png differ
diff --git a/qa/design/interface-center-api-connection.png b/qa/design/interface-center-api-connection.png
new file mode 100644
index 0000000..67297d1
Binary files /dev/null and b/qa/design/interface-center-api-connection.png differ
diff --git a/qa/design/interface-center-api-security.png b/qa/design/interface-center-api-security.png
new file mode 100644
index 0000000..6b29ac2
Binary files /dev/null and b/qa/design/interface-center-api-security.png differ
diff --git a/qa/design/interface-center-mysql-comparison.jpg b/qa/design/interface-center-mysql-comparison.jpg
new file mode 100644
index 0000000..8259099
Binary files /dev/null and b/qa/design/interface-center-mysql-comparison.jpg differ
diff --git a/qa/design/interface-center-mysql-connection.png b/qa/design/interface-center-mysql-connection.png
new file mode 100644
index 0000000..870b076
Binary files /dev/null and b/qa/design/interface-center-mysql-connection.png differ
diff --git a/qa/design/interface-center-responsive-1024-loaded.png b/qa/design/interface-center-responsive-1024-loaded.png
new file mode 100644
index 0000000..12c1746
Binary files /dev/null and b/qa/design/interface-center-responsive-1024-loaded.png differ
diff --git a/qa/design/interface-center-responsive-1024.png b/qa/design/interface-center-responsive-1024.png
new file mode 100644
index 0000000..1646217
Binary files /dev/null and b/qa/design/interface-center-responsive-1024.png differ
diff --git a/qa/design/interface-center-source-normalized.png b/qa/design/interface-center-source-normalized.png
new file mode 100644
index 0000000..3de9680
Binary files /dev/null and b/qa/design/interface-center-source-normalized.png differ
diff --git a/qa/libo-hotel-facility-rail-scroll-final.png b/qa/libo-hotel-facility-rail-scroll-final.png
deleted file mode 100644
index 022ccca..0000000
Binary files a/qa/libo-hotel-facility-rail-scroll-final.png and /dev/null differ
diff --git a/qa/libo-hotel-facility-section-final.png b/qa/libo-hotel-facility-section-final.png
deleted file mode 100644
index 3bc907f..0000000
Binary files a/qa/libo-hotel-facility-section-final.png and /dev/null differ
diff --git a/qa/libo-hotel-responsive-facilities-final.png b/qa/libo-hotel-responsive-facilities-final.png
deleted file mode 100644
index 944bf70..0000000
Binary files a/qa/libo-hotel-responsive-facilities-final.png and /dev/null differ
diff --git a/qa/libo-hotel-responsive-facilities.png b/qa/libo-hotel-responsive-facilities.png
deleted file mode 100644
index bf12da3..0000000
Binary files a/qa/libo-hotel-responsive-facilities.png and /dev/null differ
diff --git a/qa/libo-hotel-responsive-facility-state.png b/qa/libo-hotel-responsive-facility-state.png
deleted file mode 100644
index dc50c30..0000000
Binary files a/qa/libo-hotel-responsive-facility-state.png and /dev/null differ
diff --git a/qa/libo-hotel-responsive-metrics.png b/qa/libo-hotel-responsive-metrics.png
deleted file mode 100644
index 6d54b28..0000000
Binary files a/qa/libo-hotel-responsive-metrics.png and /dev/null differ
diff --git a/qa/libo-hotel-responsive-top.png b/qa/libo-hotel-responsive-top.png
deleted file mode 100644
index 6cb215c..0000000
Binary files a/qa/libo-hotel-responsive-top.png and /dev/null differ
diff --git a/qa/sql-console-entry-full-2026-08-03.png b/qa/sql-console-entry-full-2026-08-03.png
new file mode 100644
index 0000000..d4e4e3a
Binary files /dev/null and b/qa/sql-console-entry-full-2026-08-03.png differ
diff --git a/qa/sql-console-entry-toolbar-2026-08-03.png b/qa/sql-console-entry-toolbar-2026-08-03.png
new file mode 100644
index 0000000..d4e4e3a
Binary files /dev/null and b/qa/sql-console-entry-toolbar-2026-08-03.png differ
diff --git a/qa/sql-console-workspace-2026-08-03.png b/qa/sql-console-workspace-2026-08-03.png
new file mode 100644
index 0000000..7a18ab7
Binary files /dev/null and b/qa/sql-console-workspace-2026-08-03.png differ
diff --git a/requirements.txt b/requirements.txt
index b452794..21397dd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-# 智念城市知识图谱管理系统 —— Python 后端依赖
+# 智念数字资产管理系统 —— Python 后端依赖
# 安装: pip3 install -r requirements.txt
# Python 3.11+ (开发环境 3.13)
@@ -6,9 +6,11 @@ fastapi==0.136.1
uvicorn==0.47.0
starlette==1.0.0
-# 数据库 (PostgreSQL kg_admin schema)
+# 数据库(PostgreSQL:系统/图谱元数据;MySQL:独立数据中心)
psycopg==3.3.4
psycopg-pool==3.3.1
+aiomysql==0.3.2
+PyMySQL==1.1.2
# 配置 / 校验
pydantic==2.13.0
@@ -24,10 +26,6 @@ h3==4.4.2
httpx==0.28.1
requests==2.32.5
python-multipart==0.0.28
-markitdown[all]==0.1.6
-pymupdf4llm==1.27.2.3 # 结构化 PDF→Markdown(版式/表格感知,替代 markitdown 纯文本抽取)
-python-docx==1.2.0 # DOCX→Markdown 保留表格结构
-olefile==0.47 # 损坏/textutil 读不了的老 .doc 直接捞 WordDocument 流正文
# 鉴权
bcrypt==4.1.3
diff --git a/scripts/backup_mysql_encrypted.sh b/scripts/backup_mysql_encrypted.sh
new file mode 100755
index 0000000..e1d6f0b
--- /dev/null
+++ b/scripts/backup_mysql_encrypted.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+umask 077
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd -- "${SCRIPT_DIR}/.." && pwd)"
+BACKUP_ROOT="${DATA_BACKUP_ROOT:-${PROJECT_DIR}/backups/mysql}"
+PASSPHRASE_FILE="${DATA_BACKUP_PASSPHRASE_FILE:-}"
+
+if [[ -z "${PASSPHRASE_FILE}" || ! -f "${PASSPHRASE_FILE}" || ! -r "${PASSPHRASE_FILE}" ]]; then
+ echo "DATA_BACKUP_PASSPHRASE_FILE 必须指向可读的独立密钥文件" >&2
+ exit 2
+fi
+if [[ "$(wc -c < "${PASSPHRASE_FILE}" | tr -d ' ')" -lt 32 ]]; then
+ echo "备份加密口令文件内容不能少于 32 个字符" >&2
+ exit 2
+fi
+if [[ -z "${BACKUP_ROOT}" || "${BACKUP_ROOT}" == "/" ]]; then
+ echo "拒绝使用不安全的备份目录" >&2
+ exit 2
+fi
+
+for command_name in docker gzip openssl; do
+ command -v "${command_name}" >/dev/null 2>&1 || {
+ echo "缺少命令:${command_name}" >&2
+ exit 2
+ }
+done
+
+mkdir -p -- "${BACKUP_ROOT}"
+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+archive_name="mysql-all-${timestamp}.sql.gz.enc"
+archive_path="${BACKUP_ROOT}/${archive_name}"
+temporary_path="${BACKUP_ROOT}/.${archive_name}.tmp"
+checksum_path="${archive_path}.sha256"
+manifest_path="${archive_path}.json"
+
+cleanup() {
+ rm -f -- "${temporary_path}"
+}
+trap cleanup EXIT INT TERM
+
+cd -- "${PROJECT_DIR}"
+docker compose exec -T mysql sh -eu -c '
+ MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysqldump \
+ --user=root \
+ --all-databases \
+ --single-transaction \
+ --quick \
+ --routines \
+ --events \
+ --triggers \
+ --hex-blob \
+ --set-gtid-purged=OFF
+' | gzip -9 | openssl enc -aes-256-cbc -salt -pbkdf2 -iter 200000 \
+ -pass "file:${PASSPHRASE_FILE}" -out "${temporary_path}"
+
+test -s "${temporary_path}"
+mv -- "${temporary_path}" "${archive_path}"
+
+if command -v sha256sum >/dev/null 2>&1; then
+ checksum="$(sha256sum "${archive_path}" | awk '{print $1}')"
+else
+ checksum="$(shasum -a 256 "${archive_path}" | awk '{print $1}')"
+fi
+printf '%s %s\n' "${checksum}" "${archive_name}" > "${checksum_path}"
+printf '{\n "created_at": "%s",\n "file": "%s",\n "sha256": "%s",\n "encrypted": true,\n "cipher": "AES-256-CBC/PBKDF2",\n "compression": "gzip"\n}\n' \
+ "${timestamp}" "${archive_name}" "${checksum}" > "${manifest_path}"
+chmod 600 "${archive_path}" "${checksum_path}" "${manifest_path}"
+
+echo "加密备份已生成:${archive_path}"
+echo "请将 .enc、.sha256 和 .json 同步到独立的异地存储。"
diff --git a/scripts/build_travel_agency_project.py b/scripts/build_travel_agency_project.py
index 5317359..8d4cc4e 100644
--- a/scripts/build_travel_agency_project.py
+++ b/scripts/build_travel_agency_project.py
@@ -1861,7 +1861,7 @@ def write_outputs(builder: KGBuilder, schema: dict[str, Any], qa: list[dict[str,
"",
"## 页面与系统建议",
"- 已按 `project_id=travel_agency`、`tenant_id=travel_agency`、`graph_name=travel_agency` 独立建项目,避免污染城市图谱。",
- "- 建议首页保留“图谱中心”,进入项目后所有列表、图谱浏览、人工录入都默认带当前项目头。",
+ "- 建议首页保留“图谱中心”,进入项目后数据中心、图谱浏览和业务问答都默认带当前项目头。",
"- 后续做客服工作台时,建议把 `TravelDemand` 抽取表单放在左侧,把命中的 `TourProduct/TourVariant/Hotel/Restaurant/Vehicle` 放在右侧,最终生成 `ItineraryPlan`。",
"- 客服问答不要直接让模型自由编线路,应先从 `TravelDemand -> ItineraryPlan -> TourProduct/TourVariant` 找证据,再让模型组织话术。",
"",
diff --git a/scripts/evaluate_markitdown_conversion.py b/scripts/evaluate_markitdown_conversion.py
deleted file mode 100644
index a7f8ed6..0000000
--- a/scripts/evaluate_markitdown_conversion.py
+++ /dev/null
@@ -1,365 +0,0 @@
-#!/usr/bin/env python3
-"""Evaluate MarkItDown conversion quality on a local document corpus.
-
-The goal is not to prove that a converter is universally good. The goal is to
-make conversion quality measurable for this knowledge-extraction product:
-success rate, latency, information coverage, structure preservation, noise, and
-optional similarity to reference Markdown.
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import re
-import statistics
-import sys
-import time
-from dataclasses import dataclass
-from difflib import SequenceMatcher
-from pathlib import Path
-from typing import Any
-
-
-SUPPORTED_EXTENSIONS = {
- ".txt",
- ".md",
- ".markdown",
- ".csv",
- ".json",
- ".xml",
- ".html",
- ".htm",
- ".pdf",
- ".docx",
- ".doc",
- ".pptx",
- ".ppt",
- ".xlsx",
- ".xls",
- ".zip",
- ".epub",
- ".jpg",
- ".jpeg",
- ".png",
- ".gif",
- ".wav",
- ".mp3",
-}
-
-
-@dataclass
-class Case:
- case_id: str
- file_path: Path
- must_terms: list[str]
- forbidden_terms: list[str]
- expected_headings_min: int
- expected_tables_min: int
- expected_lists_min: int
- expected_links_min: int
- min_chars: int
- gold_markdown_path: Path | None
- notes: str
-
-
-def load_manifest(path: Path | None) -> dict[str, Any]:
- if not path:
- return {"cases": []}
- with path.open("r", encoding="utf-8") as f:
- data = json.load(f)
- if not isinstance(data, dict) or not isinstance(data.get("cases"), list):
- raise ValueError("Manifest must be a JSON object with a cases array")
- return data
-
-
-def discover_cases(input_dir: Path, manifest: dict[str, Any]) -> list[Case]:
- by_file = {
- str(item.get("file", "")).strip(): item
- for item in manifest.get("cases", [])
- if isinstance(item, dict) and str(item.get("file", "")).strip()
- }
- if by_file:
- files = [input_dir / rel for rel in by_file]
- else:
- files = [
- p
- for p in sorted(input_dir.rglob("*"))
- if p.is_file() and p.suffix.lower() in SUPPORTED_EXTENSIONS
- ]
-
- cases: list[Case] = []
- for file_path in files:
- rel = str(file_path.relative_to(input_dir)) if file_path.is_relative_to(input_dir) else file_path.name
- cfg = by_file.get(rel, {})
- case_id = str(cfg.get("case_id") or file_path.with_suffix("").name)
- gold = cfg.get("gold_markdown")
- cases.append(
- Case(
- case_id=case_id,
- file_path=file_path,
- must_terms=[str(v) for v in cfg.get("must_terms", []) if str(v).strip()],
- forbidden_terms=[str(v) for v in cfg.get("forbidden_terms", []) if str(v).strip()],
- expected_headings_min=int(cfg.get("expected_headings_min") or 0),
- expected_tables_min=int(cfg.get("expected_tables_min") or 0),
- expected_lists_min=int(cfg.get("expected_lists_min") or 0),
- expected_links_min=int(cfg.get("expected_links_min") or 0),
- min_chars=int(cfg.get("min_chars") or 80),
- gold_markdown_path=(input_dir / str(gold)) if gold else None,
- notes=str(cfg.get("notes") or ""),
- )
- )
- return cases
-
-
-def convert_with_markitdown(file_path: Path) -> str:
- try:
- from markitdown import MarkItDown
- except ImportError as exc:
- raise RuntimeError("MarkItDown is not installed. Run: pip install -r requirements.txt") from exc
-
- result = MarkItDown(enable_plugins=False).convert(str(file_path))
- text = getattr(result, "text_content", None) or getattr(result, "markdown", None) or ""
- return str(text).strip()
-
-
-def count_patterns(markdown: str) -> dict[str, int]:
- lines = markdown.splitlines()
- return {
- "chars": len(markdown),
- "lines": len(lines),
- "headings": sum(1 for line in lines if re.match(r"^\s{0,3}#{1,6}\s+\S", line)),
- "table_rows": sum(1 for line in lines if line.count("|") >= 2),
- "list_items": sum(1 for line in lines if re.match(r"^\s*(?:[-*+]|\d+[.)])\s+\S", line)),
- "links": len(re.findall(r"\[[^\]]+\]\([^)]+\)|https?://\S+", markdown)),
- "replacement_chars": markdown.count("\ufffd"),
- "null_chars": markdown.count("\x00"),
- "html_tags": len(re.findall(r"?[A-Za-z][^>]{0,200}>", markdown)),
- "long_lines": sum(1 for line in lines if len(line) > 500),
- }
-
-
-def term_coverage(markdown: str, terms: list[str]) -> tuple[float, list[str]]:
- if not terms:
- return 1.0, []
- haystack = markdown.lower()
- missing = [term for term in terms if term.lower() not in haystack]
- return (len(terms) - len(missing)) / len(terms), missing
-
-
-def forbidden_hits(markdown: str, terms: list[str]) -> list[str]:
- haystack = markdown.lower()
- return [term for term in terms if term.lower() in haystack]
-
-
-def min_ratio(actual: int, expected: int) -> float:
- if expected <= 0:
- return 1.0
- return min(actual / expected, 1.0)
-
-
-def score_case(metrics: dict[str, Any]) -> float:
- content_score = min(metrics["chars"] / max(metrics["min_chars"], 1), 1.0)
- structure_score = statistics.mean(
- [
- metrics["heading_score"],
- metrics["table_score"],
- metrics["list_score"],
- metrics["link_score"],
- ]
- )
- noise_penalty = min(
- 1.0,
- metrics["replacement_chars"] * 0.08
- + metrics["null_chars"] * 0.2
- + metrics["long_lines"] * 0.03
- + metrics["forbidden_hit_count"] * 0.12,
- )
- gold_similarity = metrics.get("gold_similarity")
- if gold_similarity is None:
- score = (
- 0.35 * metrics["must_term_coverage"]
- + 0.25 * content_score
- + 0.25 * structure_score
- + 0.15 * (1.0 - noise_penalty)
- )
- else:
- score = (
- 0.30 * metrics["must_term_coverage"]
- + 0.20 * content_score
- + 0.20 * structure_score
- + 0.15 * (1.0 - noise_penalty)
- + 0.15 * gold_similarity
- )
- return round(max(0.0, min(score, 1.0)), 4)
-
-
-def evaluate_case(case: Case, output_dir: Path) -> dict[str, Any]:
- started = time.perf_counter()
- output_path = output_dir / "converted" / f"{case.case_id}.markitdown.md"
- output_path.parent.mkdir(parents=True, exist_ok=True)
- result: dict[str, Any] = {
- "case_id": case.case_id,
- "file": str(case.file_path),
- "notes": case.notes,
- "success": False,
- }
-
- try:
- markdown = convert_with_markitdown(case.file_path)
- output_path.write_text(markdown + "\n", encoding="utf-8")
- counts = count_patterns(markdown)
- coverage, missing = term_coverage(markdown, case.must_terms)
- forbidden = forbidden_hits(markdown, case.forbidden_terms)
- metrics: dict[str, Any] = {
- **counts,
- "min_chars": case.min_chars,
- "must_terms": case.must_terms,
- "must_term_coverage": round(coverage, 4),
- "missing_terms": missing,
- "forbidden_terms": case.forbidden_terms,
- "forbidden_hits": forbidden,
- "forbidden_hit_count": len(forbidden),
- "expected_headings_min": case.expected_headings_min,
- "expected_tables_min": case.expected_tables_min,
- "expected_lists_min": case.expected_lists_min,
- "expected_links_min": case.expected_links_min,
- "heading_score": min_ratio(counts["headings"], case.expected_headings_min),
- "table_score": min_ratio(counts["table_rows"], case.expected_tables_min),
- "list_score": min_ratio(counts["list_items"], case.expected_lists_min),
- "link_score": min_ratio(counts["links"], case.expected_links_min),
- }
- if case.gold_markdown_path and case.gold_markdown_path.exists():
- gold = case.gold_markdown_path.read_text(encoding="utf-8")
- metrics["gold_similarity"] = round(SequenceMatcher(None, gold, markdown).ratio(), 4)
- else:
- metrics["gold_similarity"] = None
-
- result.update(
- {
- "success": True,
- "latency_ms": round((time.perf_counter() - started) * 1000),
- "markdown_path": str(output_path),
- "metrics": metrics,
- "quality_score": score_case(metrics),
- }
- )
- except Exception as exc: # noqa: BLE001
- result.update(
- {
- "latency_ms": round((time.perf_counter() - started) * 1000),
- "error": str(exc),
- "quality_score": 0.0,
- }
- )
- return result
-
-
-def aggregate(results: list[dict[str, Any]]) -> dict[str, Any]:
- scores = [float(r.get("quality_score") or 0) for r in results]
- successes = [r for r in results if r.get("success")]
- latencies = [float(r.get("latency_ms") or 0) for r in successes]
- return {
- "cases": len(results),
- "successes": len(successes),
- "failures": len(results) - len(successes),
- "success_rate": round(len(successes) / len(results), 4) if results else 0,
- "mean_quality_score": round(statistics.mean(scores), 4) if scores else 0,
- "median_quality_score": round(statistics.median(scores), 4) if scores else 0,
- "mean_latency_ms": round(statistics.mean(latencies)) if latencies else 0,
- }
-
-
-def write_markdown_report(report: dict[str, Any], path: Path) -> None:
- summary = report["summary"]
- lines = [
- "# MarkItDown Conversion Evaluation",
- "",
- "## Summary",
- "",
- f"- Cases: {summary['cases']}",
- f"- Success rate: {summary['success_rate']:.2%}",
- f"- Mean quality score: {summary['mean_quality_score']:.3f}",
- f"- Median quality score: {summary['median_quality_score']:.3f}",
- f"- Mean latency: {summary['mean_latency_ms']} ms",
- "",
- "## Cases",
- "",
- "| Case | Success | Score | Chars | Must-term coverage | Missing terms | Latency |",
- "| --- | --- | ---: | ---: | ---: | --- | ---: |",
- ]
- for item in report["results"]:
- metrics = item.get("metrics") or {}
- missing = ", ".join(metrics.get("missing_terms") or [])
- lines.append(
- "| {case} | {success} | {score:.3f} | {chars} | {coverage:.2%} | {missing} | {latency} ms |".format(
- case=item.get("case_id"),
- success="yes" if item.get("success") else "no",
- score=float(item.get("quality_score") or 0),
- chars=metrics.get("chars", 0),
- coverage=float(metrics.get("must_term_coverage") or 0),
- missing=missing.replace("|", "\\|") or "-",
- latency=item.get("latency_ms", 0),
- )
- )
- lines.append("")
- lines.append("## Interpretation")
- lines.append("")
- lines.append("- Score >= 0.85: suitable for normal extraction after spot check.")
- lines.append("- 0.70 <= score < 0.85: usable, but inspect missing terms or structure loss.")
- lines.append("- Score < 0.70: do not trust automatic extraction without fallback/OCR/manual correction.")
- path.write_text("\n".join(lines) + "\n", encoding="utf-8")
-
-
-def main() -> int:
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--input-dir", default="data/markitdown_eval/input")
- parser.add_argument("--manifest", default="data/markitdown_eval/manifest.json")
- parser.add_argument("--output-dir", default="outputs/markitdown_eval")
- parser.add_argument("--fail-under", type=float, default=0.70)
- parser.add_argument("--allow-missing-manifest", action="store_true")
- args = parser.parse_args()
-
- input_dir = Path(args.input_dir)
- manifest_path = Path(args.manifest)
- output_dir = Path(args.output_dir)
- output_dir.mkdir(parents=True, exist_ok=True)
-
- if not input_dir.exists():
- print(f"Input directory not found: {input_dir}", file=sys.stderr)
- return 2
- if not manifest_path.exists() and not args.allow_missing_manifest:
- print(
- f"Manifest not found: {manifest_path}. Pass --allow-missing-manifest to auto-discover files.",
- file=sys.stderr,
- )
- return 2
-
- manifest = load_manifest(manifest_path if manifest_path.exists() else None)
- cases = discover_cases(input_dir, manifest)
- if not cases:
- print(f"No supported files found in {input_dir}", file=sys.stderr)
- return 2
-
- results = [evaluate_case(case, output_dir) for case in cases]
- report = {
- "input_dir": str(input_dir),
- "manifest": str(manifest_path) if manifest_path.exists() else None,
- "summary": aggregate(results),
- "results": results,
- }
- json_path = output_dir / "markitdown_eval_report.json"
- md_path = output_dir / "markitdown_eval_report.md"
- json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
- write_markdown_report(report, md_path)
-
- print(f"Wrote {json_path}")
- print(f"Wrote {md_path}")
- mean_score = report["summary"]["mean_quality_score"]
- if mean_score < args.fail_under:
- print(f"Mean quality score {mean_score:.3f} is below fail-under {args.fail_under:.3f}", file=sys.stderr)
- return 1
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/scripts/migrate_data_center_postgres_to_mysql.py b/scripts/migrate_data_center_postgres_to_mysql.py
new file mode 100644
index 0000000..7ce9489
--- /dev/null
+++ b/scripts/migrate_data_center_postgres_to_mysql.py
@@ -0,0 +1,374 @@
+#!/usr/bin/env python3
+"""Copy the legacy PostgreSQL-schema Data Center into real MySQL databases.
+
+The command is dry-run by default. It never changes PostgreSQL source data and
+will not overwrite an existing MySQL target unless both ``--execute`` and
+``--replace-target`` are supplied.
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import sys
+from typing import Any
+import uuid
+
+from psycopg import sql
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+if str(REPOSITORY_ROOT) not in sys.path:
+ sys.path.insert(0, str(REPOSITORY_ROOT))
+
+from app import db
+from app.config import settings
+from app.data_platform.mysql_db import close_data_pool, get_data_conn, init_data_pool
+from app.data_platform.mysql_service import (
+ _create_physical_table,
+ delete_project_database,
+ ensure_platform_registry,
+ ensure_project_database,
+ get_project_database,
+)
+from app.data_platform.registry import TABLE_DEFINITIONS
+from app.data_platform.schema import (
+ _custom_table_from_row,
+ _field_definitions_from_rows,
+ _field_row,
+ _table_with_identity,
+)
+
+
+SYSTEM_COLUMNS = (
+ "id",
+ "tenant_id",
+ "project_id",
+ "created_at",
+ "updated_at",
+ "deleted_at",
+ "deleted_by",
+)
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--project-id", action="append", help="仅迁移指定数据库编码,可重复")
+ parser.add_argument("--execute", action="store_true", help="实际执行;未提供时只输出迁移计划")
+ parser.add_argument(
+ "--replace-target",
+ action="store_true",
+ help="目标 MySQL 数据库已存在时先删除后重建(必须同时提供 --execute)",
+ )
+ parser.add_argument("--batch-size", type=int, default=1000)
+ return parser.parse_args()
+
+
+def mysql_value(value: Any, *, json_field: bool = False) -> Any:
+ if value is None:
+ return None
+ if isinstance(value, uuid.UUID):
+ return str(value)
+ if isinstance(value, datetime) and value.tzinfo is not None:
+ return value.astimezone(timezone.utc).replace(tzinfo=None)
+ if json_field and not isinstance(value, str):
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+ return value
+
+
+async def legacy_databases(project_ids: set[str] | None) -> list[dict[str, Any]]:
+ async with db.get_conn() as conn:
+ async with conn.cursor() as cur:
+ query = sql.SQL(
+ "SELECT * FROM {}.project_databases WHERE status='ready'"
+ ).format(sql.Identifier(settings.db_schema))
+ params: tuple[Any, ...] = ()
+ if project_ids:
+ query += sql.SQL(" AND project_id = ANY(%s)")
+ params = (list(project_ids),)
+ query += sql.SQL(" ORDER BY created_at, project_id")
+ await cur.execute(query, params)
+ return [dict(row) for row in await cur.fetchall()]
+
+
+async def legacy_table_entries(
+ project_id: str,
+) -> tuple[tuple[Any, str, str], ...]:
+ """Read legacy metadata directly without running PostgreSQL DDL."""
+
+ async with db.get_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ sql.SQL(
+ "SELECT * FROM {}.project_table_overrides WHERE project_id=%s"
+ ).format(sql.Identifier(settings.db_schema)),
+ (project_id,),
+ )
+ overrides = {
+ str(row["source_code"]): dict(row) for row in await cur.fetchall()
+ }
+ await cur.execute(
+ sql.SQL(
+ """
+ SELECT * FROM {}.project_table_definitions
+ WHERE project_id=%s AND status='active'
+ ORDER BY created_at, table_code
+ """
+ ).format(sql.Identifier(settings.db_schema)),
+ (project_id,),
+ )
+ custom_rows = [dict(row) for row in await cur.fetchall()]
+
+ entries: list[tuple[Any, str, str]] = []
+ for definition in TABLE_DEFINITIONS:
+ override = overrides.get(definition.code)
+ if override and str(override["status"]) == "deleted":
+ continue
+ effective = (
+ _table_with_identity(
+ definition,
+ code=str(override["table_code"]),
+ label=str(override["label"]),
+ fields=(
+ _field_definitions_from_rows(override["fields_jsonb"])
+ if override.get("fields_jsonb") is not None
+ else definition.fields
+ ),
+ )
+ if override
+ else definition
+ )
+ entries.append((effective, "builtin", definition.code))
+ entries.extend(
+ (_custom_table_from_row(row), "custom", str(row["table_code"]))
+ for row in custom_rows
+ )
+ return tuple(entries)
+
+
+async def target_has_database(project_id: str) -> bool:
+ return await get_project_database(project_id) is not None
+
+
+async def rebuild_target_structure(
+ database: dict[str, Any],
+ entries: tuple[tuple[Any, str, str], ...],
+) -> None:
+ project_id = str(database["project_id"])
+ database_name = str(database["database_name"])
+ async with get_data_conn(database_name) as conn:
+ async with conn.cursor() as cur:
+ await cur.execute("SET FOREIGN_KEY_CHECKS=0")
+ await cur.execute("SHOW TABLES")
+ for row in await cur.fetchall():
+ table_name = str(next(iter(row.values())))
+ if not table_name.replace("_", "a").isalnum():
+ raise RuntimeError(f"目标表名不安全:{table_name}")
+ await cur.execute(f"DROP TABLE `{table_name}`")
+ await cur.execute("SET FOREIGN_KEY_CHECKS=1")
+ await conn.commit()
+ async with get_data_conn() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "DELETE FROM project_table_definitions WHERE project_id=%s",
+ (project_id,),
+ )
+ await cur.execute(
+ "DELETE FROM data_change_logs WHERE project_id=%s",
+ (project_id,),
+ )
+ for order, (definition, origin, source_code) in enumerate(entries):
+ await cur.execute(
+ """
+ INSERT INTO project_table_definitions (
+ project_id, source_code, table_code, label, group_name,
+ description, fields_json, origin, display_order,
+ allow_create, allow_update, allow_delete, created_by
+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'migration')
+ """,
+ (
+ project_id,
+ source_code,
+ definition.code,
+ definition.label,
+ definition.group,
+ definition.description,
+ json.dumps([_field_row(field) for field in definition.fields], ensure_ascii=False),
+ origin,
+ order,
+ int(definition.allow_create),
+ int(definition.allow_update),
+ int(definition.allow_delete),
+ ),
+ )
+ await conn.commit()
+ for definition, _origin, _source_code in entries:
+ await _create_physical_table(database_name, definition)
+
+
+async def copy_table_rows(
+ source_schema: str,
+ target_database: str,
+ definition: Any,
+ batch_size: int,
+) -> int:
+ business_columns = [field.code for field in definition.fields]
+ columns = [
+ "id",
+ "tenant_id",
+ "project_id",
+ *business_columns,
+ "created_at",
+ "updated_at",
+ "deleted_at",
+ "deleted_by",
+ ]
+ json_fields = {field.code for field in definition.fields if field.data_type == "json"}
+ placeholders = ", ".join(["%s"] * len(columns))
+ target_query = (
+ f"INSERT INTO `{definition.code}` "
+ f"({', '.join(f'`{column}`' for column in columns)}) VALUES ({placeholders})"
+ )
+ copied = 0
+ async with db.get_conn() as source_conn:
+ async with source_conn.cursor() as source_cur:
+ await source_cur.execute(
+ sql.SQL("SELECT {} FROM {}.{} ORDER BY created_at, id").format(
+ sql.SQL(", ").join(sql.Identifier(column) for column in columns),
+ sql.Identifier(source_schema),
+ sql.Identifier(definition.code),
+ )
+ )
+ async with get_data_conn(target_database) as target_conn:
+ async with target_conn.cursor() as target_cur:
+ while True:
+ rows = await source_cur.fetchmany(batch_size)
+ if not rows:
+ break
+ values = [
+ tuple(
+ mysql_value(row[column], json_field=column in json_fields)
+ for column in columns
+ )
+ for row in rows
+ ]
+ await target_cur.executemany(target_query, values)
+ copied += len(values)
+ await target_conn.commit()
+ return copied
+
+
+async def copy_audit_logs(source_schema: str, project_id: str, batch_size: int) -> int:
+ columns = (
+ "tenant_id",
+ "project_id",
+ "table_code",
+ "record_id",
+ "operation",
+ "before_data",
+ "after_data",
+ "actor",
+ "created_at",
+ )
+ copied = 0
+ async with db.get_conn() as source_conn:
+ async with source_conn.cursor() as source_cur:
+ await source_cur.execute(
+ sql.SQL("SELECT {} FROM {}.data_change_logs ORDER BY created_at, id").format(
+ sql.SQL(", ").join(sql.Identifier(column) for column in columns),
+ sql.Identifier(source_schema),
+ )
+ )
+ async with get_data_conn() as target_conn:
+ async with target_conn.cursor() as target_cur:
+ while True:
+ rows = await source_cur.fetchmany(batch_size)
+ if not rows:
+ break
+ values = [
+ (
+ row["tenant_id"],
+ project_id,
+ row["table_code"],
+ str(row["record_id"]),
+ row["operation"],
+ mysql_value(row["before_data"], json_field=True),
+ mysql_value(row["after_data"], json_field=True),
+ row["actor"],
+ mysql_value(row["created_at"]),
+ )
+ for row in rows
+ ]
+ await target_cur.executemany(
+ """
+ INSERT INTO data_change_logs (
+ tenant_id, project_id, table_code, record_id,
+ operation, before_data, after_data, actor, created_at
+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+ """,
+ values,
+ )
+ copied += len(values)
+ await target_conn.commit()
+ return copied
+
+
+async def migrate(args: argparse.Namespace) -> None:
+ await db.init_pool()
+ if not await init_data_pool():
+ raise RuntimeError("无法连接 DATA_MYSQL_URL 指向的 MySQL 服务")
+ await ensure_platform_registry()
+ try:
+ requested = set(args.project_id or []) or None
+ databases = await legacy_databases(requested)
+ if not databases:
+ print("没有找到待迁移的 PostgreSQL 数据中心数据库。")
+ return
+ print(f"发现 {len(databases)} 个数据库;模式:{'执行' if args.execute else '仅预览'}")
+ for source in databases:
+ project_id = str(source["project_id"])
+ entries = await legacy_table_entries(project_id)
+ print(
+ f"- {source['display_name']} ({project_id}):{len(entries)} 张表,"
+ f"{source['schema_name']} → {project_id}"
+ )
+ if not args.execute:
+ continue
+ exists = await target_has_database(project_id)
+ if exists and not args.replace_target:
+ raise RuntimeError(
+ f"目标 {project_id} 已存在;确认可覆盖后追加 --replace-target"
+ )
+ if exists:
+ target = await get_project_database(project_id)
+ await delete_project_database(project_id, str(target["database_name"]))
+ target = await ensure_project_database(
+ project_id,
+ str(source["tenant_id"]),
+ str(source["display_name"]),
+ )
+ await rebuild_target_structure(target, entries)
+ total = 0
+ for definition, _origin, _source_code in entries:
+ count = await copy_table_rows(
+ str(source["schema_name"]),
+ str(target["database_name"]),
+ definition,
+ max(100, min(10_000, args.batch_size)),
+ )
+ total += count
+ print(f" · {definition.code}: {count:,} 条")
+ audit_count = await copy_audit_logs(
+ str(source["schema_name"]),
+ project_id,
+ max(100, min(10_000, args.batch_size)),
+ )
+ print(f" 完成:业务记录 {total:,} 条,审计记录 {audit_count:,} 条")
+ finally:
+ await close_data_pool()
+ await db.close_pool()
+
+
+if __name__ == "__main__":
+ asyncio.run(migrate(parse_args()))
diff --git a/scripts/mysql_admin_account.sh b/scripts/mysql_admin_account.sh
new file mode 100755
index 0000000..7f353f5
--- /dev/null
+++ b/scripts/mysql_admin_account.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+action="${1:-}"
+account="${2:-}"
+database="${3:-}"
+permission="${4:-read}"
+
+usage() {
+ echo "用法:" >&2
+ echo " $0 create 用户名 数据库编码 read|write" >&2
+ echo " $0 revoke 用户名" >&2
+ exit 2
+}
+
+[[ "${action}" == "create" || "${action}" == "revoke" ]] || usage
+[[ "${account}" =~ ^[a-z][a-z0-9_]{2,31}$ ]] || {
+ echo "用户名只能使用 3–32 位小写字母、数字和下划线,并以字母开头" >&2
+ exit 2
+}
+
+run_mysql() {
+ docker compose exec -T mysql sh -eu -c \
+ 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql --protocol=socket --user=root --batch --skip-column-names'
+}
+
+if [[ "${action}" == "revoke" ]]; then
+ printf "DROP USER IF EXISTS '%s'@'%%';\n" "${account}" | run_mysql
+ echo "MySQL 管理账号已撤销:${account}"
+ exit 0
+fi
+
+[[ "${database}" =~ ^[A-Za-z][A-Za-z0-9_]{1,63}$ ]] || {
+ echo "数据库编码不合法" >&2
+ exit 2
+}
+[[ "${permission}" == "read" || "${permission}" == "write" ]] || usage
+
+database_exists="$(
+ printf "SELECT COUNT(*) FROM information_schema.SCHEMATA WHERE SCHEMA_NAME='%s';\n" "${database}" | run_mysql
+)"
+if [[ "${database_exists}" != "1" ]]; then
+ echo "数据库不存在:${database}" >&2
+ exit 1
+fi
+
+password="$(openssl rand -hex 24)"
+privileges="SELECT, SHOW VIEW"
+if [[ "${permission}" == "write" ]]; then
+ privileges="SELECT, INSERT, UPDATE, DELETE, SHOW VIEW"
+fi
+
+sql="$(printf \
+ "CREATE USER '%s'@'%%' IDENTIFIED BY '%s' PASSWORD EXPIRE INTERVAL 90 DAY FAILED_LOGIN_ATTEMPTS 5 PASSWORD_LOCK_TIME 1; GRANT %s ON \`%s\`.* TO '%s'@'%%';" \
+ "${account}" "${password}" "${privileges}" "${database}" "${account}")"
+printf '%s\n' "${sql}" | run_mysql
+
+echo "账号已创建(完整密码只显示本次):"
+echo " 用户名:${account}"
+echo " 数据库:${database}"
+echo " 权限:${permission}"
+echo " 密码:${password}"
+echo "请通过安全渠道交付,首次保存后清除终端历史;不要通过聊天或邮件明文发送。"
diff --git a/scripts/verify_mysql_backup.sh b/scripts/verify_mysql_backup.sh
new file mode 100755
index 0000000..713c570
--- /dev/null
+++ b/scripts/verify_mysql_backup.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+archive_path="${1:-}"
+passphrase_file="${DATA_BACKUP_PASSPHRASE_FILE:-}"
+
+if [[ -z "${archive_path}" || ! -f "${archive_path}" ]]; then
+ echo "用法:DATA_BACKUP_PASSPHRASE_FILE=/安全路径/密钥 $0 /备份/mysql-all-*.enc" >&2
+ exit 2
+fi
+if [[ -z "${passphrase_file}" || ! -r "${passphrase_file}" ]]; then
+ echo "缺少可读的 DATA_BACKUP_PASSPHRASE_FILE" >&2
+ exit 2
+fi
+
+checksum_file="${archive_path}.sha256"
+if [[ ! -f "${checksum_file}" ]]; then
+ echo "缺少校验文件:${checksum_file}" >&2
+ exit 2
+fi
+expected="$(awk 'NR==1 {print $1}' "${checksum_file}")"
+if command -v sha256sum >/dev/null 2>&1; then
+ actual="$(sha256sum "${archive_path}" | awk '{print $1}')"
+else
+ actual="$(shasum -a 256 "${archive_path}" | awk '{print $1}')"
+fi
+if [[ -z "${expected}" || "${actual}" != "${expected}" ]]; then
+ echo "备份 SHA-256 校验失败" >&2
+ exit 1
+fi
+
+openssl enc -d -aes-256-cbc -pbkdf2 -iter 200000 \
+ -pass "file:${passphrase_file}" -in "${archive_path}" | gzip -t
+
+echo "备份加密、校验和与压缩结构验证通过。"
+echo "生产环境仍应定期恢复到隔离 MySQL 实例进行完整恢复演练。"
diff --git a/tests/test_management_surface.py b/tests/test_management_surface.py
new file mode 100644
index 0000000..413e75c
--- /dev/null
+++ b/tests/test_management_surface.py
@@ -0,0 +1,67 @@
+import unittest
+
+from app.main import app
+
+
+class ManagementSurfaceTests(unittest.TestCase):
+ def test_schema_routes_remain_available(self) -> None:
+ paths = {route.path for route in app.routes}
+
+ self.assertIn("/v1/admin/ontology-schemas", paths)
+ self.assertIn("/v1/admin/ontology-schemas/current", paths)
+ self.assertIn("/v1/admin/ontology-schemas/{schema_id}", paths)
+
+ def test_retired_management_apis_are_not_registered(self) -> None:
+ paths = {route.path for route in app.routes}
+ retired_prefixes = (
+ "/v1/admin/source-profiles",
+ "/v1/admin/batches",
+ "/v1/admin/conflicts",
+ "/v1/admin/entities",
+ "/v1/admin/evidence",
+ "/v1/admin/inventory",
+ "/v1/admin/treatment-history",
+ "/v1/admin/vocabulary",
+ "/v1/admin/aligner",
+ "/v1/admin/publish-jobs",
+ "/v1/admin/acquisition-tasks",
+ "/v1/admin/manual-ingest",
+ "/v1/admin/audit",
+ "/v1/admin/audit-runs",
+ "/v1/admin/question-traces",
+ "/v1/admin/simulation-panel",
+ "/v1/admin/areas",
+ "/v1/admin/notifications",
+ "/v1/admin/agent-settings",
+ )
+
+ for prefix in retired_prefixes:
+ self.assertFalse(
+ any(path == prefix or path.startswith(f"{prefix}/") for path in paths),
+ prefix,
+ )
+
+ for path in (
+ "/v1/admin/plaza/usage",
+ "/v1/admin/plaza/alerts",
+ "/v1/admin/users/{user_id}/areas",
+ ):
+ self.assertNotIn(path, paths)
+
+ def test_retained_core_routes_remain_registered(self) -> None:
+ paths = {route.path for route in app.routes}
+
+ for path in (
+ "/v1/admin/plaza/overview",
+ "/v1/admin/plaza/user-query",
+ "/v1/admin/graph/overview",
+ "/v1/admin/users",
+ "/v1/admin/roles",
+ "/v1/admin/permission-matrix",
+ "/v1/admin/ocr/passport",
+ ):
+ self.assertIn(path, paths)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_mysql_data_center.py b/tests/test_mysql_data_center.py
new file mode 100644
index 0000000..35008ca
--- /dev/null
+++ b/tests/test_mysql_data_center.py
@@ -0,0 +1,235 @@
+import unittest
+from unittest.mock import AsyncMock, Mock, patch
+
+from app.data_platform import interface_service, mysql_service
+from app.data_platform.mysql_service import (
+ _managed_index_name,
+ _mysql_default,
+ _mysql_type,
+ project_database_name,
+ validate_console_sql,
+)
+from app.main import app
+
+
+class _FakeCursor:
+ def __init__(self) -> None:
+ self.executed: list[tuple[str, tuple[object, ...]]] = []
+ self.rowcount = 1
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, _exc_type, _exc, _traceback) -> None:
+ return None
+
+ async def execute(self, statement: str, params: tuple[object, ...] = ()) -> None:
+ self.executed.append((statement, params))
+
+ async def fetchone(self):
+ return None
+
+
+class _FakeConnection:
+ def __init__(self) -> None:
+ self.cursor_instance = _FakeCursor()
+ self.commit_count = 0
+
+ def cursor(self) -> _FakeCursor:
+ return self.cursor_instance
+
+ async def commit(self) -> None:
+ self.commit_count += 1
+
+
+class _FakeConnectionContext:
+ def __init__(self, connection: _FakeConnection) -> None:
+ self.connection = connection
+
+ async def __aenter__(self) -> _FakeConnection:
+ return self.connection
+
+ async def __aexit__(self, _exc_type, _exc, _traceback) -> None:
+ return None
+
+
+class MySQLDataCenterContractTests(unittest.TestCase):
+ def test_physical_database_name_is_stable_and_mysql_safe(self) -> None:
+ self.assertEqual(project_database_name("yunyou_libo"), "yunyou_libo")
+ long_name = "a" * 63
+ physical = project_database_name(long_name)
+ self.assertEqual(physical, long_name)
+ self.assertLessEqual(len(_managed_index_name("uq", long_name)), 64)
+
+ def test_postgresql_registry_types_translate_to_mysql(self) -> None:
+ self.assertEqual(_mysql_type("UUID"), "CHAR(36)")
+ self.assertEqual(_mysql_type("JSONB"), "JSON")
+ self.assertEqual(_mysql_type("TIMESTAMPTZ"), "DATETIME(6)")
+ self.assertEqual(_mysql_type("DOUBLE PRECISION"), "DOUBLE")
+ self.assertEqual(_mysql_type("NUMERIC(12,2)"), "DECIMAL(12,2)")
+ self.assertEqual(_mysql_default("'{}'::jsonb", "JSON"), "(JSON_OBJECT())")
+
+ def test_console_accepts_mysql_discovery_queries_and_frontend_sql(self) -> None:
+ statement, keyword = validate_console_sql("SHOW TABLES;", "demo")
+ self.assertEqual((statement, keyword), ("SHOW TABLES", "show"))
+ statement, keyword = validate_console_sql(
+ 'SELECT * FROM "hotel_profiles" WHERE deleted_at IS NULL LIMIT 100;',
+ "demo",
+ )
+ self.assertEqual(keyword, "select")
+ self.assertIn('"hotel_profiles"', statement)
+
+ def test_console_rejects_cross_database_or_privileged_sql(self) -> None:
+ for sql in (
+ "SHOW DATABASES",
+ "SELECT * FROM platform_control.api_clients",
+ "SELECT * FROM other_database.hotel_profiles",
+ "SELECT * FROM unrelated_database.hotel_profiles",
+ "SHOW TABLES FROM unrelated_database",
+ "DROP TABLE hotel_profiles",
+ ):
+ with self.subTest(sql=sql), self.assertRaises(ValueError):
+ validate_console_sql(sql, "demo")
+
+ def test_console_is_read_only_and_blocks_high_risk_queries(self) -> None:
+ for sql in (
+ "DELETE FROM hotel_profiles WHERE id='1'",
+ "UPDATE hotel_profiles SET name='x' WHERE id='1'",
+ "SELECT SLEEP(10)",
+ "SELECT * FROM hotel_profiles FOR UPDATE",
+ "SELECT id INTO @captured FROM hotel_profiles LIMIT 1",
+ "WITH changed AS (SELECT 1) DELETE FROM hotel_profiles WHERE id='1'",
+ ):
+ with self.subTest(sql=sql), self.assertRaises(ValueError):
+ validate_console_sql(sql, "demo")
+
+ def test_enabled_console_writes_still_require_scoped_update(self) -> None:
+ with patch.object(mysql_service.settings, "data_sql_console_write_enabled", True):
+ with self.assertRaisesRegex(ValueError, "WHERE"):
+ validate_console_sql("UPDATE hotel_profiles SET name='x'", "demo")
+ statement, keyword = validate_console_sql(
+ "UPDATE hotel_profiles SET name='x' WHERE id='1'",
+ "demo",
+ )
+ self.assertEqual(keyword, "update")
+ self.assertIn("WHERE id='1'", statement)
+ with self.assertRaisesRegex(ValueError, "物理删除"):
+ validate_console_sql(
+ "DELETE FROM hotel_profiles WHERE id='1'",
+ "demo",
+ )
+
+ def test_interface_center_routes_are_registered(self) -> None:
+ paths = {route.path for route in app.routes}
+ for path in (
+ "/v1/admin/interface-center/summary",
+ "/v1/admin/interface-center/clients",
+ "/v1/admin/interface-center/policies",
+ "/v1/admin/interface-center/logs",
+ "/v1/admin/interface-center/security-readiness",
+ "/v1/admin/data-platform/security/audit-logs",
+ "/v1/openapi/data/catalog",
+ "/v1/openapi/data/databases/{database_id}/tables/{table_code}/records",
+ ):
+ self.assertIn(path, paths)
+
+
+class MySQLDatabaseProvisioningTests(unittest.IsolatedAsyncioTestCase):
+ async def test_system_database_name_is_rejected(self) -> None:
+ with patch.object(mysql_service, "_ready", AsyncMock()):
+ with self.assertRaisesRegex(ValueError, "系统数据库"):
+ await mysql_service.ensure_project_database("mysql", "mysql", "MySQL")
+
+ async def test_new_database_is_created_without_template_tables(self) -> None:
+ connection = _FakeConnection()
+ database = {
+ "project_id": "empty_demo",
+ "tenant_id": "empty_demo",
+ "display_name": "Empty Demo",
+ "database_name": "empty_demo",
+ "schema_name": "empty_demo",
+ "engine": "mysql",
+ "status": "ready",
+ }
+ get_connection = Mock(
+ side_effect=lambda *_args, **_kwargs: _FakeConnectionContext(connection),
+ )
+
+ with (
+ patch.object(mysql_service, "_ready", AsyncMock()),
+ patch.object(mysql_service, "get_data_conn", get_connection),
+ patch.object(
+ mysql_service,
+ "list_project_table_definitions",
+ AsyncMock(return_value=()),
+ ),
+ patch.object(mysql_service, "_create_physical_table", AsyncMock()) as create_table,
+ patch.object(
+ mysql_service,
+ "get_project_database",
+ AsyncMock(return_value=database),
+ ),
+ ):
+ result = await mysql_service.ensure_project_database(
+ "empty_demo",
+ "empty_demo",
+ "Empty Demo",
+ )
+
+ statements = "\n".join(statement for statement, _params in connection.cursor_instance.executed)
+ self.assertEqual(result, database)
+ self.assertIn("CREATE DATABASE IF NOT EXISTS `empty_demo`", statements)
+ self.assertIn("INSERT INTO project_databases", statements)
+ self.assertNotIn("project_table_definitions", statements)
+ create_table.assert_not_awaited()
+
+
+class InterfaceCenterSecuritySummaryTests(unittest.IsolatedAsyncioTestCase):
+ async def test_dbeaver_summary_separates_ssh_and_database_endpoints(self) -> None:
+ class SummaryCursor:
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, _exc_type, _exc, _traceback) -> None:
+ return None
+
+ async def execute(self, _statement: str) -> None:
+ return None
+
+ async def fetchone(self):
+ return {
+ "active_clients": 0,
+ "active_credentials": 0,
+ "active_policies": 0,
+ "calls_24h": 0,
+ "errors_24h": 0,
+ }
+
+ class SummaryConnection:
+ def cursor(self):
+ return SummaryCursor()
+
+ with (
+ patch.object(interface_service, "_ready", AsyncMock()),
+ patch.object(
+ interface_service,
+ "get_data_conn",
+ Mock(return_value=_FakeConnectionContext(SummaryConnection())),
+ ),
+ patch.object(interface_service.settings, "data_mysql_ssh_host", "db.example.com"),
+ patch.object(interface_service.settings, "data_mysql_ssh_port", 22),
+ patch.object(interface_service.settings, "data_mysql_admin_host", "127.0.0.1"),
+ patch.object(interface_service.settings, "data_mysql_admin_port", 3307),
+ patch.object(interface_service.settings, "mysql_host_bind", "127.0.0.1"),
+ ):
+ summary = await interface_service.interface_summary()
+
+ self.assertEqual(summary["ssh_host"], "db.example.com")
+ self.assertEqual(summary["ssh_port"], 22)
+ self.assertEqual(summary["database_host"], "127.0.0.1")
+ self.assertEqual(summary["database_port"], 3307)
+ self.assertFalse(summary["mysql_publicly_bound"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_security_baseline.py b/tests/test_security_baseline.py
new file mode 100644
index 0000000..eef5d6f
--- /dev/null
+++ b/tests/test_security_baseline.py
@@ -0,0 +1,150 @@
+from datetime import datetime, timedelta, timezone
+import unittest
+from unittest.mock import AsyncMock, patch
+
+from fastapi import HTTPException
+from jose import jwt
+
+from app import db
+from app.auth import (
+ create_access_token,
+ get_current_user,
+ require_admin,
+ require_data_operator,
+)
+from app.config import settings
+from app.data_platform import interface_service
+from app.rate_limit import FixedWindowLimiter
+from app.security_baseline import enforce_security_baseline, security_readiness
+
+
+def _production_settings(**updates):
+ values = {
+ "app_environment": "production",
+ "security_strict_mode": True,
+ "auth_secret": "a" * 48,
+ "interface_api_secret": "b" * 48,
+ "auth_default_password": "S3cure-Admin-Credential-2026!",
+ "database_url": "postgresql://app:" + "p" * 32 + "@127.0.0.1:5432/kg",
+ "data_mysql_url": "mysql://app:" + "m" * 32 + "@127.0.0.1:3306/control",
+ "mysql_host_bind": "127.0.0.1",
+ "data_mysql_direct_access_enabled": True,
+ "data_mysql_ssh_tunnel_required": True,
+ "cors_allowed_origins": "https://data.example.cn",
+ "trusted_hosts": "data.example.cn",
+ "auth_algorithm": "HS256",
+ "auth_token_expire_minutes": 60,
+ "ingest_api_keys": "k" * 40,
+ }
+ values.update(updates)
+ return settings.model_copy(update=values)
+
+
+class SecurityBaselineTests(unittest.TestCase):
+ def test_safe_production_settings_have_no_critical_findings(self) -> None:
+ config = _production_settings()
+ report = security_readiness(config)
+ self.assertEqual(report["status"], "ready")
+ self.assertEqual(report["critical_count"], 0)
+ enforce_security_baseline(config)
+
+ def test_strict_mode_rejects_public_mysql_and_placeholder_secrets(self) -> None:
+ config = _production_settings(
+ mysql_host_bind="0.0.0.0",
+ auth_secret="change-me",
+ )
+ report = security_readiness(config)
+ codes = {item["code"] for item in report["findings"]}
+ self.assertIn("MYSQL_PUBLIC_BIND", codes)
+ self.assertIn("AUTH_SECRET", codes)
+ with self.assertRaisesRegex(RuntimeError, "生产安全基线检查失败"):
+ enforce_security_baseline(config)
+
+ def test_production_cors_requires_https(self) -> None:
+ report = security_readiness(
+ _production_settings(cors_allowed_origins="http://data.example.cn")
+ )
+ self.assertIn(
+ "CORS_WITHOUT_HTTPS",
+ {item["code"] for item in report["findings"]},
+ )
+
+
+class AuthenticationSecurityTests(unittest.IsolatedAsyncioTestCase):
+ async def test_token_uses_issuer_audience_and_live_database_roles(self) -> None:
+ token = create_access_token({"sub": "admin@example.com", "roles": ["collector"]})
+ claims = jwt.decode(
+ token,
+ settings.auth_secret,
+ algorithms=[settings.auth_algorithm],
+ issuer=settings.auth_issuer,
+ audience=settings.auth_audience,
+ )
+ self.assertTrue(claims["jti"])
+ self.assertEqual(claims["iss"], settings.auth_issuer)
+ with patch.object(
+ db,
+ "get_user_auth",
+ AsyncMock(
+ return_value={
+ "username": "admin@example.com",
+ "full_name": "Admin",
+ "status": "active",
+ "roles": ["admin"],
+ }
+ ),
+ ):
+ user = await get_current_user(token)
+ self.assertEqual(user["roles"], ["admin"])
+
+ async def test_disabled_account_invalidates_existing_token(self) -> None:
+ token = create_access_token({"sub": "disabled@example.com", "roles": ["admin"]})
+ with patch.object(
+ db,
+ "get_user_auth",
+ AsyncMock(return_value={"username": "disabled@example.com", "status": "disabled"}),
+ ):
+ with self.assertRaises(HTTPException) as raised:
+ await get_current_user(token)
+ self.assertEqual(raised.exception.status_code, 401)
+
+ async def test_rate_limiter_blocks_at_configured_threshold(self) -> None:
+ limiter = FixedWindowLimiter(limit=2, window_seconds=60, block_seconds=30)
+ self.assertEqual(await limiter.record("user"), 0)
+ self.assertGreater(await limiter.record("user"), 0)
+ self.assertGreater(await limiter.check("user"), 0)
+ await limiter.reset("user")
+ self.assertEqual(await limiter.check("user"), 0)
+
+ def test_admin_dependency_rejects_non_admin(self) -> None:
+ with self.assertRaises(HTTPException) as raised:
+ require_admin({"username": "collector", "roles": ["collector"]})
+ self.assertEqual(raised.exception.status_code, 403)
+
+ def test_data_center_rejects_collector_role(self) -> None:
+ with self.assertRaises(HTTPException) as raised:
+ require_data_operator({"username": "collector", "roles": ["collector"]})
+ self.assertEqual(raised.exception.status_code, 403)
+ operator = require_data_operator(
+ {"username": "operator", "roles": ["operator"]}
+ )
+ self.assertEqual(operator["username"], "operator")
+
+
+class InterfaceCredentialSecurityTests(unittest.TestCase):
+ def test_api_credentials_default_to_short_lived(self) -> None:
+ before = datetime.now(timezone.utc).replace(tzinfo=None)
+ with patch.object(interface_service.settings, "interface_api_default_expiry_days", 30):
+ expiry = interface_service._parse_expiry(None)
+ self.assertGreater(expiry, before + timedelta(days=29))
+ self.assertLess(expiry, before + timedelta(days=31))
+
+ def test_api_credentials_cannot_exceed_maximum_lifetime(self) -> None:
+ too_far = datetime.now(timezone.utc) + timedelta(days=91)
+ with patch.object(interface_service.settings, "interface_api_max_expiry_days", 90):
+ with self.assertRaisesRegex(ValueError, "最长只能签发"):
+ interface_service._parse_expiry(too_far)
+
+
+if __name__ == "__main__":
+ unittest.main()