Add Cloud Tour Libo knowledge graph platform
This commit is contained in:
320
app/api/doc_restructure.py
Normal file
320
app/api/doc_restructure.py
Normal file
@@ -0,0 +1,320 @@
|
||||
"""文档 MD 的 ②结构化重排 + ③事实级校验（供 8765 上传转换后调用）。
|
||||
|
||||
来自 doc-eval 压测验证过的逻辑，纯函数、无外部副作用：
|
||||
· restructure_md(md) 把"忠实转换"的宽表 MD 折叠成「每产品一节 + 干净明细表」的结构化 MD
|
||||
· validate(src, out) 事实级校验：数字保真(幻觉)、按产品价格不串位、内容不被删减、业务规则
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
NUM = re.compile(r"\d+(?:\.\d+)?")
|
||||
|
||||
|
||||
def _clean(s: str) -> str:
|
||||
s = (s or "").replace("<br>", " ")
|
||||
s = re.sub(r"\s+", " ", s).strip()
|
||||
n = len(s)
|
||||
if n > 20: # 折叠"整串精确对折重复"（源 Excel 合并单元格常存双份）
|
||||
for mid in range(n // 2 - 2, n // 2 + 3):
|
||||
if 0 < mid < n:
|
||||
left, right = s[:mid].strip(), s[mid:].strip()
|
||||
if left and len(left) > 10 and left == right:
|
||||
return left
|
||||
return s
|
||||
|
||||
|
||||
def parse_blocks(md: str):
|
||||
lines = md.splitlines()
|
||||
blocks, i, n, buf = [], 0, len(lines), []
|
||||
|
||||
def flush():
|
||||
if buf:
|
||||
for para in re.split(r"\n\s*\n", "\n".join(buf)):
|
||||
if para.strip():
|
||||
blocks.append(("text", para.strip()))
|
||||
buf.clear()
|
||||
|
||||
def is_row(s):
|
||||
return s.strip().startswith("|") and s.count("|") >= 2
|
||||
|
||||
def is_sep(s):
|
||||
return bool(re.match(r"^\s*\|[\s:|-]+\|\s*$", s)) and "-" in s
|
||||
|
||||
while i < n:
|
||||
if is_row(lines[i]) and i + 1 < n and is_sep(lines[i + 1]):
|
||||
flush()
|
||||
header = [c.strip() for c in lines[i].strip().strip("|").split("|")]
|
||||
j, rows = i + 2, []
|
||||
while j < n and is_row(lines[j]) and not is_sep(lines[j]):
|
||||
row = [c.strip() for c in lines[j].strip().strip("|").split("|")]
|
||||
rows.append((row + [""] * len(header))[: len(header)])
|
||||
j += 1
|
||||
blocks.append(("table", header, rows))
|
||||
i = j
|
||||
else:
|
||||
buf.append(lines[i])
|
||||
i += 1
|
||||
flush()
|
||||
return blocks
|
||||
|
||||
|
||||
def classify_columns(header, rows):
|
||||
n, doc, keys, data = len(rows), [], [], []
|
||||
for c in range(len(header)):
|
||||
vals = [r[c] for r in rows]
|
||||
nonempty = [v for v in vals if v]
|
||||
if not nonempty:
|
||||
continue
|
||||
distinct = len(set(nonempty))
|
||||
changes = sum(1 for k in range(1, n) if vals[k] != vals[k - 1])
|
||||
if distinct <= 1:
|
||||
doc.append(c)
|
||||
elif n >= 4 and changes <= max(1, int(n * 0.45)):
|
||||
keys.append(c)
|
||||
else:
|
||||
data.append(c)
|
||||
return doc, keys, data
|
||||
|
||||
|
||||
def _restructure_table(header, rows):
|
||||
doc, keys, data = classify_columns(header, rows)
|
||||
docmeta = {}
|
||||
for c in doc:
|
||||
v = next((r[c] for r in rows if r[c]), "")
|
||||
if v:
|
||||
docmeta[_clean(header[c])] = _clean(v)
|
||||
if not keys or not data:
|
||||
keep = [c for c in range(len(header)) if any(r[c] for r in rows)]
|
||||
out = ["| " + " | ".join(_clean(header[c]) for c in keep) + " |",
|
||||
"| " + " | ".join("---" for _ in keep) + " |"]
|
||||
out += ["| " + " | ".join(r[c] for c in keep) + " |" for r in rows]
|
||||
return docmeta, ["\n".join(out)]
|
||||
title_col = max(keys, key=lambda c: len({r[c] for r in rows if r[c]}))
|
||||
attr_keys = [c for c in keys if c != title_col]
|
||||
sections, cur_key, grp = [], object(), []
|
||||
|
||||
def emit(group):
|
||||
if not group:
|
||||
return
|
||||
seg = [f"## {_clean(group[0][title_col]) or '(未命名)'}"]
|
||||
for c in attr_keys:
|
||||
v = _clean(group[0][c])
|
||||
if v:
|
||||
seg.append(f"- {_clean(header[c])}: {v}")
|
||||
seg.append("")
|
||||
seg.append("| " + " | ".join(_clean(header[c]) for c in data) + " |")
|
||||
seg.append("| " + " | ".join("---" for _ in data) + " |")
|
||||
for r in group:
|
||||
seg.append("| " + " | ".join(_clean(r[c]) for c in data) + " |")
|
||||
sections.append("\n".join(seg))
|
||||
|
||||
for r in rows:
|
||||
kt = tuple(r[c] for c in keys)
|
||||
if kt != cur_key:
|
||||
emit(grp)
|
||||
grp, cur_key = [], kt
|
||||
grp.append(r)
|
||||
emit(grp)
|
||||
return docmeta, sections
|
||||
|
||||
|
||||
def restructure_md(md: str, source: str = "") -> str:
|
||||
blocks = parse_blocks(md)
|
||||
docmeta, body, seen, title = {}, [], set(), ""
|
||||
for blk in blocks:
|
||||
if blk[0] == "text":
|
||||
para = blk[1]
|
||||
key = re.sub(r"\s+", "", para)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
if not title and not para.startswith(("#", "|")):
|
||||
title = para.split("\n")[0]
|
||||
if "\n" not in para:
|
||||
continue
|
||||
if re.fullmatch(r"##\s*Sheet\d+\s*", para):
|
||||
continue
|
||||
body.append(("text", para))
|
||||
else:
|
||||
dm, secs = _restructure_table(blk[1], blk[2])
|
||||
docmeta.update(dm)
|
||||
body.append(("secs", secs))
|
||||
out = ["## 基本信息", ""]
|
||||
if source:
|
||||
out.append(f"- 来源文件: {source}")
|
||||
for k, v in docmeta.items():
|
||||
out.append(f"- {k}: {v}")
|
||||
out.append("")
|
||||
if title:
|
||||
out.append(f"# {title}\n")
|
||||
for kind, val in body:
|
||||
if kind == "text" and not val.startswith("#"):
|
||||
out.append(val + "\n")
|
||||
elif kind == "text":
|
||||
out.append(val + "\n")
|
||||
else:
|
||||
out.extend(s + "\n" for s in val)
|
||||
return "\n".join(out).strip() + "\n"
|
||||
|
||||
|
||||
# ---------- ②-LLM 语义结构化（类型自适应，用于行程单/复杂件；价格表走免费确定性）----------
|
||||
STRUCTURE_PROMPT = """你是旅游 ToB 文档结构化助手。把下面从源文件转出的原始 Markdown，重排成"结构化科学 MD"，供后期 LLM 检索入库。
|
||||
|
||||
【先判断文档类型，套用对应结构】
|
||||
· 报价单/价目表：顶部「## 基本信息」表(供应商/有效期/出发地/币种)；每个产品一个 `## 产品：<名称>` 小节(拼团类型/起订人数/景区小交通费/退费政策等)；价格用「房型 | 成人价 | 儿童价 | 单房差」表；通用规则、参考酒店各自独立小节。
|
||||
· 行程单：顶部「## 基本信息」表(产品名/天数/车型/出发地)；按天分 `## D1 <当天路线>`、`## D2 …` 小节，每天完整保留 行程详情/餐食(早中晚)/住宿/交通；费用包含、费用不含、购物场所、预订须知、温馨提示 各自独立小节。
|
||||
· 资源表(酒店/车辆/餐厅)：每个资源一个小节 + 属性。
|
||||
· 其它：用标题分层、表格如实保留。
|
||||
|
||||
【铁律，违反即事故】
|
||||
1. 完整保留原文每一条信息、逐条照搬——严禁摘要、严禁概括、严禁删减合并、严禁省略。这是入库用的全量数据；输出篇幅应与原文相当。
|
||||
2. 所有数值/价格/日期/时间/电话严格来自原文，一字不改、不新增、不计算、不脑补；看不清就原样抄。
|
||||
3. 可纠正明显的"表头误标"(如"车型"列内容其实是"N人拼小团"→改标「拼团类型」)，但只改标签、不改数值。
|
||||
4. 元数据用可见的「## 基本信息」表，不要 YAML frontmatter。
|
||||
只输出 MD，不要任何解释。"""
|
||||
|
||||
|
||||
def resolve_llm_key() -> str | None:
|
||||
"""LLM key 解析顺序：环境变量 LLM_API_KEY / DEEPSEEK_API_KEY → ~/.ark_key 文件。"""
|
||||
import os
|
||||
k = os.environ.get("LLM_API_KEY") or os.environ.get("DEEPSEEK_API_KEY")
|
||||
if k:
|
||||
return k.strip()
|
||||
try:
|
||||
with open(os.path.expanduser("~/.ark_key")) as f:
|
||||
return f.read().strip() or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def structure_llm(md: str, api_key: str) -> str:
|
||||
"""调 LLM(默认火山豆包，可用 LLM_BASE_URL/LLM_MODEL 覆盖)做类型自适应结构化，带重试退避。"""
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
base_url = os.environ.get("LLM_BASE_URL", "https://ark.cn-beijing.volces.com/api/v3")
|
||||
model = os.environ.get("LLM_MODEL", "doubao-seed-2-0-lite-260428")
|
||||
payload = {"model": model, "temperature": 0,
|
||||
"messages": [{"role": "system", "content": STRUCTURE_PROMPT},
|
||||
{"role": "user", "content": md}]}
|
||||
last = None
|
||||
for attempt in range(4):
|
||||
try:
|
||||
r = requests.post(f"{base_url}/chat/completions",
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
json=payload, timeout=180)
|
||||
r.raise_for_status()
|
||||
return r.json()["choices"][0]["message"]["content"]
|
||||
except requests.exceptions.RequestException as e:
|
||||
last = e
|
||||
if attempt < 3:
|
||||
time.sleep(2 * (attempt + 1))
|
||||
raise last
|
||||
|
||||
|
||||
# ---------- ③ 事实级校验 ----------
|
||||
def _wide_product_numbers(md: str) -> dict:
|
||||
out = defaultdict(set)
|
||||
for blk in parse_blocks(md):
|
||||
if blk[0] != "table":
|
||||
continue
|
||||
header, rows = blk[1], blk[2]
|
||||
_doc, keys, _data = classify_columns(header, rows)
|
||||
if not keys:
|
||||
continue
|
||||
tcol = max(keys, key=lambda c: len({r[c] for r in rows if r[c]}))
|
||||
for r in rows:
|
||||
for cell in r:
|
||||
out[_clean(r[tcol])].update(NUM.findall(cell))
|
||||
return out
|
||||
|
||||
|
||||
def _section_numbers(md: str) -> dict:
|
||||
out, cur = defaultdict(set), None
|
||||
for line in md.splitlines():
|
||||
m = re.match(r"^##\s+(?:产品[:：])?\s*(.+)$", line)
|
||||
if m:
|
||||
cur = _clean(m.group(1))
|
||||
elif cur and line.count("|") >= 2:
|
||||
out[cur].update(NUM.findall(line))
|
||||
return out
|
||||
|
||||
|
||||
def _has_price_table(md: str) -> bool:
|
||||
for blk in parse_blocks(md):
|
||||
if blk[0] != "table":
|
||||
continue
|
||||
_doc, keys, data = classify_columns(blk[1], blk[2])
|
||||
if keys and len(data) >= 2:
|
||||
vals = [r[c] for r in blk[2] for c in data]
|
||||
nums = [float(v) for v in vals if re.fullmatch(r"\d+(?:\.\d+)?", v)]
|
||||
if nums and sum(x >= 100 for x in nums) >= len(nums) * 0.5:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _strip_fm(md: str) -> str:
|
||||
m = re.match(r"^---\n.*?\n---\n", md, re.S)
|
||||
return md[m.end():] if m else md
|
||||
|
||||
|
||||
def validate(src_md: str, out_md: str) -> dict:
|
||||
out_body = _strip_fm(out_md)
|
||||
# 去掉"来源文件"行（文件名里的日期是元数据、不是业务数据，不参与数字校验）
|
||||
out_body = "\n".join(l for l in out_body.splitlines() if not l.lstrip().startswith("- 来源文件"))
|
||||
src_nums, out_nums = Counter(NUM.findall(src_md)), Counter(NUM.findall(out_body))
|
||||
invented = {n: c for n, c in (out_nums - src_nums).items() if n not in src_nums}
|
||||
|
||||
misassigned: dict = {}
|
||||
if _has_price_table(src_md):
|
||||
src_by = _wide_product_numbers(src_md)
|
||||
for prod, nums in _section_numbers(out_md).items():
|
||||
match = next((src_by[k] for k in src_by if k and (k in prod or prod in k)), None)
|
||||
if match is None:
|
||||
continue
|
||||
bad = {n for n in (nums - match) if float(n) >= 100 and not 1900 <= float(n) <= 2099}
|
||||
if bad:
|
||||
misassigned[prod] = sorted(bad)
|
||||
shared = {n for n, c in Counter(n for ns in misassigned.values() for n in ns).items()
|
||||
if c >= max(3, int(len(misassigned) * 0.5))}
|
||||
misassigned = {p: [n for n in ns if n not in shared]
|
||||
for p, ns in misassigned.items() if [n for n in ns if n not in shared]}
|
||||
|
||||
def _prose(md):
|
||||
return sum(len(re.sub(r"\s", "", l)) for l in md.splitlines() if l.count("|") < 2)
|
||||
|
||||
sp, op = _prose(src_md), _prose(out_body)
|
||||
content_loss = f"②正文{op}字 / ①{sp}字，疑似删减" if (sp > 300 and op < sp * 0.5) else None
|
||||
|
||||
warn = []
|
||||
for line in out_md.splitlines():
|
||||
if line.count("|") >= 3:
|
||||
cells = [c.strip() for c in line.strip("|").split("|")]
|
||||
nums = [c for c in cells if re.fullmatch(r"\d+(?:\.\d+)?", c)]
|
||||
if len(nums) >= 2 and float(nums[0]) < float(nums[1]):
|
||||
warn.append(f"成人价<儿童价? {line.strip()[:50]}")
|
||||
return {
|
||||
"passed": not invented and not misassigned and not content_loss,
|
||||
"invented": invented,
|
||||
"misassigned": misassigned,
|
||||
"content_loss": content_loss,
|
||||
"business_warn": warn[:5],
|
||||
}
|
||||
|
||||
|
||||
def build_structured(md: str, source: str = "") -> tuple[str, dict, str]:
|
||||
"""8765 用：自动路由结构化。价格表且确定性已过 → 免费确定性重排；
|
||||
行程单/复杂件/确定性没过 → 若有 key 则上 LLM(类型自适应)。返回 (结构化MD, 校验, 模式)。"""
|
||||
deterministic = restructure_md(md, source=source)
|
||||
rep = validate(md, deterministic)
|
||||
key = resolve_llm_key()
|
||||
if key and not (_has_price_table(md) and rep["passed"]):
|
||||
try:
|
||||
llm_md = structure_llm(md, key)
|
||||
if llm_md and llm_md.strip():
|
||||
return llm_md, validate(md, llm_md), "llm"
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return deterministic, rep, "deterministic"
|
||||
Reference in New Issue
Block a user