Add Cloud Tour Libo knowledge graph platform

scripts/amap_spatial_collect.py

@@ -605,6 +605,7 @@ def upsert_graph_places(graph_name: str, rows: list[dict[str, Any]]) -> None:
             "MERGE (dist)-[:PART_OF]->(city) "
             "MERGE (city)-[:PART_OF]->(prov) "
             "MERGE (c9:GeoCell {h3_id:$h3_r9}) SET c9.resolution=9 "
+            "MERGE (c9)-[:LOCATED_IN]->(dist) "
             "MERGE (p)-[:IN_H3_R9]->(c9)"
         )
         params = {

scripts/douyin_fetch_data.py

@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+"""单独拉取抖音数据的运行入口。
+
+这个文件不重写采集逻辑，而是复用系统原版:
+    app/agents/douyin_agent.py
+
+常用命令:
+    python3 scripts/douyin_login.py
+    python3 scripts/douyin_fetch_data.py --name "某某酒店" --city "贵阳"
+    python3 scripts/douyin_fetch_data.py --input hotels.txt --city "贵阳" --save-evidence
+
+输出:
+    默认写到 data/douyin_probe/fetch_<时间>.json
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import csv
+import json
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from app.agents.douyin_agent import (  # noqa: E402
+    _collect,
+    _parse_dy_comments,
+    _parse_dy_notes,
+)
+
+
+DEFAULT_OUT_DIR = ROOT / "data" / "douyin_probe"
+
+
+def _ts() -> str:
+    return datetime.now().strftime("%Y%m%d_%H%M%S")
+
+
+def _read_names(path: str | None) -> list[str]:
+    if not path:
+        return []
+    names: list[str] = []
+    for line in Path(path).read_text(encoding="utf-8").splitlines():
+        value = line.strip()
+        if value and not value.startswith("#"):
+            names.append(value)
+    return names
+
+
+def _dedupe_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    deduped: list[dict[str, Any]] = []
+    seen: set[tuple[str, str, str]] = set()
+    for record in records:
+        key = (
+            str(record.get("platform") or ""),
+            str(record.get("kind") or ""),
+            str(record.get("source_id") or record.get("url") or ""),
+        )
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(record)
+    return deduped
+
+
+def _record_for_csv(record: dict[str, Any]) -> dict[str, Any]:
+    fields = [
+        "platform",
+        "kind",
+        "source_id",
+        "url",
+        "entity_name",
+        "place_natural_key",
+        "keyword",
+        "title",
+        "content",
+        "author",
+        "author_id",
+        "likes",
+        "comments",
+        "collects",
+        "shares",
+        "publish_time",
+        "location",
+    ]
+    return {field: record.get(field, "") for field in fields}
+
+
+def _write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as file:
+        for record in records:
+            file.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+
+def _write_csv(path: Path, records: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    rows = [_record_for_csv(record) for record in records]
+    fieldnames = list(rows[0].keys()) if rows else list(_record_for_csv({}).keys())
+    with path.open("w", encoding="utf-8-sig", newline="") as file:
+        writer = csv.DictWriter(file, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def _build_keyword(name: str, city: str, suffix: str, keyword: str | None) -> str:
+    if keyword:
+        return keyword.strip()
+    parts = [city.strip(), name.strip(), suffix.strip()]
+    return " ".join(part for part in parts if part)
+
+
+def _make_tab_picker(preferred_tab: str):
+    def pick(labels: list[str]) -> str | None:
+        if not labels:
+            return None
+        if preferred_tab and preferred_tab in labels:
+            return preferred_tab
+        if "综合" in labels:
+            return "综合"
+        return labels[0]
+
+    return pick
+
+
+def fetch_one(
+    *,
+    name: str,
+    city: str,
+    suffix: str,
+    keyword: str | None,
+    tab: str,
+    deep: bool,
+    place_key: str | None,
+) -> dict[str, Any]:
+    query = _build_keyword(name, city, suffix, keyword)
+    print(f"[douyin] 开始采集: name={name} keyword={query}")
+    result = _collect(query, _make_tab_picker(tab), deep=deep)
+
+    if result.get("logged_in") is False:
+        return {
+            "ok": False,
+            "need_login": True,
+            "name": name,
+            "keyword": query,
+            "summary": "抖音未登录。请先运行 python3 scripts/douyin_login.py 完成一次登录。",
+            "notes": [],
+            "comments": [],
+            "records": [],
+            "raw_api_count": 0,
+            "api_url_count": result.get("api_url_count", 0),
+            "tabs": result.get("tabs", []),
+        }
+
+    if result.get("error"):
+        return {
+            "ok": False,
+            "need_login": False,
+            "name": name,
+            "keyword": query,
+            "summary": f"采集异常: {result.get('error')}",
+            "notes": [],
+            "comments": [],
+            "records": [],
+            "raw_api_count": 0,
+            "api_url_count": result.get("api_url_count", 0),
+            "tabs": result.get("tabs", []),
+        }
+
+    raw_api = result.get("raw_api") or []
+    notes = _parse_dy_notes(raw_api, name, query)
+    comments = _parse_dy_comments(raw_api, name, query)
+    records = _dedupe_records(notes + comments)
+    for record in records:
+        if place_key:
+            record["place_natural_key"] = place_key
+
+    return {
+        "ok": True,
+        "need_login": False,
+        "name": name,
+        "keyword": query,
+        "summary": f"视频 {len(notes)} 条，评论 {len(comments)} 条，去重后 {len(records)} 条。",
+        "notes": notes,
+        "comments": comments,
+        "records": records,
+        "raw_api_count": len(raw_api),
+        "api_url_count": result.get("api_url_count", 0),
+        "tabs": result.get("tabs", []),
+    }
+
+
+async def _save_evidence(records: list[dict[str, Any]]) -> int:
+    if not records:
+        return 0
+    from app.db import close_pool, init_pool, sa_save_evidence
+
+    await init_pool()
+    try:
+        return await sa_save_evidence(records)
+    finally:
+        await close_pool()
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="按酒店/关键词单独获取抖音视频与评论数据")
+    parser.add_argument("--name", help="单个酒店/地点名称")
+    parser.add_argument("--input", help="批量名称文件，每行一个酒店/地点名称")
+    parser.add_argument("--city", default="贵阳", help="城市前缀，默认: 贵阳")
+    parser.add_argument("--suffix", default="", help="附加搜索词，例如: 酒店 入住 评价")
+    parser.add_argument("--keyword", help="完整搜索关键词；传入后会忽略 --city/--name/--suffix 拼接")
+    parser.add_argument("--tab", default="综合", help="优先选择的抖音搜索 tab，默认: 综合")
+    parser.add_argument("--place-key", help="写入记录的 place_natural_key，单名称时可用")
+    parser.add_argument("--no-deep", action="store_true", help="只采搜索页，不进入视频详情深采评论")
+    parser.add_argument("--save-evidence", action="store_true", help="把 records 写入系统 Evidence 表")
+    parser.add_argument("--out", help="JSON 输出路径")
+    parser.add_argument("--jsonl-out", help="JSONL records 输出路径")
+    parser.add_argument("--csv-out", help="CSV records 输出路径")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    names = []
+    if args.name:
+        names.append(args.name.strip())
+    names.extend(_read_names(args.input))
+    names = [name for name in names if name]
+
+    if args.keyword and not names:
+        names = [args.keyword.strip()]
+    if not names:
+        raise SystemExit("请提供 --name、--input 或 --keyword")
+    if args.keyword and len(names) > 1:
+        raise SystemExit("--keyword 是完整单次搜索词，不能和多名称 --input 同时使用")
+    if args.place_key and len(names) > 1:
+        raise SystemExit("--place-key 只适合单个 --name 使用，批量请不要传")
+
+    all_results: list[dict[str, Any]] = []
+    all_records: list[dict[str, Any]] = []
+    for name in names:
+        result = fetch_one(
+            name=name,
+            city=args.city,
+            suffix=args.suffix,
+            keyword=args.keyword,
+            tab=args.tab,
+            deep=not args.no_deep,
+            place_key=args.place_key,
+        )
+        print(f"[douyin] {result['summary']}")
+        all_results.append(result)
+        all_records.extend(result.get("records") or [])
+
+    all_records = _dedupe_records(all_records)
+    saved = 0
+    if args.save_evidence:
+        saved = asyncio.run(_save_evidence(all_records))
+        print(f"[douyin] 已写入 Evidence: {saved}")
+
+    out_dir = DEFAULT_OUT_DIR
+    out_path = Path(args.out) if args.out else out_dir / f"fetch_{_ts()}.json"
+    jsonl_path = Path(args.jsonl_out) if args.jsonl_out else out_path.with_suffix(".jsonl")
+    csv_path = Path(args.csv_out) if args.csv_out else out_path.with_suffix(".csv")
+
+    payload = {
+        "ok": all(result.get("ok") for result in all_results),
+        "need_login": any(result.get("need_login") for result in all_results),
+        "saved_evidence": saved,
+        "total_records": len(all_records),
+        "results": all_results,
+    }
+    _write_json(out_path, payload)
+    _write_jsonl(jsonl_path, all_records)
+    _write_csv(csv_path, all_records)
+
+    print(f"[douyin] JSON:  {out_path}")
+    print(f"[douyin] JSONL: {jsonl_path}")
+    print(f"[douyin] CSV:   {csv_path}")
+
+
+if __name__ == "__main__":
+    os.chdir(ROOT)
+    main()

scripts/enrich_yunyou_libo_food_deals_reviews.py

@@ -0,0 +1,276 @@
+#!/usr/bin/env python3
+"""Attach structured Dianping deal, tag, and review payloads to Libo food nodes."""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+from falkordb import FalkorDB
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+from app.config import settings
+
+
+DEFAULT_SOURCE_DIR = Path(
+    "/Users/xuexue/Desktop/荔波小七孔源数据/大众点评/raw"
+)
+
+REVIEW_THEMES: tuple[tuple[str, tuple[str, ...]], ...] = (
+    ("口味评价", ("味道", "口味", "好吃", "鲜香", "酸汤", "烤鱼")),
+    ("服务体验", ("服务", "老板", "店员", "热情", "招待")),
+    ("环境体验", ("环境", "装修", "干净", "卫生", "民族风")),
+    ("菜品分量", ("分量", "份量", "量大", "量足")),
+    ("性价比", ("性价比", "价格", "价位", "实惠")),
+    ("停车便利", ("停车", "停车场")),
+    ("排队情况", ("排队", "等位")),
+    ("游客推荐", ("推荐", "必吃", "值得", "不踩雷")),
+)
+
+
+def clean(value: Any) -> str:
+    return "" if value is None else str(value).strip()
+
+
+def number(value: Any) -> float | int | None:
+    text = clean(value).replace("￥", "").replace("¥", "").replace(",", "")
+    if not text:
+        return None
+    try:
+        result = float(text)
+    except ValueError:
+        return None
+    return int(result) if result.is_integer() else result
+
+
+def compact(record: dict[str, Any]) -> dict[str, Any]:
+    return {
+        key: value
+        for key, value in record.items()
+        if value not in (None, "", [], {})
+    }
+
+
+def split_values(value: Any) -> list[str]:
+    return [
+        item.strip()
+        for item in re.split(r"\s*[|｜、,，]\s*", clean(value))
+        if item.strip()
+    ]
+
+
+def normalize_title(value: Any) -> str:
+    return re.sub(r"[\s【】\[\]（）()·｜|/\\\-—_]", "", clean(value))
+
+
+def load_raw_records(source_dir: Path) -> dict[str, dict[str, Any]]:
+    records: dict[str, dict[str, Any]] = {}
+    for path in sorted(source_dir.glob("*.json")):
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        if not isinstance(payload, dict):
+            continue
+        identifiers = {
+            path.stem.removeprefix("dianping_"),
+            clean(payload.get("POI_ID")),
+            clean(payload.get("店铺ID")),
+            clean(payload.get("店铺UUID")),
+        }
+        for identifier in identifiers:
+            if identifier:
+                records[identifier] = payload
+    return records
+
+
+def deal_image(
+    title: str,
+    sequence: int,
+    recommended_dishes: list[dict[str, Any]],
+    shop_image: str,
+) -> str:
+    normalized_title = normalize_title(title)
+    usable_images = []
+    for dish in recommended_dishes:
+        image_url = clean(dish.get("image"))
+        if not image_url:
+            continue
+        usable_images.append(image_url)
+        dish_name = normalize_title(dish.get("name"))
+        if dish_name and dish_name in normalized_title:
+            return image_url
+    if usable_images:
+        return usable_images[(sequence - 1) % len(usable_images)]
+    return shop_image
+
+
+def build_deals(payload: dict[str, Any]) -> list[dict[str, Any]]:
+    raw_deals = payload.get("团购信息") or []
+    if not isinstance(raw_deals, list):
+        return []
+    recommended_dishes = payload.get("推荐菜详情") or []
+    if not isinstance(recommended_dishes, list):
+        recommended_dishes = []
+    shop_image = clean(payload.get("店铺图片"))
+    deals: list[dict[str, Any]] = []
+    for sequence, item in enumerate(raw_deals, 1):
+        if not isinstance(item, dict):
+            continue
+        title = clean(item.get("title"))
+        if not title:
+            continue
+        terms = split_values(item.get("tags"))
+        if not terms:
+            terms = [
+                value
+                for value in re.split(r"\s+", clean(item.get("dishes")))
+                if value
+            ]
+        deals.append(compact({
+            "deal_id": f"{clean(payload.get('店铺ID')) or clean(payload.get('POI_ID'))}:{sequence}",
+            "title": title,
+            "price": number(item.get("price")),
+            "original_price": number(item.get("original_price")),
+            "discount": clean(item.get("discount")),
+            "terms": terms,
+            "image_url": deal_image(
+                title,
+                sequence,
+                [dish for dish in recommended_dishes if isinstance(dish, dict)],
+                shop_image,
+            ),
+        }))
+    return deals
+
+
+def build_reviews(payload: dict[str, Any]) -> list[dict[str, Any]]:
+    raw_reviews = payload.get("评论数据") or []
+    if not isinstance(raw_reviews, list):
+        return []
+    source_id = (
+        clean(payload.get("店铺ID"))
+        or clean(payload.get("POI_ID"))
+        or "dianping"
+    )
+    reviews: list[dict[str, Any]] = []
+    for sequence, item in enumerate(raw_reviews, 1):
+        if not isinstance(item, dict):
+            continue
+        content = clean(item.get("评论内容"))
+        if not content:
+            continue
+        reviews.append(compact({
+            "review_id": f"{source_id}:{sequence}",
+            "nickname": clean(item.get("用户名")) or "匿名用户",
+            "review_at": clean(item.get("评论时间")),
+            "rating_label": clean(item.get("评分")),
+            "stars": number(item.get("星级")),
+            "content": content,
+            "image_urls": split_values(item.get("评论图片")),
+            "image_count": number(item.get("图片数量")),
+        }))
+    return reviews
+
+
+def build_review_tags(
+    payload: dict[str, Any],
+    reviews: list[dict[str, Any]],
+    existing_review_tags: str,
+) -> list[dict[str, Any]]:
+    explicit = split_values(payload.get("评论标签")) or split_values(existing_review_tags)
+    if explicit:
+        return [
+            {"label": label, "count": 0}
+            for label in explicit[:10]
+        ]
+
+    theme_counts: Counter[str] = Counter()
+    for review in reviews:
+        content = clean(review.get("content"))
+        for label, keywords in REVIEW_THEMES:
+            if any(keyword in content for keyword in keywords):
+                theme_counts[label] += 1
+    return [
+        {"label": label, "count": count}
+        for label, count in theme_counts.most_common(8)
+    ]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
+    parser.add_argument("--graph-name", default="yunyou_libo")
+    args = parser.parse_args()
+
+    raw_records = load_raw_records(args.source_dir)
+    graph = FalkorDB(
+        host=settings.falkordb_host,
+        port=settings.falkordb_port,
+    ).select_graph(args.graph_name)
+    rows = graph.query(
+        "MATCH (n:FoodPlace) WHERE n.dianping_name IS NOT NULL "
+        "RETURN n.element_id,n.dianping_source_poi_id,n.dianping_shop_id,"
+        "n.dianping_shop_uuid,n.gaode_poi_id,n.dianping_review_tags"
+    ).result_set
+
+    updated_nodes = 0
+    stored_deals = 0
+    stored_reviews = 0
+    stored_tags = 0
+    missing_raw = 0
+    for row in rows:
+        element_id = clean(row[0])
+        identifiers = [clean(value) for value in row[1:5] if clean(value)]
+        existing_review_tags = clean(row[5])
+        payload = next(
+            (raw_records[identifier] for identifier in identifiers if identifier in raw_records),
+            None,
+        )
+        if payload is None:
+            missing_raw += 1
+            continue
+        deals = build_deals(payload)
+        reviews = build_reviews(payload)
+        tags = build_review_tags(payload, reviews, existing_review_tags)
+        result = graph.query(
+            "MATCH (n:FoodPlace {element_id:$element_id}) "
+            "SET n.group_buy_items_json=$deals,"
+            "n.food_review_items_json=$reviews,"
+            "n.food_review_tags_json=$tags "
+            "RETURN count(n)",
+            {
+                "element_id": element_id,
+                "deals": json.dumps(deals, ensure_ascii=False, separators=(",", ":")),
+                "reviews": json.dumps(reviews, ensure_ascii=False, separators=(",", ":")),
+                "tags": json.dumps(tags, ensure_ascii=False, separators=(",", ":")),
+            },
+        ).result_set
+        changed = int(result[0][0] if result else 0)
+        if changed:
+            updated_nodes += changed
+            stored_deals += len(deals) * changed
+            stored_reviews += len(reviews) * changed
+            stored_tags += len(tags) * changed
+
+    print(json.dumps({
+        "graph_name": args.graph_name,
+        "raw_record_identifiers": len(raw_records),
+        "graph_food_nodes": len(rows),
+        "updated_nodes": updated_nodes,
+        "missing_raw_nodes": missing_raw,
+        "stored_deals": stored_deals,
+        "stored_reviews": stored_reviews,
+        "stored_review_tags": stored_tags,
+    }, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

scripts/enrich_yunyou_libo_hotel_facility_categories.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+"""Attach categorized Ctrip facilities and verified facility photos to Hotel nodes."""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from falkordb import FalkorDB
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+from app.config import settings
+
+
+CATEGORY_ORDER = (
+    "热门设施",
+    "康体设施",
+    "交通服务",
+    "餐饮服务",
+    "娱乐活动设施",
+    "前台服务",
+    "公共区",
+    "儿童设施服务",
+    "商务服务",
+    "清洁服务",
+    "安全与安保",
+    "无障碍设施服务",
+    "网络与通讯",
+    "房型特色",
+    "设施服务",
+    "更多设施",
+)
+DEFAULT_SOURCE_ROOT = Path("/Users/xuexue/Documents/云游荔波/outputs")
+
+
+def clean(value: Any) -> str:
+    return "" if value is None else str(value).strip()
+
+
+def parse_rank(value: Any) -> int:
+    try:
+        return int(float(clean(value) or "9999"))
+    except ValueError:
+        return 9999
+
+
+def resolve_category(row: dict[str, str]) -> str:
+    category = clean(row.get("设施分类"))
+    group = clean(row.get("设施分组"))
+    if category:
+        return category
+    if group == "热门设施":
+        return "热门设施"
+    if group and group not in {"服务及设施", "更多设施"}:
+        return group
+    return "更多设施"
+
+
+def load_facility_payloads(source_root: Path) -> dict[str, dict[str, Any]]:
+    facilities: dict[str, dict[str, dict[str, dict[str, Any]]]] = defaultdict(
+        lambda: defaultdict(dict)
+    )
+    facility_images: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict)
+
+    facility_paths = sorted(source_root.glob("**/csv/携程_酒店详情_设施服务.csv"))
+    image_paths = sorted(source_root.glob("**/csv/携程_酒店详情_酒店图片.csv"))
+    if not facility_paths:
+        raise FileNotFoundError(f"未找到携程设施服务CSV：{source_root}")
+
+    for path in facility_paths:
+        with path.open("r", encoding="utf-8-sig", newline="") as stream:
+            for row in csv.DictReader(stream):
+                hotel_id = clean(row.get("酒店ID"))
+                name = clean(row.get("设施名称"))
+                if not hotel_id or not name:
+                    continue
+                category = resolve_category(row)
+                item = {
+                    "name": name,
+                    "fee": clean(row.get("收费类型")),
+                    "description": clean(row.get("设施说明")),
+                    "rank": parse_rank(row.get("排序")),
+                    "source_url": clean(row.get("来源URL")),
+                }
+                key = name.casefold()
+                current = facilities[hotel_id][category].get(key)
+                current_richness = (
+                    int(bool(current and current.get("fee")))
+                    + int(bool(current and current.get("description")))
+                    + int(bool(current and current.get("source_url")))
+                )
+                item_richness = (
+                    int(bool(item["fee"]))
+                    + int(bool(item["description"]))
+                    + int(bool(item["source_url"]))
+                )
+                if (
+                    current is None
+                    or item_richness > current_richness
+                    or (
+                        item_richness == current_richness
+                        and item["rank"] < current["rank"]
+                    )
+                ):
+                    facilities[hotel_id][category][key] = item
+
+    for path in image_paths:
+        with path.open("r", encoding="utf-8-sig", newline="") as stream:
+            for row in csv.DictReader(stream):
+                hotel_id = clean(row.get("酒店ID"))
+                image_type = clean(row.get("图片类型")).lower()
+                url = clean(row.get("图片URL"))
+                if not hotel_id or image_type != "facility" or not url:
+                    continue
+                image = {
+                    "url": url,
+                    "rank": parse_rank(row.get("图片序号")),
+                    "source_url": clean(row.get("来源URL")),
+                }
+                current = facility_images[hotel_id].get(url)
+                if current is None or image["rank"] < current["rank"]:
+                    facility_images[hotel_id][url] = image
+
+    category_rank = {category: index for index, category in enumerate(CATEGORY_ORDER)}
+    payloads: dict[str, dict[str, Any]] = {}
+    for hotel_id, grouped_categories in facilities.items():
+        category_items: dict[str, list[dict[str, Any]]] = {}
+        ordered_categories = sorted(
+            grouped_categories,
+            key=lambda category: (category_rank.get(category, 999), category),
+        )
+        for category in ordered_categories:
+            items = sorted(
+                grouped_categories[category].values(),
+                key=lambda item: (item["rank"], item["name"]),
+            )
+            if items:
+                category_items[category] = items
+        images = sorted(
+            facility_images.get(hotel_id, {}).values(),
+            key=lambda image: (image["rank"], image["url"]),
+        )[:8]
+        payloads[hotel_id] = {
+            "categories": category_items,
+            "images": images,
+        }
+    return payloads
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--source-root", type=Path, default=DEFAULT_SOURCE_ROOT)
+    parser.add_argument("--graph-name", default="yunyou_libo")
+    args = parser.parse_args()
+
+    payloads = load_facility_payloads(args.source_root)
+    graph = FalkorDB(
+        host=settings.falkordb_host,
+        port=settings.falkordb_port,
+    ).select_graph(args.graph_name)
+
+    hotel_rows = graph.query(
+        "MATCH (n:Hotel) WHERE n.ctrip_hotel_id IS NOT NULL "
+        "RETURN DISTINCT n.ctrip_hotel_id"
+    ).result_set
+    graph_hotel_ids = {clean(row[0]) for row in hotel_rows if row and clean(row[0])}
+
+    updated_hotels = 0
+    updated_facilities = 0
+    updated_images = 0
+    hotels_with_images = 0
+    for hotel_id in sorted(graph_hotel_ids & payloads.keys()):
+        payload = payloads[hotel_id]
+        serialized = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
+        result = graph.query(
+            "MATCH (n:Hotel) WHERE n.ctrip_hotel_id=$hotel_id "
+            "SET n.facility_category_items_json=$payload "
+            "RETURN count(n)",
+            {"hotel_id": hotel_id, "payload": serialized},
+        ).result_set
+        changed = int(result[0][0] if result else 0)
+        if not changed:
+            continue
+        facility_count = sum(len(items) for items in payload["categories"].values())
+        image_count = len(payload["images"])
+        updated_hotels += changed
+        updated_facilities += facility_count * changed
+        updated_images += image_count * changed
+        if image_count:
+            hotels_with_images += changed
+
+    print(json.dumps({
+        "graph_name": args.graph_name,
+        "source_hotels": len(payloads),
+        "graph_hotels_with_ctrip_id": len(graph_hotel_ids),
+        "updated_hotels": updated_hotels,
+        "stored_facilities": updated_facilities,
+        "hotels_with_facility_images": hotels_with_images,
+        "stored_facility_images": updated_images,
+    }, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

scripts/enrich_yunyou_libo_hotel_nearby_categories.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Attach categorized Ctrip nearby-place samples to yunyou_libo Hotel nodes."""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from falkordb import FalkorDB
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+from app.config import settings
+
+
+CATEGORY_ORDER = ("交通", "景点", "美食", "购物")
+DEFAULT_SOURCE_ROOT = Path("/Users/xuexue/Documents/云游荔波/outputs")
+
+
+def clean(value: Any) -> str:
+    return "" if value is None else str(value).strip()
+
+
+def load_nearby_rows(source_root: Path) -> dict[str, dict[str, list[dict[str, Any]]]]:
+    grouped: dict[str, dict[str, dict[str, dict[str, Any]]]] = defaultdict(
+        lambda: defaultdict(dict)
+    )
+    paths = sorted(source_root.glob("**/csv/携程_酒店详情_周边地点.csv"))
+    if not paths:
+        raise FileNotFoundError(f"未找到携程周边地点CSV：{source_root}")
+
+    for path in paths:
+        with path.open("r", encoding="utf-8-sig", newline="") as stream:
+            for row in csv.DictReader(stream):
+                hotel_id = clean(row.get("酒店ID"))
+                category = clean(row.get("地点分类"))
+                name = clean(row.get("地点名称"))
+                if not hotel_id or category not in CATEGORY_ORDER or not name:
+                    continue
+                place_id = clean(row.get("地点ID")) or name
+                try:
+                    rank = int(float(clean(row.get("排序")) or "999"))
+                except ValueError:
+                    rank = 999
+                item = {
+                    "name": name,
+                    "distance": clean(row.get("距离")),
+                    "travel_time": clean(row.get("交通时间")),
+                    "rank": rank,
+                }
+                current = grouped[hotel_id][category].get(place_id)
+                if current is None or item["rank"] < current["rank"]:
+                    grouped[hotel_id][category][place_id] = item
+
+    result: dict[str, dict[str, list[dict[str, Any]]]] = {}
+    for hotel_id, categories in grouped.items():
+        result[hotel_id] = {}
+        for category in CATEGORY_ORDER:
+            items = sorted(
+                categories.get(category, {}).values(),
+                key=lambda item: (item["rank"], item["name"]),
+            )
+            if items:
+                result[hotel_id][category] = items
+    return result
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--source-root", type=Path, default=DEFAULT_SOURCE_ROOT)
+    parser.add_argument("--graph-name", default="yunyou_libo")
+    args = parser.parse_args()
+
+    nearby_by_hotel = load_nearby_rows(args.source_root)
+    graph = FalkorDB(
+        host=settings.falkordb_host,
+        port=settings.falkordb_port,
+    ).select_graph(args.graph_name)
+
+    hotel_rows = graph.query(
+        "MATCH (n:Hotel) WHERE n.ctrip_hotel_id IS NOT NULL "
+        "RETURN DISTINCT n.ctrip_hotel_id"
+    ).result_set
+    graph_hotel_ids = {clean(row[0]) for row in hotel_rows if row and clean(row[0])}
+
+    updated_hotels = 0
+    updated_items = 0
+    for hotel_id in sorted(graph_hotel_ids & nearby_by_hotel.keys()):
+        categories = nearby_by_hotel[hotel_id]
+        payload = json.dumps(categories, ensure_ascii=False, separators=(",", ":"))
+        result = graph.query(
+            "MATCH (n:Hotel) WHERE n.ctrip_hotel_id=$hotel_id "
+            "SET n.nearby_category_items_json=$payload "
+            "RETURN count(n)",
+            {"hotel_id": hotel_id, "payload": payload},
+        ).result_set
+        changed = int(result[0][0] if result else 0)
+        if changed:
+            updated_hotels += changed
+            updated_items += sum(len(items) for items in categories.values()) * changed
+
+    print(json.dumps({
+        "graph_name": args.graph_name,
+        "source_hotels": len(nearby_by_hotel),
+        "graph_hotels_with_ctrip_id": len(graph_hotel_ids),
+        "updated_hotels": updated_hotels,
+        "stored_category_items": updated_items,
+    }, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

scripts/enrich_yunyou_libo_hotel_rooms_reviews.py

@@ -0,0 +1,291 @@
+#!/usr/bin/env python3
+"""Attach detailed Ctrip room, offer, and guest-review data to Libo Hotel nodes."""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sys
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from falkordb import FalkorDB
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+from app.config import settings
+
+
+DEFAULT_SOURCE_DIR = Path(
+    "/Users/xuexue/Desktop/荔波小七孔源数据/携程/CSV数据表"
+)
+
+
+def clean(value: Any) -> str:
+    return "" if value is None else str(value).strip()
+
+
+def number(value: Any) -> float | None:
+    text = clean(value).replace(",", "")
+    if not text:
+        return None
+    try:
+        result = float(text)
+    except ValueError:
+        return None
+    return int(result) if result.is_integer() else result
+
+
+def integer(value: Any) -> int | None:
+    result = number(value)
+    return int(result) if result is not None else None
+
+
+def load_csv(path: Path) -> list[dict[str, str]]:
+    with path.open("r", encoding="utf-8-sig", newline="") as stream:
+        return list(csv.DictReader(stream))
+
+
+def compact(record: dict[str, Any]) -> dict[str, Any]:
+    return {
+        key: value
+        for key, value in record.items()
+        if value not in (None, "", [], {})
+    }
+
+
+def clean_offer_text(value: Any, kind: str) -> str:
+    """Keep stable booking terms while dropping scraper-combined live inventory copy."""
+    text = clean(value)
+    if not text:
+        return ""
+    import re
+
+    patterns = {
+        "breakfast": [
+            r"无早餐",
+            r"\d+\s*份早餐",
+            r"含早餐",
+        ],
+        "cancellation_policy": [
+            r"\d{2}月\d{2}日\s*\d{2}:\d{2}前可免费取消",
+            r"限时取消",
+            r"免费取消",
+            r"不可取消",
+        ],
+        "confirmation_policy": [
+            r"立即确认",
+            r"等待确认",
+        ],
+        "payment_method": [
+            r"在线付(?:[·・]\s*可先住后付)?",
+            r"到店付",
+            r"可先住后付",
+        ],
+    }
+    for pattern in patterns.get(kind, []):
+        match = re.search(pattern, text)
+        if match:
+            return match.group(0).replace(" ", "")
+    if len(text) <= 36 and not re.search(r"仅剩\s*\d+\s*间", text):
+        return text
+    return ""
+
+
+def load_room_images(source_dir: Path) -> dict[tuple[str, str], str]:
+    result: dict[tuple[str, str], tuple[int, str]] = {}
+    for row in load_csv(source_dir / "携程_酒店详情_酒店图片.csv"):
+        if clean(row.get("图片类型")).lower() != "room":
+            continue
+        hotel_id = clean(row.get("酒店ID") or row.get("\ufeff酒店ID"))
+        room_id = clean(row.get("房型ID"))
+        image_url = clean(row.get("图片URL"))
+        if not hotel_id or not room_id or not image_url:
+            continue
+        sequence = integer(row.get("图片序号")) or 999999
+        current = result.get((hotel_id, room_id))
+        if current is None or sequence < current[0]:
+            result[(hotel_id, room_id)] = (sequence, image_url)
+    return {key: value[1] for key, value in result.items()}
+
+
+def load_room_payloads(source_dir: Path) -> dict[str, list[dict[str, Any]]]:
+    room_rows = load_csv(source_dir / "携程_酒店详情_房型.csv")
+    offer_rows = load_csv(source_dir / "携程_酒店详情_售卖方案.csv")
+    room_images = load_room_images(source_dir)
+
+    offers_by_room: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
+    for row in offer_rows:
+        hotel_id = clean(row.get("酒店ID"))
+        room_id = clean(row.get("房型ID"))
+        if not hotel_id or not room_id:
+            continue
+        offers_by_room[(hotel_id, room_id)].append(compact({
+            "offer_id": clean(row.get("售卖方案ID")),
+            "sequence": integer(row.get("方案序号")),
+            "breakfast": clean_offer_text(row.get("早餐"), "breakfast"),
+            "cancellation_policy": clean_offer_text(
+                row.get("取消政策"),
+                "cancellation_policy",
+            ),
+            "confirmation_policy": clean_offer_text(
+                row.get("确认政策"),
+                "confirmation_policy",
+            ),
+            "payment_method": clean_offer_text(
+                row.get("支付方式"),
+                "payment_method",
+            ),
+            "occupancy": integer(row.get("入住人数")),
+            "original_price": number(row.get("原价")),
+            "current_price": number(row.get("当前价")),
+            "tax_fee": number(row.get("税费")),
+            "currency": clean(row.get("币种")) or "CNY",
+            "source_url": clean(row.get("来源URL")),
+        }))
+
+    result: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in room_rows:
+        hotel_id = clean(row.get("酒店ID"))
+        room_id = clean(row.get("房型ID"))
+        name = clean(row.get("房型名称"))
+        if not hotel_id or not room_id or not name:
+            continue
+        offers = offers_by_room.get((hotel_id, room_id), [])
+        offers.sort(key=lambda item: (
+            item.get("current_price") is None,
+            item.get("current_price") or 0,
+            item.get("sequence") or 999,
+        ))
+        result[hotel_id].append(compact({
+            "room_id": room_id,
+            "name": name,
+            "price": number(row.get("房型价格")),
+            "currency": clean(row.get("币种")) or "CNY",
+            "room_url": clean(row.get("房型URL")),
+            "image_url": room_images.get((hotel_id, room_id), ""),
+            "bed_type": clean(row.get("床型")),
+            "window": clean(row.get("窗户")),
+            "area": clean(row.get("面积")),
+            "floor": clean(row.get("楼层")),
+            "capacity": integer(row.get("可住人数")),
+            "extra_bed_policy": clean(row.get("加床政策")),
+            "smoking_policy": clean(row.get("吸烟政策")),
+            "network": clean(row.get("网络")),
+            "facilities": clean(row.get("房型设施")),
+            "image_count": integer(row.get("房型图片数")),
+            "source_url": clean(row.get("来源URL")),
+            "offers": offers,
+        }))
+
+    for rooms in result.values():
+        rooms.sort(key=lambda item: (
+            item.get("price") is None,
+            item.get("price") or 0,
+            item.get("name") or "",
+        ))
+    return dict(result)
+
+
+def review_timestamp(value: str) -> float:
+    try:
+        return datetime.fromisoformat(value.replace("/", "-")).timestamp()
+    except ValueError:
+        return 0
+
+
+def load_review_payloads(source_dir: Path) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in load_csv(source_dir / "携程_酒店详情_住客点评.csv"):
+        hotel_id = clean(row.get("酒店ID") or row.get("\ufeff酒店ID"))
+        review_id = clean(row.get("点评ID"))
+        content = clean(row.get("点评内容"))
+        if not hotel_id or not review_id or not content:
+            continue
+        image_urls = [
+            item.strip()
+            for item in clean(row.get("点评图片")).split("|")
+            if item.strip()
+        ]
+        result[hotel_id].append(compact({
+            "review_id": review_id,
+            "nickname": clean(row.get("用户昵称")) or "匿名用户",
+            "review_at": clean(row.get("点评时间")),
+            "stay_at": clean(row.get("入住时间")),
+            "room_type": clean(row.get("入住房型")),
+            "travel_type": clean(row.get("出行类型")),
+            "score": number(row.get("总评分")),
+            "content": content,
+            "like_count": integer(row.get("点赞数")),
+            "reply_count": integer(row.get("回复数")),
+            "image_count": integer(row.get("图片数量")),
+            "image_urls": image_urls,
+            "source_url": clean(row.get("来源URL")),
+        }))
+
+    for reviews in result.values():
+        reviews.sort(
+            key=lambda item: review_timestamp(clean(item.get("review_at"))),
+            reverse=True,
+        )
+    return dict(result)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
+    parser.add_argument("--graph-name", default="yunyou_libo")
+    args = parser.parse_args()
+
+    rooms_by_hotel = load_room_payloads(args.source_dir)
+    reviews_by_hotel = load_review_payloads(args.source_dir)
+    graph = FalkorDB(
+        host=settings.falkordb_host,
+        port=settings.falkordb_port,
+    ).select_graph(args.graph_name)
+
+    hotel_rows = graph.query(
+        "MATCH (n:Hotel) WHERE n.ctrip_hotel_id IS NOT NULL "
+        "RETURN DISTINCT n.ctrip_hotel_id"
+    ).result_set
+    graph_hotel_ids = {clean(row[0]) for row in hotel_rows if row and clean(row[0])}
+
+    updated_hotels = 0
+    stored_rooms = 0
+    stored_reviews = 0
+    for hotel_id in sorted(graph_hotel_ids & (rooms_by_hotel.keys() | reviews_by_hotel.keys())):
+        rooms = rooms_by_hotel.get(hotel_id, [])
+        reviews = reviews_by_hotel.get(hotel_id, [])
+        result = graph.query(
+            "MATCH (n:Hotel) WHERE n.ctrip_hotel_id=$hotel_id "
+            "SET n.room_items_json=$rooms, n.review_items_json=$reviews "
+            "RETURN count(n)",
+            {
+                "hotel_id": hotel_id,
+                "rooms": json.dumps(rooms, ensure_ascii=False, separators=(",", ":")),
+                "reviews": json.dumps(reviews, ensure_ascii=False, separators=(",", ":")),
+            },
+        ).result_set
+        changed = int(result[0][0] if result else 0)
+        if changed:
+            updated_hotels += changed
+            stored_rooms += len(rooms) * changed
+            stored_reviews += len(reviews) * changed
+
+    print(json.dumps({
+        "graph_name": args.graph_name,
+        "source_hotels_with_rooms": len(rooms_by_hotel),
+        "source_hotels_with_reviews": len(reviews_by_hotel),
+        "graph_hotels_with_ctrip_id": len(graph_hotel_ids),
+        "updated_hotels": updated_hotels,
+        "stored_rooms": stored_rooms,
+        "stored_reviews": stored_reviews,
+    }, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

scripts/evaluate_markitdown_conversion.py

@@ -0,0 +1,365 @@
+#!/usr/bin/env python3
+"""Evaluate MarkItDown conversion quality on a local document corpus.
+
+The goal is not to prove that a converter is universally good. The goal is to
+make conversion quality measurable for this knowledge-extraction product:
+success rate, latency, information coverage, structure preservation, noise, and
+optional similarity to reference Markdown.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import statistics
+import sys
+import time
+from dataclasses import dataclass
+from difflib import SequenceMatcher
+from pathlib import Path
+from typing import Any
+
+
+SUPPORTED_EXTENSIONS = {
+    ".txt",
+    ".md",
+    ".markdown",
+    ".csv",
+    ".json",
+    ".xml",
+    ".html",
+    ".htm",
+    ".pdf",
+    ".docx",
+    ".doc",
+    ".pptx",
+    ".ppt",
+    ".xlsx",
+    ".xls",
+    ".zip",
+    ".epub",
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".gif",
+    ".wav",
+    ".mp3",
+}
+
+
+@dataclass
+class Case:
+    case_id: str
+    file_path: Path
+    must_terms: list[str]
+    forbidden_terms: list[str]
+    expected_headings_min: int
+    expected_tables_min: int
+    expected_lists_min: int
+    expected_links_min: int
+    min_chars: int
+    gold_markdown_path: Path | None
+    notes: str
+
+
+def load_manifest(path: Path | None) -> dict[str, Any]:
+    if not path:
+        return {"cases": []}
+    with path.open("r", encoding="utf-8") as f:
+        data = json.load(f)
+    if not isinstance(data, dict) or not isinstance(data.get("cases"), list):
+        raise ValueError("Manifest must be a JSON object with a cases array")
+    return data
+
+
+def discover_cases(input_dir: Path, manifest: dict[str, Any]) -> list[Case]:
+    by_file = {
+        str(item.get("file", "")).strip(): item
+        for item in manifest.get("cases", [])
+        if isinstance(item, dict) and str(item.get("file", "")).strip()
+    }
+    if by_file:
+        files = [input_dir / rel for rel in by_file]
+    else:
+        files = [
+            p
+            for p in sorted(input_dir.rglob("*"))
+            if p.is_file() and p.suffix.lower() in SUPPORTED_EXTENSIONS
+        ]
+
+    cases: list[Case] = []
+    for file_path in files:
+        rel = str(file_path.relative_to(input_dir)) if file_path.is_relative_to(input_dir) else file_path.name
+        cfg = by_file.get(rel, {})
+        case_id = str(cfg.get("case_id") or file_path.with_suffix("").name)
+        gold = cfg.get("gold_markdown")
+        cases.append(
+            Case(
+                case_id=case_id,
+                file_path=file_path,
+                must_terms=[str(v) for v in cfg.get("must_terms", []) if str(v).strip()],
+                forbidden_terms=[str(v) for v in cfg.get("forbidden_terms", []) if str(v).strip()],
+                expected_headings_min=int(cfg.get("expected_headings_min") or 0),
+                expected_tables_min=int(cfg.get("expected_tables_min") or 0),
+                expected_lists_min=int(cfg.get("expected_lists_min") or 0),
+                expected_links_min=int(cfg.get("expected_links_min") or 0),
+                min_chars=int(cfg.get("min_chars") or 80),
+                gold_markdown_path=(input_dir / str(gold)) if gold else None,
+                notes=str(cfg.get("notes") or ""),
+            )
+        )
+    return cases
+
+
+def convert_with_markitdown(file_path: Path) -> str:
+    try:
+        from markitdown import MarkItDown
+    except ImportError as exc:
+        raise RuntimeError("MarkItDown is not installed. Run: pip install -r requirements.txt") from exc
+
+    result = MarkItDown(enable_plugins=False).convert(str(file_path))
+    text = getattr(result, "text_content", None) or getattr(result, "markdown", None) or ""
+    return str(text).strip()
+
+
+def count_patterns(markdown: str) -> dict[str, int]:
+    lines = markdown.splitlines()
+    return {
+        "chars": len(markdown),
+        "lines": len(lines),
+        "headings": sum(1 for line in lines if re.match(r"^\s{0,3}#{1,6}\s+\S", line)),
+        "table_rows": sum(1 for line in lines if line.count("|") >= 2),
+        "list_items": sum(1 for line in lines if re.match(r"^\s*(?:[-*+]|\d+[.)])\s+\S", line)),
+        "links": len(re.findall(r"\[[^\]]+\]\([^)]+\)|https?://\S+", markdown)),
+        "replacement_chars": markdown.count("\ufffd"),
+        "null_chars": markdown.count("\x00"),
+        "html_tags": len(re.findall(r"</?[A-Za-z][^>]{0,200}>", markdown)),
+        "long_lines": sum(1 for line in lines if len(line) > 500),
+    }
+
+
+def term_coverage(markdown: str, terms: list[str]) -> tuple[float, list[str]]:
+    if not terms:
+        return 1.0, []
+    haystack = markdown.lower()
+    missing = [term for term in terms if term.lower() not in haystack]
+    return (len(terms) - len(missing)) / len(terms), missing
+
+
+def forbidden_hits(markdown: str, terms: list[str]) -> list[str]:
+    haystack = markdown.lower()
+    return [term for term in terms if term.lower() in haystack]
+
+
+def min_ratio(actual: int, expected: int) -> float:
+    if expected <= 0:
+        return 1.0
+    return min(actual / expected, 1.0)
+
+
+def score_case(metrics: dict[str, Any]) -> float:
+    content_score = min(metrics["chars"] / max(metrics["min_chars"], 1), 1.0)
+    structure_score = statistics.mean(
+        [
+            metrics["heading_score"],
+            metrics["table_score"],
+            metrics["list_score"],
+            metrics["link_score"],
+        ]
+    )
+    noise_penalty = min(
+        1.0,
+        metrics["replacement_chars"] * 0.08
+        + metrics["null_chars"] * 0.2
+        + metrics["long_lines"] * 0.03
+        + metrics["forbidden_hit_count"] * 0.12,
+    )
+    gold_similarity = metrics.get("gold_similarity")
+    if gold_similarity is None:
+        score = (
+            0.35 * metrics["must_term_coverage"]
+            + 0.25 * content_score
+            + 0.25 * structure_score
+            + 0.15 * (1.0 - noise_penalty)
+        )
+    else:
+        score = (
+            0.30 * metrics["must_term_coverage"]
+            + 0.20 * content_score
+            + 0.20 * structure_score
+            + 0.15 * (1.0 - noise_penalty)
+            + 0.15 * gold_similarity
+        )
+    return round(max(0.0, min(score, 1.0)), 4)
+
+
+def evaluate_case(case: Case, output_dir: Path) -> dict[str, Any]:
+    started = time.perf_counter()
+    output_path = output_dir / "converted" / f"{case.case_id}.markitdown.md"
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    result: dict[str, Any] = {
+        "case_id": case.case_id,
+        "file": str(case.file_path),
+        "notes": case.notes,
+        "success": False,
+    }
+
+    try:
+        markdown = convert_with_markitdown(case.file_path)
+        output_path.write_text(markdown + "\n", encoding="utf-8")
+        counts = count_patterns(markdown)
+        coverage, missing = term_coverage(markdown, case.must_terms)
+        forbidden = forbidden_hits(markdown, case.forbidden_terms)
+        metrics: dict[str, Any] = {
+            **counts,
+            "min_chars": case.min_chars,
+            "must_terms": case.must_terms,
+            "must_term_coverage": round(coverage, 4),
+            "missing_terms": missing,
+            "forbidden_terms": case.forbidden_terms,
+            "forbidden_hits": forbidden,
+            "forbidden_hit_count": len(forbidden),
+            "expected_headings_min": case.expected_headings_min,
+            "expected_tables_min": case.expected_tables_min,
+            "expected_lists_min": case.expected_lists_min,
+            "expected_links_min": case.expected_links_min,
+            "heading_score": min_ratio(counts["headings"], case.expected_headings_min),
+            "table_score": min_ratio(counts["table_rows"], case.expected_tables_min),
+            "list_score": min_ratio(counts["list_items"], case.expected_lists_min),
+            "link_score": min_ratio(counts["links"], case.expected_links_min),
+        }
+        if case.gold_markdown_path and case.gold_markdown_path.exists():
+            gold = case.gold_markdown_path.read_text(encoding="utf-8")
+            metrics["gold_similarity"] = round(SequenceMatcher(None, gold, markdown).ratio(), 4)
+        else:
+            metrics["gold_similarity"] = None
+
+        result.update(
+            {
+                "success": True,
+                "latency_ms": round((time.perf_counter() - started) * 1000),
+                "markdown_path": str(output_path),
+                "metrics": metrics,
+                "quality_score": score_case(metrics),
+            }
+        )
+    except Exception as exc:  # noqa: BLE001
+        result.update(
+            {
+                "latency_ms": round((time.perf_counter() - started) * 1000),
+                "error": str(exc),
+                "quality_score": 0.0,
+            }
+        )
+    return result
+
+
+def aggregate(results: list[dict[str, Any]]) -> dict[str, Any]:
+    scores = [float(r.get("quality_score") or 0) for r in results]
+    successes = [r for r in results if r.get("success")]
+    latencies = [float(r.get("latency_ms") or 0) for r in successes]
+    return {
+        "cases": len(results),
+        "successes": len(successes),
+        "failures": len(results) - len(successes),
+        "success_rate": round(len(successes) / len(results), 4) if results else 0,
+        "mean_quality_score": round(statistics.mean(scores), 4) if scores else 0,
+        "median_quality_score": round(statistics.median(scores), 4) if scores else 0,
+        "mean_latency_ms": round(statistics.mean(latencies)) if latencies else 0,
+    }
+
+
+def write_markdown_report(report: dict[str, Any], path: Path) -> None:
+    summary = report["summary"]
+    lines = [
+        "# MarkItDown Conversion Evaluation",
+        "",
+        "## Summary",
+        "",
+        f"- Cases: {summary['cases']}",
+        f"- Success rate: {summary['success_rate']:.2%}",
+        f"- Mean quality score: {summary['mean_quality_score']:.3f}",
+        f"- Median quality score: {summary['median_quality_score']:.3f}",
+        f"- Mean latency: {summary['mean_latency_ms']} ms",
+        "",
+        "## Cases",
+        "",
+        "| Case | Success | Score | Chars | Must-term coverage | Missing terms | Latency |",
+        "| --- | --- | ---: | ---: | ---: | --- | ---: |",
+    ]
+    for item in report["results"]:
+        metrics = item.get("metrics") or {}
+        missing = ", ".join(metrics.get("missing_terms") or [])
+        lines.append(
+            "| {case} | {success} | {score:.3f} | {chars} | {coverage:.2%} | {missing} | {latency} ms |".format(
+                case=item.get("case_id"),
+                success="yes" if item.get("success") else "no",
+                score=float(item.get("quality_score") or 0),
+                chars=metrics.get("chars", 0),
+                coverage=float(metrics.get("must_term_coverage") or 0),
+                missing=missing.replace("|", "\\|") or "-",
+                latency=item.get("latency_ms", 0),
+            )
+        )
+    lines.append("")
+    lines.append("## Interpretation")
+    lines.append("")
+    lines.append("- Score >= 0.85: suitable for normal extraction after spot check.")
+    lines.append("- 0.70 <= score < 0.85: usable, but inspect missing terms or structure loss.")
+    lines.append("- Score < 0.70: do not trust automatic extraction without fallback/OCR/manual correction.")
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input-dir", default="data/markitdown_eval/input")
+    parser.add_argument("--manifest", default="data/markitdown_eval/manifest.json")
+    parser.add_argument("--output-dir", default="outputs/markitdown_eval")
+    parser.add_argument("--fail-under", type=float, default=0.70)
+    parser.add_argument("--allow-missing-manifest", action="store_true")
+    args = parser.parse_args()
+
+    input_dir = Path(args.input_dir)
+    manifest_path = Path(args.manifest)
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if not input_dir.exists():
+        print(f"Input directory not found: {input_dir}", file=sys.stderr)
+        return 2
+    if not manifest_path.exists() and not args.allow_missing_manifest:
+        print(
+            f"Manifest not found: {manifest_path}. Pass --allow-missing-manifest to auto-discover files.",
+            file=sys.stderr,
+        )
+        return 2
+
+    manifest = load_manifest(manifest_path if manifest_path.exists() else None)
+    cases = discover_cases(input_dir, manifest)
+    if not cases:
+        print(f"No supported files found in {input_dir}", file=sys.stderr)
+        return 2
+
+    results = [evaluate_case(case, output_dir) for case in cases]
+    report = {
+        "input_dir": str(input_dir),
+        "manifest": str(manifest_path) if manifest_path.exists() else None,
+        "summary": aggregate(results),
+        "results": results,
+    }
+    json_path = output_dir / "markitdown_eval_report.json"
+    md_path = output_dir / "markitdown_eval_report.md"
+    json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+    write_markdown_report(report, md_path)
+
+    print(f"Wrote {json_path}")
+    print(f"Wrote {md_path}")
+    mean_score = report["summary"]["mean_quality_score"]
+    if mean_score < args.fail_under:
+        print(f"Mean quality score {mean_score:.3f} is below fail-under {args.fail_under:.3f}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/server_rootless_docker.sh

@@ -37,7 +37,11 @@ compose() {
   ensure_docker
   export DOCKER_HOST="unix://$DOCKER_SOCK"
   cd "$PROJECT_DIR"
-  docker compose "$@"
+  local compose_files=(-f docker-compose.yml)
+  if [ -f docker-compose.server.yml ]; then
+    compose_files+=(-f docker-compose.server.yml)
+  fi
+  docker compose "${compose_files[@]}" "$@"
 }
 
 case "${1:-up}" in

scripts/test_ocr_passport.py

@@ -0,0 +1,59 @@
+"""阿里云护照 OCR 联调脚本。
+
+用法:
+    # 测本地图片
+    python3 scripts/test_ocr_passport.py /path/to/passport.jpg
+
+    # 测公网 URL
+    python3 scripts/test_ocr_passport.py --url https://example.com/passport.jpg
+
+前置:
+    pip install -r requirements.txt
+    在项目根目录的 .env 填好:
+        ALIYUN_OCR_ACCESS_KEY_ID=
+        ALIYUN_OCR_ACCESS_KEY_SECRET=
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+# 让脚本能在仓库根目录直接运行
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from app.aliyun_ocr import OcrCallError, OcrConfigError, recognize_passport  # noqa: E402
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="阿里云护照 OCR 自检")
+    parser.add_argument("image", nargs="?", help="本地图片路径")
+    parser.add_argument("--url", help="公网图片 URL")
+    args = parser.parse_args()
+
+    if not args.image and not args.url:
+        parser.error("请提供本地图片路径或 --url")
+
+    try:
+        if args.image:
+            path = Path(args.image).expanduser().resolve()
+            if not path.is_file():
+                print(f"[FAIL] 文件不存在: {path}", file=sys.stderr)
+                return 2
+            data = recognize_passport(image_bytes=path.read_bytes())
+        else:
+            data = recognize_passport(image_url=args.url)
+    except OcrConfigError as exc:
+        print(f"[CONFIG] {exc}", file=sys.stderr)
+        return 3
+    except OcrCallError as exc:
+        print(f"[CALL]   {exc}", file=sys.stderr)
+        return 4
+
+    print(json.dumps(data, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/test_qwen_vl.py

@@ -0,0 +1,57 @@
+"""测试通义千问多模态：URL 图片 + 本地图片 两种用法。
+
+用法:
+    export DASHSCOPE_API_KEY=sk-xxxx
+    python scripts/test_qwen_vl.py                       # 跑 URL 用例
+    python scripts/test_qwen_vl.py /path/to/local.jpg    # 跑本地图片用例
+"""
+
+import os
+import sys
+from pathlib import Path
+
+from dashscope import MultiModalConversation
+
+
+def call(image_ref: str, prompt: str = "详细描述这张图片，并提取里面所有可见文字。") -> None:
+    resp = MultiModalConversation.call(
+        model="qwen-vl-max",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"image": image_ref},
+                    {"text": prompt},
+                ],
+            }
+        ],
+    )
+    if resp.status_code != 200:
+        print(f"FAIL  status={resp.status_code}  code={resp.code}  msg={resp.message}")
+        sys.exit(1)
+
+    # 兼容两种返回结构
+    content = resp.output.choices[0].message.content
+    if isinstance(content, list):
+        text = "".join(seg.get("text", "") for seg in content)
+    else:
+        text = content
+    print("OK\n" + text)
+    print(f"\n[usage] {resp.usage}")
+
+
+def main() -> None:
+    if not os.getenv("DASHSCOPE_API_KEY"):
+        sys.exit("ERROR: 请先 export DASHSCOPE_API_KEY=sk-xxxx")
+
+    if len(sys.argv) > 1:
+        path = Path(sys.argv[1]).expanduser().resolve()
+        if not path.is_file():
+            sys.exit(f"文件不存在: {path}")
+        call(f"file://{path}")
+    else:
+        call("https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg")
+
+
+if __name__ == "__main__":
+    main()