Add Cloud Tour Libo knowledge graph platform
This commit is contained in:
276
scripts/enrich_yunyou_libo_food_deals_reviews.py
Normal file
276
scripts/enrich_yunyou_libo_food_deals_reviews.py
Normal file
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Attach structured Dianping deal, tag, and review payloads to Libo food nodes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from falkordb import FalkorDB
|
||||
|
||||
ROOT_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT_DIR))
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
DEFAULT_SOURCE_DIR = Path(
|
||||
"/Users/xuexue/Desktop/荔波小七孔源数据/大众点评/raw"
|
||||
)
|
||||
|
||||
REVIEW_THEMES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
("口味评价", ("味道", "口味", "好吃", "鲜香", "酸汤", "烤鱼")),
|
||||
("服务体验", ("服务", "老板", "店员", "热情", "招待")),
|
||||
("环境体验", ("环境", "装修", "干净", "卫生", "民族风")),
|
||||
("菜品分量", ("分量", "份量", "量大", "量足")),
|
||||
("性价比", ("性价比", "价格", "价位", "实惠")),
|
||||
("停车便利", ("停车", "停车场")),
|
||||
("排队情况", ("排队", "等位")),
|
||||
("游客推荐", ("推荐", "必吃", "值得", "不踩雷")),
|
||||
)
|
||||
|
||||
|
||||
def clean(value: Any) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def number(value: Any) -> float | int | None:
|
||||
text = clean(value).replace("￥", "").replace("¥", "").replace(",", "")
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
result = float(text)
|
||||
except ValueError:
|
||||
return None
|
||||
return int(result) if result.is_integer() else result
|
||||
|
||||
|
||||
def compact(record: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
key: value
|
||||
for key, value in record.items()
|
||||
if value not in (None, "", [], {})
|
||||
}
|
||||
|
||||
|
||||
def split_values(value: Any) -> list[str]:
|
||||
return [
|
||||
item.strip()
|
||||
for item in re.split(r"\s*[|｜、,，]\s*", clean(value))
|
||||
if item.strip()
|
||||
]
|
||||
|
||||
|
||||
def normalize_title(value: Any) -> str:
|
||||
return re.sub(r"[\s【】\[\]（）()·｜|/\\\-—_]", "", clean(value))
|
||||
|
||||
|
||||
def load_raw_records(source_dir: Path) -> dict[str, dict[str, Any]]:
|
||||
records: dict[str, dict[str, Any]] = {}
|
||||
for path in sorted(source_dir.glob("*.json")):
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
identifiers = {
|
||||
path.stem.removeprefix("dianping_"),
|
||||
clean(payload.get("POI_ID")),
|
||||
clean(payload.get("店铺ID")),
|
||||
clean(payload.get("店铺UUID")),
|
||||
}
|
||||
for identifier in identifiers:
|
||||
if identifier:
|
||||
records[identifier] = payload
|
||||
return records
|
||||
|
||||
|
||||
def deal_image(
|
||||
title: str,
|
||||
sequence: int,
|
||||
recommended_dishes: list[dict[str, Any]],
|
||||
shop_image: str,
|
||||
) -> str:
|
||||
normalized_title = normalize_title(title)
|
||||
usable_images = []
|
||||
for dish in recommended_dishes:
|
||||
image_url = clean(dish.get("image"))
|
||||
if not image_url:
|
||||
continue
|
||||
usable_images.append(image_url)
|
||||
dish_name = normalize_title(dish.get("name"))
|
||||
if dish_name and dish_name in normalized_title:
|
||||
return image_url
|
||||
if usable_images:
|
||||
return usable_images[(sequence - 1) % len(usable_images)]
|
||||
return shop_image
|
||||
|
||||
|
||||
def build_deals(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
raw_deals = payload.get("团购信息") or []
|
||||
if not isinstance(raw_deals, list):
|
||||
return []
|
||||
recommended_dishes = payload.get("推荐菜详情") or []
|
||||
if not isinstance(recommended_dishes, list):
|
||||
recommended_dishes = []
|
||||
shop_image = clean(payload.get("店铺图片"))
|
||||
deals: list[dict[str, Any]] = []
|
||||
for sequence, item in enumerate(raw_deals, 1):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
title = clean(item.get("title"))
|
||||
if not title:
|
||||
continue
|
||||
terms = split_values(item.get("tags"))
|
||||
if not terms:
|
||||
terms = [
|
||||
value
|
||||
for value in re.split(r"\s+", clean(item.get("dishes")))
|
||||
if value
|
||||
]
|
||||
deals.append(compact({
|
||||
"deal_id": f"{clean(payload.get('店铺ID')) or clean(payload.get('POI_ID'))}:{sequence}",
|
||||
"title": title,
|
||||
"price": number(item.get("price")),
|
||||
"original_price": number(item.get("original_price")),
|
||||
"discount": clean(item.get("discount")),
|
||||
"terms": terms,
|
||||
"image_url": deal_image(
|
||||
title,
|
||||
sequence,
|
||||
[dish for dish in recommended_dishes if isinstance(dish, dict)],
|
||||
shop_image,
|
||||
),
|
||||
}))
|
||||
return deals
|
||||
|
||||
|
||||
def build_reviews(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
raw_reviews = payload.get("评论数据") or []
|
||||
if not isinstance(raw_reviews, list):
|
||||
return []
|
||||
source_id = (
|
||||
clean(payload.get("店铺ID"))
|
||||
or clean(payload.get("POI_ID"))
|
||||
or "dianping"
|
||||
)
|
||||
reviews: list[dict[str, Any]] = []
|
||||
for sequence, item in enumerate(raw_reviews, 1):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
content = clean(item.get("评论内容"))
|
||||
if not content:
|
||||
continue
|
||||
reviews.append(compact({
|
||||
"review_id": f"{source_id}:{sequence}",
|
||||
"nickname": clean(item.get("用户名")) or "匿名用户",
|
||||
"review_at": clean(item.get("评论时间")),
|
||||
"rating_label": clean(item.get("评分")),
|
||||
"stars": number(item.get("星级")),
|
||||
"content": content,
|
||||
"image_urls": split_values(item.get("评论图片")),
|
||||
"image_count": number(item.get("图片数量")),
|
||||
}))
|
||||
return reviews
|
||||
|
||||
|
||||
def build_review_tags(
|
||||
payload: dict[str, Any],
|
||||
reviews: list[dict[str, Any]],
|
||||
existing_review_tags: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
explicit = split_values(payload.get("评论标签")) or split_values(existing_review_tags)
|
||||
if explicit:
|
||||
return [
|
||||
{"label": label, "count": 0}
|
||||
for label in explicit[:10]
|
||||
]
|
||||
|
||||
theme_counts: Counter[str] = Counter()
|
||||
for review in reviews:
|
||||
content = clean(review.get("content"))
|
||||
for label, keywords in REVIEW_THEMES:
|
||||
if any(keyword in content for keyword in keywords):
|
||||
theme_counts[label] += 1
|
||||
return [
|
||||
{"label": label, "count": count}
|
||||
for label, count in theme_counts.most_common(8)
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
|
||||
parser.add_argument("--graph-name", default="yunyou_libo")
|
||||
args = parser.parse_args()
|
||||
|
||||
raw_records = load_raw_records(args.source_dir)
|
||||
graph = FalkorDB(
|
||||
host=settings.falkordb_host,
|
||||
port=settings.falkordb_port,
|
||||
).select_graph(args.graph_name)
|
||||
rows = graph.query(
|
||||
"MATCH (n:FoodPlace) WHERE n.dianping_name IS NOT NULL "
|
||||
"RETURN n.element_id,n.dianping_source_poi_id,n.dianping_shop_id,"
|
||||
"n.dianping_shop_uuid,n.gaode_poi_id,n.dianping_review_tags"
|
||||
).result_set
|
||||
|
||||
updated_nodes = 0
|
||||
stored_deals = 0
|
||||
stored_reviews = 0
|
||||
stored_tags = 0
|
||||
missing_raw = 0
|
||||
for row in rows:
|
||||
element_id = clean(row[0])
|
||||
identifiers = [clean(value) for value in row[1:5] if clean(value)]
|
||||
existing_review_tags = clean(row[5])
|
||||
payload = next(
|
||||
(raw_records[identifier] for identifier in identifiers if identifier in raw_records),
|
||||
None,
|
||||
)
|
||||
if payload is None:
|
||||
missing_raw += 1
|
||||
continue
|
||||
deals = build_deals(payload)
|
||||
reviews = build_reviews(payload)
|
||||
tags = build_review_tags(payload, reviews, existing_review_tags)
|
||||
result = graph.query(
|
||||
"MATCH (n:FoodPlace {element_id:$element_id}) "
|
||||
"SET n.group_buy_items_json=$deals,"
|
||||
"n.food_review_items_json=$reviews,"
|
||||
"n.food_review_tags_json=$tags "
|
||||
"RETURN count(n)",
|
||||
{
|
||||
"element_id": element_id,
|
||||
"deals": json.dumps(deals, ensure_ascii=False, separators=(",", ":")),
|
||||
"reviews": json.dumps(reviews, ensure_ascii=False, separators=(",", ":")),
|
||||
"tags": json.dumps(tags, ensure_ascii=False, separators=(",", ":")),
|
||||
},
|
||||
).result_set
|
||||
changed = int(result[0][0] if result else 0)
|
||||
if changed:
|
||||
updated_nodes += changed
|
||||
stored_deals += len(deals) * changed
|
||||
stored_reviews += len(reviews) * changed
|
||||
stored_tags += len(tags) * changed
|
||||
|
||||
print(json.dumps({
|
||||
"graph_name": args.graph_name,
|
||||
"raw_record_identifiers": len(raw_records),
|
||||
"graph_food_nodes": len(rows),
|
||||
"updated_nodes": updated_nodes,
|
||||
"missing_raw_nodes": missing_raw,
|
||||
"stored_deals": stored_deals,
|
||||
"stored_reviews": stored_reviews,
|
||||
"stored_review_tags": stored_tags,
|
||||
}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user