277 lines
9.0 KiB
Python
277 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
||
"""Attach structured Dianping deal, tag, and review payloads to Libo food nodes."""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from falkordb import FalkorDB
|
||
|
||
ROOT_DIR = Path(__file__).resolve().parents[1]
|
||
if str(ROOT_DIR) not in sys.path:
|
||
sys.path.insert(0, str(ROOT_DIR))
|
||
|
||
from app.config import settings
|
||
|
||
|
||
DEFAULT_SOURCE_DIR = Path(
|
||
"/Users/xuexue/Desktop/荔波小七孔源数据/大众点评/raw"
|
||
)
|
||
|
||
REVIEW_THEMES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||
("口味评价", ("味道", "口味", "好吃", "鲜香", "酸汤", "烤鱼")),
|
||
("服务体验", ("服务", "老板", "店员", "热情", "招待")),
|
||
("环境体验", ("环境", "装修", "干净", "卫生", "民族风")),
|
||
("菜品分量", ("分量", "份量", "量大", "量足")),
|
||
("性价比", ("性价比", "价格", "价位", "实惠")),
|
||
("停车便利", ("停车", "停车场")),
|
||
("排队情况", ("排队", "等位")),
|
||
("游客推荐", ("推荐", "必吃", "值得", "不踩雷")),
|
||
)
|
||
|
||
|
||
def clean(value: Any) -> str:
|
||
return "" if value is None else str(value).strip()
|
||
|
||
|
||
def number(value: Any) -> float | int | None:
|
||
text = clean(value).replace("￥", "").replace("¥", "").replace(",", "")
|
||
if not text:
|
||
return None
|
||
try:
|
||
result = float(text)
|
||
except ValueError:
|
||
return None
|
||
return int(result) if result.is_integer() else result
|
||
|
||
|
||
def compact(record: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
key: value
|
||
for key, value in record.items()
|
||
if value not in (None, "", [], {})
|
||
}
|
||
|
||
|
||
def split_values(value: Any) -> list[str]:
|
||
return [
|
||
item.strip()
|
||
for item in re.split(r"\s*[|｜、,，]\s*", clean(value))
|
||
if item.strip()
|
||
]
|
||
|
||
|
||
def normalize_title(value: Any) -> str:
|
||
return re.sub(r"[\s【】\[\]（）()·｜|/\\\-—_]", "", clean(value))
|
||
|
||
|
||
def load_raw_records(source_dir: Path) -> dict[str, dict[str, Any]]:
|
||
records: dict[str, dict[str, Any]] = {}
|
||
for path in sorted(source_dir.glob("*.json")):
|
||
try:
|
||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError):
|
||
continue
|
||
if not isinstance(payload, dict):
|
||
continue
|
||
identifiers = {
|
||
path.stem.removeprefix("dianping_"),
|
||
clean(payload.get("POI_ID")),
|
||
clean(payload.get("店铺ID")),
|
||
clean(payload.get("店铺UUID")),
|
||
}
|
||
for identifier in identifiers:
|
||
if identifier:
|
||
records[identifier] = payload
|
||
return records
|
||
|
||
|
||
def deal_image(
|
||
title: str,
|
||
sequence: int,
|
||
recommended_dishes: list[dict[str, Any]],
|
||
shop_image: str,
|
||
) -> str:
|
||
normalized_title = normalize_title(title)
|
||
usable_images = []
|
||
for dish in recommended_dishes:
|
||
image_url = clean(dish.get("image"))
|
||
if not image_url:
|
||
continue
|
||
usable_images.append(image_url)
|
||
dish_name = normalize_title(dish.get("name"))
|
||
if dish_name and dish_name in normalized_title:
|
||
return image_url
|
||
if usable_images:
|
||
return usable_images[(sequence - 1) % len(usable_images)]
|
||
return shop_image
|
||
|
||
|
||
def build_deals(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||
raw_deals = payload.get("团购信息") or []
|
||
if not isinstance(raw_deals, list):
|
||
return []
|
||
recommended_dishes = payload.get("推荐菜详情") or []
|
||
if not isinstance(recommended_dishes, list):
|
||
recommended_dishes = []
|
||
shop_image = clean(payload.get("店铺图片"))
|
||
deals: list[dict[str, Any]] = []
|
||
for sequence, item in enumerate(raw_deals, 1):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
title = clean(item.get("title"))
|
||
if not title:
|
||
continue
|
||
terms = split_values(item.get("tags"))
|
||
if not terms:
|
||
terms = [
|
||
value
|
||
for value in re.split(r"\s+", clean(item.get("dishes")))
|
||
if value
|
||
]
|
||
deals.append(compact({
|
||
"deal_id": f"{clean(payload.get('店铺ID')) or clean(payload.get('POI_ID'))}:{sequence}",
|
||
"title": title,
|
||
"price": number(item.get("price")),
|
||
"original_price": number(item.get("original_price")),
|
||
"discount": clean(item.get("discount")),
|
||
"terms": terms,
|
||
"image_url": deal_image(
|
||
title,
|
||
sequence,
|
||
[dish for dish in recommended_dishes if isinstance(dish, dict)],
|
||
shop_image,
|
||
),
|
||
}))
|
||
return deals
|
||
|
||
|
||
def build_reviews(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||
raw_reviews = payload.get("评论数据") or []
|
||
if not isinstance(raw_reviews, list):
|
||
return []
|
||
source_id = (
|
||
clean(payload.get("店铺ID"))
|
||
or clean(payload.get("POI_ID"))
|
||
or "dianping"
|
||
)
|
||
reviews: list[dict[str, Any]] = []
|
||
for sequence, item in enumerate(raw_reviews, 1):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
content = clean(item.get("评论内容"))
|
||
if not content:
|
||
continue
|
||
reviews.append(compact({
|
||
"review_id": f"{source_id}:{sequence}",
|
||
"nickname": clean(item.get("用户名")) or "匿名用户",
|
||
"review_at": clean(item.get("评论时间")),
|
||
"rating_label": clean(item.get("评分")),
|
||
"stars": number(item.get("星级")),
|
||
"content": content,
|
||
"image_urls": split_values(item.get("评论图片")),
|
||
"image_count": number(item.get("图片数量")),
|
||
}))
|
||
return reviews
|
||
|
||
|
||
def build_review_tags(
|
||
payload: dict[str, Any],
|
||
reviews: list[dict[str, Any]],
|
||
existing_review_tags: str,
|
||
) -> list[dict[str, Any]]:
|
||
explicit = split_values(payload.get("评论标签")) or split_values(existing_review_tags)
|
||
if explicit:
|
||
return [
|
||
{"label": label, "count": 0}
|
||
for label in explicit[:10]
|
||
]
|
||
|
||
theme_counts: Counter[str] = Counter()
|
||
for review in reviews:
|
||
content = clean(review.get("content"))
|
||
for label, keywords in REVIEW_THEMES:
|
||
if any(keyword in content for keyword in keywords):
|
||
theme_counts[label] += 1
|
||
return [
|
||
{"label": label, "count": count}
|
||
for label, count in theme_counts.most_common(8)
|
||
]
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
|
||
parser.add_argument("--graph-name", default="yunyou_libo")
|
||
args = parser.parse_args()
|
||
|
||
raw_records = load_raw_records(args.source_dir)
|
||
graph = FalkorDB(
|
||
host=settings.falkordb_host,
|
||
port=settings.falkordb_port,
|
||
).select_graph(args.graph_name)
|
||
rows = graph.query(
|
||
"MATCH (n:FoodPlace) WHERE n.dianping_name IS NOT NULL "
|
||
"RETURN n.element_id,n.dianping_source_poi_id,n.dianping_shop_id,"
|
||
"n.dianping_shop_uuid,n.gaode_poi_id,n.dianping_review_tags"
|
||
).result_set
|
||
|
||
updated_nodes = 0
|
||
stored_deals = 0
|
||
stored_reviews = 0
|
||
stored_tags = 0
|
||
missing_raw = 0
|
||
for row in rows:
|
||
element_id = clean(row[0])
|
||
identifiers = [clean(value) for value in row[1:5] if clean(value)]
|
||
existing_review_tags = clean(row[5])
|
||
payload = next(
|
||
(raw_records[identifier] for identifier in identifiers if identifier in raw_records),
|
||
None,
|
||
)
|
||
if payload is None:
|
||
missing_raw += 1
|
||
continue
|
||
deals = build_deals(payload)
|
||
reviews = build_reviews(payload)
|
||
tags = build_review_tags(payload, reviews, existing_review_tags)
|
||
result = graph.query(
|
||
"MATCH (n:FoodPlace {element_id:$element_id}) "
|
||
"SET n.group_buy_items_json=$deals,"
|
||
"n.food_review_items_json=$reviews,"
|
||
"n.food_review_tags_json=$tags "
|
||
"RETURN count(n)",
|
||
{
|
||
"element_id": element_id,
|
||
"deals": json.dumps(deals, ensure_ascii=False, separators=(",", ":")),
|
||
"reviews": json.dumps(reviews, ensure_ascii=False, separators=(",", ":")),
|
||
"tags": json.dumps(tags, ensure_ascii=False, separators=(",", ":")),
|
||
},
|
||
).result_set
|
||
changed = int(result[0][0] if result else 0)
|
||
if changed:
|
||
updated_nodes += changed
|
||
stored_deals += len(deals) * changed
|
||
stored_reviews += len(reviews) * changed
|
||
stored_tags += len(tags) * changed
|
||
|
||
print(json.dumps({
|
||
"graph_name": args.graph_name,
|
||
"raw_record_identifiers": len(raw_records),
|
||
"graph_food_nodes": len(rows),
|
||
"updated_nodes": updated_nodes,
|
||
"missing_raw_nodes": missing_raw,
|
||
"stored_deals": stored_deals,
|
||
"stored_reviews": stored_reviews,
|
||
"stored_review_tags": stored_tags,
|
||
}, ensure_ascii=False))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|