#!/usr/bin/env python3 """Attach structured Dianping deal, tag, and review payloads to Libo food nodes.""" from __future__ import annotations import argparse import json import re import sys from collections import Counter from pathlib import Path from typing import Any from falkordb import FalkorDB ROOT_DIR = Path(__file__).resolve().parents[1] if str(ROOT_DIR) not in sys.path: sys.path.insert(0, str(ROOT_DIR)) from app.config import settings DEFAULT_SOURCE_DIR = Path( "/Users/xuexue/Desktop/荔波小七孔源数据/大众点评/raw" ) REVIEW_THEMES: tuple[tuple[str, tuple[str, ...]], ...] = ( ("口味评价", ("味道", "口味", "好吃", "鲜香", "酸汤", "烤鱼")), ("服务体验", ("服务", "老板", "店员", "热情", "招待")), ("环境体验", ("环境", "装修", "干净", "卫生", "民族风")), ("菜品分量", ("分量", "份量", "量大", "量足")), ("性价比", ("性价比", "价格", "价位", "实惠")), ("停车便利", ("停车", "停车场")), ("排队情况", ("排队", "等位")), ("游客推荐", ("推荐", "必吃", "值得", "不踩雷")), ) def clean(value: Any) -> str: return "" if value is None else str(value).strip() def number(value: Any) -> float | int | None: text = clean(value).replace("¥", "").replace("¥", "").replace(",", "") if not text: return None try: result = float(text) except ValueError: return None return int(result) if result.is_integer() else result def compact(record: dict[str, Any]) -> dict[str, Any]: return { key: value for key, value in record.items() if value not in (None, "", [], {}) } def split_values(value: Any) -> list[str]: return [ item.strip() for item in re.split(r"\s*[||、,,]\s*", clean(value)) if item.strip() ] def normalize_title(value: Any) -> str: return re.sub(r"[\s【】\[\]()()·||/\\\-—_]", "", clean(value)) def load_raw_records(source_dir: Path) -> dict[str, dict[str, Any]]: records: dict[str, dict[str, Any]] = {} for path in sorted(source_dir.glob("*.json")): try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if not isinstance(payload, dict): continue identifiers = { path.stem.removeprefix("dianping_"), clean(payload.get("POI_ID")), clean(payload.get("店铺ID")), clean(payload.get("店铺UUID")), } for identifier in identifiers: if identifier: records[identifier] = payload return records def deal_image( title: str, sequence: int, recommended_dishes: list[dict[str, Any]], shop_image: str, ) -> str: normalized_title = normalize_title(title) usable_images = [] for dish in recommended_dishes: image_url = clean(dish.get("image")) if not image_url: continue usable_images.append(image_url) dish_name = normalize_title(dish.get("name")) if dish_name and dish_name in normalized_title: return image_url if usable_images: return usable_images[(sequence - 1) % len(usable_images)] return shop_image def build_deals(payload: dict[str, Any]) -> list[dict[str, Any]]: raw_deals = payload.get("团购信息") or [] if not isinstance(raw_deals, list): return [] recommended_dishes = payload.get("推荐菜详情") or [] if not isinstance(recommended_dishes, list): recommended_dishes = [] shop_image = clean(payload.get("店铺图片")) deals: list[dict[str, Any]] = [] for sequence, item in enumerate(raw_deals, 1): if not isinstance(item, dict): continue title = clean(item.get("title")) if not title: continue terms = split_values(item.get("tags")) if not terms: terms = [ value for value in re.split(r"\s+", clean(item.get("dishes"))) if value ] deals.append(compact({ "deal_id": f"{clean(payload.get('店铺ID')) or clean(payload.get('POI_ID'))}:{sequence}", "title": title, "price": number(item.get("price")), "original_price": number(item.get("original_price")), "discount": clean(item.get("discount")), "terms": terms, "image_url": deal_image( title, sequence, [dish for dish in recommended_dishes if isinstance(dish, dict)], shop_image, ), })) return deals def build_reviews(payload: dict[str, Any]) -> list[dict[str, Any]]: raw_reviews = payload.get("评论数据") or [] if not isinstance(raw_reviews, list): return [] source_id = ( clean(payload.get("店铺ID")) or clean(payload.get("POI_ID")) or "dianping" ) reviews: list[dict[str, Any]] = [] for sequence, item in enumerate(raw_reviews, 1): if not isinstance(item, dict): continue content = clean(item.get("评论内容")) if not content: continue reviews.append(compact({ "review_id": f"{source_id}:{sequence}", "nickname": clean(item.get("用户名")) or "匿名用户", "review_at": clean(item.get("评论时间")), "rating_label": clean(item.get("评分")), "stars": number(item.get("星级")), "content": content, "image_urls": split_values(item.get("评论图片")), "image_count": number(item.get("图片数量")), })) return reviews def build_review_tags( payload: dict[str, Any], reviews: list[dict[str, Any]], existing_review_tags: str, ) -> list[dict[str, Any]]: explicit = split_values(payload.get("评论标签")) or split_values(existing_review_tags) if explicit: return [ {"label": label, "count": 0} for label in explicit[:10] ] theme_counts: Counter[str] = Counter() for review in reviews: content = clean(review.get("content")) for label, keywords in REVIEW_THEMES: if any(keyword in content for keyword in keywords): theme_counts[label] += 1 return [ {"label": label, "count": count} for label, count in theme_counts.most_common(8) ] def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR) parser.add_argument("--graph-name", default="yunyou_libo") args = parser.parse_args() raw_records = load_raw_records(args.source_dir) graph = FalkorDB( host=settings.falkordb_host, port=settings.falkordb_port, ).select_graph(args.graph_name) rows = graph.query( "MATCH (n:FoodPlace) WHERE n.dianping_name IS NOT NULL " "RETURN n.element_id,n.dianping_source_poi_id,n.dianping_shop_id," "n.dianping_shop_uuid,n.gaode_poi_id,n.dianping_review_tags" ).result_set updated_nodes = 0 stored_deals = 0 stored_reviews = 0 stored_tags = 0 missing_raw = 0 for row in rows: element_id = clean(row[0]) identifiers = [clean(value) for value in row[1:5] if clean(value)] existing_review_tags = clean(row[5]) payload = next( (raw_records[identifier] for identifier in identifiers if identifier in raw_records), None, ) if payload is None: missing_raw += 1 continue deals = build_deals(payload) reviews = build_reviews(payload) tags = build_review_tags(payload, reviews, existing_review_tags) result = graph.query( "MATCH (n:FoodPlace {element_id:$element_id}) " "SET n.group_buy_items_json=$deals," "n.food_review_items_json=$reviews," "n.food_review_tags_json=$tags " "RETURN count(n)", { "element_id": element_id, "deals": json.dumps(deals, ensure_ascii=False, separators=(",", ":")), "reviews": json.dumps(reviews, ensure_ascii=False, separators=(",", ":")), "tags": json.dumps(tags, ensure_ascii=False, separators=(",", ":")), }, ).result_set changed = int(result[0][0] if result else 0) if changed: updated_nodes += changed stored_deals += len(deals) * changed stored_reviews += len(reviews) * changed stored_tags += len(tags) * changed print(json.dumps({ "graph_name": args.graph_name, "raw_record_identifiers": len(raw_records), "graph_food_nodes": len(rows), "updated_nodes": updated_nodes, "missing_raw_nodes": missing_raw, "stored_deals": stored_deals, "stored_reviews": stored_reviews, "stored_review_tags": stored_tags, }, ensure_ascii=False)) if __name__ == "__main__": main()