Add Cloud Tour Libo knowledge graph platform
This commit is contained in:
291
scripts/enrich_yunyou_libo_hotel_rooms_reviews.py
Normal file
291
scripts/enrich_yunyou_libo_hotel_rooms_reviews.py
Normal file
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Attach detailed Ctrip room, offer, and guest-review data to Libo Hotel nodes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from falkordb import FalkorDB
|
||||
|
||||
ROOT_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT_DIR))
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
DEFAULT_SOURCE_DIR = Path(
|
||||
"/Users/xuexue/Desktop/荔波小七孔源数据/携程/CSV数据表"
|
||||
)
|
||||
|
||||
|
||||
def clean(value: Any) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def number(value: Any) -> float | None:
|
||||
text = clean(value).replace(",", "")
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
result = float(text)
|
||||
except ValueError:
|
||||
return None
|
||||
return int(result) if result.is_integer() else result
|
||||
|
||||
|
||||
def integer(value: Any) -> int | None:
|
||||
result = number(value)
|
||||
return int(result) if result is not None else None
|
||||
|
||||
|
||||
def load_csv(path: Path) -> list[dict[str, str]]:
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as stream:
|
||||
return list(csv.DictReader(stream))
|
||||
|
||||
|
||||
def compact(record: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
key: value
|
||||
for key, value in record.items()
|
||||
if value not in (None, "", [], {})
|
||||
}
|
||||
|
||||
|
||||
def clean_offer_text(value: Any, kind: str) -> str:
|
||||
"""Keep stable booking terms while dropping scraper-combined live inventory copy."""
|
||||
text = clean(value)
|
||||
if not text:
|
||||
return ""
|
||||
import re
|
||||
|
||||
patterns = {
|
||||
"breakfast": [
|
||||
r"无早餐",
|
||||
r"\d+\s*份早餐",
|
||||
r"含早餐",
|
||||
],
|
||||
"cancellation_policy": [
|
||||
r"\d{2}月\d{2}日\s*\d{2}:\d{2}前可免费取消",
|
||||
r"限时取消",
|
||||
r"免费取消",
|
||||
r"不可取消",
|
||||
],
|
||||
"confirmation_policy": [
|
||||
r"立即确认",
|
||||
r"等待确认",
|
||||
],
|
||||
"payment_method": [
|
||||
r"在线付(?:[·・]\s*可先住后付)?",
|
||||
r"到店付",
|
||||
r"可先住后付",
|
||||
],
|
||||
}
|
||||
for pattern in patterns.get(kind, []):
|
||||
match = re.search(pattern, text)
|
||||
if match:
|
||||
return match.group(0).replace(" ", "")
|
||||
if len(text) <= 36 and not re.search(r"仅剩\s*\d+\s*间", text):
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def load_room_images(source_dir: Path) -> dict[tuple[str, str], str]:
|
||||
result: dict[tuple[str, str], tuple[int, str]] = {}
|
||||
for row in load_csv(source_dir / "携程_酒店详情_酒店图片.csv"):
|
||||
if clean(row.get("图片类型")).lower() != "room":
|
||||
continue
|
||||
hotel_id = clean(row.get("酒店ID") or row.get("\ufeff酒店ID"))
|
||||
room_id = clean(row.get("房型ID"))
|
||||
image_url = clean(row.get("图片URL"))
|
||||
if not hotel_id or not room_id or not image_url:
|
||||
continue
|
||||
sequence = integer(row.get("图片序号")) or 999999
|
||||
current = result.get((hotel_id, room_id))
|
||||
if current is None or sequence < current[0]:
|
||||
result[(hotel_id, room_id)] = (sequence, image_url)
|
||||
return {key: value[1] for key, value in result.items()}
|
||||
|
||||
|
||||
def load_room_payloads(source_dir: Path) -> dict[str, list[dict[str, Any]]]:
|
||||
room_rows = load_csv(source_dir / "携程_酒店详情_房型.csv")
|
||||
offer_rows = load_csv(source_dir / "携程_酒店详情_售卖方案.csv")
|
||||
room_images = load_room_images(source_dir)
|
||||
|
||||
offers_by_room: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in offer_rows:
|
||||
hotel_id = clean(row.get("酒店ID"))
|
||||
room_id = clean(row.get("房型ID"))
|
||||
if not hotel_id or not room_id:
|
||||
continue
|
||||
offers_by_room[(hotel_id, room_id)].append(compact({
|
||||
"offer_id": clean(row.get("售卖方案ID")),
|
||||
"sequence": integer(row.get("方案序号")),
|
||||
"breakfast": clean_offer_text(row.get("早餐"), "breakfast"),
|
||||
"cancellation_policy": clean_offer_text(
|
||||
row.get("取消政策"),
|
||||
"cancellation_policy",
|
||||
),
|
||||
"confirmation_policy": clean_offer_text(
|
||||
row.get("确认政策"),
|
||||
"confirmation_policy",
|
||||
),
|
||||
"payment_method": clean_offer_text(
|
||||
row.get("支付方式"),
|
||||
"payment_method",
|
||||
),
|
||||
"occupancy": integer(row.get("入住人数")),
|
||||
"original_price": number(row.get("原价")),
|
||||
"current_price": number(row.get("当前价")),
|
||||
"tax_fee": number(row.get("税费")),
|
||||
"currency": clean(row.get("币种")) or "CNY",
|
||||
"source_url": clean(row.get("来源URL")),
|
||||
}))
|
||||
|
||||
result: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in room_rows:
|
||||
hotel_id = clean(row.get("酒店ID"))
|
||||
room_id = clean(row.get("房型ID"))
|
||||
name = clean(row.get("房型名称"))
|
||||
if not hotel_id or not room_id or not name:
|
||||
continue
|
||||
offers = offers_by_room.get((hotel_id, room_id), [])
|
||||
offers.sort(key=lambda item: (
|
||||
item.get("current_price") is None,
|
||||
item.get("current_price") or 0,
|
||||
item.get("sequence") or 999,
|
||||
))
|
||||
result[hotel_id].append(compact({
|
||||
"room_id": room_id,
|
||||
"name": name,
|
||||
"price": number(row.get("房型价格")),
|
||||
"currency": clean(row.get("币种")) or "CNY",
|
||||
"room_url": clean(row.get("房型URL")),
|
||||
"image_url": room_images.get((hotel_id, room_id), ""),
|
||||
"bed_type": clean(row.get("床型")),
|
||||
"window": clean(row.get("窗户")),
|
||||
"area": clean(row.get("面积")),
|
||||
"floor": clean(row.get("楼层")),
|
||||
"capacity": integer(row.get("可住人数")),
|
||||
"extra_bed_policy": clean(row.get("加床政策")),
|
||||
"smoking_policy": clean(row.get("吸烟政策")),
|
||||
"network": clean(row.get("网络")),
|
||||
"facilities": clean(row.get("房型设施")),
|
||||
"image_count": integer(row.get("房型图片数")),
|
||||
"source_url": clean(row.get("来源URL")),
|
||||
"offers": offers,
|
||||
}))
|
||||
|
||||
for rooms in result.values():
|
||||
rooms.sort(key=lambda item: (
|
||||
item.get("price") is None,
|
||||
item.get("price") or 0,
|
||||
item.get("name") or "",
|
||||
))
|
||||
return dict(result)
|
||||
|
||||
|
||||
def review_timestamp(value: str) -> float:
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("/", "-")).timestamp()
|
||||
except ValueError:
|
||||
return 0
|
||||
|
||||
|
||||
def load_review_payloads(source_dir: Path) -> dict[str, list[dict[str, Any]]]:
|
||||
result: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in load_csv(source_dir / "携程_酒店详情_住客点评.csv"):
|
||||
hotel_id = clean(row.get("酒店ID") or row.get("\ufeff酒店ID"))
|
||||
review_id = clean(row.get("点评ID"))
|
||||
content = clean(row.get("点评内容"))
|
||||
if not hotel_id or not review_id or not content:
|
||||
continue
|
||||
image_urls = [
|
||||
item.strip()
|
||||
for item in clean(row.get("点评图片")).split("|")
|
||||
if item.strip()
|
||||
]
|
||||
result[hotel_id].append(compact({
|
||||
"review_id": review_id,
|
||||
"nickname": clean(row.get("用户昵称")) or "匿名用户",
|
||||
"review_at": clean(row.get("点评时间")),
|
||||
"stay_at": clean(row.get("入住时间")),
|
||||
"room_type": clean(row.get("入住房型")),
|
||||
"travel_type": clean(row.get("出行类型")),
|
||||
"score": number(row.get("总评分")),
|
||||
"content": content,
|
||||
"like_count": integer(row.get("点赞数")),
|
||||
"reply_count": integer(row.get("回复数")),
|
||||
"image_count": integer(row.get("图片数量")),
|
||||
"image_urls": image_urls,
|
||||
"source_url": clean(row.get("来源URL")),
|
||||
}))
|
||||
|
||||
for reviews in result.values():
|
||||
reviews.sort(
|
||||
key=lambda item: review_timestamp(clean(item.get("review_at"))),
|
||||
reverse=True,
|
||||
)
|
||||
return dict(result)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
|
||||
parser.add_argument("--graph-name", default="yunyou_libo")
|
||||
args = parser.parse_args()
|
||||
|
||||
rooms_by_hotel = load_room_payloads(args.source_dir)
|
||||
reviews_by_hotel = load_review_payloads(args.source_dir)
|
||||
graph = FalkorDB(
|
||||
host=settings.falkordb_host,
|
||||
port=settings.falkordb_port,
|
||||
).select_graph(args.graph_name)
|
||||
|
||||
hotel_rows = graph.query(
|
||||
"MATCH (n:Hotel) WHERE n.ctrip_hotel_id IS NOT NULL "
|
||||
"RETURN DISTINCT n.ctrip_hotel_id"
|
||||
).result_set
|
||||
graph_hotel_ids = {clean(row[0]) for row in hotel_rows if row and clean(row[0])}
|
||||
|
||||
updated_hotels = 0
|
||||
stored_rooms = 0
|
||||
stored_reviews = 0
|
||||
for hotel_id in sorted(graph_hotel_ids & (rooms_by_hotel.keys() | reviews_by_hotel.keys())):
|
||||
rooms = rooms_by_hotel.get(hotel_id, [])
|
||||
reviews = reviews_by_hotel.get(hotel_id, [])
|
||||
result = graph.query(
|
||||
"MATCH (n:Hotel) WHERE n.ctrip_hotel_id=$hotel_id "
|
||||
"SET n.room_items_json=$rooms, n.review_items_json=$reviews "
|
||||
"RETURN count(n)",
|
||||
{
|
||||
"hotel_id": hotel_id,
|
||||
"rooms": json.dumps(rooms, ensure_ascii=False, separators=(",", ":")),
|
||||
"reviews": json.dumps(reviews, ensure_ascii=False, separators=(",", ":")),
|
||||
},
|
||||
).result_set
|
||||
changed = int(result[0][0] if result else 0)
|
||||
if changed:
|
||||
updated_hotels += changed
|
||||
stored_rooms += len(rooms) * changed
|
||||
stored_reviews += len(reviews) * changed
|
||||
|
||||
print(json.dumps({
|
||||
"graph_name": args.graph_name,
|
||||
"source_hotels_with_rooms": len(rooms_by_hotel),
|
||||
"source_hotels_with_reviews": len(reviews_by_hotel),
|
||||
"graph_hotels_with_ctrip_id": len(graph_hotel_ids),
|
||||
"updated_hotels": updated_hotels,
|
||||
"stored_rooms": stored_rooms,
|
||||
"stored_reviews": stored_reviews,
|
||||
}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user