292 lines
10 KiB
Python
292 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Attach detailed Ctrip room, offer, and guest-review data to Libo Hotel nodes."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import sys
|
|
from collections import defaultdict
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from falkordb import FalkorDB
|
|
|
|
ROOT_DIR = Path(__file__).resolve().parents[1]
|
|
if str(ROOT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(ROOT_DIR))
|
|
|
|
from app.config import settings
|
|
|
|
|
|
DEFAULT_SOURCE_DIR = Path(
|
|
"/Users/xuexue/Desktop/荔波小七孔源数据/携程/CSV数据表"
|
|
)
|
|
|
|
|
|
def clean(value: Any) -> str:
|
|
return "" if value is None else str(value).strip()
|
|
|
|
|
|
def number(value: Any) -> float | None:
|
|
text = clean(value).replace(",", "")
|
|
if not text:
|
|
return None
|
|
try:
|
|
result = float(text)
|
|
except ValueError:
|
|
return None
|
|
return int(result) if result.is_integer() else result
|
|
|
|
|
|
def integer(value: Any) -> int | None:
|
|
result = number(value)
|
|
return int(result) if result is not None else None
|
|
|
|
|
|
def load_csv(path: Path) -> list[dict[str, str]]:
|
|
with path.open("r", encoding="utf-8-sig", newline="") as stream:
|
|
return list(csv.DictReader(stream))
|
|
|
|
|
|
def compact(record: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
key: value
|
|
for key, value in record.items()
|
|
if value not in (None, "", [], {})
|
|
}
|
|
|
|
|
|
def clean_offer_text(value: Any, kind: str) -> str:
|
|
"""Keep stable booking terms while dropping scraper-combined live inventory copy."""
|
|
text = clean(value)
|
|
if not text:
|
|
return ""
|
|
import re
|
|
|
|
patterns = {
|
|
"breakfast": [
|
|
r"无早餐",
|
|
r"\d+\s*份早餐",
|
|
r"含早餐",
|
|
],
|
|
"cancellation_policy": [
|
|
r"\d{2}月\d{2}日\s*\d{2}:\d{2}前可免费取消",
|
|
r"限时取消",
|
|
r"免费取消",
|
|
r"不可取消",
|
|
],
|
|
"confirmation_policy": [
|
|
r"立即确认",
|
|
r"等待确认",
|
|
],
|
|
"payment_method": [
|
|
r"在线付(?:[·・]\s*可先住后付)?",
|
|
r"到店付",
|
|
r"可先住后付",
|
|
],
|
|
}
|
|
for pattern in patterns.get(kind, []):
|
|
match = re.search(pattern, text)
|
|
if match:
|
|
return match.group(0).replace(" ", "")
|
|
if len(text) <= 36 and not re.search(r"仅剩\s*\d+\s*间", text):
|
|
return text
|
|
return ""
|
|
|
|
|
|
def load_room_images(source_dir: Path) -> dict[tuple[str, str], str]:
|
|
result: dict[tuple[str, str], tuple[int, str]] = {}
|
|
for row in load_csv(source_dir / "携程_酒店详情_酒店图片.csv"):
|
|
if clean(row.get("图片类型")).lower() != "room":
|
|
continue
|
|
hotel_id = clean(row.get("酒店ID") or row.get("\ufeff酒店ID"))
|
|
room_id = clean(row.get("房型ID"))
|
|
image_url = clean(row.get("图片URL"))
|
|
if not hotel_id or not room_id or not image_url:
|
|
continue
|
|
sequence = integer(row.get("图片序号")) or 999999
|
|
current = result.get((hotel_id, room_id))
|
|
if current is None or sequence < current[0]:
|
|
result[(hotel_id, room_id)] = (sequence, image_url)
|
|
return {key: value[1] for key, value in result.items()}
|
|
|
|
|
|
def load_room_payloads(source_dir: Path) -> dict[str, list[dict[str, Any]]]:
|
|
room_rows = load_csv(source_dir / "携程_酒店详情_房型.csv")
|
|
offer_rows = load_csv(source_dir / "携程_酒店详情_售卖方案.csv")
|
|
room_images = load_room_images(source_dir)
|
|
|
|
offers_by_room: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
|
for row in offer_rows:
|
|
hotel_id = clean(row.get("酒店ID"))
|
|
room_id = clean(row.get("房型ID"))
|
|
if not hotel_id or not room_id:
|
|
continue
|
|
offers_by_room[(hotel_id, room_id)].append(compact({
|
|
"offer_id": clean(row.get("售卖方案ID")),
|
|
"sequence": integer(row.get("方案序号")),
|
|
"breakfast": clean_offer_text(row.get("早餐"), "breakfast"),
|
|
"cancellation_policy": clean_offer_text(
|
|
row.get("取消政策"),
|
|
"cancellation_policy",
|
|
),
|
|
"confirmation_policy": clean_offer_text(
|
|
row.get("确认政策"),
|
|
"confirmation_policy",
|
|
),
|
|
"payment_method": clean_offer_text(
|
|
row.get("支付方式"),
|
|
"payment_method",
|
|
),
|
|
"occupancy": integer(row.get("入住人数")),
|
|
"original_price": number(row.get("原价")),
|
|
"current_price": number(row.get("当前价")),
|
|
"tax_fee": number(row.get("税费")),
|
|
"currency": clean(row.get("币种")) or "CNY",
|
|
"source_url": clean(row.get("来源URL")),
|
|
}))
|
|
|
|
result: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for row in room_rows:
|
|
hotel_id = clean(row.get("酒店ID"))
|
|
room_id = clean(row.get("房型ID"))
|
|
name = clean(row.get("房型名称"))
|
|
if not hotel_id or not room_id or not name:
|
|
continue
|
|
offers = offers_by_room.get((hotel_id, room_id), [])
|
|
offers.sort(key=lambda item: (
|
|
item.get("current_price") is None,
|
|
item.get("current_price") or 0,
|
|
item.get("sequence") or 999,
|
|
))
|
|
result[hotel_id].append(compact({
|
|
"room_id": room_id,
|
|
"name": name,
|
|
"price": number(row.get("房型价格")),
|
|
"currency": clean(row.get("币种")) or "CNY",
|
|
"room_url": clean(row.get("房型URL")),
|
|
"image_url": room_images.get((hotel_id, room_id), ""),
|
|
"bed_type": clean(row.get("床型")),
|
|
"window": clean(row.get("窗户")),
|
|
"area": clean(row.get("面积")),
|
|
"floor": clean(row.get("楼层")),
|
|
"capacity": integer(row.get("可住人数")),
|
|
"extra_bed_policy": clean(row.get("加床政策")),
|
|
"smoking_policy": clean(row.get("吸烟政策")),
|
|
"network": clean(row.get("网络")),
|
|
"facilities": clean(row.get("房型设施")),
|
|
"image_count": integer(row.get("房型图片数")),
|
|
"source_url": clean(row.get("来源URL")),
|
|
"offers": offers,
|
|
}))
|
|
|
|
for rooms in result.values():
|
|
rooms.sort(key=lambda item: (
|
|
item.get("price") is None,
|
|
item.get("price") or 0,
|
|
item.get("name") or "",
|
|
))
|
|
return dict(result)
|
|
|
|
|
|
def review_timestamp(value: str) -> float:
|
|
try:
|
|
return datetime.fromisoformat(value.replace("/", "-")).timestamp()
|
|
except ValueError:
|
|
return 0
|
|
|
|
|
|
def load_review_payloads(source_dir: Path) -> dict[str, list[dict[str, Any]]]:
|
|
result: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for row in load_csv(source_dir / "携程_酒店详情_住客点评.csv"):
|
|
hotel_id = clean(row.get("酒店ID") or row.get("\ufeff酒店ID"))
|
|
review_id = clean(row.get("点评ID"))
|
|
content = clean(row.get("点评内容"))
|
|
if not hotel_id or not review_id or not content:
|
|
continue
|
|
image_urls = [
|
|
item.strip()
|
|
for item in clean(row.get("点评图片")).split("|")
|
|
if item.strip()
|
|
]
|
|
result[hotel_id].append(compact({
|
|
"review_id": review_id,
|
|
"nickname": clean(row.get("用户昵称")) or "匿名用户",
|
|
"review_at": clean(row.get("点评时间")),
|
|
"stay_at": clean(row.get("入住时间")),
|
|
"room_type": clean(row.get("入住房型")),
|
|
"travel_type": clean(row.get("出行类型")),
|
|
"score": number(row.get("总评分")),
|
|
"content": content,
|
|
"like_count": integer(row.get("点赞数")),
|
|
"reply_count": integer(row.get("回复数")),
|
|
"image_count": integer(row.get("图片数量")),
|
|
"image_urls": image_urls,
|
|
"source_url": clean(row.get("来源URL")),
|
|
}))
|
|
|
|
for reviews in result.values():
|
|
reviews.sort(
|
|
key=lambda item: review_timestamp(clean(item.get("review_at"))),
|
|
reverse=True,
|
|
)
|
|
return dict(result)
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
|
|
parser.add_argument("--graph-name", default="yunyou_libo")
|
|
args = parser.parse_args()
|
|
|
|
rooms_by_hotel = load_room_payloads(args.source_dir)
|
|
reviews_by_hotel = load_review_payloads(args.source_dir)
|
|
graph = FalkorDB(
|
|
host=settings.falkordb_host,
|
|
port=settings.falkordb_port,
|
|
).select_graph(args.graph_name)
|
|
|
|
hotel_rows = graph.query(
|
|
"MATCH (n:Hotel) WHERE n.ctrip_hotel_id IS NOT NULL "
|
|
"RETURN DISTINCT n.ctrip_hotel_id"
|
|
).result_set
|
|
graph_hotel_ids = {clean(row[0]) for row in hotel_rows if row and clean(row[0])}
|
|
|
|
updated_hotels = 0
|
|
stored_rooms = 0
|
|
stored_reviews = 0
|
|
for hotel_id in sorted(graph_hotel_ids & (rooms_by_hotel.keys() | reviews_by_hotel.keys())):
|
|
rooms = rooms_by_hotel.get(hotel_id, [])
|
|
reviews = reviews_by_hotel.get(hotel_id, [])
|
|
result = graph.query(
|
|
"MATCH (n:Hotel) WHERE n.ctrip_hotel_id=$hotel_id "
|
|
"SET n.room_items_json=$rooms, n.review_items_json=$reviews "
|
|
"RETURN count(n)",
|
|
{
|
|
"hotel_id": hotel_id,
|
|
"rooms": json.dumps(rooms, ensure_ascii=False, separators=(",", ":")),
|
|
"reviews": json.dumps(reviews, ensure_ascii=False, separators=(",", ":")),
|
|
},
|
|
).result_set
|
|
changed = int(result[0][0] if result else 0)
|
|
if changed:
|
|
updated_hotels += changed
|
|
stored_rooms += len(rooms) * changed
|
|
stored_reviews += len(reviews) * changed
|
|
|
|
print(json.dumps({
|
|
"graph_name": args.graph_name,
|
|
"source_hotels_with_rooms": len(rooms_by_hotel),
|
|
"source_hotels_with_reviews": len(reviews_by_hotel),
|
|
"graph_hotels_with_ctrip_id": len(graph_hotel_ids),
|
|
"updated_hotels": updated_hotels,
|
|
"stored_rooms": stored_rooms,
|
|
"stored_reviews": stored_reviews,
|
|
}, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|