duanshuwen/WonderQ-Project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

数据迁移bug修复v2

Browse Source
This commit is contained in:
andy
2026-08-25 12:06:42 +08:00
parent 202444353e
commit a7255d8bb8
7 changed files with 426 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
WonderQ-Admin/alembic/versions/0017_home_content.py
View File

@@ -50,6 +50,8 @@ SEED_TEAM_BUILDINGS = (
"description": "洞穴、瀑降与协作，适合 10-30 人。",
"image": "https://dimg04.c-ctrip.com/images/1mh0412000njfr1ot9453_W_640_10000.jpg?proc=autoorient",
"demandKeyword": "户外团建",
"detailSubtitle": "洞穴、瀑降与协作，适合 10-30 人。",
"detailParagraphs": ["洞穴、瀑降与协作，适合 10-30 人。"],
"isActive": True,
"sortOrder": 0,
},
@@ -60,6 +62,8 @@ SEED_TEAM_BUILDINGS = (
"description": "溯溪与分组协作，兼顾参与感与安全。",
"image": "https://dimg04.c-ctrip.com/images/0EQ5712000ca7t504EC0E_W_640_10000.jpg?proc=autoorient",
"demandKeyword": "峡谷团建",
"detailSubtitle": "溯溪与分组协作，兼顾参与感与安全。",
"detailParagraphs": ["溯溪与分组协作，兼顾参与感与安全。"],
"isActive": True,
"sortOrder": 1,
},
@@ -70,6 +74,8 @@ SEED_TEAM_BUILDINGS = (
"description": "夜游、长桌宴与文化体验，适合团建收尾。",
"image": "https://p6.itc.cn/q_70/images03/20200918/df728d2b79d943da869333e2ea2c92c8.jpeg",
"demandKeyword": "贵州团建",
"detailSubtitle": "夜游、长桌宴与文化体验，适合团建收尾。",
"detailParagraphs": ["夜游、长桌宴与文化体验，适合团建收尾。"],
"isActive": True,
"sortOrder": 2,
},
@@ -80,6 +86,7 @@ SEED_WILD_ARCHIVES = (
"id": "hundred-meter-descent",
"title": "百米自降",
"image": "https://genk.mediacdn.vn/139269124445442048/2024/4/27/10-23-sinkhole-1714189653945948438879.jpg",
"images": ["https://genk.mediacdn.vn/139269124445442048/2024/4/27/10-23-sinkhole-1714189653945948438879.jpg"],
"demandKeyword": "悬崖瀑降",
"isActive": True,
"sortOrder": 0,
@@ -88,6 +95,7 @@ SEED_WILD_ARCHIVES = (
"id": "shilong-cave",
"title": "石龙洞",
"image": "https://www.zurnal24.si/media/img/5e/d5/9526a56dba168aa136f3.jpeg",
"images": ["https://www.zurnal24.si/media/img/5e/d5/9526a56dba168aa136f3.jpeg"],
"demandKeyword": "地心探险",
"isActive": True,
"sortOrder": 1,
@@ -96,6 +104,7 @@ SEED_WILD_ARCHIVES = (
"id": "cliff-current",
"title": "绝壁迎流",
"image": "https://q9.itc.cn/q_70/images03/20250810/b62f5afc191a4947a66e6b32721b0235.jpeg",
"images": ["https://q9.itc.cn/q_70/images03/20250810/b62f5afc191a4947a66e6b32721b0235.jpeg"],
"demandKeyword": "峡谷探险",
"isActive": True,
"sortOrder": 2,
@@ -104,6 +113,7 @@ SEED_WILD_ARCHIVES = (
"id": "canyon-streaming",
"title": "峡谷溯溪",
"image": "https://dimg04.c-ctrip.com/images/0EQ5712000ca7t504EC0E_W_640_10000.jpg?proc=autoorient",
"images": ["https://dimg04.c-ctrip.com/images/0EQ5712000ca7t504EC0E_W_640_10000.jpg?proc=autoorient"],
"demandKeyword": "峡谷溯溪",
"isActive": True,
"sortOrder": 3,
@@ -120,12 +130,34 @@ def _create_table(table_name: str, columns: list[sa.Column]) -> None:
op.create_index(f"ix_{table_name}_sortOrder", table_name, ["sortOrder"])
def _prepare_seed_rows(
rows: tuple[dict, ...],
column_names: set[str],
timestamp: datetime,
) -> list[dict]:
return [
{
key: value
for key, value in {**row, "createdAt": timestamp, "updatedAt": timestamp}.items()
if key in column_names
}
for row in rows
]
def _seed_table(table_name: str, columns: list[sa.Column], rows: tuple[dict, ...]) -> None:
table = sa.table(table_name, *[sa.column(column.name, column.type) for column in columns])
if context.is_offline_mode():
seed_columns = [sa.column(column.name, column.type) for column in columns]
else:
seed_columns = [
sa.column(column["name"], column["type"])
for column in inspect(op.get_bind()).get_columns(table_name)
]
table = sa.table(table_name, *seed_columns)
timestamp = datetime(2026, 1, 1)
op.bulk_insert(
table,
[{**row, "createdAt": timestamp, "updatedAt": timestamp} for row in rows],
_prepare_seed_rows(rows, {column.name for column in seed_columns}, timestamp),
)

28
WonderQ-Admin/alembic/versions/0019_home_wild_archive_images.py
View File

@@ -6,6 +6,7 @@ Revises: 0018_home_wanfa_recommendations
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
from sqlalchemy.dialects import postgresql
@@ -16,18 +17,25 @@ depends_on = None
def upgrade() -> None:
op.add_column(
"HomeWildArchive",
sa.Column(
"images",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
server_default=sa.text("'[]'::jsonb"),
),
)
bind = op.get_bind()
inspector = inspect(bind)
if "HomeWildArchive" not in set(inspector.get_table_names()):
return
column_names = {column["name"] for column in inspector.get_columns("HomeWildArchive")}
if "images" not in column_names:
op.add_column(
"HomeWildArchive",
sa.Column(
"images",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
server_default=sa.text("'[]'::jsonb"),
),
)
op.execute(
sa.text(
'UPDATE "HomeWildArchive" SET "images" = jsonb_build_array("image") WHERE jsonb_array_length("images") = 0'
'UPDATE "HomeWildArchive" SET "images" = jsonb_build_array("image") '
'WHERE "images" IS NULL OR jsonb_array_length("images") = 0'
)
)
op.alter_column("HomeWildArchive", "images", server_default=None)

41
WonderQ-Admin/alembic/versions/0020_home_team_building_details.py
View File

@@ -6,6 +6,7 @@ Revises: 0019_home_wild_archive_images
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
from sqlalchemy.dialects import postgresql
@@ -16,30 +17,38 @@ depends_on = None
def upgrade() -> None:
op.add_column(
"HomeTeamBuilding",
sa.Column("detailSubtitle", sa.String(), nullable=False, server_default=""),
)
op.add_column(
"HomeTeamBuilding",
sa.Column(
"detailParagraphs",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
server_default=sa.text("'[]'::jsonb"),
),
)
bind = op.get_bind()
inspector = inspect(bind)
if "HomeTeamBuilding" not in set(inspector.get_table_names()):
return
column_names = {column["name"] for column in inspector.get_columns("HomeTeamBuilding")}
if "detailSubtitle" not in column_names:
op.add_column(
"HomeTeamBuilding",
sa.Column("detailSubtitle", sa.String(), nullable=False, server_default=""),
)
if "detailParagraphs" not in column_names:
op.add_column(
"HomeTeamBuilding",
sa.Column(
"detailParagraphs",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
server_default=sa.text("'[]'::jsonb"),
),
)
op.execute(
sa.text(
'UPDATE "HomeTeamBuilding" '
'SET "detailSubtitle" = CASE '
'WHEN "detailSubtitle" = \'\' THEN "description" '
'WHEN "detailSubtitle" IS NULL OR "detailSubtitle" = \'\' THEN "description" '
'ELSE "detailSubtitle" END, '
'"detailParagraphs" = CASE '
'WHEN jsonb_array_length("detailParagraphs") = 0 '
'WHEN "detailParagraphs" IS NULL OR jsonb_array_length("detailParagraphs") = 0 '
'THEN jsonb_build_array("description") '
'ELSE "detailParagraphs" END '
'WHERE "detailSubtitle" = \'\' OR jsonb_array_length("detailParagraphs") = 0'
'WHERE "detailSubtitle" IS NULL OR "detailSubtitle" = \'\' '
'OR "detailParagraphs" IS NULL OR jsonb_array_length("detailParagraphs") = 0'
)
)
op.alter_column("HomeTeamBuilding", "detailSubtitle", server_default=None)

14
WonderQ-Admin/alembic/versions/0021_detail_records.py
View File

@@ -8,6 +8,7 @@ from datetime import datetime
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
from sqlalchemy.dialects import postgresql
@@ -17,7 +18,7 @@ branch_labels = None
depends_on = None
def upgrade() -> None:
def _create_detail_table() -> None:
op.create_table(
"DetailRecord",
sa.Column("id", sa.String(), nullable=False),
@@ -42,6 +43,12 @@ def upgrade() -> None:
op.create_index("ix_DetailRecord_isActive", "DetailRecord", ["isActive"])
op.create_index("ix_DetailRecord_sortOrder", "DetailRecord", ["sortOrder"])
def upgrade() -> None:
bind = op.get_bind()
if "DetailRecord" not in set(inspect(bind).get_table_names()):
_create_detail_table()
detail_table = sa.table(
"DetailRecord",
sa.column("id", sa.String()),
@@ -61,7 +68,10 @@ def upgrade() -> None:
sa.column("createdAt", sa.DateTime()),
sa.column("updatedAt", sa.DateTime()),
)
routes = op.get_bind().execute(
if bind.execute(sa.select(detail_table.c.id).limit(1)).first() is not None:
return
routes = bind.execute(
sa.text('SELECT "id", "title", "subtitle", "image", "sortOrder" FROM "WanfaRoute" ORDER BY "sortOrder" ASC')
).mappings().all()
timestamp = datetime(2026, 1, 1)

9
WonderQ-Admin/alembic/versions/0023_detail_concierge_advisor.py
View File

@@ -6,6 +6,7 @@ Revises: 0022_opaque_ids
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
revision = "0023_detail_concierge_advisor"
@@ -15,7 +16,13 @@ depends_on = None
def upgrade() -> None:
op.add_column("DetailRecord", sa.Column("conciergeAdvisorId", sa.String(), nullable=True))
bind = op.get_bind()
inspector = inspect(bind)
if "DetailRecord" not in set(inspector.get_table_names()):
return
column_names = {column["name"] for column in inspector.get_columns("DetailRecord")}
if "conciergeAdvisorId" not in column_names:
op.add_column("DetailRecord", sa.Column("conciergeAdvisorId", sa.String(), nullable=True))
def downgrade() -> None:

126
WonderQ-Admin/alembic/versions/0024_vehicle_demand.py
View File

@@ -2,6 +2,7 @@
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
from sqlalchemy.dialects import postgresql
@@ -11,34 +12,107 @@ branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("Lead", sa.Column("leadType", sa.String(), nullable=False, server_default="general"))
op.add_column("Lead", sa.Column("contactName", sa.String(), nullable=True))
op.add_column("Lead", sa.Column("customerId", sa.String(), nullable=True))
op.add_column("Lead", sa.Column("vehicleDemand", postgresql.JSONB(astext_type=sa.Text()), nullable=True))
op.create_index("ix_Lead_leadType", "Lead", ["leadType"])
op.create_index("ix_Lead_customerId", "Lead", ["customerId"])
op.create_foreign_key("fk_Lead_customerId_Customer", "Lead", "Customer", ["customerId"], ["id"], ondelete="SET NULL")
def _ensure_lead_schema(bind) -> None:
inspector = inspect(bind)
table_names = set(inspector.get_table_names())
if "Lead" not in table_names:
return
column_names = {column["name"] for column in inspector.get_columns("Lead")}
missing_columns = {
"leadType": sa.Column("leadType", sa.String(), nullable=False, server_default="general"),
"contactName": sa.Column("contactName", sa.String(), nullable=True),
"customerId": sa.Column("customerId", sa.String(), nullable=True),
"vehicleDemand": sa.Column(
"vehicleDemand",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
),
}
for column_name, column in missing_columns.items():
if column_name not in column_names:
op.add_column("Lead", column)
column_names.add(column_name)
index_names = {index["name"] for index in inspector.get_indexes("Lead") if index.get("name")}
if "ix_Lead_leadType" not in index_names:
op.create_index("ix_Lead_leadType", "Lead", ["leadType"])
if "ix_Lead_customerId" not in index_names:
op.create_index("ix_Lead_customerId", "Lead", ["customerId"])
customer_foreign_key_exists = any(
foreign_key.get("constrained_columns") == ["customerId"]
and foreign_key.get("referred_table") == "Customer"
and foreign_key.get("referred_columns") == ["id"]
for foreign_key in inspector.get_foreign_keys("Lead")
)
if not customer_foreign_key_exists and "Customer" in table_names:
op.create_foreign_key(
"fk_Lead_customerId_Customer",
"Lead",
"Customer",
["customerId"],
["id"],
ondelete="SET NULL",
)
op.alter_column("Lead", "leadType", server_default=None)
op.create_table(
"VehicleServiceConfig",
sa.Column("id", sa.String(), nullable=False),
sa.Column("introTitle", sa.String(), nullable=False),
sa.Column("intro", sa.Text(), nullable=False),
sa.Column("serviceSections", postgresql.JSONB(astext_type=sa.Text()), nullable=False, server_default=sa.text("'[]'::jsonb")),
sa.Column("advantages", postgresql.JSONB(astext_type=sa.Text()), nullable=False, server_default=sa.text("'[]'::jsonb")),
sa.Column("processSteps", postgresql.JSONB(astext_type=sa.Text()), nullable=False, server_default=sa.text("'[]'::jsonb")),
sa.Column("isActive", sa.Boolean(), nullable=False, server_default=sa.text("true")),
sa.Column("createdAt", sa.DateTime(), nullable=False),
sa.Column("updatedAt", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_VehicleServiceConfig_isActive", "VehicleServiceConfig", ["isActive"])
op.alter_column("VehicleServiceConfig", "serviceSections", server_default=None)
op.alter_column("VehicleServiceConfig", "advantages", server_default=None)
op.alter_column("VehicleServiceConfig", "processSteps", server_default=None)
op.alter_column("VehicleServiceConfig", "isActive", server_default=None)
def _ensure_vehicle_service_config(bind) -> None:
inspector = inspect(bind)
table_names = set(inspector.get_table_names())
if "VehicleServiceConfig" not in table_names:
op.create_table(
"VehicleServiceConfig",
sa.Column("id", sa.String(), nullable=False),
sa.Column("introTitle", sa.String(), nullable=False),
sa.Column("intro", sa.Text(), nullable=False),
sa.Column(
"serviceSections",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
server_default=sa.text("'[]'::jsonb"),
),
sa.Column(
"advantages",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
server_default=sa.text("'[]'::jsonb"),
),
sa.Column(
"processSteps",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
server_default=sa.text("'[]'::jsonb"),
),
sa.Column("isActive", sa.Boolean(), nullable=False, server_default=sa.text("true")),
sa.Column("createdAt", sa.DateTime(), nullable=False),
sa.Column("updatedAt", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_VehicleServiceConfig_isActive", "VehicleServiceConfig", ["isActive"])
column_names = {"advantages", "isActive", "processSteps", "serviceSections"}
else:
index_names = {
index["name"]
for index in inspector.get_indexes("VehicleServiceConfig")
if index.get("name")
}
if "ix_VehicleServiceConfig_isActive" not in index_names:
op.create_index("ix_VehicleServiceConfig_isActive", "VehicleServiceConfig", ["isActive"])
column_names = {
column["name"] for column in inspector.get_columns("VehicleServiceConfig")
}
for column_name in ("serviceSections", "advantages", "processSteps", "isActive"):
if column_name in column_names:
op.alter_column("VehicleServiceConfig", column_name, server_default=None)
def upgrade() -> None:
bind = op.get_bind()
_ensure_lead_schema(bind)
_ensure_vehicle_service_config(bind)
def downgrade() -> None:

229
WonderQ-Admin/tests/test_bootstrap_migration_compatibility.py Normal file
View File

@@ -0,0 +1,229 @@
from datetime import datetime
from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path
import pytest
import sqlalchemy as sa
VERSIONS_DIR = Path(__file__).parents[1] / "alembic" / "versions"
def load_migration(filename: str):
path = VERSIONS_DIR / filename
spec = spec_from_file_location(f"bootstrap_compatibility_{path.stem}", path)
assert spec and spec.loader
migration = module_from_spec(spec)
spec.loader.exec_module(migration)
return migration
class CurrentSchemaInspector:
tables = {
"Customer",
"DetailRecord",
"HomeTeamBuilding",
"HomeWildArchive",
"Lead",
"VehicleServiceConfig",
}
columns = {
"DetailRecord": {"conciergeAdvisorId"},
"HomeTeamBuilding": {"detailParagraphs", "detailSubtitle"},
"HomeWildArchive": {"images"},
"Lead": {"contactName", "customerId", "leadType", "vehicleDemand"},
"VehicleServiceConfig": {
"advantages",
"intro",
"introTitle",
"isActive",
"processSteps",
"serviceSections",
},
}
def get_table_names(self) -> list[str]:
return sorted(self.tables)
def get_columns(self, table_name: str) -> list[dict[str, object]]:
return [
{"name": name, "type": sa.String()}
for name in sorted(self.columns.get(table_name, set()))
]
def get_indexes(self, table_name: str) -> list[dict[str, object]]:
indexes = {
"Lead": {"ix_Lead_customerId", "ix_Lead_leadType"},
"VehicleServiceConfig": {"ix_VehicleServiceConfig_isActive"},
}
return [{"name": name} for name in sorted(indexes.get(table_name, set()))]
def get_foreign_keys(self, table_name: str) -> list[dict[str, object]]:
if table_name != "Lead":
return []
return [
{
"name": "Lead_customerId_fkey",
"constrained_columns": ["customerId"],
"referred_table": "Customer",
"referred_columns": ["id"],
}
]
class ExistingRowsResult:
def first(self):
return ("existing",)
def mappings(self):
return self
def all(self) -> list[object]:
return []
class ExistingRowsBind:
def execute(self, _statement) -> ExistingRowsResult:
return ExistingRowsResult()
def record_operation(monkeypatch, migration, operation: str, events: list[tuple[str, ...]]) -> None:
monkeypatch.setattr(
migration.op,
operation,
lambda *args, **_kwargs: events.append((operation, *(str(arg) for arg in args))),
)
def test_home_content_seed_rows_cover_current_non_null_detail_columns():
migration = load_migration("0017_home_content.py")
timestamp = datetime(2026, 1, 1)
team_rows = migration._prepare_seed_rows(
migration.SEED_TEAM_BUILDINGS,
{"id", "description", "detailSubtitle", "detailParagraphs", "createdAt", "updatedAt"},
timestamp,
)
archive_rows = migration._prepare_seed_rows(
migration.SEED_WILD_ARCHIVES,
{"id", "image", "images", "createdAt", "updatedAt"},
timestamp,
)
assert team_rows[0]["detailSubtitle"] == team_rows[0]["description"]
assert team_rows[0]["detailParagraphs"] == [team_rows[0]["description"]]
assert archive_rows[0]["images"] == [archive_rows[0]["image"]]
def test_home_content_online_seed_reflects_current_table_columns(monkeypatch):
migration = load_migration("0017_home_content.py")
inspector = CurrentSchemaInspector()
inspector.tables = {"HomeTeamBuilding"}
inspector.columns = {
"HomeTeamBuilding": {
"createdAt",
"demandKeyword",
"description",
"detailParagraphs",
"detailSubtitle",
"id",
"image",
"isActive",
"sortOrder",
"tag",
"title",
"updatedAt",
}
}
inserted: dict[str, object] = {}
monkeypatch.setattr(migration.context, "is_offline_mode", lambda: False)
monkeypatch.setattr(migration, "inspect", lambda _bind: inspector, raising=False)
monkeypatch.setattr(migration.op, "get_bind", lambda: object())
monkeypatch.setattr(
migration.op,
"bulk_insert",
lambda table, rows: inserted.update(table=table, rows=rows),
)
historical_columns = [
sa.Column("id", sa.String()),
sa.Column("description", sa.Text()),
]
migration._seed_table(
"HomeTeamBuilding",
historical_columns,
migration.SEED_TEAM_BUILDINGS,
)
inserted_table = inserted["table"]
inserted_rows = inserted["rows"]
assert set(inserted_table.c.keys()) == inspector.columns["HomeTeamBuilding"]
assert inserted_rows[0]["detailSubtitle"] == inserted_rows[0]["description"]
assert inserted_rows[0]["detailParagraphs"] == [inserted_rows[0]["description"]]
@pytest.mark.parametrize(
("filename", "table_name", "column_names"),
[
("0019_home_wild_archive_images.py", "HomeWildArchive", {"images"}),
(
"0020_home_team_building_details.py",
"HomeTeamBuilding",
{"detailParagraphs", "detailSubtitle"},
),
("0023_detail_concierge_advisor.py", "DetailRecord", {"conciergeAdvisorId"}),
],
)
def test_column_migrations_do_not_add_columns_already_in_current_schema(
monkeypatch,
filename: str,
table_name: str,
column_names: set[str],
):
migration = load_migration(filename)
inspector = CurrentSchemaInspector()
inspector.tables = {table_name}
inspector.columns = {table_name: column_names}
added_columns: list[tuple[str, ...]] = []
monkeypatch.setattr(migration, "inspect", lambda _bind: inspector, raising=False)
monkeypatch.setattr(migration.op, "get_bind", lambda: object())
record_operation(monkeypatch, migration, "add_column", added_columns)
monkeypatch.setattr(migration.op, "execute", lambda *_args, **_kwargs: None)
monkeypatch.setattr(migration.op, "alter_column", lambda *_args, **_kwargs: None)
migration.upgrade()
assert added_columns == []
def test_detail_record_migration_reuses_populated_current_table(monkeypatch):
migration = load_migration("0021_detail_records.py")
inspector = CurrentSchemaInspector()
events: list[tuple[str, ...]] = []
monkeypatch.setattr(migration, "inspect", lambda _bind: inspector, raising=False)
monkeypatch.setattr(migration.op, "get_bind", lambda: ExistingRowsBind())
for operation in ("bulk_insert", "create_index", "create_table"):
record_operation(monkeypatch, migration, operation, events)
migration.upgrade()
assert events == []
def test_vehicle_demand_migration_reuses_current_schema_objects(monkeypatch):
migration = load_migration("0024_vehicle_demand.py")
inspector = CurrentSchemaInspector()
events: list[tuple[str, ...]] = []
monkeypatch.setattr(migration, "inspect", lambda _bind: inspector, raising=False)
monkeypatch.setattr(migration.op, "get_bind", lambda: object())
for operation in ("add_column", "create_foreign_key", "create_index", "create_table"):
record_operation(monkeypatch, migration, operation, events)
monkeypatch.setattr(migration.op, "alter_column", lambda *_args, **_kwargs: None)
migration.upgrade()
assert events == []