Files
Cloud-Tour-to-Libo/app/data_platform/record_service.py
T

710 lines
26 KiB
Python

from __future__ import annotations
import json
import uuid
from decimal import Decimal
from typing import Any
from fastapi import HTTPException
from psycopg import sql
from psycopg.types.json import Jsonb
from app.config import settings
from app.data_platform.registry import FieldDefinition, TableDefinition
from app.data_platform.schema import (
get_project_database,
list_project_table_definitions,
list_project_table_entries,
resolve_project_table_entry,
)
from app.db import get_conn
def _json_safe(value: Any) -> Any:
if isinstance(value, Decimal):
return float(value)
if isinstance(value, uuid.UUID):
return str(value)
if hasattr(value, "isoformat"):
return value.isoformat()
return value
def _row_json(row: dict[str, Any] | None) -> dict[str, Any] | None:
if row is None:
return None
return {key: _json_safe(value) for key, value in dict(row).items()}
async def _table_or_404(project_id: str, table_code: str) -> TableDefinition:
entry = await resolve_project_table_entry(project_id, table_code)
if entry:
return entry[0]
raise HTTPException(404, "数据表未注册")
async def _project_table_definitions(project_id: str) -> tuple[TableDefinition, ...]:
return await list_project_table_definitions(project_id)
async def _database_or_404(project_id: str) -> dict[str, Any]:
database = await get_project_database(project_id)
if database:
return database
raise HTTPException(404, "关系数据库不存在或已收藏")
def _coerce_value(field: FieldDefinition, value: Any) -> Any:
if value in ("", None):
return None
if field.data_type == "json":
if isinstance(value, str):
try:
value = json.loads(value)
except json.JSONDecodeError as exc:
raise HTTPException(400, f"{field.label}不是合法 JSON") from exc
return Jsonb(value)
if field.data_type == "boolean":
if isinstance(value, bool):
return value
return str(value).lower() in {"true", "1", "yes", "是"}
if field.data_type == "number":
if "INTEGER" in field.sql_type:
try:
return int(value)
except (TypeError, ValueError) as exc:
raise HTTPException(400, f"{field.label}必须是整数") from exc
try:
return Decimal(str(value))
except Exception as exc:
raise HTTPException(400, f"{field.label}必须是数字") from exc
return value
def _validated_payload(
table: TableDefinition,
body: dict[str, Any],
*,
create: bool,
) -> dict[str, Any]:
definitions = {field.code: field for field in table.fields}
unknown = sorted(set(body) - set(definitions))
if unknown:
raise HTTPException(400, f"不允许的字段:{', '.join(unknown)}")
payload: dict[str, Any] = {}
for code, value in body.items():
field = definitions[code]
if not field.editable:
continue
payload[code] = _coerce_value(field, value)
if create:
missing = [
field.label
for field in table.fields
if field.required and payload.get(field.code) in (None, "")
]
if missing:
raise HTTPException(400, f"缺少必填字段:{', '.join(missing)}")
return payload
async def list_databases() -> list[dict[str, Any]]:
admin_schema = sql.Identifier(settings.db_schema)
async with get_conn() as conn:
async with conn.cursor() as cur:
await cur.execute(
sql.SQL(
"""
SELECT d.*
FROM {}.project_databases d
WHERE d.status='ready'
ORDER BY
CASE WHEN d.project_id='yunyou_libo' THEN 0 ELSE 1 END,
d.created_at DESC
"""
).format(admin_schema)
)
databases = await cur.fetchall()
result: list[dict[str, Any]] = []
for database in databases:
schema_name = str(database["schema_name"])
definitions = await _project_table_definitions(str(database["project_id"]))
table_counts: list[int] = []
latest_updates: list[Any] = []
for table in definitions:
await cur.execute(
sql.SQL(
"SELECT count(*) AS count, max(updated_at) AS updated_at "
"FROM {}.{} WHERE deleted_at IS NULL"
).format(
sql.Identifier(schema_name),
sql.Identifier(table.code),
)
)
summary = await cur.fetchone()
table_counts.append(int(summary["count"] or 0))
if summary["updated_at"]:
latest_updates.append(summary["updated_at"])
row = _row_json(dict(database)) or {}
row.update(
{
"table_count": len(definitions),
"record_count": sum(table_counts),
"updated_at": _json_safe(max(latest_updates)) if latest_updates else _json_safe(database["updated_at"]),
}
)
result.append(row)
return result
async def list_tables(project_id: str) -> dict[str, Any]:
database = await _database_or_404(project_id)
schema_name = str(database["schema_name"])
entries = await list_project_table_entries(project_id)
async with get_conn() as conn:
async with conn.cursor() as cur:
tables: list[dict[str, Any]] = []
for definition, origin, _source_code in entries:
await cur.execute(
sql.SQL(
"SELECT count(*) AS count, max(updated_at) AS updated_at "
"FROM {}.{} WHERE deleted_at IS NULL"
).format(
sql.Identifier(schema_name),
sql.Identifier(definition.code),
)
)
summary = await cur.fetchone()
item = definition.as_dict()
item.update(
{
"record_count": int(summary["count"] or 0),
"updated_at": _json_safe(summary["updated_at"]),
"is_custom": origin == "custom",
"origin": origin,
}
)
tables.append(item)
return {"database": _row_json(database), "tables": tables}
async def inspect_table(project_id: str, table_code: str) -> dict[str, Any]:
"""Return the physical PostgreSQL structure and foreign-key relationships."""
table = await _table_or_404(project_id, table_code)
database = await _database_or_404(project_id)
schema_name = str(database["schema_name"])
definitions = await _project_table_definitions(project_id)
table_labels = {definition.code: definition.label for definition in definitions}
field_definitions = {field.code: field for field in table.fields}
system_labels = {
"id": "记录 ID",
"tenant_id": "租户 ID",
"project_id": "数据库编码",
"created_at": "创建时间",
"updated_at": "更新时间",
"deleted_at": "删除时间",
"deleted_by": "删除人",
}
action_labels = {
"a": "NO ACTION",
"r": "RESTRICT",
"c": "CASCADE",
"n": "SET NULL",
"d": "SET DEFAULT",
}
async with get_conn() as conn:
async with conn.cursor() as cur:
await cur.execute(
"""
SELECT
a.attnum AS ordinal_position,
a.attname AS name,
format_type(a.atttypid, a.atttypmod) AS sql_type,
NOT a.attnotnull AS nullable,
pg_get_expr(ad.adbin, ad.adrelid) AS default_value,
col_description(c.oid, a.attnum) AS comment
FROM pg_attribute a
JOIN pg_class c ON c.oid = a.attrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_attrdef ad
ON ad.adrelid = c.oid AND ad.adnum = a.attnum
WHERE n.nspname = %s
AND c.relname = %s
AND a.attnum > 0
AND NOT a.attisdropped
ORDER BY a.attnum
""",
(schema_name, table.code),
)
physical_columns = await cur.fetchall()
await cur.execute(
"""
SELECT
con.conname AS name,
CASE con.contype
WHEN 'p' THEN 'primary_key'
WHEN 'u' THEN 'unique'
WHEN 'f' THEN 'foreign_key'
WHEN 'c' THEN 'check'
ELSE con.contype::text
END AS type,
ARRAY(
SELECT att.attname
FROM unnest(con.conkey) WITH ORDINALITY AS key(attnum, ord)
JOIN pg_attribute att
ON att.attrelid = con.conrelid
AND att.attnum = key.attnum
ORDER BY key.ord
) AS columns,
pg_get_constraintdef(con.oid, true) AS definition
FROM pg_constraint con
JOIN pg_class rel ON rel.oid = con.conrelid
JOIN pg_namespace ns ON ns.oid = rel.relnamespace
WHERE ns.nspname = %s AND rel.relname = %s
ORDER BY
CASE con.contype WHEN 'p' THEN 0 WHEN 'f' THEN 1 ELSE 2 END,
con.conname
""",
(schema_name, table.code),
)
constraint_rows = await cur.fetchall()
await cur.execute(
"""
SELECT
idx.relname AS name,
i.indisunique AS unique,
i.indisprimary AS primary,
pg_get_indexdef(i.indexrelid) AS definition
FROM pg_index i
JOIN pg_class rel ON rel.oid = i.indrelid
JOIN pg_namespace ns ON ns.oid = rel.relnamespace
JOIN pg_class idx ON idx.oid = i.indexrelid
WHERE ns.nspname = %s AND rel.relname = %s
ORDER BY i.indisprimary DESC, i.indisunique DESC, idx.relname
""",
(schema_name, table.code),
)
index_rows = await cur.fetchall()
await cur.execute(
"""
SELECT
con.conname AS name,
src_ns.nspname AS source_schema,
src.relname AS source_table,
ARRAY(
SELECT att.attname
FROM unnest(con.conkey) WITH ORDINALITY AS key(attnum, ord)
JOIN pg_attribute att
ON att.attrelid = con.conrelid
AND att.attnum = key.attnum
ORDER BY key.ord
) AS source_columns,
tgt_ns.nspname AS target_schema,
tgt.relname AS target_table,
ARRAY(
SELECT att.attname
FROM unnest(con.confkey) WITH ORDINALITY AS key(attnum, ord)
JOIN pg_attribute att
ON att.attrelid = con.confrelid
AND att.attnum = key.attnum
ORDER BY key.ord
) AS target_columns,
con.confupdtype::text AS on_update,
con.confdeltype::text AS on_delete
FROM pg_constraint con
JOIN pg_class src ON src.oid = con.conrelid
JOIN pg_namespace src_ns ON src_ns.oid = src.relnamespace
JOIN pg_class tgt ON tgt.oid = con.confrelid
JOIN pg_namespace tgt_ns ON tgt_ns.oid = tgt.relnamespace
WHERE con.contype = 'f'
AND (
(src_ns.nspname = %s AND src.relname = %s)
OR
(tgt_ns.nspname = %s AND tgt.relname = %s)
)
ORDER BY con.conname
""",
(schema_name, table.code, schema_name, table.code),
)
relationship_rows = await cur.fetchall()
constraints = [
{
"name": str(row["name"]),
"type": str(row["type"]),
"columns": list(row["columns"] or []),
"definition": str(row["definition"] or ""),
}
for row in constraint_rows
]
primary_columns = {
column
for constraint in constraints
if constraint["type"] == "primary_key"
for column in constraint["columns"]
}
unique_columns = {
column
for constraint in constraints
if constraint["type"] in {"primary_key", "unique"}
and len(constraint["columns"]) == 1
for column in constraint["columns"]
}
foreign_columns = {
column
for constraint in constraints
if constraint["type"] == "foreign_key"
for column in constraint["columns"]
}
columns: list[dict[str, Any]] = []
for row in physical_columns:
code = str(row["name"])
definition = field_definitions.get(code)
columns.append(
{
"ordinal_position": int(row["ordinal_position"]),
"name": code,
"label": definition.label if definition else system_labels.get(code, code),
"data_type": definition.data_type if definition else "system",
"sql_type": str(row["sql_type"]),
"nullable": bool(row["nullable"]),
"default": row["default_value"],
"primary_key": code in primary_columns,
"unique": code in unique_columns,
"foreign_key": code in foreign_columns,
"editable": bool(definition.editable) if definition else False,
"structure_editable": bool(definition) and code not in system_labels,
"searchable": bool(definition.searchable) if definition else False,
"sortable": bool(definition.sortable) if definition else False,
"visible_in_list": bool(definition.visible_in_list) if definition else False,
"comment": row["comment"],
}
)
relationships = []
for row in relationship_rows:
source_table = str(row["source_table"])
target_table = str(row["target_table"])
relationships.append(
{
"name": str(row["name"]),
"direction": "outgoing" if source_table == table.code else "incoming",
"source_schema": str(row["source_schema"]),
"source_table": source_table,
"source_table_label": table_labels.get(source_table, source_table),
"source_columns": list(row["source_columns"] or []),
"target_schema": str(row["target_schema"]),
"target_table": target_table,
"target_table_label": table_labels.get(target_table, target_table),
"target_columns": list(row["target_columns"] or []),
"on_update": action_labels.get(str(row["on_update"]), str(row["on_update"])),
"on_delete": action_labels.get(str(row["on_delete"]), str(row["on_delete"])),
}
)
indexes = [
{
"name": str(row["name"]),
"unique": bool(row["unique"]),
"primary": bool(row["primary"]),
"definition": str(row["definition"] or ""),
}
for row in index_rows
]
return {
"table": {
"schema_name": schema_name,
"table_code": table.code,
"label": table.label,
"description": table.description,
"column_count": len(columns),
"index_count": len(indexes),
"constraint_count": len(constraints),
"primary_key": list(primary_columns),
},
"columns": columns,
"constraints": constraints,
"indexes": indexes,
"relationships": relationships,
}
async def list_records(
project_id: str,
table_code: str,
*,
page: int,
page_size: int,
search: str | None,
sort_field: str | None,
sort_order: str,
) -> dict[str, Any]:
table = await _table_or_404(project_id, table_code)
database = await _database_or_404(project_id)
schema_name = str(database["schema_name"])
page = max(1, page)
page_size = max(1, min(5000, page_size))
offset = (page - 1) * page_size
allowed_sort = {"id", "created_at", "updated_at", *(field.code for field in table.fields if field.sortable)}
order_field = sort_field if sort_field in allowed_sort else "updated_at"
order_keyword = sql.SQL("ASC") if sort_order.lower() == "asc" else sql.SQL("DESC")
where_parts: list[sql.Composable] = [sql.SQL("deleted_at IS NULL")]
params: list[Any] = []
searchable = [field for field in table.fields if field.searchable]
if search and searchable:
pattern = f"%{search.strip()}%"
where_parts.append(
sql.SQL("(")
+ sql.SQL(" OR ").join(
sql.SQL("{}::text ILIKE %s").format(sql.Identifier(field.code))
for field in searchable
)
+ sql.SQL(")")
)
params.extend(pattern for _ in searchable)
where_clause = sql.SQL(" AND ").join(where_parts)
columns = ["id", *(field.code for field in table.fields), "created_at", "updated_at"]
async with get_conn() as conn:
async with conn.cursor() as cur:
await cur.execute(
sql.SQL("SELECT count(*) AS count FROM {}.{} WHERE {}").format(
sql.Identifier(schema_name),
sql.Identifier(table.code),
where_clause,
),
params,
)
total = int((await cur.fetchone())["count"])
await cur.execute(
sql.SQL(
"SELECT {} FROM {}.{} WHERE {} ORDER BY {} {} LIMIT %s OFFSET %s"
).format(
sql.SQL(", ").join(sql.Identifier(column) for column in columns),
sql.Identifier(schema_name),
sql.Identifier(table.code),
where_clause,
sql.Identifier(order_field),
order_keyword,
),
[*params, page_size, offset],
)
rows = [_row_json(dict(row)) for row in await cur.fetchall()]
return {
"table": table.as_dict(),
"items": rows,
"page": page,
"page_size": page_size,
"total": total,
}
async def _write_audit(
cur,
*,
schema_name: str,
tenant_id: str,
project_id: str,
table_code: str,
record_id: uuid.UUID,
operation: str,
before_data: dict[str, Any] | None,
after_data: dict[str, Any] | None,
actor: str,
) -> None:
await cur.execute(
sql.SQL(
"""
INSERT INTO {}.data_change_logs (
tenant_id, project_id, table_code, record_id, operation,
before_data, after_data, actor
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
"""
).format(sql.Identifier(schema_name)),
(
tenant_id,
project_id,
table_code,
record_id,
operation,
Jsonb(before_data) if before_data is not None else None,
Jsonb(after_data) if after_data is not None else None,
actor,
),
)
async def create_record(
project_id: str,
table_code: str,
body: dict[str, Any],
actor: str,
) -> dict[str, Any]:
table = await _table_or_404(project_id, table_code)
if not table.allow_create:
raise HTTPException(403, "该表不允许新增")
database = await _database_or_404(project_id)
payload = _validated_payload(table, body, create=True)
record_id = uuid.uuid4()
schema_name = str(database["schema_name"])
tenant_id = str(database["tenant_id"])
columns = ["id", "tenant_id", "project_id", *payload.keys()]
values = [record_id, tenant_id, project_id, *payload.values()]
returning = ["id", *(field.code for field in table.fields), "created_at", "updated_at"]
async with get_conn() as conn:
async with conn.cursor() as cur:
await cur.execute(
sql.SQL("INSERT INTO {}.{} ({}) VALUES ({}) RETURNING {}").format(
sql.Identifier(schema_name),
sql.Identifier(table.code),
sql.SQL(", ").join(sql.Identifier(column) for column in columns),
sql.SQL(", ").join(sql.Placeholder() for _ in values),
sql.SQL(", ").join(sql.Identifier(column) for column in returning),
),
values,
)
row = _row_json(dict(await cur.fetchone()))
await _write_audit(
cur,
schema_name=schema_name,
tenant_id=tenant_id,
project_id=project_id,
table_code=table_code,
record_id=record_id,
operation="create",
before_data=None,
after_data=row,
actor=actor,
)
await conn.commit()
return row or {}
async def update_record(
project_id: str,
table_code: str,
record_id: str,
body: dict[str, Any],
actor: str,
) -> dict[str, Any]:
table = await _table_or_404(project_id, table_code)
if not table.allow_update:
raise HTTPException(403, "该表不允许修改")
try:
record_uuid = uuid.UUID(record_id)
except ValueError as exc:
raise HTTPException(400, "记录 ID 格式错误") from exc
database = await _database_or_404(project_id)
payload = _validated_payload(table, body, create=False)
if not payload:
raise HTTPException(400, "没有可修改字段")
schema_name = str(database["schema_name"])
tenant_id = str(database["tenant_id"])
returning = ["id", *(field.code for field in table.fields), "created_at", "updated_at"]
async with get_conn() as conn:
async with conn.cursor() as cur:
await cur.execute(
sql.SQL("SELECT {} FROM {}.{} WHERE id=%s AND deleted_at IS NULL").format(
sql.SQL(", ").join(sql.Identifier(column) for column in returning),
sql.Identifier(schema_name),
sql.Identifier(table.code),
),
(record_uuid,),
)
before = _row_json(await cur.fetchone())
if not before:
raise HTTPException(404, "记录不存在")
assignments = [
sql.SQL("{}=%s").format(sql.Identifier(column))
for column in payload
]
if any(field.code == "version" for field in table.fields):
assignments.append(sql.SQL("version=version+1"))
assignments.append(sql.SQL("updated_at=now()"))
await cur.execute(
sql.SQL("UPDATE {}.{} SET {} WHERE id=%s RETURNING {}").format(
sql.Identifier(schema_name),
sql.Identifier(table.code),
sql.SQL(", ").join(assignments),
sql.SQL(", ").join(sql.Identifier(column) for column in returning),
),
[*payload.values(), record_uuid],
)
after = _row_json(await cur.fetchone())
await _write_audit(
cur,
schema_name=schema_name,
tenant_id=tenant_id,
project_id=project_id,
table_code=table_code,
record_id=record_uuid,
operation="update",
before_data=before,
after_data=after,
actor=actor,
)
await conn.commit()
return after or {}
async def delete_record(
project_id: str,
table_code: str,
record_id: str,
actor: str,
) -> dict[str, bool]:
table = await _table_or_404(project_id, table_code)
if not table.allow_delete:
raise HTTPException(403, "该表不允许删除")
try:
record_uuid = uuid.UUID(record_id)
except ValueError as exc:
raise HTTPException(400, "记录 ID 格式错误") from exc
database = await _database_or_404(project_id)
schema_name = str(database["schema_name"])
tenant_id = str(database["tenant_id"])
columns = ["id", *(field.code for field in table.fields), "created_at", "updated_at"]
async with get_conn() as conn:
async with conn.cursor() as cur:
await cur.execute(
sql.SQL("SELECT {} FROM {}.{} WHERE id=%s AND deleted_at IS NULL").format(
sql.SQL(", ").join(sql.Identifier(column) for column in columns),
sql.Identifier(schema_name),
sql.Identifier(table.code),
),
(record_uuid,),
)
before = _row_json(await cur.fetchone())
if not before:
raise HTTPException(404, "记录不存在")
await cur.execute(
sql.SQL(
"UPDATE {}.{} SET deleted_at=now(), deleted_by=%s, updated_at=now() WHERE id=%s"
).format(
sql.Identifier(schema_name),
sql.Identifier(table.code),
),
(actor, record_uuid),
)
await _write_audit(
cur,
schema_name=schema_name,
tenant_id=tenant_id,
project_id=project_id,
table_code=table_code,
record_id=record_uuid,
operation="delete",
before_data=before,
after_data=None,
actor=actor,
)
await conn.commit()
return {"ok": True}