feat: add historical data management

This commit is contained in:
wangxuming
2026-07-22 10:18:49 +08:00
parent 66951b4dc3
commit c6044d972c
23 changed files with 3389 additions and 12 deletions

View File

@@ -15,6 +15,7 @@ const maintenanceLockID int64 = 733081337591911
// transaction while trying to clean an entire historical table.
type CleanupStats struct {
PersonalDataPurged int64
HistoryAnonymized int64
IdempotencyDeleted int64
SessionsDeleted int64
AuditDeleted int64
@@ -83,6 +84,31 @@ func RunMaintenanceOnce(ctx context.Context, db *sql.DB, now time.Time, batchSiz
if err != nil {
return rollback(fmt.Errorf("purge personal data: %w", err))
}
stats.HistoryAnonymized, err = boundedUpdate(ctx, tx, `
WITH candidates AS (
SELECT id
FROM queue_tickets
WHERE history_anonymized_at IS NULL
AND joined_at <= ($1::timestamptz - interval '90 days')
ORDER BY joined_at, id
LIMIT $2
FOR UPDATE SKIP LOCKED
)
UPDATE queue_tickets AS ticket
SET phone_ciphertext = NULL,
phone_nonce = NULL,
phone_hmac = NULL,
last_name_ciphertext = NULL,
last_name_nonce = NULL,
honorific = '游客',
personal_data_purged_at = COALESCE(ticket.personal_data_purged_at, $1),
history_anonymized_at = $1,
updated_at = $1
FROM candidates
WHERE ticket.id = candidates.id`, now, batchSize)
if err != nil {
return rollback(fmt.Errorf("mark historical data anonymized: %w", err))
}
stats.IdempotencyDeleted, err = boundedDelete(ctx, tx, `
WITH candidates AS (
SELECT id
@@ -160,9 +186,10 @@ func StartMaintenance(ctx context.Context, db *sql.DB, logger *slog.Logger, inte
logger.Error("database maintenance failed", "error", err)
return
}
if stats.PersonalDataPurged+stats.IdempotencyDeleted+stats.SessionsDeleted+stats.AuditDeleted > 0 {
if stats.PersonalDataPurged+stats.HistoryAnonymized+stats.IdempotencyDeleted+stats.SessionsDeleted+stats.AuditDeleted > 0 {
logger.Info("database maintenance completed",
"personal_data_purged", stats.PersonalDataPurged,
"history_anonymized", stats.HistoryAnonymized,
"idempotency_deleted", stats.IdempotencyDeleted,
"sessions_deleted", stats.SessionsDeleted,
"audit_deleted", stats.AuditDeleted,