Initial commit

This commit is contained in:
wangxuming
2026-07-12 15:53:24 +08:00
commit 68d61700d5
252 changed files with 23291 additions and 0 deletions

View File

@@ -0,0 +1,81 @@
package database
import (
"context"
"database/sql"
"fmt"
"log/slog"
"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
type OpenOptions struct {
MaxOpenConns int
MaxIdleConns int
ConnMaxIdleTime time.Duration
ConnMaxLifetime time.Duration
}
func defaultOpenOptions() OpenOptions {
return OpenOptions{MaxOpenConns: 20, MaxIdleConns: 5, ConnMaxIdleTime: 5 * time.Minute, ConnMaxLifetime: 30 * time.Minute}
}
func normalizeOpenOptions(options []OpenOptions) (OpenOptions, error) {
if len(options) == 0 {
return defaultOpenOptions(), nil
}
value := options[0]
if value.MaxOpenConns < 1 || value.MaxIdleConns < 0 || value.MaxIdleConns > value.MaxOpenConns {
return OpenOptions{}, fmt.Errorf("invalid database connection pool limits")
}
if value.ConnMaxIdleTime <= 0 || value.ConnMaxLifetime <= 0 {
return OpenOptions{}, fmt.Errorf("database connection lifetimes must be positive")
}
return value, nil
}
func Open(ctx context.Context, dsn string, appLogger *slog.Logger, options ...OpenOptions) (*gorm.DB, error) {
poolOptions, err := normalizeOpenOptions(options)
if err != nil {
return nil, err
}
db, err := gorm.Open(postgres.New(postgres.Config{DSN: dsn}), &gorm.Config{
Logger: newStructuredGORMLogger(appLogger),
TranslateError: true,
})
if err != nil {
return nil, fmt.Errorf("open postgres: %w", err)
}
sqlDB, err := db.DB()
if err != nil {
return nil, fmt.Errorf("get sql db: %w", err)
}
sqlDB.SetMaxOpenConns(poolOptions.MaxOpenConns)
sqlDB.SetMaxIdleConns(poolOptions.MaxIdleConns)
sqlDB.SetConnMaxIdleTime(poolOptions.ConnMaxIdleTime)
sqlDB.SetConnMaxLifetime(poolOptions.ConnMaxLifetime)
pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if err := sqlDB.PingContext(pingCtx); err != nil {
_ = sqlDB.Close()
return nil, fmt.Errorf("ping postgres: %w", err)
}
appLogger.Info("database connected")
return db, nil
}
func Close(db *gorm.DB) error {
sqlDB, err := db.DB()
if err != nil {
return err
}
return sqlDB.Close()
}
func SQLDB(db *gorm.DB) (*sql.DB, error) {
return db.DB()
}

View File

@@ -0,0 +1,144 @@
package database
import (
"context"
"database/sql"
"io"
"log/slog"
"os"
"strings"
"testing"
"time"
"github.com/google/uuid"
)
func TestPostgresMigrationAndMaintenanceIntegration(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("TEST_DATABASE_URL"))
if dsn == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
db, err := Open(ctx, dsn, logger)
if err != nil {
t.Fatal(err)
}
defer Close(db)
sqlDB, err := SQLDB(db)
if err != nil {
t.Fatal(err)
}
if err := Migrate(ctx, sqlDB, logger); err != nil {
t.Fatal(err)
}
if err := Migrate(ctx, sqlDB, logger); err != nil {
t.Fatalf("second migration run must be idempotent: %v", err)
}
if err := SchemaReady(ctx, sqlDB); err != nil {
t.Fatalf("schema readiness check failed after migration: %v", err)
}
var migrationCount int
if err := sqlDB.QueryRowContext(ctx, `SELECT count(*) FROM schema_migrations`).Scan(&migrationCount); err != nil {
t.Fatal(err)
}
if migrationCount < 8 {
t.Fatalf("migration count = %d, want at least 8", migrationCount)
}
var nullable string
if err := sqlDB.QueryRowContext(ctx, `
SELECT is_nullable
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = 'queue_tickets' AND column_name = 'phone_hmac'`).Scan(&nullable); err != nil {
t.Fatal(err)
}
if nullable != "YES" {
t.Fatalf("phone_hmac is_nullable = %q, want YES after purge migration", nullable)
}
if _, err := sqlDB.ExecContext(ctx, `UPDATE users SET password_hash = $1, updated_at = now() WHERE username = $2`, "rotated-hash", "xqkwljtadmin"); err != nil {
t.Fatalf("protected super admin password rotation should be allowed: %v", err)
}
if _, err := sqlDB.ExecContext(ctx, `UPDATE users SET role = 'STAFF' WHERE username = $1`, "xqkwljtadmin"); err == nil {
t.Fatal("protected super admin role change should be rejected")
}
fixture := newMaintenanceFixture(t, ctx, sqlDB)
stats, err := RunMaintenanceOnce(ctx, sqlDB, time.Now().UTC(), 100)
if err != nil {
t.Fatal(err)
}
if stats.PersonalDataPurged != 1 || stats.IdempotencyDeleted != 1 || stats.SessionsDeleted != 1 || stats.AuditDeleted != 1 {
t.Fatalf("maintenance stats = %#v, want one row in each category", stats)
}
var phoneCiphertext, phoneNonce, phoneHMAC, purgedAt any
if err := sqlDB.QueryRowContext(ctx, `
SELECT phone_ciphertext, phone_nonce, phone_hmac, personal_data_purged_at
FROM queue_tickets WHERE id = $1`, fixture.ticketID).
Scan(&phoneCiphertext, &phoneNonce, &phoneHMAC, &purgedAt); err != nil {
t.Fatal(err)
}
if phoneCiphertext != nil || phoneNonce != nil || phoneHMAC != nil || purgedAt == nil {
t.Fatalf("purged ticket fields = %#v/%#v/%#v/%#v", phoneCiphertext, phoneNonce, phoneHMAC, purgedAt)
}
}
type maintenanceFixture struct {
userID string
projectID string
ticketID string
}
func newMaintenanceFixture(t *testing.T, ctx context.Context, db *sql.DB) maintenanceFixture {
t.Helper()
now := time.Now().UTC().Truncate(time.Microsecond)
userID, projectID, sessionID, ticketID := uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()
code := "MNT" + strings.ToUpper(strings.ReplaceAll(userID[:5], "-", ""))
if _, err := db.ExecContext(ctx, `
INSERT INTO users (id, username, password_hash, role, active)
VALUES ($1, $2, $3, 'ADMIN', true)`, userID, "maintenance_"+userID[:8], "not-a-real-hash"); err != nil {
t.Fatal(err)
}
if _, err := db.ExecContext(ctx, `
INSERT INTO projects (
id, code, name, status, timezone, ticket_prefix, call_batch_size,
grace_period_minutes, eta_mode, average_batch_interval_seconds,
continuous_rate_per_minute, eta_buffer_minutes, device_simulation_mode
) VALUES ($1, $2, 'Maintenance Test', 'ENDED', 'Asia/Shanghai', 'M', 1, 0,
'FIXED_BATCH', 60, 1, 0, 'DISABLED')`, projectID, code); err != nil {
t.Fatal(err)
}
if _, err := db.ExecContext(ctx, `
INSERT INTO queue_sessions (id, project_id, business_date, status, next_ticket_number, revision, opened_at, closed_at)
VALUES ($1, $2, '2020-01-01', 'ENDED', 2, 1, $3, $3)`, sessionID, projectID, now.Add(-48*time.Hour)); err != nil {
t.Fatal(err)
}
if _, err := db.ExecContext(ctx, `
INSERT INTO queue_tickets (
id, project_id, queue_session_id, ticket_number, display_number,
public_token_hash, phone_ciphertext, phone_nonce, phone_hmac, honorific,
status, joined_at, completed_at, personal_data_purge_at, created_by
) VALUES ($1, $2, $3, 1, '00001', $4, $5, $6, $7, '女士', 'COMPLETED',
$8, $8, $9, $10)`, ticketID, projectID, sessionID, strings.Repeat("a", 64), []byte("ciphertext"), make([]byte, 12), strings.Repeat("b", 64), now.Add(-72*time.Hour), now.Add(-time.Hour), userID); err != nil {
t.Fatal(err)
}
if _, err := db.ExecContext(ctx, `
INSERT INTO idempotency_keys (project_id, user_id, scope, key, request_hash, response_code, response_body, expires_at)
VALUES ($1, $2, 'TEST', $3, $4, 200, '{}', $5)`, projectID, userID, "expired-"+userID, strings.Repeat("c", 64), now.Add(-time.Hour)); err != nil {
t.Fatal(err)
}
if _, err := db.ExecContext(ctx, `
INSERT INTO auth_sessions (id, user_id, token_hash, expires_at, last_seen_at)
VALUES ($1, $2, $3, $4, $4)`, uuid.NewString(), userID, strings.Repeat("d", 64), now.Add(-time.Hour)); err != nil {
t.Fatal(err)
}
if _, err := db.ExecContext(ctx, `
INSERT INTO audit_entries (project_id, actor_user_id, action, entity_type, details, request_id, user_agent, retain_until)
VALUES ($1, $2, 'TEST', 'TEST', '{}', $3, 'integration-test', $4)`, projectID, userID, "request-"+userID, now.Add(-time.Hour)); err != nil {
t.Fatal(err)
}
return maintenanceFixture{userID: userID, projectID: projectID, ticketID: ticketID}
}

View File

@@ -0,0 +1,57 @@
package database
import (
"context"
"errors"
"log/slog"
"time"
"gorm.io/gorm"
gormlogger "gorm.io/gorm/logger"
)
type structuredGORMLogger struct {
logger *slog.Logger
level gormlogger.LogLevel
slowThreshold time.Duration
}
func newStructuredGORMLogger(logger *slog.Logger) gormlogger.Interface {
return structuredGORMLogger{logger: logger, level: gormlogger.Warn, slowThreshold: 500 * time.Millisecond}
}
func (l structuredGORMLogger) LogMode(level gormlogger.LogLevel) gormlogger.Interface {
l.level = level
return l
}
func (l structuredGORMLogger) Info(_ context.Context, message string, args ...any) {
if l.level >= gormlogger.Info {
l.logger.Info("gorm", "message", message, "args", args)
}
}
func (l structuredGORMLogger) Warn(_ context.Context, message string, args ...any) {
if l.level >= gormlogger.Warn {
l.logger.Warn("gorm", "message", message, "args", args)
}
}
func (l structuredGORMLogger) Error(_ context.Context, message string, args ...any) {
if l.level >= gormlogger.Error {
l.logger.Error("gorm", "message", message, "args", args)
}
}
func (l structuredGORMLogger) Trace(_ context.Context, begin time.Time, query func() (sql string, rowsAffected int64), err error) {
elapsed := time.Since(begin)
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) && l.level >= gormlogger.Error {
_, rows := query()
l.logger.Error("database query failed", "error", err, "rows", rows, "duration_ms", elapsed.Milliseconds())
return
}
if elapsed > l.slowThreshold && l.level >= gormlogger.Warn {
_, rows := query()
l.logger.Warn("slow database query", "rows", rows, "duration_ms", elapsed.Milliseconds())
}
}

View File

@@ -0,0 +1,184 @@
package database
import (
"context"
"database/sql"
"fmt"
"log/slog"
"time"
)
const maintenanceLockID int64 = 733081337591911
// CleanupStats describes rows handled by one bounded maintenance pass.
// Counts are deliberately bounded so a busy API pod cannot hold a long
// transaction while trying to clean an entire historical table.
type CleanupStats struct {
PersonalDataPurged int64
IdempotencyDeleted int64
SessionsDeleted int64
AuditDeleted int64
}
// RunMaintenanceOnce purges expired personal data and bounded operational
// records. A PostgreSQL advisory lock makes it safe to run from every API pod
// without duplicate work when the service is horizontally scaled.
func RunMaintenanceOnce(ctx context.Context, db *sql.DB, now time.Time, batchSize int) (CleanupStats, error) {
if db == nil {
return CleanupStats{}, fmt.Errorf("maintenance database is nil")
}
if batchSize < 1 {
return CleanupStats{}, fmt.Errorf("maintenance batch size must be positive")
}
conn, err := db.Conn(ctx)
if err != nil {
return CleanupStats{}, fmt.Errorf("acquire maintenance connection: %w", err)
}
defer conn.Close()
var acquired bool
if err := conn.QueryRowContext(ctx, `SELECT pg_try_advisory_lock($1)`, maintenanceLockID).Scan(&acquired); err != nil {
return CleanupStats{}, fmt.Errorf("acquire maintenance lock: %w", err)
}
if !acquired {
return CleanupStats{}, nil
}
defer func() {
_, _ = conn.ExecContext(context.Background(), `SELECT pg_advisory_unlock($1)`, maintenanceLockID)
}()
tx, err := conn.BeginTx(ctx, nil)
if err != nil {
return CleanupStats{}, fmt.Errorf("begin maintenance transaction: %w", err)
}
rollback := func(cause error) (CleanupStats, error) {
_ = tx.Rollback()
return CleanupStats{}, cause
}
stats := CleanupStats{}
stats.PersonalDataPurged, err = boundedUpdate(ctx, tx, `
WITH candidates AS (
SELECT id
FROM queue_tickets
WHERE personal_data_purge_at <= $1
AND personal_data_purged_at IS NULL
AND status IN ('COMPLETED', 'MISSED', 'CANCELED')
ORDER BY personal_data_purge_at, id
LIMIT $2
FOR UPDATE SKIP LOCKED
)
UPDATE queue_tickets AS ticket
SET phone_ciphertext = NULL,
phone_nonce = NULL,
phone_hmac = NULL,
last_name_ciphertext = NULL,
last_name_nonce = NULL,
honorific = '游客',
personal_data_purged_at = $1,
updated_at = $1
FROM candidates
WHERE ticket.id = candidates.id`, now, batchSize)
if err != nil {
return rollback(fmt.Errorf("purge personal data: %w", err))
}
stats.IdempotencyDeleted, err = boundedDelete(ctx, tx, `
WITH candidates AS (
SELECT id
FROM idempotency_keys
WHERE expires_at <= $1
ORDER BY expires_at, id
LIMIT $2
)
DELETE FROM idempotency_keys AS record
USING candidates
WHERE record.id = candidates.id`, now, batchSize)
if err != nil {
return rollback(fmt.Errorf("delete expired idempotency keys: %w", err))
}
stats.SessionsDeleted, err = boundedDelete(ctx, tx, `
WITH candidates AS (
SELECT id
FROM auth_sessions
WHERE expires_at <= $1
OR (revoked_at IS NOT NULL AND revoked_at <= $1 - interval '30 days')
ORDER BY expires_at, id
LIMIT $2
)
DELETE FROM auth_sessions AS session
USING candidates
WHERE session.id = candidates.id`, now, batchSize)
if err != nil {
return rollback(fmt.Errorf("delete expired sessions: %w", err))
}
stats.AuditDeleted, err = boundedDelete(ctx, tx, `
WITH candidates AS (
SELECT id
FROM audit_entries
WHERE retain_until <= $1
ORDER BY retain_until, id
LIMIT $2
)
DELETE FROM audit_entries AS entry
USING candidates
WHERE entry.id = candidates.id`, now, batchSize)
if err != nil {
return rollback(fmt.Errorf("delete expired audit entries: %w", err))
}
if err := tx.Commit(); err != nil {
return CleanupStats{}, fmt.Errorf("commit maintenance transaction: %w", err)
}
return stats, nil
}
func boundedUpdate(ctx context.Context, tx *sql.Tx, query string, now time.Time, batchSize int) (int64, error) {
result, err := tx.ExecContext(ctx, query, now, batchSize)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
func boundedDelete(ctx context.Context, tx *sql.Tx, query string, now time.Time, batchSize int) (int64, error) {
return boundedUpdate(ctx, tx, query, now, batchSize)
}
// StartMaintenance starts a bounded periodic cleanup loop. It returns
// immediately; cancellation of ctx stops the loop during graceful shutdown.
func StartMaintenance(ctx context.Context, db *sql.DB, logger *slog.Logger, interval time.Duration, batchSize int) {
if interval <= 0 {
return
}
if logger == nil {
logger = slog.Default()
}
go func() {
run := func() {
stats, err := RunMaintenanceOnce(ctx, db, time.Now().UTC(), batchSize)
if err != nil {
logger.Error("database maintenance failed", "error", err)
return
}
if stats.PersonalDataPurged+stats.IdempotencyDeleted+stats.SessionsDeleted+stats.AuditDeleted > 0 {
logger.Info("database maintenance completed",
"personal_data_purged", stats.PersonalDataPurged,
"idempotency_deleted", stats.IdempotencyDeleted,
"sessions_deleted", stats.SessionsDeleted,
"audit_deleted", stats.AuditDeleted,
)
}
}
run()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
run()
}
}
}()
}

View File

@@ -0,0 +1,170 @@
package database
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"io/fs"
"log/slog"
"sort"
"strconv"
"strings"
"calllinesystem/server/migrations"
)
const migrationLockID int64 = 733081337591910
func Migrate(ctx context.Context, db *sql.DB, logger *slog.Logger) error {
conn, err := db.Conn(ctx)
if err != nil {
return fmt.Errorf("acquire migration connection: %w", err)
}
defer conn.Close()
if _, err := conn.ExecContext(ctx, `SELECT pg_advisory_lock($1)`, migrationLockID); err != nil {
return fmt.Errorf("lock migrations: %w", err)
}
defer func() {
_, _ = conn.ExecContext(context.Background(), `SELECT pg_advisory_unlock($1)`, migrationLockID)
}()
if _, err := conn.ExecContext(ctx, `
CREATE TABLE IF NOT EXISTS schema_migrations (
version bigint PRIMARY KEY,
name text NOT NULL,
checksum char(64) NOT NULL,
applied_at timestamptz NOT NULL DEFAULT now()
)`); err != nil {
return fmt.Errorf("create schema_migrations: %w", err)
}
applied, err := readApplied(ctx, conn)
if err != nil {
return err
}
entries, err := fs.ReadDir(migrations.Files, ".")
if err != nil {
return fmt.Errorf("read embedded migrations: %w", err)
}
var names []string
for _, entry := range entries {
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".up.sql") {
names = append(names, entry.Name())
}
}
sort.Strings(names)
for _, name := range names {
version, err := migrationVersion(name)
if err != nil {
return err
}
body, err := migrations.Files.ReadFile(name)
if err != nil {
return fmt.Errorf("read migration %s: %w", name, err)
}
sumBytes := sha256.Sum256(body)
checksum := hex.EncodeToString(sumBytes[:])
if existing, ok := applied[version]; ok {
if existing != checksum {
return fmt.Errorf("migration %d checksum changed", version)
}
continue
}
tx, err := conn.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("begin migration %s: %w", name, err)
}
if _, err = tx.ExecContext(ctx, string(body)); err == nil {
_, err = tx.ExecContext(ctx,
`INSERT INTO schema_migrations (version, name, checksum) VALUES ($1, $2, $3)`,
version, name, checksum,
)
}
if err != nil {
_ = tx.Rollback()
return fmt.Errorf("apply migration %s: %w", name, err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("commit migration %s: %w", name, err)
}
logger.Info("database migration applied", "version", version, "name", name)
}
return nil
}
// LatestVersion returns the highest embedded migration version. It is used by
// readiness checks so an API Pod cannot report ready against an older schema.
func LatestVersion() (int64, error) {
entries, err := fs.ReadDir(migrations.Files, ".")
if err != nil {
return 0, fmt.Errorf("read embedded migrations: %w", err)
}
var latest int64
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".up.sql") {
continue
}
version, err := migrationVersion(entry.Name())
if err != nil {
return 0, err
}
if version > latest {
latest = version
}
}
if latest == 0 {
return 0, fmt.Errorf("no embedded migrations found")
}
return latest, nil
}
// SchemaReady verifies that all embedded migrations have been applied.
func SchemaReady(ctx context.Context, db *sql.DB) error {
latest, err := LatestVersion()
if err != nil {
return err
}
var applied int64
if err := db.QueryRowContext(ctx, `SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(&applied); err != nil {
return fmt.Errorf("read applied migration version: %w", err)
}
if applied < latest {
return fmt.Errorf("database schema version %d is behind embedded version %d", applied, latest)
}
return nil
}
func readApplied(ctx context.Context, conn *sql.Conn) (map[int64]string, error) {
rows, err := conn.QueryContext(ctx, `SELECT version, checksum FROM schema_migrations ORDER BY version`)
if err != nil {
return nil, fmt.Errorf("read schema migrations: %w", err)
}
defer rows.Close()
applied := make(map[int64]string)
for rows.Next() {
var version int64
var checksum string
if err := rows.Scan(&version, &checksum); err != nil {
return nil, fmt.Errorf("scan schema migration: %w", err)
}
applied[version] = checksum
}
return applied, rows.Err()
}
func migrationVersion(name string) (int64, error) {
prefix, _, ok := strings.Cut(name, "_")
if !ok {
return 0, fmt.Errorf("invalid migration filename %q", name)
}
version, err := strconv.ParseInt(prefix, 10, 64)
if err != nil || version <= 0 {
return 0, fmt.Errorf("invalid migration version in %q", name)
}
return version, nil
}