增加支付逻辑

This commit is contained in:
andy committed 2026-10-12 13:57:23 +08:00
1 parent 4ac786aafb
commit dfb2c91b8d
77 files changed
+6370 -45

No files matched your search

+272
View File
@@ -0,0 +1,272 @@
// Package migrations applies the canonical SQL migrations before the API starts.
// SQL in sql/ is generated from database/migrations by scripts/sync-go-migrations.mjs.
package migrations
import (
"context"
"crypto/sha256"
"embed"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"log"
"sort"
"strconv"
"strings"
"time"
"git.nianxx.cn/wangxuming/NianAIGC/backend/internal/postgres"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
//go:embed sql/*.sql
var sqlFiles embed.FS
const migrationLockID int64 = 7308731946202608
type migration struct {
version string
checksum string
sql string
}
// Run is a no-op unless DATABASE_AUTO_MIGRATE=true. Its context covers
// connection, advisory-lock waiting, every migration, and role provisioning.
func Run(ctx context.Context, getenv func(string) string) error {
if getenv == nil {
return errors.New("environment getter is required")
}
switch strings.ToLower(strings.TrimSpace(getenv("DATABASE_AUTO_MIGRATE"))) {
case "", "false":
return nil
case "true":
default:
return errors.New("DATABASE_AUTO_MIGRATE must be true or false")
}
if strings.ToLower(strings.TrimSpace(getenv("ZHINIAN_DATA_BACKEND"))) != "postgres" {
return errors.New("DATABASE_AUTO_MIGRATE requires ZHINIAN_DATA_BACKEND=postgres")
}
timeout := 180 * time.Second
if raw := strings.TrimSpace(getenv("DATABASE_MIGRATION_TIMEOUT_SECONDS")); raw != "" {
seconds, err := strconv.ParseInt(raw, 10, 64)
if err != nil || seconds < 1 || seconds > 86400 {
return errors.New("DATABASE_MIGRATION_TIMEOUT_SECONDS must be an integer from 1 to 86400")
}
timeout = time.Duration(seconds) * time.Second
}
role := strings.TrimSpace(getenv("DATABASE_APP_ROLE"))
if role == "" && strings.EqualFold(strings.TrimSpace(getenv("NODE_ENV")), "production") {
return errors.New("DATABASE_APP_ROLE is required in production")
}
if role != "" && !rolePattern.MatchString(role) {
return errors.New("DATABASE_APP_ROLE must match [a-z_][a-z0-9_]{0,62}")
}
all, err := discover(sqlFiles)
if err != nil {
return err
}
migrationURL := strings.TrimSpace(getenv("DATABASE_MIGRATION_URL"))
appURL := strings.TrimSpace(getenv("DATABASE_URL"))
if migrationURL == "" {
migrationURL = appURL
}
if migrationURL == "" {
return errors.New("DATABASE_MIGRATION_URL or DATABASE_URL is required")
}
// Reuse the application's URI validation and forced transport policy.
config, err := postgres.ParseConfig(func(key string) string {
if key == "DATABASE_URL" {
return migrationURL
}
return getenv(key)
}, nil)
if err != nil {
return fmt.Errorf("migration database configuration: %w", err)
}
pgconfig, err := pgx.ParseConfig(config.DatabaseURL)
if err != nil {
return errors.New("invalid migration database connection configuration")
}
if appURL != "" && strings.TrimSpace(getenv("DATABASE_MIGRATION_URL")) != "" {
appConfig, err := postgres.ParseConfig(func(key string) string {
if key == "DATABASE_URL" {
return appURL
}
return getenv(key)
}, nil)
if err != nil {
return fmt.Errorf("application database configuration: %w", err)
}
appPGConfig, err := pgx.ParseConfig(appConfig.DatabaseURL)
if err != nil {
return errors.New("invalid application database connection configuration")
}
if !strings.EqualFold(pgconfig.Host, appPGConfig.Host) || pgconfig.Port != appPGConfig.Port || pgconfig.Database != appPGConfig.Database {
return errors.New("DATABASE_MIGRATION_URL must target the same host, port, and database as DATABASE_URL")
}
}
pgconfig.RuntimeParams["application_name"] = "zhinian-migrate"
pgconfig.ConnectTimeout = config.ConnectionTimeout
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
log.Printf("zhinian-api checking database migrations (%d bundled versions)", len(all))
conn, err := pgx.ConnectConfig(ctx, pgconfig)
if err != nil {
return safeDBError("connect", "", err)
}
defer conn.Close(context.Background())
log.Print("zhinian-api waiting for database migration lock")
if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockID); err != nil {
return safeDBError("acquire migration lock", "", err)
}
// Session-level lock is released when conn closes, including on timeout.
if err := apply(ctx, conn, all, role); err != nil {
return err
}
log.Printf("zhinian-api database migrations ready (%d versions)", len(all))
return nil
}
func apply(ctx context.Context, conn *pgx.Conn, all []migration, role string) error {
if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
checksum text NOT NULL,
applied_at timestamptz NOT NULL DEFAULT now()
)`); err != nil {
return safeDBError("create migration tracking table", "", err)
}
rows, err := conn.Query(ctx, "SELECT version, checksum FROM schema_migrations")
if err != nil {
return safeDBError("read migration history", "", err)
}
applied := make(map[string]string)
for rows.Next() {
var version, checksum string
if err := rows.Scan(&version, &checksum); err != nil {
rows.Close()
return safeDBError("read migration history", "", err)
}
applied[version] = checksum
}
err = rows.Err()
rows.Close()
if err != nil {
return safeDBError("read migration history", "", err)
}
known := make(map[string]string, len(all))
for _, item := range all {
known[item.version] = item.checksum
}
for version, checksum := range applied {
expected, exists := known[version]
if !exists {
return fmt.Errorf("applied migration %q is absent from bundle", version)
}
if checksum != expected {
return fmt.Errorf("applied migration %q checksum differs from bundle", version)
}
}
for _, item := range all {
if _, exists := applied[item.version]; exists {
continue
}
body, err := withoutOuterTransaction(item.sql)
if err != nil {
return fmt.Errorf("migration %s: %w", item.version, err)
}
tx, err := conn.Begin(ctx)
if err != nil {
return safeDBError("begin migration", item.version, err)
}
// PostgreSQL's simple protocol permits multi-statement migration files.
_, err = tx.Exec(ctx, body, pgx.QueryExecModeSimpleProtocol)
if err == nil {
_, err = tx.Exec(ctx, "INSERT INTO schema_migrations(version, checksum) VALUES ($1, $2)", item.version, item.checksum)
}
if err == nil {
err = tx.Commit(ctx)
}
if err != nil {
_ = tx.Rollback(context.Background())
return safeDBError("apply migration", item.version, err)
}
log.Printf("zhinian-api applied database migration %s", item.version)
}
if role != "" {
if err := provisionRole(ctx, conn, role); err != nil {
return err
}
}
return nil
}
func discover(fsys fs.FS) ([]migration, error) {
entries, err := fs.ReadDir(fsys, "sql")
if err != nil {
return nil, errors.New("read embedded migration bundle")
}
var all []migration
for _, entry := range entries {
name := entry.Name()
if entry.IsDir() || !strings.HasSuffix(name, ".sql") {
continue
}
data, err := fs.ReadFile(fsys, "sql/"+name)
if err != nil {
return nil, fmt.Errorf("read embedded migration %s", name)
}
checksum := sha256.Sum256(data)
all = append(all, migration{strings.TrimSuffix(name, ".sql"), hex.EncodeToString(checksum[:]), string(data)})
}
sort.Slice(all, func(i, j int) bool { return all[i].version < all[j].version })
if len(all) == 0 {
return nil, errors.New("embedded migration bundle is empty")
}
return all, nil
}
// The canonical SQL may contain an explicit outer BEGIN/COMMIT. Strip only
// when both commands are the first and last non-comment statements; otherwise
// our transaction would commit before recording the checksum.
func withoutOuterTransaction(sql string) (string, error) {
first, last := -1, -1
lines := strings.SplitAfter(sql, "\n")
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if trimmed != "" && !strings.HasPrefix(trimmed, "--") {
if first < 0 {
first = i
}
last = i
}
}
if first < 0 {
return "", errors.New("empty migration SQL")
}
begin := strings.EqualFold(strings.TrimSpace(lines[first]), "begin;")
commit := strings.EqualFold(strings.TrimSpace(lines[last]), "commit;")
if begin != commit {
return "", errors.New("outer transaction must contain both BEGIN and COMMIT")
}
if !begin {
return sql, nil
}
lines[first], lines[last] = "", ""
return strings.Join(lines, ""), nil
}
func safeDBError(operation, version string, err error) error {
if version != "" {
operation += " " + version
}
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
return fmt.Errorf("%s failed (SQLSTATE %s)", operation, pgErr.Code)
}
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
return fmt.Errorf("%s timed out or was canceled", operation)
}
return fmt.Errorf("%s failed (%T)", operation, err)
}
@@ -0,0 +1,162 @@
package migrations
import (
"context"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/jackc/pgx/v5"
)
// Requires an independently disposable database with no valuable contents.
// It never consults the application's DATABASE_URL or .env files.
func TestIntegrationRunOnDisposableDatabase(t *testing.T) {
dsn := os.Getenv("NIAN_TEST_MIGRATION_DATABASE_URL")
if dsn == "" {
t.Skip("set NIAN_TEST_MIGRATION_DATABASE_URL for disposable PostgreSQL integration test")
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
conn, err := pgx.Connect(ctx, dsn)
if err != nil {
t.Fatal("connect disposable database:", err)
}
defer conn.Close(context.Background())
if _, err := conn.Exec(ctx, `DO $$ BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'nian_migration_test') THEN
CREATE ROLE nian_migration_test NOLOGIN;
END IF;
END $$`); err != nil {
t.Fatal("create disposable test role:", err)
}
all, err := discover(sqlFiles)
if err != nil {
t.Fatal(err)
}
// Repair only markers this test might have left if an earlier assertion
// stopped execution before normal cleanup.
var trackingTable *string
if err := conn.QueryRow(ctx, "SELECT to_regclass('public.schema_migrations')::text").Scan(&trackingTable); err != nil {
t.Fatal(err)
}
if trackingTable != nil {
if _, err := conn.Exec(ctx, "UPDATE schema_migrations SET checksum=$1 WHERE version=$2 AND checksum='test-mismatch'", all[0].checksum, all[0].version); err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, "DELETE FROM schema_migrations WHERE version='test_unknown'"); err != nil {
t.Fatal(err)
}
}
env := func(key string) string {
switch key {
case "DATABASE_AUTO_MIGRATE":
return "true"
case "ZHINIAN_DATA_BACKEND":
return "postgres"
case "DATABASE_MIGRATION_URL", "DATABASE_URL":
return dsn
case "DATABASE_APP_ROLE":
return "nian_migration_test"
case "NODE_ENV":
return "production"
default:
return ""
}
}
var wg sync.WaitGroup
results := make(chan error, 2)
for i := 0; i < 2; i++ {
wg.Add(1)
go func() {
defer wg.Done()
results <- Run(ctx, env)
}()
}
wg.Wait()
close(results)
for err := range results {
if err != nil {
t.Fatal("concurrent migration:", err)
}
}
if err := Run(ctx, env); err != nil {
t.Fatal("repeat migration:", err)
}
var count int
if err := conn.QueryRow(ctx, "SELECT count(*) FROM schema_migrations").Scan(&count); err != nil || count != len(all) {
t.Fatalf("recorded migrations: count=%d, err=%v", count, err)
}
var privilege bool
if err := conn.QueryRow(ctx, "SELECT has_table_privilege('nian_migration_test', 'public.payment_orders', 'SELECT,INSERT,UPDATE')").Scan(&privilege); err != nil || !privilege {
t.Fatalf("payment order privileges: granted=%v, err=%v", privilege, err)
}
// A mismatch must prevent any later migration or startup.
first := all[0]
if _, err := conn.Exec(ctx, "UPDATE schema_migrations SET checksum='test-mismatch' WHERE version=$1", first.version); err != nil {
t.Fatal(err)
}
defer func() {
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cleanupCancel()
_, _ = conn.Exec(cleanupCtx, "UPDATE schema_migrations SET checksum=$1 WHERE version=$2", first.checksum, first.version)
_, _ = conn.Exec(cleanupCtx, "DELETE FROM schema_migrations WHERE version='test_unknown'")
}()
if err := Run(ctx, env); err == nil || !strings.Contains(err.Error(), "checksum differs") {
t.Fatalf("expected checksum rejection, got %v", err)
}
if _, err := conn.Exec(ctx, "UPDATE schema_migrations SET checksum=$1 WHERE version=$2", first.checksum, first.version); err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, "INSERT INTO schema_migrations(version, checksum) VALUES ('test_unknown', 'test')"); err != nil {
t.Fatal(err)
}
if err := Run(ctx, env); err == nil || !strings.Contains(err.Error(), "absent from bundle") {
t.Fatalf("expected unknown-version rejection, got %v", err)
}
if _, err := conn.Exec(ctx, "DELETE FROM schema_migrations WHERE version='test_unknown'"); err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockID); err != nil {
t.Fatal(err)
}
defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", migrationLockID)
shortCtx, shortCancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer shortCancel()
if err := Run(shortCtx, env); err == nil || !strings.Contains(err.Error(), "timed out") {
t.Fatalf("expected advisory-lock timeout, got %v", err)
}
if _, err := conn.Exec(ctx, "SELECT pg_advisory_unlock($1)", migrationLockID); err != nil {
t.Fatal(err)
}
// The outer BEGIN/COMMIT in a legacy migration must not commit its DDL
// before the version/checksum row is inserted by apply.
const syntheticVersion = "9999_atomicity_test"
const syntheticTable = "public.nian_migration_atomicity_test"
if _, err := conn.Exec(ctx, "DROP TABLE IF EXISTS "+syntheticTable); err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, "DELETE FROM schema_migrations WHERE version=$1", syntheticVersion); err != nil {
t.Fatal(err)
}
synthetic := migration{
version: syntheticVersion,
sql: "-- synthetic migration\nBEGIN;\nCREATE TABLE " + syntheticTable + " (id integer PRIMARY KEY);\nSELECT 1 / 0;\nCOMMIT;\n",
}
if err := apply(ctx, conn, append(all, synthetic), ""); err == nil || !strings.Contains(err.Error(), syntheticVersion) {
t.Fatalf("expected synthetic migration failure with version, got %v", err)
}
var objectName *string
if err := conn.QueryRow(ctx, "SELECT to_regclass($1)::text", syntheticTable).Scan(&objectName); err != nil {
t.Fatal(err)
}
if objectName != nil {
t.Fatalf("failed migration left DDL object %s", *objectName)
}
var recorded int
if err := conn.QueryRow(ctx, "SELECT count(*) FROM schema_migrations WHERE version=$1", syntheticVersion).Scan(&recorded); err != nil || recorded != 0 {
t.Fatalf("failed migration recorded version: count=%d, err=%v", recorded, err)
}
}
@@ -0,0 +1,97 @@
package migrations
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"strings"
"testing"
"testing/fstest"
)
func TestEmbeddedBundleMatchesCanonicalSQL(t *testing.T) {
all, err := discover(sqlFiles)
if err != nil {
t.Fatal(err)
}
canonical, err := filepath.Glob("../../../database/migrations/*.sql")
if err != nil {
t.Fatal(err)
}
if len(canonical) != len(all) {
t.Fatalf("canonical count %d differs from embedded count %d", len(canonical), len(all))
}
for i, path := range canonical {
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
version := strings.TrimSuffix(filepath.Base(path), ".sql")
if all[i].version != version || !bytes.Equal(data, []byte(all[i].sql)) {
t.Fatalf("embedded migration %s differs from canonical SQL", version)
}
checksum := sha256.Sum256(data)
if all[i].checksum != hex.EncodeToString(checksum[:]) {
t.Fatalf("checksum differs for %s", version)
}
}
}
func TestDiscoverMigrations(t *testing.T) {
fsys := fstest.MapFS{
"sql/0002.sql": {Data: []byte("select 2;\n")},
"sql/0001.sql": {Data: []byte("select 1;\n")},
}
all, err := discover(fsys)
if err != nil || len(all) != 2 || all[0].version != "0001" || all[1].version != "0002" {
t.Fatalf("discover: %+v, %v", all, err)
}
}
func TestOuterTransactionRemoval(t *testing.T) {
for _, tc := range []struct {
in, want string
}{
{"-- title\n\nBEGIN;\nselect 1;\nCOMMIT;\n", "-- title\n\nselect 1;\n"},
{"do $$ begin raise notice 'x'; end $$;", "do $$ begin raise notice 'x'; end $$;"},
} {
got, err := withoutOuterTransaction(tc.in)
if err != nil || got != tc.want {
t.Fatalf("got %q, %v; want %q", got, err, tc.want)
}
}
for _, sql := range []string{"begin;\nselect 1;", "select 1;\ncommit;"} {
if _, err := withoutOuterTransaction(sql); err == nil {
t.Fatalf("accepted unmatched transaction: %q", sql)
}
}
}
func TestRunDisabledAndConfiguration(t *testing.T) {
getter := func(mapper map[string]string) func(string) string {
return func(key string) string { return mapper[key] }
}
if err := Run(context.Background(), getter(nil)); err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
env map[string]string
want string
}{
{map[string]string{"DATABASE_AUTO_MIGRATE": "yes"}, "must be true or false"},
{map[string]string{"DATABASE_AUTO_MIGRATE": "true", "ZHINIAN_DATA_BACKEND": "local"}, "requires ZHINIAN_DATA_BACKEND=postgres"},
{map[string]string{"DATABASE_AUTO_MIGRATE": "true", "ZHINIAN_DATA_BACKEND": "postgres", "DATABASE_MIGRATION_TIMEOUT_SECONDS": "0"}, "must be an integer"},
{map[string]string{"DATABASE_AUTO_MIGRATE": "true", "ZHINIAN_DATA_BACKEND": "postgres", "NODE_ENV": "production"}, "DATABASE_APP_ROLE is required"},
{map[string]string{"DATABASE_AUTO_MIGRATE": "true", "ZHINIAN_DATA_BACKEND": "postgres"}, "DATABASE_MIGRATION_URL or DATABASE_URL is required"},
{map[string]string{"DATABASE_AUTO_MIGRATE": "true", "ZHINIAN_DATA_BACKEND": "postgres", "DATABASE_APP_ROLE": "bad;role"}, "DATABASE_APP_ROLE must match"},
{map[string]string{"DATABASE_AUTO_MIGRATE": "true", "ZHINIAN_DATA_BACKEND": "postgres", "DATABASE_MIGRATION_URL": "postgres://admin:secret@localhost/db1", "DATABASE_URL": "postgres://app:secret@localhost/db2"}, "must target the same"},
} {
err := Run(context.Background(), getter(tc.env))
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("env %+v: got %v, want %q", tc.env, err, tc.want)
}
}
}
+90
View File
@@ -0,0 +1,90 @@
package migrations
import (
"context"
"fmt"
"regexp"
"strings"
"github.com/jackc/pgx/v5"
)
var rolePattern = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,62}$`)
var appTables = []struct {
name string
privileges string
}{
{"assets", "SELECT, INSERT, DELETE"},
{"generation_jobs", "SELECT, INSERT, UPDATE, DELETE"},
{"seedream_layer_compositions", "SELECT, INSERT, UPDATE, DELETE"},
{"usage_events", "SELECT, INSERT, UPDATE"},
{"projects", "SELECT, UPDATE"},
{"image_templates", "SELECT, INSERT, UPDATE, DELETE"},
{"platform_organizations", "SELECT, INSERT, UPDATE, DELETE"},
{"platform_users", "SELECT, INSERT, UPDATE, DELETE"},
{"platform_account_migrations", "SELECT, INSERT, UPDATE"},
{"platform_runtime_settings", "SELECT, INSERT, UPDATE"},
{"billing_price_rules", "SELECT, INSERT, UPDATE"},
{"billing_wallets", "SELECT, INSERT, UPDATE"},
{"billing_ledger", "SELECT, INSERT"},
{"payment_orders", "SELECT, INSERT, UPDATE"},
{"payment_refunds", "SELECT, INSERT, UPDATE"},
}
func provisionRole(ctx context.Context, conn *pgx.Conn, role string) error {
// rolePattern has already validated the identifier. Quote it consistently.
quotedRole := `"` + role + `"`
managed := make([]string, len(appTables))
for i, table := range appTables {
managed[i] = `public."` + table.name + `"`
}
tx, err := conn.Begin(ctx)
if err != nil {
return safeDBError("begin application role provisioning", "", err)
}
commands := []string{
"REVOKE CREATE ON SCHEMA public FROM PUBLIC",
"GRANT USAGE ON SCHEMA public TO " + quotedRole,
"REVOKE ALL ON TABLE " + strings.Join(managed, ", ") + " FROM " + quotedRole,
}
for _, table := range appTables {
commands = append(commands, fmt.Sprintf("GRANT %s ON TABLE public.\"%s\" TO %s", table.privileges, table.name, quotedRole))
}
commands = append(commands,
"REVOKE ALL ON FUNCTION public.claim_generation_jobs(text, integer, integer) FROM PUBLIC",
"REVOKE ALL ON FUNCTION public.billing_post_wallet_entry(text, text, text, text, text, bigint, text, text, text, jsonb) FROM PUBLIC",
"GRANT EXECUTE ON FUNCTION public.claim_generation_jobs(text, integer, integer) TO "+quotedRole,
"GRANT EXECUTE ON FUNCTION public.billing_post_wallet_entry(text, text, text, text, text, bigint, text, text, text, jsonb) TO "+quotedRole,
)
for _, command := range commands {
if _, err := tx.Exec(ctx, command); err != nil {
_ = tx.Rollback(context.Background())
return safeDBError("provision application role", "", err)
}
}
if err := tx.Commit(ctx); err != nil {
return safeDBError("commit application role privileges", "", err)
}
tableNames := make([]string, len(appTables))
privileges := make([]string, len(appTables))
for i, table := range appTables {
tableNames[i] = "public." + table.name
privileges[i] = strings.ReplaceAll(table.privileges, " ", "")
}
var tablesReady, claimReady, billingReady bool
err = conn.QueryRow(ctx, `SELECT
bool_and(has_table_privilege($1, table_name, privileges)) AS tables_ready,
has_function_privilege($1, 'public.claim_generation_jobs(text,integer,integer)', 'EXECUTE') AS claim_ready,
has_function_privilege($1,
'public.billing_post_wallet_entry(text,text,text,text,text,bigint,text,text,text,jsonb)',
'EXECUTE') AS billing_ready
FROM unnest($2::text[], $3::text[]) AS required(table_name, privileges)`, role, tableNames, privileges).Scan(&tablesReady, &claimReady, &billingReady)
if err != nil {
return safeDBError("verify application role privileges", "", err)
}
if !tablesReady || !claimReady || !billingReady {
return fmt.Errorf("application role privilege verification failed")
}
return nil
}
@@ -0,0 +1,467 @@
create table if not exists assets (
id text primary key,
owner_id text not null,
kind text not null,
name text not null,
url text not null,
storage_path text,
source text not null,
tags text[] not null default '{}',
metadata jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists generation_jobs (
id text primary key,
owner_id text not null,
external_client_id text,
capability text not null,
provider text not null,
req_key text not null,
status text not null,
prompt text,
input_asset_ids text[] not null default '{}',
input_urls text[] not null default '{}',
output_asset_ids text[] not null default '{}',
provider_task_id text,
request_payload jsonb not null default '{}'::jsonb,
response_payload jsonb,
error jsonb,
retry_of text,
idempotency_key text,
idempotency_fingerprint text,
priority integer not null default 0,
attempts integer not null default 0,
max_attempts integer not null default 3,
scheduled_at timestamptz not null default now(),
locked_at timestamptz,
locked_by text,
started_at timestamptz,
completed_at timestamptz,
webhook_url text,
webhook_attempts integer not null default 0,
webhook_last_status jsonb,
usage_context jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
alter table generation_jobs add column if not exists external_client_id text;
alter table generation_jobs add column if not exists idempotency_key text;
alter table generation_jobs add column if not exists idempotency_fingerprint text;
alter table generation_jobs add column if not exists priority integer not null default 0;
alter table generation_jobs add column if not exists attempts integer not null default 0;
alter table generation_jobs add column if not exists max_attempts integer not null default 3;
alter table generation_jobs add column if not exists scheduled_at timestamptz not null default now();
alter table generation_jobs add column if not exists locked_at timestamptz;
alter table generation_jobs add column if not exists locked_by text;
alter table generation_jobs add column if not exists started_at timestamptz;
alter table generation_jobs add column if not exists completed_at timestamptz;
alter table generation_jobs add column if not exists webhook_url text;
alter table generation_jobs add column if not exists webhook_attempts integer not null default 0;
alter table generation_jobs add column if not exists webhook_last_status jsonb;
alter table generation_jobs add column if not exists usage_context jsonb;
create table if not exists usage_events (
id text primary key,
owner_id text not null,
job_id text not null,
source text not null default 'platform',
capability text not null,
provider text,
req_key text,
account_username text,
account_display_name text,
tenant_id text,
organization_id text,
organization_name text,
quantity integer not null default 1,
estimated_unit text not null default 'job',
created_at timestamptz not null default now()
);
alter table usage_events add column if not exists source text;
alter table usage_events add column if not exists provider text;
alter table usage_events add column if not exists req_key text;
alter table usage_events add column if not exists account_username text;
alter table usage_events add column if not exists account_display_name text;
alter table usage_events add column if not exists tenant_id text;
alter table usage_events add column if not exists organization_id text;
alter table usage_events add column if not exists organization_name text;
alter table usage_events add column if not exists quantity integer not null default 1;
alter table usage_events add column if not exists estimated_unit text not null default 'job';
update usage_events as usage
set source = case
when jobs.external_client_id is not null or usage.owner_id like 'api:%' then 'api'
else 'platform'
end,
provider = coalesce(usage.provider, jobs.provider),
req_key = coalesce(usage.req_key, jobs.req_key),
quantity = 1,
estimated_unit = 'job'
from generation_jobs as jobs
where jobs.id = usage.job_id;
update usage_events
set source = case when owner_id like 'api:%' then 'api' else 'platform' end,
quantity = 1,
estimated_unit = 'job'
where source is null;
alter table usage_events alter column source set default 'platform';
alter table usage_events alter column source set not null;
alter table usage_events alter column estimated_unit set default 'job';
alter table usage_events drop constraint if exists usage_events_job_id_fkey;
create table if not exists projects (
id text primary key,
owner_id text not null,
name text not null,
brief text not null default '',
type text not null default 'custom',
asset_ids text[] not null default '{}',
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists image_templates (
id text primary key,
owner_id text not null,
name text not null,
description text,
prompt text not null,
preview_image_url text,
settings jsonb not null default '{}'::jsonb,
sort_order integer not null default 0,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create index if not exists assets_owner_created_idx on assets(owner_id, created_at desc);
create index if not exists generation_jobs_owner_created_idx on generation_jobs(owner_id, created_at desc);
create index if not exists generation_jobs_status_idx on generation_jobs(status);
create index if not exists generation_jobs_claim_idx on generation_jobs(status, scheduled_at, locked_at, priority desc);
create index if not exists generation_jobs_external_client_idx on generation_jobs(owner_id, external_client_id, created_at desc);
create unique index if not exists generation_jobs_idempotency_idx
on generation_jobs(owner_id, external_client_id, idempotency_key)
where external_client_id is not null and idempotency_key is not null;
create index if not exists usage_events_owner_created_idx on usage_events(owner_id, created_at desc);
do $$
begin
if exists (
select 1
from usage_events
group by job_id
having count(*) > 1
) then
raise exception using
errcode = '23505',
message = 'USAGE_EVENTS_DUPLICATE_JOB_ID: back up and clean duplicate usage_events.job_id rows before retrying this migration';
end if;
end;
$$;
create unique index if not exists usage_events_job_id_idx on usage_events(job_id);
create index if not exists usage_events_source_created_idx on usage_events(source, created_at desc);
create index if not exists usage_events_organization_created_idx on usage_events(organization_id, created_at desc);
create index if not exists image_templates_owner_sort_idx on image_templates(owner_id, sort_order asc, updated_at desc);
create or replace function claim_generation_jobs(
p_worker_id text,
p_limit integer default 1,
p_lock_timeout_seconds integer default 300
)
returns setof generation_jobs
language plpgsql
set search_path = public, pg_temp
as $$
declare
v_now timestamptz := now();
begin
return query
with candidates as (
select id
from generation_jobs
where status in ('queued', 'running')
and coalesce(scheduled_at, created_at) <= v_now
and (
locked_at is null
or locked_at < v_now - make_interval(secs => p_lock_timeout_seconds)
)
order by coalesce(priority, 0) desc, coalesce(scheduled_at, created_at) asc, created_at asc
limit greatest(1, least(coalesce(p_limit, 1), 20))
for update skip locked
),
updated as (
update generation_jobs
set locked_at = v_now,
locked_by = p_worker_id,
started_at = coalesce(generation_jobs.started_at, v_now),
updated_at = v_now
where id in (select id from candidates)
returning generation_jobs.*
)
select * from updated;
end;
$$;
revoke all on function claim_generation_jobs(text, integer, integer) from public;
create table if not exists platform_organizations (
id text primary key,
name text not null unique,
status text not null default 'active' check (status in ('active', 'disabled')),
archive_owner_id text not null unique,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists platform_users (
id text primary key,
phone text not null unique,
display_name text not null,
role text not null default 'user' check (role in ('super_admin', 'organization_admin', 'user')),
organization_id text references platform_organizations(id) on delete set null,
status text not null default 'active' check (status in ('active', 'disabled')),
password_hash text not null,
password_salt text not null,
failed_login_count integer not null default 0,
locked_until timestamptz,
session_version integer not null default 1,
last_login_at timestamptz,
legacy_subject text,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists platform_account_migrations (
id text primary key,
legacy_owner_id text not null unique,
legacy_phone text,
platform_user_id text not null,
created_at timestamptz not null default now()
);
create index if not exists platform_users_organization_idx on platform_users(organization_id, created_at desc);
create index if not exists platform_users_status_idx on platform_users(status, role);
create index if not exists platform_account_migrations_user_idx on platform_account_migrations(platform_user_id);
alter table generation_jobs add column if not exists billing jsonb;
alter table usage_events add column if not exists charged_amount_fen bigint;
alter table usage_events add column if not exists currency text;
create table if not exists billing_price_rules (
id text primary key,
provider text not null check (provider in ('volcengine-visual', 'evolink', 'seedance', 'bailian', 'mock')),
capability text not null check (capability in ('image.generate', 'video.generate')),
req_key text,
variant_key text,
unit text not null check (unit in ('request', 'image', 'video_second')),
standard_unit_price_fen bigint not null check (standard_unit_price_fen >= 0),
markup_multiplier numeric(12, 4) not null default 1.0000 check (markup_multiplier >= 1),
enabled boolean not null default true,
conditions jsonb not null default '{}'::jsonb,
quantity_source text check (quantity_source in ('request', 'image_count', 'duration')),
priority integer not null default 0,
note text,
source jsonb,
parameter_dimensions jsonb not null default '[]'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
alter table billing_price_rules add column if not exists variant_key text;
alter table billing_price_rules add column if not exists source jsonb;
alter table billing_price_rules add column if not exists conditions jsonb not null default '{}'::jsonb;
alter table billing_price_rules add column if not exists quantity_source text;
alter table billing_price_rules add column if not exists priority integer not null default 0;
alter table billing_price_rules add column if not exists parameter_dimensions jsonb not null default '[]'::jsonb;
alter table billing_price_rules drop constraint if exists billing_price_rules_quantity_source_check;
alter table billing_price_rules add constraint billing_price_rules_quantity_source_check check (quantity_source is null or quantity_source in ('request', 'image_count', 'duration'));
drop index if exists billing_price_rules_match_idx;
create unique index if not exists billing_price_rules_match_idx
on billing_price_rules(provider, capability, coalesce(req_key, ''), coalesce(variant_key, ''), coalesce(conditions, '{}'::jsonb));
create table if not exists billing_wallets (
organization_id text primary key references platform_organizations(id) on delete restrict,
balance_fen bigint not null default 0 check (balance_fen >= 0),
total_recharged_fen bigint not null default 0 check (total_recharged_fen >= 0),
total_charged_fen bigint not null default 0 check (total_charged_fen >= 0),
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table if not exists billing_ledger (
id text primary key,
organization_id text not null references platform_organizations(id) on delete restrict,
account_id text,
job_id text,
kind text not null check (kind in ('recharge', 'charge', 'refund', 'adjustment')),
delta_fen bigint not null check (delta_fen <> 0),
balance_after_fen bigint not null check (balance_after_fen >= 0),
currency text not null default 'CNY' check (currency = 'CNY'),
idempotency_key text not null,
description text not null,
metadata jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now()
);
do $$
declare
v_constraint record;
v_index record;
begin
for v_constraint in
select constraint_schema, constraint_name
from information_schema.table_constraints
where table_schema = current_schema()
and table_name = 'billing_ledger'
and constraint_type = 'UNIQUE'
and array(
select key_column_usage.column_name::text
from information_schema.key_column_usage
where key_column_usage.constraint_schema = table_constraints.constraint_schema
and key_column_usage.constraint_name = table_constraints.constraint_name
and key_column_usage.table_name = table_constraints.table_name
order by key_column_usage.ordinal_position
) = array['idempotency_key']::text[]
loop
execute format('alter table %I.%I drop constraint %I', current_schema(), 'billing_ledger', v_constraint.constraint_name);
end loop;
for v_index in
select indexes.schemaname, indexes.indexname
from pg_indexes as indexes
join pg_class as index_class on index_class.relname = indexes.indexname
join pg_namespace as index_namespace
on index_namespace.oid = index_class.relnamespace
and index_namespace.nspname = indexes.schemaname
join pg_index as index_metadata on index_metadata.indexrelid = index_class.oid
join pg_attribute as indexed_column
on indexed_column.attrelid = index_metadata.indrelid
and indexed_column.attnum = index_metadata.indkey[0]
left join pg_constraint as backing_constraint on backing_constraint.conindid = index_class.oid
where indexes.schemaname = current_schema()
and indexes.tablename = 'billing_ledger'
and index_metadata.indisunique
and index_metadata.indnkeyatts = 1
and indexed_column.attname = 'idempotency_key'
and backing_constraint.oid is null
loop
execute format('drop index %I.%I', v_index.schemaname, v_index.indexname);
end loop;
end;
$$;
create unique index if not exists billing_ledger_organization_idempotency_idx
on billing_ledger(organization_id, idempotency_key);
create index if not exists billing_ledger_organization_created_idx on billing_ledger(organization_id, created_at desc);
create index if not exists billing_ledger_account_created_idx on billing_ledger(account_id, created_at desc);
create index if not exists billing_ledger_job_idx on billing_ledger(job_id);
create or replace function billing_post_wallet_entry(
p_ledger_id text,
p_organization_id text,
p_account_id text,
p_job_id text,
p_kind text,
p_delta_fen bigint,
p_currency text,
p_idempotency_key text,
p_description text,
p_metadata jsonb
)
returns table (
ledger_id text,
balance_after_fen bigint,
balance_fen bigint,
total_recharged_fen bigint,
total_charged_fen bigint,
created_at timestamptz,
updated_at timestamptz,
delta_fen bigint
)
language plpgsql
set search_path = public, pg_temp
as $$
declare
v_existing billing_ledger%rowtype;
v_wallet billing_wallets%rowtype;
v_entry billing_ledger%rowtype;
v_account_id text;
begin
if p_currency <> 'CNY' then
raise exception 'BILLING_UNSUPPORTED_CURRENCY';
end if;
if p_delta_fen = 0 then
raise exception 'BILLING_ZERO_DELTA';
end if;
v_account_id := case when p_kind in ('recharge', 'adjustment') then null else p_account_id end;
perform pg_advisory_xact_lock(
hashtextextended(jsonb_build_array(p_organization_id, p_idempotency_key)::text, 0)
);
select * into v_existing
from billing_ledger
where organization_id = p_organization_id
and idempotency_key = p_idempotency_key;
if found then
if v_existing.account_id is distinct from v_account_id
or v_existing.job_id is distinct from p_job_id
or v_existing.kind is distinct from p_kind
or v_existing.delta_fen is distinct from p_delta_fen
or v_existing.currency is distinct from p_currency
then
raise exception using
errcode = 'P0001',
message = 'BILLING_IDEMPOTENCY_PAYLOAD_MISMATCH';
end if;
select * into v_wallet from billing_wallets where organization_id = p_organization_id;
return query select v_existing.id, v_existing.balance_after_fen, v_wallet.balance_fen,
v_wallet.total_recharged_fen, v_wallet.total_charged_fen, v_existing.created_at,
v_wallet.updated_at, v_existing.delta_fen;
return;
end if;
insert into billing_wallets(organization_id)
values (p_organization_id)
on conflict (organization_id) do nothing;
select * into v_wallet from billing_wallets
where organization_id = p_organization_id
for update;
if p_delta_fen < 0 and v_wallet.balance_fen < abs(p_delta_fen) then
raise exception 'BILLING_INSUFFICIENT_BALANCE';
end if;
update billing_wallets
set balance_fen = v_wallet.balance_fen + p_delta_fen,
total_recharged_fen = v_wallet.total_recharged_fen + case when p_kind = 'recharge' and p_delta_fen > 0 then p_delta_fen else 0 end,
total_charged_fen = v_wallet.total_charged_fen + case when p_kind = 'charge' and p_delta_fen < 0 then abs(p_delta_fen) else 0 end,
updated_at = now()
where organization_id = p_organization_id
returning * into v_wallet;
insert into billing_ledger(
id, organization_id, account_id, job_id, kind, delta_fen,
balance_after_fen, currency, idempotency_key, description, metadata
) values (
p_ledger_id, p_organization_id,
v_account_id,
p_job_id, p_kind, p_delta_fen,
v_wallet.balance_fen, p_currency, p_idempotency_key, p_description, coalesce(p_metadata, '{}'::jsonb)
) returning * into v_entry;
return query select v_entry.id, v_entry.balance_after_fen, v_wallet.balance_fen,
v_wallet.total_recharged_fen, v_wallet.total_charged_fen, v_entry.created_at,
v_wallet.updated_at, v_entry.delta_fen;
end;
$$;
revoke all on function billing_post_wallet_entry(text, text, text, text, text, bigint, text, text, text, jsonb) from public;
@@ -0,0 +1,69 @@
-- Recoverable generation finalization and charge-before-dispatch fencing.
-- Additive so the previous release remains rollback-compatible.
alter table public.generation_jobs
add column if not exists dispatch_ready_at timestamptz;
alter table public.generation_jobs
add column if not exists finalized_at timestamptz;
alter table public.generation_jobs
add column if not exists provider_dispatch_started_at timestamptz;
-- Existing rows were created before dispatch fencing and are therefore ready.
update public.generation_jobs
set dispatch_ready_at = coalesce(dispatch_ready_at, created_at, now())
where dispatch_ready_at is null;
-- Rows already terminal before this migration have completed their legacy
-- finalization path and must not be replayed into billing/webhook side effects.
update public.generation_jobs
set finalized_at = coalesce(finalized_at, completed_at, updated_at, now())
where finalized_at is null
and status in ('succeeded', 'failed', 'expired', 'cancelled');
create index if not exists generation_jobs_finalize_claim_idx
on public.generation_jobs(finalized_at, scheduled_at, locked_at, priority desc);
create or replace function public.claim_generation_jobs(
p_worker_id text,
p_limit integer default 1,
p_lock_timeout_seconds integer default 300
)
returns setof public.generation_jobs
language plpgsql
set search_path = public, pg_temp
as $$
declare
v_now timestamptz := now();
begin
return query
with candidates as (
select id
from public.generation_jobs
where dispatch_ready_at is not null
and finalized_at is null
and status in ('queued', 'running', 'succeeded', 'failed', 'expired', 'cancelled')
and coalesce(scheduled_at, created_at) <= v_now
and (
locked_at is null
or locked_at < v_now - make_interval(secs => p_lock_timeout_seconds)
)
order by coalesce(priority, 0) desc, coalesce(scheduled_at, created_at) asc, created_at asc
limit greatest(1, least(coalesce(p_limit, 1), 20))
for update skip locked
),
updated as (
update public.generation_jobs
set locked_at = v_now,
locked_by = p_worker_id,
started_at = coalesce(public.generation_jobs.started_at, v_now),
updated_at = v_now
where id in (select id from candidates)
returning public.generation_jobs.*
)
select * from updated;
end;
$$;
revoke all on function public.claim_generation_jobs(text, integer, integer) from public;
@@ -0,0 +1,12 @@
-- Mutable provider configuration is relational runtime state. Values are
-- intentionally plaintext in this first functional migration; at-rest
-- encryption and key rotation are separate hardening work.
create table if not exists platform_runtime_settings (
setting_key text primary key,
setting_value text not null,
revision bigint not null default 1 check (revision > 0),
updated_at timestamptz not null default now()
);
create index if not exists platform_runtime_settings_updated_at_idx
on platform_runtime_settings(updated_at desc);
@@ -0,0 +1,14 @@
-- Allow the independently routed Ark Seedream image provider in the durable
-- billing catalog. Keep the historical migration immutable and widen only the
-- existing provider constraint.
begin;
alter table public.billing_price_rules
drop constraint if exists billing_price_rules_provider_check;
alter table public.billing_price_rules
add constraint billing_price_rules_provider_check
check (provider in ('volcengine-visual', 'evolink', 'seedance', 'seedream', 'bailian', 'mock'));
commit;
@@ -0,0 +1,14 @@
create table if not exists public.seedream_layer_compositions (
job_id text primary key references public.generation_jobs(id) on delete cascade,
owner_id text not null,
base_asset_id text not null,
base_visible boolean not null default true,
layers jsonb not null default '[]'::jsonb,
version bigint not null default 1 check (version > 0),
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint seedream_layer_compositions_layers_array_check check (jsonb_typeof(layers) = 'array')
);
create index if not exists seedream_layer_compositions_owner_updated_idx
on public.seedream_layer_compositions(owner_id, updated_at desc);
@@ -0,0 +1,13 @@
-- Allow MiniMax H3 rules in the durable billing catalog. Runtime provider
-- credentials already use the generic platform_runtime_settings table.
begin;
alter table public.billing_price_rules
drop constraint if exists billing_price_rules_provider_check;
alter table public.billing_price_rules
add constraint billing_price_rules_provider_check
check (provider in ('volcengine-visual', 'evolink', 'seedance', 'seedream', 'bailian', 'minimax', 'mock'));
commit;
@@ -0,0 +1,32 @@
-- Orders are written before calling WeChat. Keep them when a user or
-- organization is deleted so paid money cannot lose its audit trail.
create table if not exists public.payment_orders (
id text primary key check (length(id) between 1 and 32 and id ~ '^[A-Za-z0-9_-]+$'),
organization_id text not null references public.platform_organizations(id) on delete restrict,
account_id text not null references public.platform_users(id) on delete restrict,
request_key text not null check (length(request_key) between 1 and 64 and request_key ~ '^[A-Za-z0-9_-]+$'),
mch_id text not null check (length(mch_id) between 1 and 64),
app_id text not null check (length(app_id) between 1 and 64),
amount_fen bigint not null check (amount_fen between 1 and 1000000),
currency text not null check (currency = 'CNY'),
status text not null default 'pending' check (status in ('pending', 'paid', 'closed')),
code_url text,
transaction_id text,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
expires_at timestamptz not null,
paid_at timestamptz,
constraint payment_orders_paid_fields_check check (
(status = 'paid' and transaction_id is not null and length(transaction_id) > 0 and paid_at is not null)
or (status <> 'paid' and transaction_id is null and paid_at is null)
),
constraint payment_orders_expiry_check check (expires_at > created_at),
constraint payment_orders_org_request_unique unique (organization_id, request_key)
);
create unique index if not exists payment_orders_provider_transaction_unique
on public.payment_orders(mch_id, transaction_id) where transaction_id is not null;
create index if not exists payment_orders_org_created_idx
on public.payment_orders(organization_id, created_at desc);
create index if not exists payment_orders_pending_check_idx
on public.payment_orders(updated_at, created_at) where status = 'pending';
@@ -0,0 +1,34 @@
-- Reserve each requested refund against its original, successful payment.
-- A successful external refund can exceed the current wallet balance.
alter table public.billing_wallets drop constraint if exists billing_wallets_balance_fen_check;
alter table public.billing_ledger drop constraint if exists billing_ledger_balance_after_fen_check;
alter table public.billing_ledger drop constraint if exists billing_ledger_kind_check;
alter table public.billing_ledger add constraint billing_ledger_kind_check
check (kind in ('recharge', 'charge', 'refund', 'adjustment', 'recharge_refund'));
create table public.payment_refunds (
id text primary key check (length(id) between 1 and 32 and id ~ '^[A-Za-z0-9_-]+$'),
order_id text not null references public.payment_orders(id) on delete restrict,
organization_id text not null references public.platform_organizations(id) on delete restrict,
operator_id text not null references public.platform_users(id) on delete restrict,
request_key text not null check (length(request_key) between 1 and 64 and request_key ~ '^[A-Za-z0-9_-]+$'),
mch_id text not null check (length(mch_id) between 1 and 64),
transaction_id text not null check (length(transaction_id) > 0),
provider_id text,
amount_fen bigint not null check (amount_fen > 0),
total_fen bigint not null check (total_fen > 0 and total_fen >= amount_fen),
currency text not null check (currency = 'CNY'),
reason text not null,
status text not null default 'pending' check (status in ('pending', 'processing', 'success', 'closed', 'abnormal', 'rejected')),
failure_reason text not null default '',
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
succeeded_at timestamptz,
constraint payment_refunds_request_unique unique (order_id, request_key),
constraint payment_refunds_success_check check ((status = 'success') = (succeeded_at is not null))
);
create unique index payment_refunds_provider_unique on public.payment_refunds(mch_id, provider_id)
where provider_id is not null;
create index payment_refunds_order_idx on public.payment_refunds(order_id, created_at desc, id desc);
create index payment_refunds_pending_idx on public.payment_refunds(updated_at, created_at, id)
where status in ('pending', 'processing', 'abnormal');